import {z} from 'zod' export type GetApiApiOrgIdExperimentsCompareArgs = {org_id: string, baselineId: string, candidateId: string, threshold?: (number | null) | undefined, outcomeDimensions?: (string | null) | undefined}; export type GetApiApiOrgIdExperimentsCompareOptions = {baseUrl: string, Authorization?: string | undefined, fetch?: typeof fetch | undefined}; export const experimentComparisonCountsBody = z.object({baselineRows: z.number().int().gte(0), candidateRows: z.number().int().gte(0), commonRows: z.number().int().gte(0), improved: z.number().int().gte(0), inconclusive: z.number().int().gte(0), missing: z.number().int().gte(0), new: z.number().int().gte(0), regressed: z.number().int().gte(0), unchanged: z.number().int().gte(0)}); export const experimentComparisonAssignmentBody = z.enum(["regressed", "improved", "unchanged", "descriptive", "baseline_only", "candidate_only", "unavailable"]); export const experimentComparisonScoreDataTypeBody = z.enum(["numeric", "categorical", "boolean"]); export const experimentComparisonDimensionKindBody = z.enum(["score", "cost", "latency"]); export const experimentComparisonSummaryDimensionBody = z.object({assignment: experimentComparisonAssignmentBody, baseline: z.number().nullable().optional(), baselineLabel: z.string().nullable().optional(), baselineOnlyRowCount: z.number().int().gte(0), baselineSampleCount: z.number().int().gte(0), canAffectOutcome: z.boolean(), candidate: z.number().nullable().optional(), candidateLabel: z.string().nullable().optional(), candidateOnlyRowCount: z.number().int().gte(0), candidateSampleCount: z.number().int().gte(0), comparableRowCount: z.number().int().gte(0), dataType: experimentComparisonScoreDataTypeBody.nullable().optional(), delta: z.number().nullable().optional(), gating: z.boolean(), id: z.string(), kind: experimentComparisonDimensionKindBody, name: z.string(), normalized: z.boolean(), orientedDelta: z.number().nullable().optional(), scoreConfigId: z.string().nullable().optional(), scoreConfigName: z.string().nullable().optional(), scoreConfigVersion: z.string().nullable().optional()}); export const experimentComparisonBucketBody = z.enum(["regressed", "improved", "unchanged", "inconclusive", "new", "missing"]); export const experimentComparisonDimensionBody = z.object({assignment: experimentComparisonAssignmentBody, baseline: z.number().nullable().optional(), baselineDispersion: z.number().nullable().optional(), baselineLabel: z.string().nullable().optional(), baselineSampleCount: z.number().int().gte(0), canAffectOutcome: z.boolean(), candidate: z.number().nullable().optional(), candidateDispersion: z.number().nullable().optional(), candidateLabel: z.string().nullable().optional(), candidateSampleCount: z.number().int().gte(0), dataType: experimentComparisonScoreDataTypeBody.nullable().optional(), delta: z.number().nullable().optional(), gating: z.boolean(), id: z.string(), kind: experimentComparisonDimensionKindBody, name: z.string(), normalized: z.boolean(), orientedDelta: z.number().nullable().optional(), scoreConfigId: z.string().nullable().optional(), scoreConfigName: z.string().nullable().optional(), scoreConfigVersion: z.string().nullable().optional(), withinNoise: z.boolean()}); export const experimentComparisonTrialBody = z.object({baselineOutput: z.unknown(), candidateOutput: z.unknown(), trialIndex: z.number().int().gte(0)}); export const experimentComparisonRowBody = z.object({baselineRowId: z.string().nullable().optional(), bucket: experimentComparisonBucketBody, candidateRowId: z.string().nullable().optional(), dimensions: z.array(experimentComparisonDimensionBody), input: z.unknown(), logicalId: z.string(), trials: z.array(experimentComparisonTrialBody)}); export const experimentComparisonResponseBody = z.object({assignmentRule: z.string(), baselineId: z.string(), baselineScoringStatus: z.string(), candidateId: z.string(), candidateScoringStatus: z.string(), counts: experimentComparisonCountsBody, datasetId: z.string(), dimensions: z.array(experimentComparisonSummaryDimensionBody), outcomeDimensions: z.array(z.string()), partial: z.boolean(), rows: z.array(experimentComparisonRowBody), threshold: z.number()}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * Compare two Experiments over the honest intersection of their pinned rows. * * GET /api/{org_id}/experiments/compare */ export const getApiApiOrgIdExperimentsCompare = async (args: GetApiApiOrgIdExperimentsCompareArgs, options: GetApiApiOrgIdExperimentsCompareOptions) => { const {baseUrl, fetch: fetchFn = fetch, Authorization} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('accept', "application/json") if (args.baselineId != null) query.append("baselineId", String(args.baselineId)) if (args.candidateId != null) query.append("candidateId", String(args.candidateId)) if (args.threshold != null) query.append("threshold", String(args.threshold)) if (args.outcomeDimensions != null) query.append("outcomeDimensions", String(args.outcomeDimensions)) if (Authorization !== undefined) headers.set("Authorization", Authorization) const url = `${baseUrl.replace(/\/+$/, '')}/api/${encodeURIComponent(String(args.org_id))}/experiments/compare${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "GET", headers}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(experimentComparisonResponseBody, await res.json()) };