import {z} from 'zod' export type GetApiApiOrgIdExperimentsExperimentIdArgs = {org_id: string, experiment_id: string, sampleSize?: (number | null) | undefined, resultPage?: (number | null) | undefined, resultPageSize?: (number | null) | undefined}; export type GetApiApiOrgIdExperimentsExperimentIdOptions = {baseUrl: string, Authorization?: string | undefined, fetch?: typeof fetch | undefined}; export const datasetItemSourceBody = z.enum(["trace", "annotation", "manual", "playground"]); export const datasetSnapshotFilterBody = z.object({logicalIds: z.array(z.string()).optional(), metadata: z.record(z.string(), z.unknown()).optional(), sources: z.array(datasetItemSourceBody).optional(), tags: z.array(z.string()).optional()}); export const experimentStatusBody = z.enum(["pending", "running", "completed", "failed", "cancelled"]); export const pinnedExperimentScorerBody = z.object({id: z.string(), version: z.number().int()}); export const promptMessageBody = z.object({content: z.string(), role: z.string()}); export const promptRefOverridesBody = z.object({model: z.string().nullable().optional(), params: z.unknown(), responseFormat: z.unknown(), tools: z.unknown()}); export const remoteTaskOverridesBody = z.object({maxConcurrency: z.number().int().gte(0).nullable().optional(), timeoutMs: z.number().int().gte(0).nullable().optional()}); export const experimentTaskBody = z.union([z.object({messages: z.array(promptMessageBody), model: z.string().nullable().optional(), params: z.unknown(), provider_id: z.string(), type: z.literal("inline_prompt")}), z.object({id: z.string(), params_overrides: promptRefOverridesBody.nullable().optional(), provider_id: z.string(), type: z.literal("prompt_ref"), version: z.number().int()}), z.object({overrides: remoteTaskOverridesBody.nullable().optional(), task_ref: z.string(), type: z.literal("remote")}), z.object({config: z.unknown(), task_fingerprint: z.string(), type: z.literal("sdk")})]); export const experimentFailureCauseBody = z.object({kind: z.string().nullable().optional(), message: z.string(), slotCount: z.number().int().gte(0)}); export const experimentAggregateSummaryBody = z.object({costIncomplete: z.boolean(), errorTaskSlots: z.number().int().gte(0), failureCauseCount: z.number().int().gte(0), failureCauses: z.array(experimentFailureCauseBody), incomplete: z.boolean(), incompleteScoreDimensions: z.number().int().gte(0), incompleteTaskSlots: z.number().int().gte(0), p50LatencyMs: z.number().int().gte(0).nullable().optional(), scoringCost: z.number().nullable().optional(), taskCost: z.number(), totalCost: z.number()}); export const experimentProgressBody = z.object({completed: z.number().int().gte(0), skipped: z.number().int().gte(0), total: z.number().int().gte(0)}); export const experimentScoreOutcomesBody = z.object({completed: z.number().int().gte(0), failed: z.number().int().gte(0), pending: z.number().int().gte(0), scored: z.number().int().gte(0), skipped: z.number().int().gte(0), total: z.number().int().gte(0), unscored: z.number().int().gte(0)}); export const experimentScoreSummaryBody = z.object({errorCount: z.number().int().gte(0), name: z.string(), noReferenceCount: z.number().int().gte(0), noTraceCount: z.number().int().gte(0), pendingCount: z.number().int().gte(0), sampleCount: z.number().int().gte(0), scoreConfigId: z.string().nullable().optional(), scoreConfigName: z.string().nullable().optional(), scoreConfigVersion: z.number().int().nullable().optional(), scorerId: z.string(), scorerVersion: z.number().int(), skippedCount: z.number().int().gte(0), value: z.unknown()}); export const experimentSummaryStatusBody = z.enum(["pending", "running", "scoring", "completed", "cancelled", "execution_failed", "scoring_failed"]); export const experimentTaskOutcomesBody = z.object({failed: z.number().int().gte(0), pending: z.number().int().gte(0), skipped: z.number().int().gte(0), succeeded: z.number().int().gte(0), total: z.number().int().gte(0)}); export const experimentViewResponseBody = z.object({createdAt: z.number().int(), createdBy: z.string(), datasetFilter: datasetSnapshotFilterBody.nullable().optional(), datasetId: z.string(), datasetName: z.string().nullable().optional(), datasetVersion: z.number().int(), datasetWriteVersion: z.number().int(), deadlineAt: z.number().int(), description: z.string().nullable().optional(), executionCompletedAt: z.number().int().nullable().optional(), executionStatus: experimentStatusBody, executionStatusReason: z.string().nullable().optional(), id: z.string(), idempotencyKey: z.string().nullable().optional(), isBaseline: z.boolean(), lifecycleVersion: z.number().int(), metadata: z.unknown(), name: z.string(), orgId: z.string(), promptContentHash: z.string().nullable().optional(), promptId: z.string().nullable().optional(), promptName: z.string().nullable().optional(), promptVersion: z.number().int().nullable().optional(), retryCount: z.number().int().gte(0), scorers: z.array(pinnedExperimentScorerBody), task: experimentTaskBody, trialCount: z.number().int().gte(0), aggregateSummary: experimentAggregateSummaryBody.nullable().optional(), executionProgress: experimentProgressBody.nullable().optional(), scoreOutcomes: experimentScoreOutcomesBody.nullable().optional(), scoreSummaries: z.array(experimentScoreSummaryBody).nullable().optional(), scoringProgress: experimentProgressBody.nullable().optional(), scoringStatus: z.string().nullable().optional(), status: experimentSummaryStatusBody.nullable().optional(), taskOutcomes: experimentTaskOutcomesBody.nullable().optional()}).nullable(); export const experimentScorerApplicabilityBody = z.object({eligibleRowCount: z.number().int().gte(0), eligibleSlotCount: z.number().int().gte(0), noReferenceRowCount: z.number().int().gte(0), noReferenceSlotCount: z.number().int().gte(0), scorerId: z.string(), scorerVersion: z.number().int()}); export const experimentApplicabilityPreviewBody = z.object({eligibleScoringDimensionCount: z.number().int().gte(0), eligibleTaskSlotCount: z.number().int().gte(0), fullySkippedRowCount: z.number().int().gte(0), fullySkippedSlotCount: z.number().int().gte(0), partiallySkippedRowCount: z.number().int().gte(0), partiallySkippedSlotCount: z.number().int().gte(0), scorerApplicability: z.array(experimentScorerApplicabilityBody), totalSlotCount: z.number().int().gte(0)}); export const experimentCostDimensionBody = z.object({callCount: z.number().int().gte(0), estimatedCost: z.number().nullable().optional(), inputTokensPerCall: z.number().int(), model: z.string().nullable().optional(), outputTokensPerCall: z.number().int(), scorerId: z.string().nullable().optional(), scorerVersion: z.number().int().nullable().optional()}); export const experimentCostEstimateBody = z.object({confirmationRequired: z.boolean(), currency: z.string(), dimensions: z.array(experimentCostDimensionBody), estimatedCost: z.number().nullable().optional(), incomplete: z.boolean(), slotCount: z.number().int().gte(0), taskCostEstimated: z.boolean(), warningThreshold: z.number()}); export const experimentSlotBody = z.object({expectedOutput: z.unknown(), input: z.unknown(), logicalId: z.string(), metadata: z.unknown(), rowId: z.string(), trialIndex: z.number().int().gte(0)}); export const experimentPreviewResponseBody = z.object({applicability: experimentApplicabilityPreviewBody, costEstimate: experimentCostEstimateBody.nullable().optional(), datasetId: z.string(), datasetVersion: z.number().int(), pinnedScorers: z.array(pinnedExperimentScorerBody), rowCount: z.number().int().gte(0), sampleSlots: z.array(experimentSlotBody), slotCount: z.number().int().gte(0), trialCount: z.number().int().gte(0)}); export const experimentClientScoreSummaryBody = z.object({clientScorerKey: z.string(), sampleCount: z.number().int().gte(0), scoreConfigId: z.string(), scoreConfigName: z.string(), value: z.unknown()}); export const experimentDispersionSummaryBody = z.object({highDispersionRowCount: z.number().int().gte(0), threshold: z.number()}); export const experimentResultPaginationBody = z.object({hasMore: z.boolean(), page: z.number().int().gte(0), pageSize: z.number().int().gte(0), totalSlots: z.number().int().gte(0)}); export const experimentDimensionDispersionBody = z.object({consensus: z.unknown(), normalized: z.number().nullable().optional(), outlierTrialIndex: z.number().int().gte(0).nullable().optional(), rawStd: z.number().nullable().optional(), scorerId: z.string(), scorerVersion: z.number().int(), trialCount: z.number().int().gte(0)}); export const experimentRowDispersionBody = z.object({dimensions: z.array(experimentDimensionDispersionBody), high: z.boolean(), logicalId: z.string(), maxNormalized: z.number().nullable().optional(), outlierTrialIndex: z.number().int().gte(0).nullable().optional(), rowId: z.string()}); export const experimentSkipSummaryBody = z.object({fullySkippedSlots: z.number().int().gte(0), noReferenceDimensions: z.number().int().gte(0), noTraceDimensions: z.number().int().gte(0), partiallySkippedSlots: z.number().int().gte(0), skippedDimensions: z.number().int().gte(0)}); export const experimentResultScoreStatusBody = z.enum(["pending", "in_progress", "success", "skipped", "error"]); export const experimentResultScoreBody = z.object({score: z.unknown(), scorerId: z.string(), scorerVersion: z.number().int(), status: experimentResultScoreStatusBody}); export const experimentSlotStatusBody = z.enum(["pending", "running", "scoring", "completed", "skipped", "task_failed", "score_failed"]); export const experimentResultTaskStatusBody = z.enum(["pending", "queued", "in_progress", "ok", "skipped", "error"]); export const experimentResultSlotBody = z.object({clientScores: z.array(z.unknown()), execution: z.unknown(), expectedOutput: z.unknown(), input: z.unknown(), logicalId: z.string(), rowId: z.string(), scores: z.array(experimentResultScoreBody), status: experimentSlotStatusBody, taskStatus: experimentResultTaskStatusBody, trialIndex: z.number().int().gte(0)}); export const experimentResultsResponseBody = z.object({clientScoreSummaries: z.array(experimentClientScoreSummaryBody), dispersionSummary: experimentDispersionSummaryBody, executions: z.array(z.unknown()), pagination: experimentResultPaginationBody, rowDispersions: z.array(experimentRowDispersionBody), scores: z.array(z.unknown()), skipSummary: experimentSkipSummaryBody, slots: z.array(experimentResultSlotBody)}); export const getApiApiOrgIdExperimentsExperimentIdResponse = z.object({experiment: experimentViewResponseBody, preview: experimentPreviewResponseBody, results: experimentResultsResponseBody}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * GET /api/{org_id}/experiments/{experiment_id} */ export const getApiApiOrgIdExperimentsExperimentId = async (args: GetApiApiOrgIdExperimentsExperimentIdArgs, options: GetApiApiOrgIdExperimentsExperimentIdOptions) => { const {baseUrl, fetch: fetchFn = fetch, Authorization} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('accept', "application/json") if (args.sampleSize != null) query.append("sampleSize", String(args.sampleSize)) if (args.resultPage != null) query.append("resultPage", String(args.resultPage)) if (args.resultPageSize != null) query.append("resultPageSize", String(args.resultPageSize)) if (Authorization !== undefined) headers.set("Authorization", Authorization) const url = `${baseUrl.replace(/\/+$/, '')}/api/${encodeURIComponent(String(args.org_id))}/experiments/${encodeURIComponent(String(args.experiment_id))}${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "GET", headers}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(getApiApiOrgIdExperimentsExperimentIdResponse, await res.json()) };