import {z} from 'zod' export type GetApiApiOrgIdExperimentsArgs = {org_id: string, includeSummary?: (boolean | null) | undefined, datasetId?: (string | null) | undefined, promptId?: (string | null) | undefined, promptVersion?: (number | null) | undefined, contentHash?: (string | null) | undefined}; export type GetApiApiOrgIdExperimentsOptions = {baseUrl: string, Authorization?: string | undefined, fetch?: typeof fetch | undefined}; export const datasetItemSourceBody = z.enum(["trace", "annotation", "manual", "playground"]); export const datasetSnapshotFilterBody = z.object({logicalIds: z.array(z.string()).optional(), metadata: z.record(z.string(), z.unknown()).optional(), sources: z.array(datasetItemSourceBody).optional(), tags: z.array(z.string()).optional()}); export const experimentStatusBody = z.enum(["pending", "running", "completed", "failed", "cancelled"]); export const pinnedExperimentScorerBody = z.object({id: z.string(), version: z.number().int()}); export const promptMessageBody = z.object({content: z.string(), role: z.string()}); export const promptRefOverridesBody = z.object({model: z.string().nullable().optional(), params: z.unknown(), responseFormat: z.unknown(), tools: z.unknown()}); export const remoteTaskOverridesBody = z.object({maxConcurrency: z.number().int().gte(0).nullable().optional(), timeoutMs: z.number().int().gte(0).nullable().optional()}); export const experimentTaskBody = z.union([z.object({messages: z.array(promptMessageBody), model: z.string().nullable().optional(), params: z.unknown(), provider_id: z.string(), type: z.literal("inline_prompt")}), z.object({id: z.string(), params_overrides: promptRefOverridesBody.nullable().optional(), provider_id: z.string(), type: z.literal("prompt_ref"), version: z.number().int()}), z.object({overrides: remoteTaskOverridesBody.nullable().optional(), task_ref: z.string(), type: z.literal("remote")}), z.object({config: z.unknown(), task_fingerprint: z.string(), type: z.literal("sdk")})]); export const experimentFailureCauseBody = z.object({kind: z.string().nullable().optional(), message: z.string(), slotCount: z.number().int().gte(0)}); export const experimentAggregateSummaryBody = z.object({costIncomplete: z.boolean(), errorTaskSlots: z.number().int().gte(0), failureCauseCount: z.number().int().gte(0), failureCauses: z.array(experimentFailureCauseBody), incomplete: z.boolean(), incompleteScoreDimensions: z.number().int().gte(0), incompleteTaskSlots: z.number().int().gte(0), p50LatencyMs: z.number().int().gte(0).nullable().optional(), scoringCost: z.number().nullable().optional(), taskCost: z.number(), totalCost: z.number()}); export const experimentProgressBody = z.object({completed: z.number().int().gte(0), skipped: z.number().int().gte(0), total: z.number().int().gte(0)}); export const experimentScoreOutcomesBody = z.object({completed: z.number().int().gte(0), failed: z.number().int().gte(0), pending: z.number().int().gte(0), scored: z.number().int().gte(0), skipped: z.number().int().gte(0), total: z.number().int().gte(0), unscored: z.number().int().gte(0)}); export const experimentScoreSummaryBody = z.object({errorCount: z.number().int().gte(0), name: z.string(), noReferenceCount: z.number().int().gte(0), noTraceCount: z.number().int().gte(0), pendingCount: z.number().int().gte(0), sampleCount: z.number().int().gte(0), scoreConfigId: z.string().nullable().optional(), scoreConfigName: z.string().nullable().optional(), scoreConfigVersion: z.number().int().nullable().optional(), scorerId: z.string(), scorerVersion: z.number().int(), skippedCount: z.number().int().gte(0), value: z.unknown()}); export const experimentSummaryStatusBody = z.enum(["pending", "running", "scoring", "completed", "cancelled", "execution_failed", "scoring_failed"]); export const experimentTaskOutcomesBody = z.object({failed: z.number().int().gte(0), pending: z.number().int().gte(0), skipped: z.number().int().gte(0), succeeded: z.number().int().gte(0), total: z.number().int().gte(0)}); export const experimentViewResponseBody = z.object({createdAt: z.number().int(), createdBy: z.string(), datasetFilter: datasetSnapshotFilterBody.nullable().optional(), datasetId: z.string(), datasetName: z.string().nullable().optional(), datasetVersion: z.number().int(), datasetWriteVersion: z.number().int(), deadlineAt: z.number().int(), description: z.string().nullable().optional(), executionCompletedAt: z.number().int().nullable().optional(), executionStatus: experimentStatusBody, executionStatusReason: z.string().nullable().optional(), id: z.string(), idempotencyKey: z.string().nullable().optional(), isBaseline: z.boolean(), lifecycleVersion: z.number().int(), metadata: z.unknown(), name: z.string(), orgId: z.string(), promptContentHash: z.string().nullable().optional(), promptId: z.string().nullable().optional(), promptName: z.string().nullable().optional(), promptVersion: z.number().int().nullable().optional(), retryCount: z.number().int().gte(0), scorers: z.array(pinnedExperimentScorerBody), task: experimentTaskBody, trialCount: z.number().int().gte(0), aggregateSummary: experimentAggregateSummaryBody.nullable().optional(), executionProgress: experimentProgressBody.nullable().optional(), scoreOutcomes: experimentScoreOutcomesBody.nullable().optional(), scoreSummaries: z.array(experimentScoreSummaryBody).nullable().optional(), scoringProgress: experimentProgressBody.nullable().optional(), scoringStatus: z.string().nullable().optional(), status: experimentSummaryStatusBody.nullable().optional(), taskOutcomes: experimentTaskOutcomesBody.nullable().optional()}).nullable(); export const getApiApiOrgIdExperimentsResponse = z.object({list: z.array(experimentViewResponseBody)}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * GET /api/{org_id}/experiments */ export const getApiApiOrgIdExperiments = async (args: GetApiApiOrgIdExperimentsArgs, options: GetApiApiOrgIdExperimentsOptions) => { const {baseUrl, fetch: fetchFn = fetch, Authorization} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('accept', "application/json") if (args.includeSummary != null) query.append("includeSummary", String(args.includeSummary)) if (args.datasetId != null) query.append("datasetId", String(args.datasetId)) if (args.promptId != null) query.append("promptId", String(args.promptId)) if (args.promptVersion != null) query.append("promptVersion", String(args.promptVersion)) if (args.contentHash != null) query.append("contentHash", String(args.contentHash)) if (Authorization !== undefined) headers.set("Authorization", Authorization) const url = `${baseUrl.replace(/\/+$/, '')}/api/${encodeURIComponent(String(args.org_id))}/experiments${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "GET", headers}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(getApiApiOrgIdExperimentsResponse, await res.json()) };