import {z} from 'zod' export type CreateApiApiV2ProjectsProjectIdTestRunsArgs = {project_id: string, body: {dataset_name: string, include_outputs?: (boolean | null) | undefined, test_run_name?: (string | null) | undefined, test_run_description?: (string | null) | undefined, flavor_name?: (string | null) | undefined, target_evaluation_ids?: (Array | null) | undefined}}; export type CreateApiApiV2ProjectsProjectIdTestRunsOptions = {baseUrl: string, BearerAuth?: string | undefined, fetch?: typeof fetch | undefined}; export const inputVariable: z.ZodType = z.union([z.string(), z.number().int(), z.boolean(), z.number(), z.record(z.string(), z.lazy(() => inputVariable)), z.array(z.lazy(() => inputVariable))]); export const textBlockInfo = z.object({text: z.string(), type: z.literal("text").optional()}); export const jSONValue: z.ZodType = z.union([z.boolean(), z.number().int(), z.number(), z.string(), z.record(z.string(), z.lazy(() => jSONValue)), z.array(z.lazy(() => jSONValue))]).nullable(); export const toolCallBlockInfo = z.object({id: z.string(), name: z.string(), arguments: jSONValue, thought_signature: z.string().nullable().optional(), type: z.literal("tool_call").optional()}); export const sessionMediaReference = z.object({asset_id: z.string(), session_id: z.string(), media_type: z.enum(["audio", "video", "image", "file"]), filename: z.string().nullable().optional(), kind: z.literal("session_asset").optional()}); export const testCaseMediaReference = z.object({asset_id: z.string(), test_case_id: z.string(), media_type: z.enum(["audio", "video", "image", "file"]), filename: z.string().nullable().optional(), kind: z.literal("test_case_asset").optional()}); export const uploadedMediaReference = z.object({base64Data: z.string(), contentType: z.string(), media_type: z.enum(["audio", "video", "image", "file"]), filename: z.string().nullable().optional(), kind: z.literal("uploaded_file").optional()}); export const downloadedMediaReference = z.object({base64Data: z.string(), contentType: z.string(), media_type: z.enum(["audio", "video", "image", "file"]), asset_id: z.string().nullable().optional(), session_id: z.string().nullable().optional(), test_case_id: z.string().nullable().optional(), filename: z.string().nullable().optional(), kind: z.literal("downloaded_file").optional()}); export const imageBlockInfo = z.object({media_reference: z.discriminatedUnion("kind", [sessionMediaReference, testCaseMediaReference, uploadedMediaReference, downloadedMediaReference]), type: z.literal("image").optional()}); export const audioBlockInfo = z.object({media_reference: z.discriminatedUnion("kind", [sessionMediaReference, testCaseMediaReference, uploadedMediaReference, downloadedMediaReference]), type: z.literal("audio").optional()}); export const fileBlockInfo = z.object({media_reference: z.discriminatedUnion("kind", [sessionMediaReference, testCaseMediaReference, uploadedMediaReference, downloadedMediaReference]), type: z.literal("file").optional()}); export const toolResultBlockInfo: z.ZodType = z.object({tool_call_id: z.string(), content: z.array(z.discriminatedUnion("type", [textBlockInfo, z.lazy(() => toolResultBlockInfo), toolCallBlockInfo, imageBlockInfo, audioBlockInfo, fileBlockInfo])), type: z.literal("tool_result").optional()}); export const userMessageInfo = z.object({content: z.array(z.discriminatedUnion("type", [textBlockInfo, toolResultBlockInfo, toolCallBlockInfo, imageBlockInfo, audioBlockInfo, fileBlockInfo])), role: z.literal("user").optional()}); export const systemMessageInfo = z.object({content: z.string(), role: z.literal("system").optional()}); export const developerMessageInfo = z.object({content: z.string(), role: z.literal("developer").optional()}); export const assistantMessageInfo = z.object({content: z.array(z.discriminatedUnion("type", [textBlockInfo, toolResultBlockInfo, toolCallBlockInfo, imageBlockInfo, audioBlockInfo, fileBlockInfo])), role: z.literal("assistant").optional()}); export const textBlock = z.object({text: z.string(), type: z.literal("text").optional()}); export const toolCallBlock = z.object({id: z.string(), name: z.string(), arguments: jSONValue, thought_signature: z.string().nullable().optional(), type: z.literal("tool_call").optional()}); export const imageBlock = z.object({media_id: z.string(), type: z.literal("image").optional()}); export const audioBlock = z.object({media_id: z.string(), type: z.literal("audio").optional()}); export const fileBlock = z.object({media_id: z.string(), filename: z.string().nullable().optional(), type: z.literal("file").optional()}); export const toolResultBlock: z.ZodType = z.object({tool_call_id: z.string(), content: z.union([z.string(), z.array(z.discriminatedUnion("type", [textBlock, z.lazy(() => toolResultBlock), toolCallBlock, imageBlock, audioBlock, fileBlock]))]), type: z.literal("tool_result").optional()}); export const userMessage = z.object({content: z.union([z.string(), z.array(z.discriminatedUnion("type", [textBlock, toolResultBlock, toolCallBlock, imageBlock, audioBlock, fileBlock]))]), role: z.literal("user").optional()}); export const assistantMessage = z.object({content: z.union([z.string(), z.array(z.discriminatedUnion("type", [textBlock, toolResultBlock, toolCallBlock, imageBlock, audioBlock, fileBlock]))]), role: z.literal("assistant").optional()}); export const systemMessage = z.object({content: z.string(), role: z.literal("system").optional()}); export const developerMessage = z.object({content: z.string(), role: z.literal("developer").optional()}); export const testCaseInfo = z.object({id: z.string(), values: z.record(z.string(), inputVariable), output: z.string().nullable(), output_message: z.union([z.union([userMessageInfo, systemMessageInfo, developerMessageInfo, assistantMessageInfo]), userMessage, assistantMessage, systemMessage, developerMessage]).nullable(), history: z.array(z.union([z.union([userMessageInfo, systemMessageInfo, developerMessageInfo, assistantMessageInfo]), userMessage, assistantMessage, systemMessage, developerMessage])).nullable(), metadata: z.record(z.string(), z.union([z.string(), z.number().int(), z.number(), z.boolean()])).nullable()}); export const traceTestCaseInfo = z.object({test_case_id: z.string(), input: jSONValue, output: jSONValue.nullable(), custom_metadata: z.record(z.string(), z.union([z.string(), z.number(), z.number().int(), z.boolean()])).nullable().optional(), test_case_type: z.literal("trace").optional()}); export const createApiApiV2ProjectsProjectIdTestRunsResponse = z.object({test_run_id: z.string(), test_run_name: z.string(), test_run_description: z.string(), test_cases: z.array(testCaseInfo).nullable(), trace_test_cases: z.array(traceTestCaseInfo).nullable()}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * Create Test Run * * POST /api/v2/projects/{project_id}/test-runs */ export const createApiApiV2ProjectsProjectIdTestRuns = async (args: CreateApiApiV2ProjectsProjectIdTestRunsArgs, options: CreateApiApiV2ProjectsProjectIdTestRunsOptions) => { const {baseUrl, fetch: fetchFn = fetch, BearerAuth} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('content-type', "application/json") headers.set('accept', "application/json") if (BearerAuth !== undefined) headers.set('authorization', `Bearer ${BearerAuth}`) const url = `${baseUrl.replace(/\/+$/, '')}/api/v2/projects/${encodeURIComponent(String(args.project_id))}/test-runs${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "POST", headers, body: JSON.stringify(args.body)}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(createApiApiV2ProjectsProjectIdTestRunsResponse, await res.json()) };