import {z} from 'zod' export type GetApiV2ScorersScorerIdVersionsArgs = {scorer_id: unknown, run_id?: (unknown | null) | undefined, starting_token?: number | undefined, limit?: number | undefined}; export type GetApiV2ScorersScorerIdVersionsOptions = {baseUrl?: string | undefined, APIKeyHeader?: string | undefined, fetch?: typeof fetch | undefined}; export const fewShotExample = z.object({generation_prompt_and_response: z.string(), evaluating_response: z.string()}); export const chainPollTemplate = z.object({metric_system_prompt: z.string().nullable().optional(), metric_description: z.string().nullable().optional(), value_field_name: z.string().optional(), explanation_field_name: z.string().optional(), template: z.string(), metric_few_shot_examples: z.array(fewShotExample).optional(), response_schema: z.record(z.string(), z.unknown()).nullable().optional()}); export const nodeType = z.enum(["chain", "chat", "http", "llm", "retriever", "tool", "agent", "workflow", "trace", "session"]); export const outputTypeEnum = z.enum(["boolean", "categorical", "count", "discrete", "freeform", "percentage", "multilabel", "retrieved_chunk_list_boolean", "boolean_multilabel", "boolean_probability"]); export const multimodalCapability = z.enum(["vision", "audio"]); export const generatedScorerConfiguration = z.object({model_alias: z.string().optional(), num_judges: z.number().int().gte(1).lte(10).optional(), output_type: outputTypeEnum.optional(), scoreable_node_types: z.array(z.string()).optional(), cot_enabled: z.boolean().optional(), ground_truth: z.boolean().optional(), multimodal_capabilities: z.array(multimodalCapability).nullable().optional()}); export const generatedScorerResponse = z.object({id: z.string(), name: z.string(), instructions: z.string().nullable().optional(), chain_poll_template: chainPollTemplate, user_prompt: z.string().nullable().optional(), created_by: z.string(), created_at: z.string(), updated_at: z.string(), scoreable_node_types: z.array(nodeType).nullable(), scorer_configuration: generatedScorerConfiguration}); export const dataTypeOptions = z.enum(["unknown", "text", "label", "floating_point", "integer", "timestamp", "milli_seconds", "boolean", "uuid", "percentage", "dollars", "array", "template_label", "thumb_rating_percentage", "user_id", "text_offsets", "segments", "hallucination_segments", "thumb_rating", "score_rating", "star_rating", "tags_rating", "choice_rating", "thumb_rating_aggregate", "score_rating_aggregate", "star_rating_aggregate", "tags_rating_aggregate", "choice_rating_aggregate"]); export const createUpdateRegisteredScorerResponse = z.object({id: z.string(), name: z.string(), score_type: z.string().nullable(), created_at: z.string(), updated_at: z.string(), created_by: z.string(), data_type: dataTypeOptions.nullable(), scoreable_node_types: z.array(z.string()).nullable()}); export const lunaInputTypeEnum = z.enum(["span", "trace_object", "trace_input_output_only"]); export const lunaOutputTypeEnum = z.enum(["float", "string", "string_list", "bool_list"]); export const scorerName = z.enum(["action_completion_luna", "action_advancement_luna", "agentic_session_success", "agentic_session_success", "action_completion_vision", "action_completion_audio", "agentic_workflow_success", "agentic_workflow_success", "agent_efficiency", "agent_flow", "agent_flow_vision", "agent_flow_audio", "chunk_attribution_utilization_luna", "chunk_attribution_utilization", "chunk_relevance", "chunk_relevance_luna", "context_precision", "precision_at_k", "completeness_luna", "completeness", "context_adherence", "context_adherence_luna", "context_adherence_vision_luna", "context_adherence_vision", "context_adherence_audio", "context_relevance", "context_relevance_luna", "conversation_quality", "conversation_quality_vision", "conversation_quality_audio", "unsafe_output", "correctness", "correctness_vision", "correctness_audio", "ground_truth_adherence", "ground_truth_adherence_vision", "ground_truth_adherence_audio", "visual_fidelity", "visual_quality", "input_pii", "input_pii_vision_luna", "input_pii_gpt", "input_pii_vision", "input_pii_audio", "input_sexist", "input_sexist", "input_sexist_vision", "input_sexist_audio", "input_sexist_luna", "input_sexist_luna", "input_sexist_vision_luna", "input_tone", "input_tone_gpt", "input_toxicity", "input_toxicity_luna", "input_toxicity_vision_luna", "input_toxicity_vision", "input_toxicity_audio", "instruction_adherence", "instruction_adherence_vision", "instruction_adherence_audio", "output_pii", "output_pii_vision_luna", "output_pii_gpt", "output_pii_vision", "output_pii_audio", "output_sexist", "output_sexist", "output_sexist_vision", "output_sexist_audio", "output_sexist_luna", "output_sexist_luna", "output_sexist_vision_luna", "output_tone", "output_tone_gpt", "output_toxicity", "output_toxicity_luna", "output_toxicity_vision_luna", "output_toxicity_vision", "output_toxicity_audio", "prompt_injection", "prompt_injection_vision", "prompt_injection_audio", "prompt_injection_luna", "prompt_injection_vision_luna", "reasoning_coherence", "reasoning_coherence_vision", "reasoning_coherence_audio", "sql_efficiency", "sql_adherence", "sql_injection", "sql_correctness", "tool_error_rate", "tool_error_rate_luna", "tool_selection_quality", "tool_selection_quality_vision", "tool_selection_quality_audio", "tool_selection_quality_luna", "user_intent_change", "user_intent_change_vision", "user_intent_change_audio", "interruption_detection"]); export const fineTunedScorerResponse = z.object({id: z.string(), name: z.string(), lora_task_id: z.number().int(), lora_weights_path: z.string().nullable().optional(), prompt: z.string(), luna_input_type: lunaInputTypeEnum.nullable().optional(), luna_output_type: lunaOutputTypeEnum.nullable().optional(), class_name_to_vocab_ix: z.union([z.record(z.string(), z.array(z.number().int())), z.record(z.string(), z.number().int())]).nullable().optional(), executor: scorerName.nullable().optional(), created_at: z.string(), updated_at: z.string(), created_by: z.string()}); export const luna3ScorerResponse = z.object({instruction: z.string().min(1).max(10000), adapter_path: z.string().min(1).max(1024), class_name_to_vocab_ix: z.record(z.string(), z.array(z.number().int())), threshold: z.number().nullable().optional(), id: z.string(), organization_id: z.string(), created_by: z.string().nullable().optional(), created_at: z.string(), updated_at: z.string()}); export const decisionChoice = z.object({label: z.string(), description: z.string().nullable().optional()}); export const decisionLevel = z.object({level: z.number().int().gte(0).lte(9), label: z.string(), description: z.string().nullable().optional()}); export const decisionBooleanCriteria = z.object({true: z.string(), false: z.string()}); export const decisionScorerDB = z.object({id: z.string(), organization_id: z.string(), created_by: z.string().nullable().optional(), question: z.string(), answer_options: z.union([z.array(z.string()), z.array(decisionChoice), z.array(decisionLevel), decisionBooleanCriteria]).nullable().optional(), created_at: z.string()}); export const baseScorerVersionResponse = z.object({id: z.string(), version: z.number().int(), scorer_id: z.string(), generated_scorer: generatedScorerResponse.nullable().optional(), registered_scorer: createUpdateRegisteredScorerResponse.nullable().optional(), finetuned_scorer: fineTunedScorerResponse.nullable().optional(), luna3_scorer: luna3ScorerResponse.nullable().optional(), decision_scorer: decisionScorerDB.nullable().optional(), model_name: z.string().nullable().optional(), num_judges: z.number().int().nullable().optional(), cot_enabled: z.boolean().nullable().optional(), runtime: z.string().nullable().optional(), code_sha256: z.string().nullable().optional(), created_at: z.string(), updated_at: z.string(), chain_poll_template: chainPollTemplate.nullable().optional(), scoreable_node_types: z.array(z.string()).nullable().optional(), output_type: outputTypeEnum.nullable().optional(), allowed_model: z.boolean().nullable().optional(), created_by: z.string().nullable().optional(), validation_task_id: z.string().nullable().optional(), request_id: z.string().nullable().optional()}); export const listScorerVersionsResponse = z.object({starting_token: z.number().int().optional(), limit: z.number().int().optional(), paginated: z.boolean().optional(), next_starting_token: z.number().int().nullable().optional(), versions: z.array(baseScorerVersionResponse).optional()}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * List All Versions For Scorer * * GET /v2/scorers/{scorer_id}/versions * * Also accepts auth: ClassicAPIKeyHeader, OAuth2PasswordBearer, HTTPBasic */ export const getApiV2ScorersScorerIdVersions = async (args: GetApiV2ScorersScorerIdVersionsArgs, options?: GetApiV2ScorersScorerIdVersionsOptions) => { const {baseUrl = "https://api.galileo.ai", fetch: fetchFn = fetch, APIKeyHeader} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('accept', "application/json") if (args.run_id != null) query.append("run_id", String(args.run_id)) if (args.starting_token != null) query.append("starting_token", String(args.starting_token)) if (args.limit != null) query.append("limit", String(args.limit)) if (APIKeyHeader !== undefined) headers.set("Splunk-AO-API-Key", APIKeyHeader) const url = `${baseUrl.replace(/\/+$/, '')}/v2/scorers/${encodeURIComponent(String(args.scorer_id))}/versions${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "GET", headers}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(listScorerVersionsResponse, await res.json()) };