import {z} from 'zod' export type GoogleCloudAiplatformV1AudioTranscriptionConfigLanguageAuto = Record; export type GoogleCloudAiplatformV1AudioTranscriptionConfigLanguageHints = {languageCodes?: Array | undefined}; export type GoogleCloudAiplatformV1AudioTranscriptionConfig = {adaptationPhrases?: Array | undefined, customVocabulary?: Array | undefined, diarization?: boolean | undefined, languageAuto?: GoogleCloudAiplatformV1AudioTranscriptionConfigLanguageAuto | undefined, languageCodes?: Array | undefined, languageHints?: GoogleCloudAiplatformV1AudioTranscriptionConfigLanguageHints | undefined, mode?: ('MODE_UNSPECIFIED' | 'VERBATIM' | 'SMART') | undefined, sourceText?: string | undefined, wordTimestamp?: boolean | undefined}; export type GoogleCloudAiplatformV1ImageConfigImageOutputOptions = {compressionQuality?: number | undefined, mimeType?: string | undefined}; export type GoogleCloudAiplatformV1ImageConfig = {aspectRatio?: string | undefined, imageOutputOptions?: GoogleCloudAiplatformV1ImageConfigImageOutputOptions | undefined, imageSize?: string | undefined, personGeneration?: ('PERSON_GENERATION_UNSPECIFIED' | 'ALLOW_ALL' | 'ALLOW_ADULT' | 'ALLOW_NONE') | undefined, prominentPeople?: ('PROMINENT_PEOPLE_UNSPECIFIED' | 'ALLOW_PROMINENT_PEOPLE' | 'BLOCK_PROMINENT_PEOPLE') | undefined}; export type GoogleCloudAiplatformV1AudioResponseFormat = {bitRate?: number | undefined, delivery?: ('DELIVERY_UNSPECIFIED' | 'INLINE' | 'URI') | undefined, mimeType?: ('MIME_TYPE_UNSPECIFIED' | 'AUDIO_MP3' | 'AUDIO_OGG_OPUS' | 'AUDIO_L16' | 'AUDIO_WAV' | 'AUDIO_ALAW' | 'AUDIO_MULAW') | undefined, sampleRate?: number | undefined}; export type GoogleCloudAiplatformV1ImageResponseFormat = {aspectRatio?: ('ASPECT_RATIO_UNSPECIFIED' | 'ASPECT_RATIO_ONE_BY_ONE' | 'ASPECT_RATIO_TWO_BY_THREE' | 'ASPECT_RATIO_THREE_BY_TWO' | 'ASPECT_RATIO_THREE_BY_FOUR' | 'ASPECT_RATIO_FOUR_BY_THREE' | 'ASPECT_RATIO_FOUR_BY_FIVE' | 'ASPECT_RATIO_FIVE_BY_FOUR' | 'ASPECT_RATIO_NINE_BY_SIXTEEN' | 'ASPECT_RATIO_SIXTEEN_BY_NINE' | 'ASPECT_RATIO_TWENTY_ONE_BY_NINE' | 'ASPECT_RATIO_ONE_BY_EIGHT' | 'ASPECT_RATIO_EIGHT_BY_ONE' | 'ASPECT_RATIO_ONE_BY_FOUR' | 'ASPECT_RATIO_FOUR_BY_ONE') | undefined, delivery?: ('DELIVERY_UNSPECIFIED' | 'INLINE' | 'URI') | undefined, imageSize?: ('IMAGE_SIZE_UNSPECIFIED' | 'IMAGE_SIZE_FIVE_TWELVE' | 'IMAGE_SIZE_ONE_K' | 'IMAGE_SIZE_TWO_K' | 'IMAGE_SIZE_FOUR_K') | undefined, mimeType?: ('MIME_TYPE_UNSPECIFIED' | 'IMAGE_JPEG') | undefined}; export type GoogleCloudAiplatformV1TextResponseFormat = {mimeType?: ('MIME_TYPE_UNSPECIFIED' | 'APPLICATION_JSON' | 'TEXT_PLAIN') | undefined, schema: unknown}; export type GoogleCloudAiplatformV1VideoResponseFormat = {aspectRatio?: ('ASPECT_RATIO_UNSPECIFIED' | 'ASPECT_RATIO_SIXTEEN_BY_NINE' | 'ASPECT_RATIO_NINE_BY_SIXTEEN') | undefined, delivery?: ('DELIVERY_UNSPECIFIED' | 'INLINE' | 'URI') | undefined, duration?: unknown | undefined, gcsUri?: string | undefined, resolution?: string | undefined}; export type GoogleCloudAiplatformV1ResponseFormat = {audio?: GoogleCloudAiplatformV1AudioResponseFormat | undefined, image?: GoogleCloudAiplatformV1ImageResponseFormat | undefined, text?: GoogleCloudAiplatformV1TextResponseFormat | undefined, video?: GoogleCloudAiplatformV1VideoResponseFormat | undefined}; export type GoogleCloudAiplatformV1Schema = {additionalProperties: unknown, anyOf?: Array | undefined, default: unknown, defs?: {[key: string]: GoogleCloudAiplatformV1Schema} | undefined, description?: string | undefined, enum?: Array | undefined, example: unknown, format?: string | undefined, items?: GoogleCloudAiplatformV1Schema | undefined, maxItems?: unknown | undefined, maxLength?: unknown | undefined, maxProperties?: unknown | undefined, maximum?: number | undefined, minItems?: unknown | undefined, minLength?: unknown | undefined, minProperties?: unknown | undefined, minimum?: number | undefined, nullable?: boolean | undefined, pattern?: string | undefined, properties?: {[key: string]: GoogleCloudAiplatformV1Schema} | undefined, propertyOrdering?: Array | undefined, ref?: string | undefined, required?: Array | undefined, title?: string | undefined, type?: ('TYPE_UNSPECIFIED' | 'STRING' | 'NUMBER' | 'INTEGER' | 'BOOLEAN' | 'ARRAY' | 'OBJECT' | 'NULL') | undefined}; export type GoogleCloudAiplatformV1GenerationConfigRoutingConfigAutoRoutingMode = {modelRoutingPreference?: ('UNKNOWN' | 'PRIORITIZE_QUALITY' | 'BALANCED' | 'PRIORITIZE_COST') | undefined}; export type GoogleCloudAiplatformV1GenerationConfigRoutingConfigManualRoutingMode = {modelName?: string | undefined}; export type GoogleCloudAiplatformV1GenerationConfigRoutingConfig = {autoMode?: GoogleCloudAiplatformV1GenerationConfigRoutingConfigAutoRoutingMode | undefined, manualMode?: GoogleCloudAiplatformV1GenerationConfigRoutingConfigManualRoutingMode | undefined}; export type GoogleCloudAiplatformV1PrebuiltVoiceConfig = {voiceName?: string | undefined}; export type GoogleCloudAiplatformV1ReplicatedVoiceConfig = {mimeType?: string | undefined, voiceSampleAudio?: string | undefined}; export type GoogleCloudAiplatformV1VoiceConfig = {prebuiltVoiceConfig?: GoogleCloudAiplatformV1PrebuiltVoiceConfig | undefined, replicatedVoiceConfig?: GoogleCloudAiplatformV1ReplicatedVoiceConfig | undefined, voice?: string | undefined}; export type GoogleCloudAiplatformV1SpeakerVoiceConfig = {speaker?: string | undefined, voiceConfig?: GoogleCloudAiplatformV1VoiceConfig | undefined}; export type GoogleCloudAiplatformV1MultiSpeakerVoiceConfig = {speakerVoiceConfigs?: Array | undefined}; export type GoogleCloudAiplatformV1SpeechConfig = {languageCode?: string | undefined, multiSpeakerVoiceConfig?: GoogleCloudAiplatformV1MultiSpeakerVoiceConfig | undefined, voiceConfig?: GoogleCloudAiplatformV1VoiceConfig | undefined}; export type GoogleCloudAiplatformV1GenerationConfigThinkingConfig = {includeThoughts?: boolean | undefined, thinkingBudget?: number | undefined, thinkingLevel?: ('THINKING_LEVEL_UNSPECIFIED' | 'LOW' | 'MEDIUM' | 'HIGH' | 'MINIMAL') | undefined}; export type GoogleCloudAiplatformV1TranslationConfig = {echoTargetLanguage?: boolean | undefined, targetLanguageCode?: string | undefined}; export type GoogleCloudAiplatformV1GenerationConfig = {audioTimestamp?: boolean | undefined, audioTranscriptionConfig?: GoogleCloudAiplatformV1AudioTranscriptionConfig | undefined, candidateCount?: number | undefined, enableAffectiveDialog?: boolean | undefined, frequencyPenalty?: number | undefined, imageConfig?: GoogleCloudAiplatformV1ImageConfig | undefined, logprobs?: number | undefined, maxOutputTokens?: number | undefined, mediaResolution?: ('MEDIA_RESOLUTION_UNSPECIFIED' | 'MEDIA_RESOLUTION_LOW' | 'MEDIA_RESOLUTION_MEDIUM' | 'MEDIA_RESOLUTION_HIGH') | undefined, presencePenalty?: number | undefined, responseFormat?: Array | undefined, responseJsonSchema: unknown, responseLogprobs?: boolean | undefined, responseMimeType?: string | undefined, responseModalities?: (Array<'MODALITY_UNSPECIFIED' | 'TEXT' | 'IMAGE' | 'AUDIO' | 'VIDEO'>) | undefined, responseSchema?: GoogleCloudAiplatformV1Schema | undefined, routingConfig?: GoogleCloudAiplatformV1GenerationConfigRoutingConfig | undefined, seed?: number | undefined, speechConfig?: GoogleCloudAiplatformV1SpeechConfig | undefined, stopSequences?: Array | undefined, temperature?: number | undefined, thinkingConfig?: GoogleCloudAiplatformV1GenerationConfigThinkingConfig | undefined, topK?: number | undefined, topP?: number | undefined, translationConfig?: GoogleCloudAiplatformV1TranslationConfig | undefined}; export type GoogleCloudAiplatformV1AutoraterConfig = {autoraterModel?: string | undefined, flipEnabled?: boolean | undefined, generationConfig?: GoogleCloudAiplatformV1GenerationConfig | undefined, samplingCount?: number | undefined}; export type GoogleCloudAiplatformV1BigQuerySource = {inputUri?: string | undefined}; export type GoogleCloudAiplatformV1GcsSource = {uris?: Array | undefined}; export type GoogleCloudAiplatformV1EvaluationDataset = {bigquerySource?: GoogleCloudAiplatformV1BigQuerySource | undefined, gcsSource?: GoogleCloudAiplatformV1GcsSource | undefined}; export type GoogleCloudAiplatformV1BleuSpec = {useEffectiveOrder?: boolean | undefined}; export type GoogleCloudAiplatformV1ComputationBasedMetricSpec = {parameters?: {[key: string]: unknown} | undefined, type?: ('COMPUTATION_BASED_METRIC_TYPE_UNSPECIFIED' | 'EXACT_MATCH' | 'BLEU' | 'ROUGE') | undefined}; export type GoogleCloudAiplatformV1CustomCodeExecutionSpec = {evaluationFunction?: string | undefined}; export type GoogleCloudAiplatformV1ExactMatchSpec = Record; export type GoogleCloudAiplatformV1PredefinedMetricSpec = {metricSpecName?: string | undefined, metricSpecParameters?: {[key: string]: unknown} | undefined}; export type GoogleCloudAiplatformV1EvaluationParserConfigCustomCodeParserConfig = {parsingFunction?: string | undefined}; export type GoogleCloudAiplatformV1EvaluationParserConfig = {customCodeParserConfig?: GoogleCloudAiplatformV1EvaluationParserConfigCustomCodeParserConfig | undefined}; export type GoogleCloudAiplatformV1RubricGenerationSpec = {modelConfig?: GoogleCloudAiplatformV1AutoraterConfig | undefined, promptTemplate?: string | undefined, rubricContentType?: ('RUBRIC_CONTENT_TYPE_UNSPECIFIED' | 'PROPERTY' | 'NL_QUESTION_ANSWER' | 'PYTHON_CODE_ASSERTION') | undefined, rubricTypeOntology?: Array | undefined}; export type GoogleCloudAiplatformV1LLMBasedMetricSpec = {additionalConfig?: {[key: string]: unknown} | undefined, judgeAutoraterConfig?: GoogleCloudAiplatformV1AutoraterConfig | undefined, metricPromptTemplate?: string | undefined, predefinedRubricGenerationSpec?: GoogleCloudAiplatformV1PredefinedMetricSpec | undefined, resultParserConfig?: GoogleCloudAiplatformV1EvaluationParserConfig | undefined, rubricGenerationSpec?: GoogleCloudAiplatformV1RubricGenerationSpec | undefined, rubricGroupKey?: string | undefined, systemInstruction?: string | undefined}; export type GoogleCloudAiplatformV1MetricMetadataScoreRange = {description?: string | undefined, max?: number | undefined, min?: number | undefined, step?: number | undefined}; export type GoogleCloudAiplatformV1MetricMetadata = {otherMetadata?: {[key: string]: unknown} | undefined, scoreRange?: GoogleCloudAiplatformV1MetricMetadataScoreRange | undefined, title?: string | undefined}; export type GoogleCloudAiplatformV1CustomOutputFormatConfig = {returnRawOutput?: boolean | undefined}; export type GoogleCloudAiplatformV1PairwiseMetricSpec = {baselineResponseFieldName?: string | undefined, candidateResponseFieldName?: string | undefined, customOutputFormatConfig?: GoogleCloudAiplatformV1CustomOutputFormatConfig | undefined, metricPromptTemplate?: string | undefined, systemInstruction?: string | undefined}; export type GoogleCloudAiplatformV1PointwiseMetricSpec = {customOutputFormatConfig?: GoogleCloudAiplatformV1CustomOutputFormatConfig | undefined, metricPromptTemplate?: string | undefined, systemInstruction?: string | undefined}; export type GoogleCloudAiplatformV1RougeSpec = {rougeType?: string | undefined, splitSummaries?: boolean | undefined, useStemmer?: boolean | undefined}; export type GoogleCloudAiplatformV1Metric = {aggregationMetrics?: (Array<'AGGREGATION_METRIC_UNSPECIFIED' | 'AVERAGE' | 'MODE' | 'STANDARD_DEVIATION' | 'VARIANCE' | 'MINIMUM' | 'MAXIMUM' | 'MEDIAN' | 'PERCENTILE_P90' | 'PERCENTILE_P95' | 'PERCENTILE_P99'>) | undefined, bleuSpec?: GoogleCloudAiplatformV1BleuSpec | undefined, computationBasedMetricSpec?: GoogleCloudAiplatformV1ComputationBasedMetricSpec | undefined, customCodeExecutionSpec?: GoogleCloudAiplatformV1CustomCodeExecutionSpec | undefined, exactMatchSpec?: GoogleCloudAiplatformV1ExactMatchSpec | undefined, llmBasedMetricSpec?: GoogleCloudAiplatformV1LLMBasedMetricSpec | undefined, metadata?: GoogleCloudAiplatformV1MetricMetadata | undefined, pairwiseMetricSpec?: GoogleCloudAiplatformV1PairwiseMetricSpec | undefined, pointwiseMetricSpec?: GoogleCloudAiplatformV1PointwiseMetricSpec | undefined, predefinedMetricSpec?: GoogleCloudAiplatformV1PredefinedMetricSpec | undefined, rougeSpec?: GoogleCloudAiplatformV1RougeSpec | undefined}; export type GoogleCloudAiplatformV1GcsDestination = {outputUriPrefix?: string | undefined}; export type GoogleCloudAiplatformV1OutputConfig = {gcsDestination?: GoogleCloudAiplatformV1GcsDestination | undefined}; export type GoogleCloudAiplatformV1EvaluateDatasetRequest = {autoraterConfig?: GoogleCloudAiplatformV1AutoraterConfig | undefined, dataset?: GoogleCloudAiplatformV1EvaluationDataset | undefined, location?: string | undefined, metrics?: Array | undefined, outputConfig?: GoogleCloudAiplatformV1OutputConfig | undefined}; export type CreateApiV1ProjectsProjectsIdLocationsLocationsIdEvaluateDatasetArgs = {projectsId: string, locationsId: string, body: GoogleCloudAiplatformV1EvaluateDatasetRequest}; export type CreateApiV1ProjectsProjectsIdLocationsLocationsIdEvaluateDatasetOptions = {baseUrl?: string | undefined, Oauth2?: string | undefined, Oauth2c?: string | undefined, fetch?: typeof fetch | undefined}; export const googleRpcStatus = z.object({code: z.number().int().optional(), details: z.array(z.record(z.string(), z.unknown())).optional(), message: z.string().optional()}); export const googleLongrunningOperation = z.object({done: z.boolean().optional(), error: googleRpcStatus.optional(), metadata: z.record(z.string(), z.unknown()).optional(), name: z.string().optional(), response: z.record(z.string(), z.unknown()).optional()}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * Evaluates a dataset based on a set of given metrics. * * POST /v1/projects/{projectsId}/locations/{locationsId}:evaluateDataset * * Also accepts auth: Oauth2 + Oauth2c */ export const createApiV1ProjectsProjectsIdLocationsLocationsIdEvaluateDataset = async (args: CreateApiV1ProjectsProjectsIdLocationsLocationsIdEvaluateDatasetArgs, options?: CreateApiV1ProjectsProjectsIdLocationsLocationsIdEvaluateDatasetOptions) => { const {baseUrl = "https://aiplatform.googleapis.com", fetch: fetchFn = fetch, Oauth2, Oauth2c} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('content-type', "application/json") headers.set('accept', "application/json") if (Oauth2 !== undefined) headers.set('authorization', `Bearer ${Oauth2}`) if (Oauth2c !== undefined) headers.set('authorization', `Bearer ${Oauth2c}`) const url = `${baseUrl.replace(/\/+$/, '')}/v1/projects/${encodeURIComponent(String(args.projectsId))}/locations/${encodeURIComponent(String(args.locationsId))}:evaluateDataset${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "POST", headers, body: JSON.stringify(args.body)}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(googleLongrunningOperation, await res.json()) };