import {z} from 'zod'

export type EvalAuthoringCommitRequest = {duplicatePolicy?: ('block' | 'warn' | 'create_anyway') | undefined, overrideReason?: string | undefined};

export type CreateApiProjectsProjectIdEvalSuitesSuiteIdAuthoringJobIdCommitArgs = {projectId: string, suiteId: string, jobId: string, body: EvalAuthoringCommitRequest};

export type CreateApiProjectsProjectIdEvalSuitesSuiteIdAuthoringJobIdCommitOptions = {baseUrl?: string | undefined, bearerAuth?: string | undefined, fetch?: typeof fetch | undefined};

export const evalTestStep = z.object({id: z.string().min(1), kind: z.enum(["prompt", "toolCall", "interact", "assert"]), prompt: z.string().optional(), serverName: z.string().optional(), toolName: z.string().optional(), arguments: z.record(z.string(), z.unknown()).optional(), action: z.record(z.string(), z.unknown()).optional(), assertion: z.record(z.string(), z.unknown()).optional()}).and(z.record(z.string(), z.unknown()));

export const modelSelection = z.object({modelId: z.string(), source: z.enum(["hosted", "org", "local"]), connectionRef: z.object({kind: z.enum(["orgProvider", "localProvider"]), id: z.string().optional(), providerKey: z.string().optional(), customProviderName: z.string().optional()}).optional(), nativeModelId: z.string().optional(), settings: z.object({reasoningEffort: z.enum(["none", "minimal", "low", "medium", "high", "xhigh", "max"]).optional(), temperature: z.number().gte(0).lte(2).optional()}).optional(), fallback: z.object({provider: z.enum(["none", "openrouter"]), model: z.literal("none")})});

export const legacyModelSelection = z.object({source: z.literal("legacy"), modelId: z.string(), provider: z.string().optional()});

export const requestedModelSelection = z.union([modelSelection, legacyModelSelection]);

export const evalAssertion = z.union([z.object({type: z.literal("toolDescriptionsPresent"), minLength: z.number().int().gt(0).lte(9007199254740991).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolAnnotationsPresent"), require: z.array(z.enum(["readOnlyHint", "destructiveHint"])).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolNamesUnique"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("noDeprecatedToolExposed"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolInputSchemasWellFormed"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolOutputSchemasPresent"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolCalledWith"), toolName: z.string().min(1), args: z.object({args: z.record(z.string(), z.unknown()), argumentMatching: z.enum(["exact", "partial", "ignore"]).optional()}), minCount: z.number().int().gt(0).lte(9007199254740991).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolCalledAtLeastOnce"), toolName: z.string().min(1), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolNeverCalled"), toolName: z.string().min(1), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("onlyToolsCalled"), toolNames: z.array(z.string().min(1)), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("firstToolWas"), toolName: z.string().min(1), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("responseContains"), needle: z.string().min(1), caseSensitive: z.boolean().optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("responseMatches"), pattern: z.string().min(1), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("noToolErrors"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("finalAssistantMessageNonEmpty"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("tokenBudgetUnder"), tokens: z.number().int().gt(0).lte(9007199254740991), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("widgetRendered"), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("widgetRenderLatencyUnder"), ms: z.number().int().gt(0).lte(9007199254740991), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("widgetNoConsoleErrors"), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("turnCountUnder"), turns: z.number().int().gt(0).lte(9007199254740991), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("noEndingQuestion"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolLatencyUnder"), ms: z.number().int().gt(0).lte(9007199254740991), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolResultContains"), needle: z.string().min(1).max(1000), caseSensitive: z.boolean().optional(), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolResultMatchesSchema"), schema: z.unknown(), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolResultSizeUnder"), maxBytes: z.number().int().gt(0).lte(9007199254740991), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("argumentsMatchToolSchema"), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolInputMatches"), toolName: z.string().min(1), patterns: z.array(z.string().min(1).max(512)), flags: z.enum(["i", "m", "s", "im", "is", "ms", "ims"]).optional(), path: z.string().min(2).max(257).optional(), min: z.number().int().gte(0).lte(9007199254740991).optional(), max: z.number().int().gte(0).lte(9007199254740991).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolResultMatches"), toolName: z.string().min(1).optional(), patterns: z.array(z.string().min(1).max(512)), flags: z.enum(["i", "m", "s", "im", "is", "ms", "ims"]).optional(), path: z.string().min(2).max(257).optional(), min: z.number().int().gte(0).lte(9007199254740991).optional(), max: z.number().int().gte(0).lte(9007199254740991).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("noRepeatedIdenticalCall"), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolCallCountUnder"), count: z.number().int().gt(0).lte(9007199254740991), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolCalledBefore"), toolName: z.string().min(1), beforeToolName: z.string().min(1), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("noDeprecatedToolCalled"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("noDestructiveToolCalled"), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("toolErrorNamesInput"), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()}), z.object({type: z.literal("fullPageHasContinuation"), toolName: z.string().min(1).optional(), role: z.enum(["gating", "advisory", "required"]).optional(), severity: z.literal("warn").optional()})]);

export const evalCaseImportClaim = z.object({status: z.enum(["exact", "approximated", "unsupported", "unresolved"]), sourceCaseKey: z.string().min(1).max(512).optional(), note: z.string().min(1).max(2000).optional()});

export const evalCaseSource = z.object({format: z.literal("markdown"), method: z.literal("ai"), fileName: z.string().min(1).max(255), fileHash: z.string(), excerpt: z.string().min(1).max(102400), startLine: z.number().int().gte(1), endLine: z.number().int().gte(1), extractorVersion: z.string().min(1).max(100)});

export const suppressedSuiteStandardCheckIds = z.array(z.enum(["discovery.description", "discovery.annotations", "discovery.collisions", "discovery.deprecated", "call.schema", "response.schema", "selection.hops", "call.parameters", "call.repeated", "response.performance", "response.size", "response.errors", "response.recovery", "response.pagination", "userValue.turns"]));

export const evalCase = z.object({judge: z.object({enabled: z.boolean().optional()}).optional(), id: z.string(), declaredId: z.string().optional(), title: z.string(), steps: z.array(evalTestStep), expectedOutput: z.string().optional(), iterations: z.number().int().gte(1).lte(10), repetitions: z.number().int().gte(1).optional(), passThreshold: z.number().gte(0).lte(1).optional(), isNegative: z.boolean(), scenario: z.string().optional(), intent: z.string().min(1).max(64).optional(), kind: z.enum(["capability", "regression"]).optional(), models: z.array(z.object({model: z.string(), provider: z.string().optional(), selection: requestedModelSelection.optional(), selectionOrigin: z.literal("backfill").optional()})), matchOptions: z.object({}).optional(), checks: z.object({mode: z.enum(["inherit", "replace", "extend"]).optional(), list: z.array(evalAssertion).optional()}).optional(), import: evalCaseImportClaim.optional(), source: evalCaseSource.optional(), createdAt: z.number().nullable().optional(), updatedAt: z.number().nullable().optional(), suppressedSuiteStandardCheckIds: suppressedSuiteStandardCheckIds.optional()});

export const evalCaseImported = z.object({generationModel: z.string(), created: z.array(evalCase), counts: z.object({normal: z.number().int().optional(), negative: z.number().int().optional()}), skipped: z.array(z.object({})).optional(), reviewUrl: z.string().optional()});

export const evalAuthoringJobAccepted = z.object({jobId: z.string(), status: z.enum(["pending", "completed", "failed", "cancelled"]), error: z.string().optional(), generationModel: z.string().optional(), created: z.array(evalCase).optional(), counts: z.object({normal: z.number().int().optional(), negative: z.number().int().optional()}).optional()});

export const createApiProjectsProjectIdEvalSuitesSuiteIdAuthoringJobIdCommitResponse = z.union([evalCaseImported, evalAuthoringJobAccepted]);

export const validateResponse = async <Output>(
  schema: {
    readonly '~standard': {
      readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray<PropertyKey | { readonly key: PropertyKey }> | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray<PropertyKey | { readonly key: PropertyKey }> | undefined }> }>
    }
  },
  value: unknown
): Promise<Output> => {
  const result = await schema['~standard'].validate(value)

  if (result.issues !== undefined) {
    const detail = result.issues
      .map(({ message, path }) =>
        path?.length
          ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}`
          : message
      )
      .join('; ')

    throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } })
  }

  return result.value
};

/**
 * Write a finished authoring job's cases into the suite
 * 
 * POST /projects/{projectId}/eval-suites/{suiteId}/authoring/{jobId}/commit
 */
export const createApiProjectsProjectIdEvalSuitesSuiteIdAuthoringJobIdCommit = async (args: CreateApiProjectsProjectIdEvalSuitesSuiteIdAuthoringJobIdCommitArgs, options?: CreateApiProjectsProjectIdEvalSuitesSuiteIdAuthoringJobIdCommitOptions) => {
const {baseUrl = "https://app.mcpjam.com/api/v1", fetch: fetchFn = fetch, bearerAuth} = options ?? {}

const query = new URLSearchParams()
const headers = new Headers()
headers.set('content-type', "application/json")
headers.set('accept', "application/json")

if (bearerAuth !== undefined) headers.set('authorization', `Bearer ${bearerAuth}`)

const url = `${baseUrl.replace(/\/+$/, '')}/projects/${encodeURIComponent(String(args.projectId))}/eval-suites/${encodeURIComponent(String(args.suiteId))}/authoring/${encodeURIComponent(String(args.jobId))}/commit${query.size > 0 ? `?${query}` : ''}`

const res = await fetchFn(url, {method: "POST", headers, body: JSON.stringify(args.body)})

if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}})

return validateResponse(createApiProjectsProjectIdEvalSuitesSuiteIdAuthoringJobIdCommitResponse, await res.json())
};