import {z} from 'zod'

export type IdentifierDTO = {id?: number | undefined, uuid?: string | undefined};

export type LoadDefDTO = {readonly?: boolean | undefined, passwordManager?: string | undefined, alias?: string | undefined, query?: string | undefined, queryHistory?: string | undefined, key?: string | undefined, expression?: string | undefined, addDateColumn?: boolean | undefined, zeroFillNull?: boolean | undefined, replaceNulls?: string | undefined, stringMode?: boolean | undefined, operator?: string | undefined, dateColumn?: string | undefined, transform?: string | undefined, filter?: string | undefined, filterNot?: string | undefined, sample?: number | undefined, backRun?: number | undefined, backRunBin?: ('MINUTE' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined, unionLookBack?: boolean | undefined, unionLookBackMinRow?: number | undefined, cache?: boolean | undefined, dateFormat?: string | undefined, timeFormat?: string | undefined, timestamp?: boolean | undefined, filePath?: string | undefined, fileQuery?: string | undefined, fullFile?: boolean | undefined, fileHeader?: string | undefined, inferSchema?: boolean | undefined, fileType?: string | undefined, delimiter?: string | undefined, fileCharSet?: string | undefined, skipLines?: number | undefined, avroSchema?: string | undefined, xmlRowTag?: string | undefined, flatten?: boolean | undefined, handleMaps?: boolean | undefined, handleMixedJson?: boolean | undefined, multiLine?: boolean | undefined, lib?: string | undefined, additionalLib?: string | undefined, driverName?: string | undefined, connectionName?: string | undefined, connectionUrl?: string | undefined, userName?: string | undefined, password?: string | undefined, connectionProperties?: {[key: string]: string} | undefined, hiveNative?: boolean | undefined, hiveNativeHWC?: boolean | undefined, useSql?: boolean | undefined, columnName?: string | undefined, lowerBound?: string | undefined, upperBound?: string | undefined, numPartitions?: number | undefined, partitionNumber?: number | undefined, escapeWithBackTick?: boolean | undefined, escapeWithSingleQuote?: boolean | undefined, escapeWithDoubleQuote?: boolean | undefined, escapeCharacter?: string | undefined, checkHeader?: boolean | undefined, archiveConnection?: string | undefined, coreFetchMode?: boolean | undefined, archiveBreaks?: boolean | undefined, ruleSerial?: boolean | undefined};

export type PushDownDefDTO = {dataset?: string | undefined, connectionName?: string | undefined, maxConnections?: number | undefined, sourceQuery?: string | undefined, backRuns?: number | undefined, backRunBin?: string | undefined, dateFormatType?: string | undefined, threads?: number | undefined, manualSourceQuery?: boolean | undefined, key?: string | undefined, sourceOutputSchema?: string | undefined, sourceBreakRules?: boolean | undefined, sourceBreakOutliers?: boolean | undefined, sourceBreakDupes?: boolean | undefined, sourceBreakShapes?: boolean | undefined, sourceBreakResults?: boolean | undefined, sqlLoggingToggle?: boolean | undefined, unionLookback?: boolean | undefined, unionLookbackMin?: number | undefined};

export type OutlierDefDTO = {id?: number | undefined, dataset?: string | undefined, only?: boolean | undefined, lookback?: number | undefined, key?: Array<string> | undefined, include?: Array<string> | undefined, exclude?: Array<string> | undefined, dateColumn?: string | undefined, timeColumn?: string | undefined, timeBin?: ('MIN' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined, timeBinQuery?: string | undefined, categorical?: boolean | undefined, by?: ('MIN' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined, limit?: number | undefined, minHistory?: number | undefined, historyLimit?: number | undefined, score?: number | undefined, aggFunc?: string | undefined, aggQuery?: string | undefined, query?: string | undefined, q1?: number | undefined, q3?: number | undefined, categoricalColumnConcatenation?: boolean | undefined, limitCategorical?: number | undefined, measurementUnit?: string | undefined, multiplierUpper?: number | undefined, multiplierLower?: number | undefined, record?: boolean | undefined, filter?: string | undefined, combine?: boolean | undefined, categoricalConfidenceType?: string | undefined, categoricalTopN?: number | undefined, categoricalBottomN?: number | undefined, categoricalMaxConfidence?: number | undefined, categoricalMaxFrequencyPercentile?: number | undefined, categoricalMinFrequency?: number | undefined, categoricalMinVariance?: number | undefined, categoricalMaxCategoryN?: number | undefined, categoricalParallel?: boolean | undefined, categoricalAlgorithm?: string | undefined, categoricalAlgorithmParameters?: {[key: string]: string} | undefined, categoricalAlgorithmParametersString?: string | undefined};

export type PatternDefDTO = {id?: number | undefined, dataset?: string | undefined, only?: boolean | undefined, lookback?: number | undefined, key?: Array<string> | undefined, dateColumn?: string | undefined, include?: Array<string> | undefined, exclude?: Array<string> | undefined, score?: number | undefined, minSupport?: number | undefined, confidence?: number | undefined, limit?: number | undefined, query?: string | undefined, filter?: string | undefined, timeBin?: ('MINUTE' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined};

export type DupeDefDTO = {on?: boolean | undefined, only?: boolean | undefined, include?: Array<string> | undefined, exclude?: Array<string> | undefined, depth?: number | undefined, lowerBound?: number | undefined, upperBound?: number | undefined, approximate?: number | undefined, limitPerDupe?: number | undefined, checkHeader?: boolean | undefined, filter?: string | undefined, ignoreCase?: boolean | undefined, score?: number | undefined, limit?: number | undefined};

export type ProfileDefDTO = {on?: boolean | undefined, only?: boolean | undefined, include?: Array<string> | undefined, exclude?: Array<string> | undefined, correlation?: boolean | undefined, histogram?: boolean | undefined, semantic?: boolean | undefined, dataConceptId?: number | undefined, limit?: number | undefined, histogramLimit?: number | undefined, score?: number | undefined, shape?: boolean | undefined, shapeTotalScore?: number | undefined, shapeSensitivity?: number | undefined, shapeMaxPerCol?: number | undefined, shapeMaxColSize?: number | undefined, shapeGranular?: boolean | undefined, behavioralDimension?: string | undefined, behavioralDimensionGroup?: string | undefined, behavioralValueColumn?: string | undefined, behaviorScoreOff?: boolean | undefined, behaviorLookback?: number | undefined, behaviorMinSupport?: number | undefined, profilePushDown?: Array<string> | undefined, behaviorRowCheck?: boolean | undefined, behaviorTimeCheck?: boolean | undefined, behaviorMinValueCheck?: boolean | undefined, behaviorMaxValueCheck?: boolean | undefined, behaviorMeanValueCheck?: boolean | undefined, behaviorNullCheck?: boolean | undefined, behaviorEmptyCheck?: boolean | undefined, behaviorUniqueCheck?: boolean | undefined, profileStringLength?: boolean | undefined, detectStringNumerics?: boolean | undefined, detectTopnBotn?: boolean | undefined, detectScalePrecision?: boolean | undefined, adaptiveTier?: string | undefined, advancedTier?: boolean | undefined, filter?: string | undefined};

export type SourceDefDTO = {on?: boolean | undefined, only?: boolean | undefined, validateValues?: boolean | undefined, key?: Array<string> | undefined, include?: Array<string> | undefined, exclude?: Array<string> | undefined, includeSrc?: Array<string> | undefined, excludeSrc?: Array<string> | undefined, map?: {[key: string]: string} | undefined, score?: number | undefined, limit?: number | undefined, sourcePushDownCount?: boolean | undefined, checkType?: boolean | undefined, checkCase?: boolean | undefined, validateSchemaOrder?: boolean | undefined, matches?: boolean | undefined, validateValuesThreshold?: number | undefined, validateValuesThresholdStrictDownScore?: boolean | undefined, validateValuesShowAll?: boolean | undefined, validateValuesIgnoreNull?: boolean | undefined, validateValuesIgnoreEmpty?: boolean | undefined, validateValuesIgnorePrecision?: boolean | undefined, validateValuesTrim?: boolean | undefined, validateValuesShowMissingKeys?: boolean | undefined, validateSrcJoinOnly?: boolean | undefined, validateValuesFilter?: string | undefined, dataset?: string | undefined, driverName?: string | undefined, user?: string | undefined, password?: string | undefined, passwordManager?: string | undefined, connectionName?: string | undefined, connectionUrl?: string | undefined, query?: string | undefined, lib?: string | undefined, connectionProperties?: {[key: string]: string} | undefined, filePath?: string | undefined, fileQuery?: string | undefined, fullFile?: boolean | undefined, header?: string | undefined, skipLines?: number | undefined, inferSchema?: boolean | undefined, fileType?: string | undefined, delimiter?: string | undefined, fileCharSet?: string | undefined, avroSchema?: string | undefined, xmlRowTag?: string | undefined, flatten?: boolean | undefined, handleMaps?: boolean | undefined, handleMixedJson?: boolean | undefined, multiLine?: boolean | undefined, filterCols?: Array<string> | undefined};

export type RuleDefDTO = {on?: boolean | undefined, only?: boolean | undefined, lib?: string | undefined, name?: string | undefined, absoluteScoring?: boolean | undefined, ruleBreakPreviewLimit?: number | undefined};

export type Connection = {dataset?: string | undefined, url?: string | undefined, user?: string | undefined, password?: string | undefined, query?: string | undefined, filePath?: string | undefined, driver?: string | undefined, driverProps?: {[key: string]: string} | undefined};

export type ColMatchDefDTO = {colMatchParallelProcesses?: number | undefined, colMatchDurationMins?: number | undefined, colMatchBatchSize?: number | undefined, level?: string | undefined, fuzzyDistance?: number | undefined, connectionList?: Array<Connection> | undefined};

export type SparkDefDTO = {numExecutors?: number | undefined, driverMemory?: string | undefined, executorMemory?: string | undefined, executorCores?: number | undefined, conf?: string | undefined, queue?: string | undefined, master?: string | undefined, principal?: string | undefined, keyTab?: string | undefined, deployMode?: string | undefined, jars?: Array<string> | undefined, packages?: Array<string> | undefined, files?: Array<string> | undefined};

export type EnvDefDTO = {dataset?: string | undefined, jdbcPrincipal?: string | undefined, jdbcKeyTab?: string | undefined};

export type RecordDefDTO = {on?: boolean | undefined, in?: string | undefined, notIn?: string | undefined, include?: Array<string> | undefined, percDeltaLimit?: number | undefined, score?: number | undefined, dataset?: string | undefined, dateColumn?: string | undefined, keyColumn?: string | undefined, timeBin?: ('MIN' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined, by?: ('MIN' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined};

export type KafkaDefDTO = {kafkaGroup?: string | undefined, kafkaBroker?: string | undefined, kafkaPort?: string | undefined, kafkaTopic?: string | undefined, kafkaKeySerde?: string | undefined, kafkaValSerde?: string | undefined, kafkaSaslEnabled?: boolean | undefined, kafkaSaslService?: string | undefined, kafkaSslEnabled?: boolean | undefined, streamSchemaRegistryUrl?: string | undefined, streamSchemaRegistryUser?: string | undefined, streamSchemaRegistryPassword?: string | undefined, streamInterval?: string | undefined, streamSchema?: string | undefined, streamFormat?: string | undefined, streamMaxLull?: string | undefined};

export type StreamDefDTO = {type?: 'Kafka' | undefined, kafkaDef?: KafkaDefDTO | undefined};

export type OutlierDefColumnSensitivitySettingsDTO = {name?: string | undefined, weight?: number | undefined, unit?: ('DEFAULT' | 'UNIT_0_001' | 'UNIT_0_01' | 'UNIT_0_1' | 'UNIT_1_0' | 'UNIT_10_0' | 'UNIT_100_0' | 'UNIT_1000_0' | 'UNIT_10000_0' | 'UNIT_100000_0' | 'UNIT_1000000_0' | 'UNIT_10000000_0') | undefined};

export type AbstractOutlierDefDTO = {columnSensitivitySettings?: Array<OutlierDefColumnSensitivitySettingsDTO> | undefined, configurations?: (Array<{id?: number | undefined, key?: Array<string> | undefined, include?: Array<string> | undefined, exclude?: Array<string> | undefined, timeColumn?: string | undefined, timeBinQuery?: string | undefined, limit?: number | undefined, minHistory?: number | undefined, historyLimit?: number | undefined, score?: number | undefined, aggFunc?: string | undefined, aggQuery?: string | undefined, query?: string | undefined, categoricalColumnConcatenation?: boolean | undefined, limitCategorical?: number | undefined, measurementUnit?: string | undefined, multiplierUpper?: number | undefined, multiplierLower?: number | undefined, record?: boolean | undefined, filter?: string | undefined, combine?: boolean | undefined, categoricalConfidenceType?: string | undefined, categoricalTopN?: number | undefined, categoricalBottomN?: number | undefined, categoricalMaxConfidence?: number | undefined, categoricalMaxFrequencyPercentile?: number | undefined, categoricalMinFrequency?: number | undefined, categoricalMinVariance?: number | undefined, categoricalMaxCategoryN?: number | undefined, categoricalParallel?: boolean | undefined, categoricalAlgorithm?: string | undefined, categoricalAlgorithmParameters?: {[key: string]: string} | undefined, categoricalAlgorithmParametersString?: string | undefined}>) | undefined, only?: boolean | undefined, lookback?: number | undefined, dateColumn?: string | undefined, timeBin?: ('MIN' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined, by?: ('MIN' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined, q1?: number | undefined, q3?: number | undefined, categorical?: boolean | undefined} | {columnSensitivitySettings?: Array<OutlierDefColumnSensitivitySettingsDTO> | undefined, configurations?: (Array<{id?: number | undefined, dataset?: string | undefined, only?: boolean | undefined, lookback?: number | undefined, key?: Array<string> | undefined, include?: Array<string> | undefined, exclude?: Array<string> | undefined, dateColumn?: string | undefined, timeColumn?: string | undefined, timeBin?: ('MIN' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined, timeBinQuery?: string | undefined, categorical?: boolean | undefined, by?: ('MIN' | 'HOUR' | 'DAY' | 'WEEK' | 'MONTH' | 'QUARTER' | 'YEAR') | undefined, limit?: number | undefined, minHistory?: number | undefined, historyLimit?: number | undefined, score?: number | undefined, aggFunc?: string | undefined, aggQuery?: string | undefined, query?: string | undefined, q1?: number | undefined, q3?: number | undefined, categoricalColumnConcatenation?: boolean | undefined, limitCategorical?: number | undefined, measurementUnit?: string | undefined, multiplierUpper?: number | undefined, multiplierLower?: number | undefined, record?: boolean | undefined, filter?: string | undefined, combine?: boolean | undefined, categoricalConfidenceType?: string | undefined, categoricalTopN?: number | undefined, categoricalBottomN?: number | undefined, categoricalMaxConfidence?: number | undefined, categoricalMaxFrequencyPercentile?: number | undefined, categoricalMinFrequency?: number | undefined, categoricalMinVariance?: number | undefined, categoricalMaxCategoryN?: number | undefined, categoricalParallel?: boolean | undefined, categoricalAlgorithm?: string | undefined, categoricalAlgorithmParameters?: {[key: string]: string} | undefined, categoricalAlgorithmParametersString?: string | undefined}>) | undefined};

export type ShapeColumnSettingsDTO = {enabled?: boolean | undefined, name?: string | undefined, type?: string | undefined};

export type ShapeDefDTO = {enabled?: boolean | undefined, totalScore?: number | undefined, sensitivity?: number | undefined, maxPerCol?: number | undefined, maxColSize?: number | undefined, granular?: boolean | undefined, columnSettings?: Array<ShapeColumnSettingsDTO> | undefined};

export type Identifier = {id?: number | undefined, uuid?: string | undefined};

export type AbstractJobScheduleConfigurationDTO = {enabled?: boolean | undefined, dataset: string, agentId: Identifier, runDateFormat: string, scheduleFrequency?: ('DAILY' | 'MONTHLY' | 'QUARTERLY') | undefined, updatedAt?: string | undefined, scheduleTime?: string | undefined};

export type DatasetDefDTO = {dataset?: string | undefined, runId?: string | undefined, runIdEnd?: string | undefined, runState?: ('DRAFT' | 'DRY' | 'CERTIFIED') | undefined, passFail?: number | undefined, passFailLimit?: number | undefined, jobDescription?: string | undefined, jobId?: IdentifierDTO | undefined, coreMaxActiveConnections?: number | undefined, linkId?: Array<string> | undefined, licenseKey?: string | undefined, logFile?: string | undefined, logLevel?: string | undefined, hootOnly?: boolean | undefined, prettyPrint?: boolean | undefined, useTemplate?: boolean | undefined, parallel?: boolean | undefined, plan?: boolean | undefined, dataPreviewOff?: boolean | undefined, datasetSafeOff?: boolean | undefined, obslimit?: number | undefined, pgUser?: string | undefined, pgPassword?: string | undefined, host?: string | undefined, port?: string | undefined, user?: string | undefined, alertEmail?: string | undefined, scheduleTime?: string | undefined, schemaScore?: number | undefined, optionAppend?: string | undefined, keyDelimiter?: string | undefined, agentId?: IdentifierDTO | undefined, load?: LoadDefDTO | undefined, pushdown?: PushDownDefDTO | undefined, outliers?: Array<OutlierDefDTO> | undefined, patterns?: Array<PatternDefDTO> | undefined, dupe?: DupeDefDTO | undefined, profile?: ProfileDefDTO | undefined, source?: SourceDefDTO | undefined, rule?: RuleDefDTO | undefined, colMatch?: ColMatchDefDTO | undefined, spark?: SparkDefDTO | undefined, env?: EnvDefDTO | undefined, record?: RecordDefDTO | undefined, stream?: StreamDefDTO | undefined, pipeline?: Array<string> | undefined, runDate?: string | undefined, runDateEnd?: string | undefined, metaTags?: Array<string> | undefined, outlierConfiguration?: AbstractOutlierDefDTO | undefined, shape?: ShapeDefDTO | undefined, jobSchedule?: AbstractJobScheduleConfigurationDTO | undefined, connectionType?: ('PULLUP' | 'PUSHDOWN') | undefined};

export type CreateApiV3DatasetDefsArgs = {body: DatasetDefDTO};

export type CreateApiV3DatasetDefsOptions = {baseUrl: string, basicAuth?: {username: string, password: string} | undefined, fetch?: typeof fetch | undefined};

export const identifierDTO = z.object({id: z.number().int().optional(), uuid: z.string().optional()});

export const loadDefDTO = z.object({readonly: z.boolean().optional(), passwordManager: z.string().optional(), alias: z.string().optional(), query: z.string().optional(), queryHistory: z.string().optional(), key: z.string().optional(), expression: z.string().optional(), addDateColumn: z.boolean().optional(), zeroFillNull: z.boolean().optional(), replaceNulls: z.string().optional(), stringMode: z.boolean().optional(), operator: z.string().optional(), dateColumn: z.string().optional(), transform: z.string().optional(), filter: z.string().optional(), filterNot: z.string().optional(), sample: z.number().optional(), backRun: z.number().int().optional(), backRunBin: z.enum(["MINUTE", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional(), unionLookBack: z.boolean().optional(), unionLookBackMinRow: z.number().int().optional(), cache: z.boolean().optional(), dateFormat: z.string().optional(), timeFormat: z.string().optional(), timestamp: z.boolean().optional(), filePath: z.string().optional(), fileQuery: z.string().optional(), fullFile: z.boolean().optional(), fileHeader: z.string().optional(), inferSchema: z.boolean().optional(), fileType: z.string().optional(), delimiter: z.string().optional(), fileCharSet: z.string().optional(), skipLines: z.number().int().optional(), avroSchema: z.string().optional(), xmlRowTag: z.string().optional(), flatten: z.boolean().optional(), handleMaps: z.boolean().optional(), handleMixedJson: z.boolean().optional(), multiLine: z.boolean().optional(), lib: z.string().optional(), additionalLib: z.string().optional(), driverName: z.string().optional(), connectionName: z.string().optional(), connectionUrl: z.string().optional(), userName: z.string().optional(), password: z.string().optional(), connectionProperties: z.record(z.string(), z.string()).optional(), hiveNative: z.boolean().optional(), hiveNativeHWC: z.boolean().optional(), useSql: z.boolean().optional(), columnName: z.string().optional(), lowerBound: z.string().optional(), upperBound: z.string().optional(), numPartitions: z.number().int().optional(), partitionNumber: z.number().int().optional(), escapeWithBackTick: z.boolean().optional(), escapeWithSingleQuote: z.boolean().optional(), escapeWithDoubleQuote: z.boolean().optional(), escapeCharacter: z.string().optional(), checkHeader: z.boolean().optional(), archiveConnection: z.string().optional(), coreFetchMode: z.boolean().optional(), archiveBreaks: z.boolean().optional(), ruleSerial: z.boolean().optional()});

export const pushDownDefDTO = z.object({dataset: z.string().optional(), connectionName: z.string().optional(), maxConnections: z.number().int().optional(), sourceQuery: z.string().optional(), backRuns: z.number().int().optional(), backRunBin: z.string().optional(), dateFormatType: z.string().optional(), threads: z.number().int().optional(), manualSourceQuery: z.boolean().optional(), key: z.string().optional(), sourceOutputSchema: z.string().optional(), sourceBreakRules: z.boolean().optional(), sourceBreakOutliers: z.boolean().optional(), sourceBreakDupes: z.boolean().optional(), sourceBreakShapes: z.boolean().optional(), sourceBreakResults: z.boolean().optional(), sqlLoggingToggle: z.boolean().optional(), unionLookback: z.boolean().optional(), unionLookbackMin: z.number().int().optional()});

export const outlierDefDTO = z.object({id: z.number().int().optional(), dataset: z.string().optional(), only: z.boolean().optional(), lookback: z.number().int().optional(), key: z.array(z.string()).optional(), include: z.array(z.string()).optional(), exclude: z.array(z.string()).optional(), dateColumn: z.string().optional(), timeColumn: z.string().optional(), timeBin: z.enum(["MIN", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional(), timeBinQuery: z.string().optional(), categorical: z.boolean().optional(), by: z.enum(["MIN", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional(), limit: z.number().int().optional(), minHistory: z.number().int().optional(), historyLimit: z.number().int().optional(), score: z.number().optional(), aggFunc: z.string().optional(), aggQuery: z.string().optional(), query: z.string().optional(), q1: z.number().optional(), q3: z.number().optional(), categoricalColumnConcatenation: z.boolean().optional(), limitCategorical: z.number().int().optional(), measurementUnit: z.string().optional(), multiplierUpper: z.number().optional(), multiplierLower: z.number().optional(), record: z.boolean().optional(), filter: z.string().optional(), combine: z.boolean().optional(), categoricalConfidenceType: z.string().optional(), categoricalTopN: z.number().int().optional(), categoricalBottomN: z.number().int().optional(), categoricalMaxConfidence: z.number().optional(), categoricalMaxFrequencyPercentile: z.number().optional(), categoricalMinFrequency: z.number().int().optional(), categoricalMinVariance: z.number().optional(), categoricalMaxCategoryN: z.number().int().optional(), categoricalParallel: z.boolean().optional(), categoricalAlgorithm: z.string().optional(), categoricalAlgorithmParameters: z.record(z.string(), z.string()).optional(), categoricalAlgorithmParametersString: z.string().optional()});

export const patternDefDTO = z.object({id: z.number().int().optional(), dataset: z.string().optional(), only: z.boolean().optional(), lookback: z.number().int().optional(), key: z.array(z.string()).optional(), dateColumn: z.string().optional(), include: z.array(z.string()).optional(), exclude: z.array(z.string()).optional(), score: z.number().optional(), minSupport: z.number().optional(), confidence: z.number().optional(), limit: z.number().int().optional(), query: z.string().optional(), filter: z.string().optional(), timeBin: z.enum(["MINUTE", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional()});

export const dupeDefDTO = z.object({on: z.boolean().optional(), only: z.boolean().optional(), include: z.array(z.string()).optional(), exclude: z.array(z.string()).optional(), depth: z.number().int().optional(), lowerBound: z.number().int().optional(), upperBound: z.number().int().optional(), approximate: z.number().int().optional(), limitPerDupe: z.number().int().optional(), checkHeader: z.boolean().optional(), filter: z.string().optional(), ignoreCase: z.boolean().optional(), score: z.number().optional(), limit: z.number().int().optional()});

export const profileDefDTO = z.object({on: z.boolean().optional(), only: z.boolean().optional(), include: z.array(z.string()).optional(), exclude: z.array(z.string()).optional(), correlation: z.boolean().optional(), histogram: z.boolean().optional(), semantic: z.boolean().optional(), dataConceptId: z.number().int().optional(), limit: z.number().int().optional(), histogramLimit: z.number().int().optional(), score: z.number().optional(), shape: z.boolean().optional(), shapeTotalScore: z.number().int().optional(), shapeSensitivity: z.number().optional(), shapeMaxPerCol: z.number().int().optional(), shapeMaxColSize: z.number().int().optional(), shapeGranular: z.boolean().optional(), behavioralDimension: z.string().optional(), behavioralDimensionGroup: z.string().optional(), behavioralValueColumn: z.string().optional(), behaviorScoreOff: z.boolean().optional(), behaviorLookback: z.number().int().optional(), behaviorMinSupport: z.number().int().optional(), profilePushDown: z.array(z.string()).optional(), behaviorRowCheck: z.boolean().optional(), behaviorTimeCheck: z.boolean().optional(), behaviorMinValueCheck: z.boolean().optional(), behaviorMaxValueCheck: z.boolean().optional(), behaviorMeanValueCheck: z.boolean().optional(), behaviorNullCheck: z.boolean().optional(), behaviorEmptyCheck: z.boolean().optional(), behaviorUniqueCheck: z.boolean().optional(), profileStringLength: z.boolean().optional(), detectStringNumerics: z.boolean().optional(), detectTopnBotn: z.boolean().optional(), detectScalePrecision: z.boolean().optional(), adaptiveTier: z.string().optional(), advancedTier: z.boolean().optional(), filter: z.string().optional()});

export const sourceDefDTO = z.object({on: z.boolean().optional(), only: z.boolean().optional(), validateValues: z.boolean().optional(), key: z.array(z.string()).optional(), include: z.array(z.string()).optional(), exclude: z.array(z.string()).optional(), includeSrc: z.array(z.string()).optional(), excludeSrc: z.array(z.string()).optional(), map: z.record(z.string(), z.string()).optional(), score: z.number().optional(), limit: z.number().int().optional(), sourcePushDownCount: z.boolean().optional(), checkType: z.boolean().optional(), checkCase: z.boolean().optional(), validateSchemaOrder: z.boolean().optional(), matches: z.boolean().optional(), validateValuesThreshold: z.number().optional(), validateValuesThresholdStrictDownScore: z.boolean().optional(), validateValuesShowAll: z.boolean().optional(), validateValuesIgnoreNull: z.boolean().optional(), validateValuesIgnoreEmpty: z.boolean().optional(), validateValuesIgnorePrecision: z.boolean().optional(), validateValuesTrim: z.boolean().optional(), validateValuesShowMissingKeys: z.boolean().optional(), validateSrcJoinOnly: z.boolean().optional(), validateValuesFilter: z.string().optional(), dataset: z.string().optional(), driverName: z.string().optional(), user: z.string().optional(), password: z.string().optional(), passwordManager: z.string().optional(), connectionName: z.string().optional(), connectionUrl: z.string().optional(), query: z.string().optional(), lib: z.string().optional(), connectionProperties: z.record(z.string(), z.string()).optional(), filePath: z.string().optional(), fileQuery: z.string().optional(), fullFile: z.boolean().optional(), header: z.string().optional(), skipLines: z.number().int().optional(), inferSchema: z.boolean().optional(), fileType: z.string().optional(), delimiter: z.string().optional(), fileCharSet: z.string().optional(), avroSchema: z.string().optional(), xmlRowTag: z.string().optional(), flatten: z.boolean().optional(), handleMaps: z.boolean().optional(), handleMixedJson: z.boolean().optional(), multiLine: z.boolean().optional(), filterCols: z.array(z.string()).optional()});

export const ruleDefDTO = z.object({on: z.boolean().optional(), only: z.boolean().optional(), lib: z.string().optional(), name: z.string().optional(), absoluteScoring: z.boolean().optional(), ruleBreakPreviewLimit: z.number().int().optional()});

export const connection = z.object({dataset: z.string().optional(), url: z.string().optional(), user: z.string().optional(), password: z.string().optional(), query: z.string().optional(), filePath: z.string().optional(), driver: z.string().optional(), driverProps: z.record(z.string(), z.string()).optional()});

export const colMatchDefDTO = z.object({colMatchParallelProcesses: z.number().int().optional(), colMatchDurationMins: z.number().int().optional(), colMatchBatchSize: z.number().int().optional(), level: z.string().optional(), fuzzyDistance: z.number().int().optional(), connectionList: z.array(connection).optional()});

export const sparkDefDTO = z.object({numExecutors: z.number().int().optional(), driverMemory: z.string().optional(), executorMemory: z.string().optional(), executorCores: z.number().int().optional(), conf: z.string().optional(), queue: z.string().optional(), master: z.string().optional(), principal: z.string().optional(), keyTab: z.string().optional(), deployMode: z.string().optional(), jars: z.array(z.string()).optional(), packages: z.array(z.string()).optional(), files: z.array(z.string()).optional()});

export const envDefDTO = z.object({dataset: z.string().optional(), jdbcPrincipal: z.string().optional(), jdbcKeyTab: z.string().optional()});

export const recordDefDTO = z.object({on: z.boolean().optional(), in: z.string().optional(), notIn: z.string().optional(), include: z.array(z.string()).optional(), percDeltaLimit: z.number().optional(), score: z.number().optional(), dataset: z.string().optional(), dateColumn: z.string().optional(), keyColumn: z.string().optional(), timeBin: z.enum(["MIN", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional(), by: z.enum(["MIN", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional()});

export const kafkaDefDTO = z.object({kafkaGroup: z.string().optional(), kafkaBroker: z.string().optional(), kafkaPort: z.string().optional(), kafkaTopic: z.string().optional(), kafkaKeySerde: z.string().optional(), kafkaValSerde: z.string().optional(), kafkaSaslEnabled: z.boolean().optional(), kafkaSaslService: z.string().optional(), kafkaSslEnabled: z.boolean().optional(), streamSchemaRegistryUrl: z.string().optional(), streamSchemaRegistryUser: z.string().optional(), streamSchemaRegistryPassword: z.string().optional(), streamInterval: z.string().optional(), streamSchema: z.string().optional(), streamFormat: z.string().optional(), streamMaxLull: z.string().optional()});

export const streamDefDTO = z.object({type: z.literal("Kafka").optional(), kafkaDef: kafkaDefDTO.optional()});

export const outlierDefColumnSensitivitySettingsDTO = z.object({name: z.string().optional(), weight: z.number().optional(), unit: z.enum(["DEFAULT", "UNIT_0_001", "UNIT_0_01", "UNIT_0_1", "UNIT_1_0", "UNIT_10_0", "UNIT_100_0", "UNIT_1000_0", "UNIT_10000_0", "UNIT_100000_0", "UNIT_1000000_0", "UNIT_10000000_0"]).optional()});

export const abstractOutlierDefDTO = z.union([z.object({columnSensitivitySettings: z.array(outlierDefColumnSensitivitySettingsDTO).optional(), configurations: z.array(z.object({id: z.number().int().optional(), key: z.array(z.string()).optional(), include: z.array(z.string()).optional(), exclude: z.array(z.string()).optional(), timeColumn: z.string().optional(), timeBinQuery: z.string().optional(), limit: z.number().int().optional(), minHistory: z.number().int().optional(), historyLimit: z.number().int().optional(), score: z.number().optional(), aggFunc: z.string().optional(), aggQuery: z.string().optional(), query: z.string().optional(), categoricalColumnConcatenation: z.boolean().optional(), limitCategorical: z.number().int().optional(), measurementUnit: z.string().optional(), multiplierUpper: z.number().optional(), multiplierLower: z.number().optional(), record: z.boolean().optional(), filter: z.string().optional(), combine: z.boolean().optional(), categoricalConfidenceType: z.string().optional(), categoricalTopN: z.number().int().optional(), categoricalBottomN: z.number().int().optional(), categoricalMaxConfidence: z.number().optional(), categoricalMaxFrequencyPercentile: z.number().optional(), categoricalMinFrequency: z.number().int().optional(), categoricalMinVariance: z.number().optional(), categoricalMaxCategoryN: z.number().int().optional(), categoricalParallel: z.boolean().optional(), categoricalAlgorithm: z.string().optional(), categoricalAlgorithmParameters: z.record(z.string(), z.string()).optional(), categoricalAlgorithmParametersString: z.string().optional()})).optional(), only: z.boolean().optional(), lookback: z.number().int().optional(), dateColumn: z.string().optional(), timeBin: z.enum(["MIN", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional(), by: z.enum(["MIN", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional(), q1: z.number().optional(), q3: z.number().optional(), categorical: z.boolean().optional()}), z.object({columnSensitivitySettings: z.array(outlierDefColumnSensitivitySettingsDTO).optional(), configurations: z.array(z.object({id: z.number().int().optional(), dataset: z.string().optional(), only: z.boolean().optional(), lookback: z.number().int().optional(), key: z.array(z.string()).optional(), include: z.array(z.string()).optional(), exclude: z.array(z.string()).optional(), dateColumn: z.string().optional(), timeColumn: z.string().optional(), timeBin: z.enum(["MIN", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional(), timeBinQuery: z.string().optional(), categorical: z.boolean().optional(), by: z.enum(["MIN", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]).optional(), limit: z.number().int().optional(), minHistory: z.number().int().optional(), historyLimit: z.number().int().optional(), score: z.number().optional(), aggFunc: z.string().optional(), aggQuery: z.string().optional(), query: z.string().optional(), q1: z.number().optional(), q3: z.number().optional(), categoricalColumnConcatenation: z.boolean().optional(), limitCategorical: z.number().int().optional(), measurementUnit: z.string().optional(), multiplierUpper: z.number().optional(), multiplierLower: z.number().optional(), record: z.boolean().optional(), filter: z.string().optional(), combine: z.boolean().optional(), categoricalConfidenceType: z.string().optional(), categoricalTopN: z.number().int().optional(), categoricalBottomN: z.number().int().optional(), categoricalMaxConfidence: z.number().optional(), categoricalMaxFrequencyPercentile: z.number().optional(), categoricalMinFrequency: z.number().int().optional(), categoricalMinVariance: z.number().optional(), categoricalMaxCategoryN: z.number().int().optional(), categoricalParallel: z.boolean().optional(), categoricalAlgorithm: z.string().optional(), categoricalAlgorithmParameters: z.record(z.string(), z.string()).optional(), categoricalAlgorithmParametersString: z.string().optional()})).optional()})]);

export const shapeColumnSettingsDTO = z.object({enabled: z.boolean().optional(), name: z.string().optional(), type: z.string().optional()});

export const shapeDefDTO = z.object({enabled: z.boolean().optional(), totalScore: z.number().int().optional(), sensitivity: z.number().optional(), maxPerCol: z.number().int().optional(), maxColSize: z.number().int().optional(), granular: z.boolean().optional(), columnSettings: z.array(shapeColumnSettingsDTO).optional()});

export const identifier = z.object({id: z.number().int().optional(), uuid: z.string().optional()});

export const abstractJobScheduleConfigurationDTO = z.object({enabled: z.boolean().optional(), dataset: z.string(), agentId: identifier, runDateFormat: z.string(), scheduleFrequency: z.enum(["DAILY", "MONTHLY", "QUARTERLY"]).optional(), updatedAt: z.string().optional(), scheduleTime: z.string().optional()});

export const datasetDefDTO = z.object({dataset: z.string().optional(), runId: z.string().optional(), runIdEnd: z.string().optional(), runState: z.enum(["DRAFT", "DRY", "CERTIFIED"]).optional(), passFail: z.number().int().optional(), passFailLimit: z.number().int().optional(), jobDescription: z.string().optional(), jobId: identifierDTO.optional(), coreMaxActiveConnections: z.number().int().optional(), linkId: z.array(z.string()).optional(), licenseKey: z.string().optional(), logFile: z.string().optional(), logLevel: z.string().optional(), hootOnly: z.boolean().optional(), prettyPrint: z.boolean().optional(), useTemplate: z.boolean().optional(), parallel: z.boolean().optional(), plan: z.boolean().optional(), dataPreviewOff: z.boolean().optional(), datasetSafeOff: z.boolean().optional(), obslimit: z.number().int().optional(), pgUser: z.string().optional(), pgPassword: z.string().optional(), host: z.string().optional(), port: z.string().optional(), user: z.string().optional(), alertEmail: z.string().optional(), scheduleTime: z.string().optional(), schemaScore: z.number().optional(), optionAppend: z.string().optional(), keyDelimiter: z.string().optional(), agentId: identifierDTO.optional(), load: loadDefDTO.optional(), pushdown: pushDownDefDTO.optional(), outliers: z.array(outlierDefDTO).optional(), patterns: z.array(patternDefDTO).optional(), dupe: dupeDefDTO.optional(), profile: profileDefDTO.optional(), source: sourceDefDTO.optional(), rule: ruleDefDTO.optional(), colMatch: colMatchDefDTO.optional(), spark: sparkDefDTO.optional(), env: envDefDTO.optional(), record: recordDefDTO.optional(), stream: streamDefDTO.optional(), pipeline: z.array(z.string()).optional(), runDate: z.string().optional(), runDateEnd: z.string().optional(), metaTags: z.array(z.string()).optional(), outlierConfiguration: abstractOutlierDefDTO.optional(), shape: shapeDefDTO.optional(), jobSchedule: abstractJobScheduleConfigurationDTO.optional(), connectionType: z.enum(["PULLUP", "PUSHDOWN"]).optional()});

export const validateResponse = async <Output>(
  schema: {
    readonly '~standard': {
      readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray<PropertyKey | { readonly key: PropertyKey }> | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray<PropertyKey | { readonly key: PropertyKey }> | undefined }> }>
    }
  },
  value: unknown
): Promise<Output> => {
  const result = await schema['~standard'].validate(value)

  if (result.issues !== undefined) {
    const detail = result.issues
      .map(({ message, path }) =>
        path?.length
          ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}`
          : message
      )
      .join('; ')

    throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } })
  }

  return result.value
};

/**
 * Create or update the dataset definition of a DQ Job
 * 
 * POST /v3/datasetDefs
 * 
 * Also accepts auth: jwtAuth, oauth2, oauth2
 */
export const createApiV3DatasetDefs = async (args: CreateApiV3DatasetDefsArgs, options: CreateApiV3DatasetDefsOptions) => {
const {baseUrl, fetch: fetchFn = fetch, basicAuth} = options ?? {}

const query = new URLSearchParams()
const headers = new Headers()
headers.set('content-type', "application/json")
headers.set('accept', "application/json")

if (basicAuth !== undefined) headers.set('authorization', `Basic ${btoa(`${basicAuth.username}:${basicAuth.password}`)}`)

const url = `${baseUrl.replace(/\/+$/, '')}/v3/datasetDefs${query.size > 0 ? `?${query}` : ''}`

const res = await fetchFn(url, {method: "POST", headers, body: JSON.stringify(args.body)})

if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}})

return validateResponse(datasetDefDTO, await res.json())
};