import {z} from 'zod' export type AddObjectsToBatchRequest = {object_ids: Array}; export type CreateApiV1BucketsBucketIdentifierBatchesBatchIdObjectsArgs = {bucket_identifier: string, batch_id: string, skip_validation?: boolean | undefined, body: AddObjectsToBatchRequest}; export type CreateApiV1BucketsBucketIdentifierBatchesBatchIdObjectsOptions = {baseUrl?: string | undefined, BearerAuth?: string | undefined, NamespaceHeader?: string | undefined, fetch?: typeof fetch | undefined}; export const taskStatusEnum = z.enum(["PENDING", "QUEUED", "IN_PROGRESS", "PROCESSING", "COMPLETED", "COMPLETED_WITH_ERRORS", "FAILED", "CANCELED", "INTERRUPTED", "UNKNOWN", "SKIPPED", "DRAFT", "ACTIVE", "ARCHIVED", "SUSPENDED", "DEACTIVATED"]); export const dedupStrategy = z.enum(["skip", "replace", "force"]); export const failureCategory = z.enum(["timeout", "infrastructure", "orphaned", "pipeline", "mvs_durability", "validation", "unknown"]); export const failedObjectRecord = z.object({object_id: z.string(), error: z.string(), error_type: z.enum(["transient", "permanent", "resource"]), timestamp: z.string()}); export const batchType = z.enum(["BUCKET", "COLLECTION"]); export const batchMetadata = z.object({campaign_id: z.string().nullable().optional(), source: z.string().nullable().optional(), tags: z.array(z.string()).nullable().optional(), notes: z.string().nullable().optional()}).and(z.record(z.string(), z.unknown())); export const errorCategory = z.enum(["dependency", "authentication", "validation", "runtime", "network", "resource"]); export const batchErrorDetail = z.object({error_type: errorCategory, message: z.string(), component: z.string().nullable().optional(), stage: z.string().nullable().optional(), traceback: z.string().nullable().optional(), timestamp: z.string().optional(), affected_document_ids: z.array(z.string()).optional(), affected_count: z.number().int().gte(1).optional(), recovery_suggestion: z.string().nullable().optional(), metadata: z.record(z.string(), z.unknown()).optional()}); export const submissionParams = z.object({entrypoint: z.string().nullable().optional(), deployment_mode: z.string().nullable().optional(), requires_gpu: z.boolean().nullable().optional(), num_cpus: z.number().nullable().optional(), num_gpus: z.number().nullable().optional(), memory_bytes: z.number().int().nullable().optional(), priority: z.number().int().nullable().optional(), plugin_archives: z.array(z.string()).nullable().optional(), plugin_dependencies: z.array(z.string()).nullable().optional(), image_uri: z.string().nullable().optional(), extractor_name: z.string().nullable().optional(), extractor_version: z.string().nullable().optional(), env_vars_keys: z.array(z.string()).nullable().optional(), manifest_key: z.string().nullable().optional(), submitted_at: z.string().nullable().optional()}); export const extractorJobInfo = z.object({rows_duplicate_id: z.number().int().nullable().optional(), rows_collapsed: z.number().int().nullable().optional(), first_duplicate_key: z.string().nullable().optional(), rows_skipped_by_stages: z.number().int().nullable().optional(), first_stage_error: z.string().nullable().optional(), extractor_type: z.string(), collection_ids: z.array(z.string()).optional(), extractor_id: z.string().nullable().optional(), ray_job_id: z.string().nullable().optional(), celery_task_id: z.string().nullable().optional(), callback_job_id: z.string().nullable().optional(), execution_mode: z.string().nullable().optional(), status: taskStatusEnum.optional(), started_at: z.string().nullable().optional(), completed_at: z.string().nullable().optional(), duration_ms: z.number().nullable().optional(), documents_written: z.number().int().nullable().optional(), warnings: z.array(z.string()).optional(), pages_dropped: z.number().int().nullable().optional(), pages_dropped_reasons: z.record(z.string(), z.number().int()).nullable().optional(), segments_dropped: z.number().int().nullable().optional(), segments_dropped_reasons: z.record(z.string(), z.number().int()).nullable().optional(), documents_skipped: z.number().int().nullable().optional(), errors: z.array(batchErrorDetail).optional(), error: z.string().nullable().optional(), last_activity_at: z.string().nullable().optional(), ray_job_status: z.string().nullable().optional(), submission_params: submissionParams.nullable().optional()}); export const infraEventType = z.enum(["oom", "preemption", "node_failure", "ray_bug", "rolling_update", "unknown"]); export const infrastructureDetail = z.object({event_type: infraEventType, detected_at: z.string().nullable().optional(), raw_signal: z.string().nullable().optional(), node_id: z.string().nullable().optional(), pod_name: z.string().nullable().optional()}); export const tierAuditInfo = z.object({tier_num: z.number().int(), submitted: z.number().int().nullable().optional(), processed: z.number().int().optional(), failed: z.number().int().optional(), skipped: z.number().int().optional(), lost: z.number().int().optional(), balanced: z.boolean().optional(), notes: z.array(z.string()).optional(), durability: z.enum(["confirmed", "lagging", "untracked", "frontier_unknown"]).nullable().optional()}); export const tierTaskInfo = z.object({scoped_sources: z.number().int().nullable().optional(), skipped_collection_reasons: z.record(z.string(), z.string()).nullable().optional(), replace_cleanup_skipped_collections: z.array(z.string()).nullable().optional(), tier_num: z.number().int().gte(0), task_id: z.string().nullable().optional(), status: taskStatusEnum.optional(), collection_ids: z.array(z.string()).optional(), extractor_jobs: z.array(extractorJobInfo).optional(), source_type: z.string(), source_collection_ids: z.array(z.string()).nullable().optional(), parent_task_id: z.string().nullable().optional(), started_at: z.string().nullable().optional(), completed_at: z.string().nullable().optional(), duration_ms: z.number().nullable().optional(), errors: z.array(batchErrorDetail).optional(), error_summary: z.record(z.string(), z.number().int()).nullable().optional(), performance: z.record(z.string(), z.unknown()).nullable().optional(), ray_job_id: z.string().nullable().optional(), ray_job_started_at: z.string().nullable().optional(), ray_job_completed_at: z.string().nullable().optional(), requires_gpu: z.boolean().nullable().optional(), worker_groups: z.array(z.string()).nullable().optional(), celery_task_id: z.string().nullable().optional(), source_documents_fetched: z.number().int().nullable().optional(), documents_after_source_filter: z.number().int().nullable().optional(), documents_missing_input_fields: z.number().int().nullable().optional(), documents_submitted_to_engine: z.number().int().nullable().optional(), documents_written: z.number().int().nullable().optional(), documents_replaced: z.number().int().nullable().optional(), documents_deleted: z.number().int().nullable().optional(), documents_before_processing: z.number().int().nullable().optional(), last_activity_at: z.string().nullable().optional(), ray_job_status: z.string().nullable().optional(), ray_job_logs: z.string().nullable().optional(), ray_job_logs_captured_at: z.string().nullable().optional(), submission_params: submissionParams.nullable().optional(), infrastructure_events: z.array(infrastructureDetail).optional(), audit: tierAuditInfo.nullable().optional(), audit_override_reason: z.string().nullable().optional()}); export const batchStageInfo = z.object({name: z.string(), index: z.number().int(), total: z.number().int(), stage_elapsed_seconds: z.number().optional(), sub_stage: z.string().nullable().optional(), sub_stage_elapsed_seconds: z.number().nullable().optional()}); export const batchPhaseDetail = z.object({processed: z.number().int().optional(), total: z.number().int().nullable().optional(), status: z.enum(["pending", "active", "done", "error"]).optional()}); export const batchPhases = z.object({dispatch: batchPhaseDetail.optional(), extraction: batchPhaseDetail.optional(), write: batchPhaseDetail.optional()}); export const batchStepInfo = z.object({name: z.string(), index: z.number().int(), total: z.number().int(), step_elapsed_seconds: z.number()}); export const batchStageTiming = z.object({name: z.string(), index: z.number().int(), total: z.number().int(), started_at: z.number(), ended_at: z.number(), duration_seconds: z.number()}); export const batchChunkStats = z.object({unit: z.string(), total_chunks: z.number().int().optional(), mean_units_per_chunk: z.number().nullable().optional()}); export const batchExtractorProgress = z.object({processed: z.number().int().optional(), seconds_since_update: z.number().nullable().optional()}); export const batchProgress = z.object({processed: z.number().int().optional(), total: z.number().int().nullable().optional(), percent: z.number().nullable().optional(), items_per_second: z.number().nullable().optional(), eta_seconds: z.number().nullable().optional(), batch_count: z.number().int().optional(), errors: z.number().int().optional(), first_error: z.string().nullable().optional(), extraction_first_error: z.string().nullable().optional(), documents_skipped: z.number().int().optional(), objects_extracted: z.number().int().nullable().optional(), objects_skipped: z.number().int().nullable().optional(), objects_failed: z.number().int().nullable().optional(), current_stage: batchStageInfo.nullable().optional(), phases: batchPhases.nullable().optional(), active_step: batchStepInfo.nullable().optional(), overshoot_percent: z.number().nullable().optional(), queue_position: z.number().int().nullable().optional(), queue_reason: z.string().nullable().optional(), stage_history: z.array(batchStageTiming).nullable().optional(), documents_written: z.number().int().nullable().optional(), chunk_stats: batchChunkStats.nullable().optional(), first_record_at: z.number().nullable().optional(), seconds_to_first_record: z.number().nullable().optional(), extractors: z.record(z.string(), batchExtractorProgress).nullable().optional(), status_warnings: z.array(z.string()).optional()}); export const batchCost = z.object({credits_consumed: z.number().int().gte(0).optional(), cost_usd: z.number().gte(0).optional(), credit_rate_usd: z.number().optional(), metered_model_usd: z.number().gte(0).nullable().optional()}); export const batchModel = z.object({batch_id: z.string().optional(), bucket_id: z.string(), namespace_id: z.string().nullable().optional(), batch_name: z.string().max(256).nullable().optional(), status: taskStatusEnum.optional(), queue_context: z.record(z.string(), z.unknown()).nullable().optional(), object_ids: z.array(z.string()).optional(), dedup_strategy: dedupStrategy.optional(), effective_dedup_strategy: dedupStrategy.nullable().optional(), submitted_by_key_id: z.string().nullable().optional(), submitted_by_key_prefix: z.string().nullable().optional(), submitted_by_is_internal: z.boolean().nullable().optional(), dedup_audit: z.record(z.string(), z.unknown()).nullable().optional(), collection_ids: z.array(z.string()).nullable().optional(), error: z.string().nullable().optional(), failure_reason: z.string().nullable().optional(), error_summary: z.record(z.string(), z.number().int()).nullable().optional(), failure_category: failureCategory.nullable().optional(), failed_objects: z.array(failedObjectRecord).optional(), failed_object_count: z.number().int().gte(0).optional(), partial_objects: z.array(z.record(z.string(), z.unknown())).nullable().optional(), partial_object_count: z.number().int().gte(0).nullable().optional(), unaccounted_object_count: z.number().int().gte(0).nullable().optional(), type: batchType.optional(), manifest_key: z.string().nullable().optional(), task_id: z.string().nullable().optional(), loaded_object_ids: z.array(z.string()).nullable().optional(), internal_metadata: z.record(z.string(), z.unknown()).nullable().optional(), metadata: batchMetadata.optional(), tier_tasks: z.array(tierTaskInfo).optional(), current_tier: z.number().int().gte(0).nullable().optional(), total_tiers: z.number().int().gte(1).optional(), dag_tiers: z.array(z.array(z.string())).nullable().optional(), created_at: z.string().optional(), progress: batchProgress.nullable().optional(), documents_written: z.number().int().nullable().optional(), documents_replaced: z.number().int().nullable().optional(), documents_deleted: z.number().int().nullable().optional(), pages_dropped: z.number().int().nullable().optional(), pages_dropped_reasons: z.record(z.string(), z.number().int()).nullable().optional(), segments_dropped: z.number().int().nullable().optional(), segments_dropped_reasons: z.record(z.string(), z.number().int()).nullable().optional(), status_diagnostics: z.record(z.string(), z.unknown()).optional(), documents_resolvable: z.boolean().optional(), documents_resolvable_error: z.string().nullable().optional(), health: z.string().nullable().optional(), cost: batchCost.nullable().optional(), last_activity_at: z.string().nullable().optional(), retry_count: z.number().int().gte(0).optional(), max_retries: z.number().int().gte(0).optional(), last_retry_at: z.string().nullable().optional(), retry_reason: z.string().nullable().optional(), node_death_resubmit_count: z.number().int().gte(0).optional(), deploy_resubmit_count: z.number().int().gte(0).optional(), quota_block_first_seen_at: z.string().nullable().optional(), webhook_url: z.string().nullable().optional(), updated_at: z.string().optional(), status_message: z.string().nullable().optional(), estimated_completion: z.string().nullable().optional(), max_node_death_resubmits: z.number().int(), max_deploy_resubmits: z.number().int(), lifecycle: z.string().nullable(), lifecycle_reason: z.string().nullable()}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * Add Objects to Batch * * POST /v1/buckets/{bucket_identifier}/batches/{batch_id}/objects */ export const createApiV1BucketsBucketIdentifierBatchesBatchIdObjects = async (args: CreateApiV1BucketsBucketIdentifierBatchesBatchIdObjectsArgs, options?: CreateApiV1BucketsBucketIdentifierBatchesBatchIdObjectsOptions) => { const {baseUrl = "https://api.mixpeek.com", fetch: fetchFn = fetch, BearerAuth, NamespaceHeader} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('content-type', "application/json") headers.set('accept', "application/json") if (args.skip_validation != null) query.append("skip_validation", String(args.skip_validation)) if (BearerAuth !== undefined) headers.set('authorization', `Bearer ${BearerAuth}`) if (NamespaceHeader !== undefined) headers.set("X-Namespace", NamespaceHeader) const url = `${baseUrl.replace(/\/+$/, '')}/v1/buckets/${encodeURIComponent(String(args.bucket_identifier))}/batches/${encodeURIComponent(String(args.batch_id))}/objects${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "POST", headers, body: JSON.stringify(args.body)}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(batchModel, await res.json()) };