import {z} from 'zod' export type LeaseWorkItemRequest = {currentWorkerTime?: unknown | undefined, location?: string | undefined, projectNumber?: unknown | undefined, requestedLeaseDuration?: unknown | undefined, unifiedWorkerRequest?: {[key: string]: unknown} | undefined, workItemTypes?: Array | undefined, workerCapabilities?: Array | undefined, workerId?: string | undefined}; export type CreateApiV1b3ProjectsProjectIdJobsJobIdWorkItemsLeaseArgs = {projectId: string, jobId: string, body: LeaseWorkItemRequest}; export type CreateApiV1b3ProjectsProjectIdJobsJobIdWorkItemsLeaseOptions = {baseUrl?: string | undefined, Oauth2?: string | undefined, Oauth2c?: string | undefined, fetch?: typeof fetch | undefined}; export const instructionInput = z.object({outputNum: z.number().int().optional(), producerInstructionIndex: z.number().int().optional()}); export const flattenInstruction = z.object({inputs: z.array(instructionInput).optional()}); export const instructionOutput = z.object({codec: z.record(z.string(), z.unknown()).optional(), name: z.string().optional(), onlyCountKeyBytes: z.boolean().optional(), onlyCountValueBytes: z.boolean().optional(), originalName: z.string().optional(), systemName: z.string().optional()}); export const multiOutputInfo = z.object({tag: z.string().optional()}); export const sourceMetadata = z.object({estimatedSizeBytes: z.string().optional(), infinite: z.boolean().optional(), producesSortedKeys: z.boolean().optional()}); export const source = z.object({baseSpecs: z.array(z.record(z.string(), z.unknown())).optional(), codec: z.record(z.string(), z.unknown()).optional(), doesNotNeedSplitting: z.boolean().optional(), metadata: sourceMetadata.optional(), spec: z.record(z.string(), z.unknown()).optional()}); export const sideInputInfo = z.object({kind: z.record(z.string(), z.unknown()).optional(), sources: z.array(source).optional(), tag: z.string().optional()}); export const parDoInstruction = z.object({input: instructionInput.optional(), multiOutputInfos: z.array(multiOutputInfo).optional(), numOutputs: z.number().int().optional(), sideInputs: z.array(sideInputInfo).optional(), userFn: z.record(z.string(), z.unknown()).optional()}); export const partialGroupByKeyInstruction = z.object({input: instructionInput.optional(), inputElementCodec: z.record(z.string(), z.unknown()).optional(), originalCombineValuesInputStoreName: z.string().optional(), originalCombineValuesStepName: z.string().optional(), sideInputs: z.array(sideInputInfo).optional(), valueCombiningFn: z.record(z.string(), z.unknown()).optional()}); export const readInstruction = z.object({source: source.optional()}); export const sink = z.object({codec: z.record(z.string(), z.unknown()).optional(), spec: z.record(z.string(), z.unknown()).optional()}); export const writeInstruction = z.object({input: instructionInput.optional(), sink: sink.optional()}); export const parallelInstruction = z.object({flatten: flattenInstruction.optional(), name: z.string().optional(), originalName: z.string().optional(), outputs: z.array(instructionOutput).optional(), parDo: parDoInstruction.optional(), partialGroupByKey: partialGroupByKeyInstruction.optional(), read: readInstruction.optional(), systemName: z.string().optional(), write: writeInstruction.optional()}); export const mapTask = z.object({counterPrefix: z.string().optional(), instructions: z.array(parallelInstruction).optional(), stageName: z.string().optional(), systemName: z.string().optional()}); export const package = z.object({location: z.string().optional(), name: z.string().optional(), sha256: z.string().optional()}); export const seqMapTaskOutputInfo = z.object({sink: sink.optional(), tag: z.string().optional()}); export const seqMapTask = z.object({inputs: z.array(sideInputInfo).optional(), name: z.string().optional(), outputInfos: z.array(seqMapTaskOutputInfo).optional(), stageName: z.string().optional(), systemName: z.string().optional(), userFn: z.record(z.string(), z.unknown()).optional()}); export const shellTask = z.object({command: z.string().optional(), exitCode: z.number().int().optional()}); export const sourceGetMetadataRequest = z.object({source: source.optional()}); export const sourceSplitOptions = z.object({desiredBundleSizeBytes: z.string().optional(), desiredShardSizeBytes: z.string().optional()}); export const sourceSplitRequest = z.object({options: sourceSplitOptions.optional(), source: source.optional()}); export const sourceOperationRequest = z.object({getMetadata: sourceGetMetadataRequest.optional(), name: z.string().optional(), originalName: z.string().optional(), split: sourceSplitRequest.optional(), stageName: z.string().optional(), systemName: z.string().optional()}); export const keyRangeDataDiskAssignment = z.object({dataDisk: z.string().optional(), end: z.string().optional(), start: z.string().optional()}); export const streamingComputationRanges = z.object({computationId: z.string().optional(), rangeAssignments: z.array(keyRangeDataDiskAssignment).optional()}); export const mountedDataDisk = z.object({dataDisk: z.string().optional()}); export const streamingComputationTask = z.object({computationRanges: z.array(streamingComputationRanges).optional(), dataDisks: z.array(mountedDataDisk).optional(), taskType: z.enum(["STREAMING_COMPUTATION_TASK_UNKNOWN", "STREAMING_COMPUTATION_TASK_STOP", "STREAMING_COMPUTATION_TASK_START"]).optional()}); export const streamingOperationalLimits = z.object({maxBagElementBytes: z.string().optional(), maxGlobalDataBytes: z.string().optional(), maxKeyBytes: z.string().optional(), maxProductionOutputBytes: z.string().optional(), maxSortedListElementBytes: z.string().optional(), maxSourceStateBytes: z.string().optional(), maxTagBytes: z.string().optional(), maxValueBytes: z.string().optional()}); export const streamingComputationConfig = z.object({computationId: z.string().optional(), instructions: z.array(parallelInstruction).optional(), stageName: z.string().optional(), systemName: z.string().optional(), transformUserNameToStateFamily: z.record(z.string(), z.string()).optional()}); export const streamingConfigTask = z.object({commitStreamChunkSizeBytes: z.string().optional(), getDataStreamChunkSizeBytes: z.string().optional(), maxWorkItemCommitBytes: z.string().optional(), operationalLimits: streamingOperationalLimits.optional(), streamingComputationConfigs: z.array(streamingComputationConfig).optional(), streamingEngineStateTagEncodingVersion: z.number().int().optional(), userStepToStateFamilyNameMap: z.record(z.string(), z.string()).optional(), userWorkerRunnerV1Settings: z.string().optional(), userWorkerRunnerV2Settings: z.string().optional(), windmillServiceEndpoint: z.string().optional(), windmillServicePort: z.string().optional()}); export const streamingApplianceSnapshotConfig = z.object({importStateEndpoint: z.string().optional(), snapshotId: z.string().optional()}); export const customSourceLocation = z.object({stateful: z.boolean().optional()}); export const pubsubLocation = z.object({dropLateData: z.boolean().optional(), dynamicDestinations: z.boolean().optional(), idLabel: z.string().optional(), subscription: z.string().optional(), timestampLabel: z.string().optional(), topic: z.string().optional(), trackingSubscription: z.string().optional(), withAttributes: z.boolean().optional()}); export const streamingSideInputLocation = z.object({stateFamily: z.string().optional(), tag: z.string().optional()}); export const streamingStageLocation = z.object({streamId: z.string().optional()}); export const streamLocation = z.object({customSourceLocation: customSourceLocation.optional(), pubsubLocation: pubsubLocation.optional(), sideInputLocation: streamingSideInputLocation.optional(), streamingStageLocation: streamingStageLocation.optional()}); export const keyRangeLocation = z.object({dataDisk: z.string().optional(), deliveryEndpoint: z.string().optional(), deprecatedPersistentDirectory: z.string().optional(), end: z.string().optional(), start: z.string().optional()}); export const stateFamilyConfig = z.object({isRead: z.boolean().optional(), stateFamily: z.string().optional()}); export const computationTopology = z.object({computationId: z.string().optional(), inputs: z.array(streamLocation).optional(), keyRanges: z.array(keyRangeLocation).optional(), outputs: z.array(streamLocation).optional(), stateFamilies: z.array(stateFamilyConfig).optional(), systemStageName: z.string().optional()}); export const dataDiskAssignment = z.object({dataDisks: z.array(z.string()).optional(), vmInstance: z.string().optional()}); export const topologyConfig = z.object({computations: z.array(computationTopology).optional(), dataDiskAssignments: z.array(dataDiskAssignment).optional(), forwardingKeyBits: z.number().int().optional(), persistentStateVersion: z.number().int().optional(), userStageToComputationNameMap: z.record(z.string(), z.string()).optional()}); export const streamingSetupTask = z.object({drain: z.boolean().optional(), receiveWorkPort: z.number().int().optional(), snapshotConfig: streamingApplianceSnapshotConfig.optional(), streamingComputationTopology: topologyConfig.optional(), workerHarnessPort: z.number().int().optional()}); export const workItem = z.object({configuration: z.string().optional(), id: z.string().optional(), initialReportIndex: z.string().optional(), jobId: z.string().optional(), leaseExpireTime: z.string().optional(), mapTask: mapTask.optional(), packages: z.array(package).optional(), projectId: z.string().optional(), reportStatusInterval: z.string().optional(), seqMapTask: seqMapTask.optional(), shellTask: shellTask.optional(), sourceOperationTask: sourceOperationRequest.optional(), streamingComputationTask: streamingComputationTask.optional(), streamingConfigTask: streamingConfigTask.optional(), streamingSetupTask: streamingSetupTask.optional()}); export const leaseWorkItemResponse = z.object({unifiedWorkerResponse: z.record(z.string(), z.unknown()).optional(), workItems: z.array(workItem).optional()}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * Leases a dataflow WorkItem to run. * * POST /v1b3/projects/{projectId}/jobs/{jobId}/workItems:lease * * Also accepts auth: Oauth2 + Oauth2c */ export const createApiV1b3ProjectsProjectIdJobsJobIdWorkItemsLease = async (args: CreateApiV1b3ProjectsProjectIdJobsJobIdWorkItemsLeaseArgs, options?: CreateApiV1b3ProjectsProjectIdJobsJobIdWorkItemsLeaseOptions) => { const {baseUrl = "https://dataflow.googleapis.com", fetch: fetchFn = fetch, Oauth2, Oauth2c} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('content-type', "application/json") headers.set('accept', "application/json") if (Oauth2 !== undefined) headers.set('authorization', `Bearer ${Oauth2}`) if (Oauth2c !== undefined) headers.set('authorization', `Bearer ${Oauth2c}`) const url = `${baseUrl.replace(/\/+$/, '')}/v1b3/projects/${encodeURIComponent(String(args.projectId))}/jobs/${encodeURIComponent(String(args.jobId))}/workItems:lease${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "POST", headers, body: JSON.stringify(args.body)}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(leaseWorkItemResponse, await res.json()) };