import {z} from 'zod' export type FetchStealthMode = 'none' | 'auto' | 'ultra'; export type FetchOutputMarkdown = {type: 'markdown'}; export type FetchOutputHtml = {type: 'html'}; export type FetchOutputLinks = {type: 'links'}; export type FetchScreenshotFormat = 'jpeg' | 'png' | 'webp'; export type FetchOutputScreenshot = {fullPage?: boolean | undefined, format?: FetchScreenshotFormat | undefined, cropToContent?: boolean | undefined, cropToContentMaxHeight?: number | undefined, cropToContentMinHeight?: number | undefined, type: 'screenshot'}; export type FetchOutputJson = {schema?: Record | undefined, prompt?: string | undefined, type: 'json'}; export type FetchOutputBranding = {type: 'branding'}; export type FetchSanitizeMode = 'none' | 'basic' | 'advanced'; export type FetchStorageStateOptions = {localStorage?: {[key: string]: string} | undefined, sessionStorage?: {[key: string]: string} | undefined}; export type FetchOutputOptions = {formats?: (Array) | undefined, sanitize?: FetchSanitizeMode | undefined, includeSelectors?: Array | undefined, excludeSelectors?: Array | undefined, storageState?: FetchStorageStateOptions | undefined}; export type ScreenConfig = {width?: number | undefined, height?: number | undefined}; export type FetchBrowserLocationOptions = {country?: string | undefined, state?: string | undefined, city?: string | undefined}; export type FetchBrowserOptions = {screen?: ScreenConfig | undefined, profileId?: string | undefined, solveCaptchas?: string | undefined, location?: FetchBrowserLocationOptions | undefined}; export type FetchWaitUntil = 'load' | 'domcontentloaded' | 'networkidle'; export type FetchNavigationOptions = {waitUntil?: FetchWaitUntil | undefined, timeoutMs?: number | undefined, waitFor?: number | undefined}; export type FetchCacheOptions = {maxAgeSeconds?: number | undefined}; export type WebCrawlOptions = {maxPages?: number | undefined, followLinks?: boolean | undefined, ignoreSitemap?: boolean | undefined, excludePatterns?: Array | undefined, includePatterns?: Array | undefined}; export type StartWebCrawlJobParams = {url: string, stealth?: FetchStealthMode | undefined, outputs?: FetchOutputOptions | undefined, browser?: FetchBrowserOptions | undefined, navigation?: FetchNavigationOptions | undefined, cache?: FetchCacheOptions | undefined, crawlOptions?: WebCrawlOptions | undefined}; export type CreateApiApiWebCrawlArgs = {body: StartWebCrawlJobParams}; export type CreateApiApiWebCrawlOptions = {baseUrl?: string | undefined, ApiKeyAuth?: string | undefined, fetch?: typeof fetch | undefined}; export const createApiApiWebCrawlResponse = z.object({jobId: z.string()}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * Start a web crawl job * * POST /api/web/crawl */ export const createApiApiWebCrawl = async (args: CreateApiApiWebCrawlArgs, options?: CreateApiApiWebCrawlOptions) => { const {baseUrl = "https://api.hyperbrowser.ai", fetch: fetchFn = fetch, ApiKeyAuth} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('content-type', "application/json") headers.set('accept', "application/json") if (ApiKeyAuth !== undefined) headers.set("x-api-key", ApiKeyAuth) const url = `${baseUrl.replace(/\/+$/, '')}/api/web/crawl${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "POST", headers, body: JSON.stringify(args.body)}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(createApiApiWebCrawlResponse, await res.json()) };