export type GetApiContextDevWebScrapeSitemapOptions = {baseUrl?: string | undefined, fetch?: typeof fetch | undefined};

/**
 * Crawls the sitemap of the given domain and returns all discovered page URLs. Supports sitemap index files (recursive), parallel fetching with concurrency control, deduplication, and filters out non-page resources (images, PDFs, etc.).
 * 
 * GET /context-dev/web/scrape/sitemap
 */
export const getApiContextDevWebScrapeSitemap = async (options?: GetApiContextDevWebScrapeSitemapOptions) => {
const {baseUrl = "https://mpp.orthogonal.com", fetch: fetchFn = fetch} = options ?? {}

const query = new URLSearchParams()
const headers = new Headers()

const url = `${baseUrl.replace(/\/+$/, '')}/context-dev/web/scrape/sitemap${query.size > 0 ? `?${query}` : ''}`

const res = await fetchFn(url, {method: "GET", headers})

if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}})

return undefined
};