import {z} from 'zod' export type DashboardWidgetType = 'LINE' | 'AREA' | 'BAR' | 'STACKED_BAR' | 'TABLE' | 'BIG_NUMBER'; export type DashboardWidgetUnit = 'COUNT' | 'PERCENT' | 'SCORE' | 'SECONDS' | 'USD' | 'MILLISECONDS'; export type DashboardWidgetMode = 'TIME_SERIES' | 'DIMENSION_SERIES'; export type DashboardWidgetBucketMode = 'SERIES' | 'RANGE'; export type DashboardDimension = 'project' | 'trace_name' | 'span_name' | 'model' | 'type' | 'thread_id' | 'test_case_id' | 'test_run_id' | 'end_user' | 'source' | 'annotator' | 'name' | 'error' | 'prompt_alias' | 'tag' | 'label' | 'evaluation_model' | 'prompt_version' | 'prompt_label' | 'prompt_commit_hash' | 'metadata' | 'classifier'; export type DashboardTopKOrderBy = 'count' | 'avg_latency' | 'p50_latency' | 'p90_latency' | 'p99_latency' | 'error_rate' | 'pass_rate' | 'failure_rate' | 'input_cost' | 'output_cost' | 'total_cost' | 'avg_cost' | 'input_tokens' | 'output_tokens' | 'total_tokens' | 'count_distinct_endUserId' | 'count_distinct_threadId' | 'count_distinct_model' | 'count_distinct_projectId' | 'count_distinct_error' | 'count_distinct_metadata' | 'error_count' | 'avg_score' | 'stddev_score' | 'avg_rating' | 'created_at' | 'start_time' | 'dimension'; export type DashboardTopKDirection = 'asc' | 'desc'; export type DashboardTopK = {limit?: number | undefined, orderBy?: DashboardTopKOrderBy | undefined, direction?: DashboardTopKDirection | undefined}; export type DashboardLayout = {x?: number | undefined, y?: number | undefined, w?: number | undefined, h?: number | undefined}; export type DashboardLineColor = 'AMBER' | 'VIOLET' | 'EMERALD' | 'BLUE' | 'PINK' | 'CYAN' | 'ROSE' | 'LIME' | 'TEAL' | 'ORANGE'; export type DashboardDataModel = 'TRACE' | 'SPAN' | 'LLM_SPAN' | 'AGENT_SPAN' | 'RETRIEVER_SPAN' | 'TOOL_SPAN' | 'CUSTOM_SPAN' | 'THREAD' | 'END_USER' | 'METRIC_DATA' | 'ANNOTATION'; export type DashboardAggregation = 'COUNT' | 'ERROR_RATE' | 'PASS_RATE' | 'UNIQUE_END_USERS' | 'UNIQUE_THREADS' | 'UNIQUE_USERS' | 'UNIQUE_METADATA_VALUES' | 'AVG_LATENCY' | 'P50_LATENCY' | 'P90_LATENCY' | 'P99_LATENCY' | 'TOTAL_COST' | 'AVG_COST' | 'INPUT_COST' | 'OUTPUT_COST' | 'AVG_COST_PER_USER' | 'INPUT_TOKENS' | 'OUTPUT_TOKENS' | 'TOTAL_TOKENS' | 'ERROR_COUNT' | 'NEW_USERS' | 'RETENTION' | 'AVG_SCORE' | 'FAILURE_RATE' | 'AVG_RATING'; export type DashboardFilterOperator = 'AND' | 'OR'; export type DashboardFilterCategory = string; export type DashboardFilterCondition = 'Is' | 'Is not' | 'Is equal to' | 'Does not equal' | 'Is less than' | 'Is equal or less than' | 'Is greater than' | 'Is equal or greater than' | 'Has' | 'Has not' | 'Contains' | 'Contains only' | 'Does not contain' | 'Has increased by more than' | 'Has increased by less than' | 'Has decreased by more than' | 'Has decreased by less than' | 'Has changed from'; export type DashboardFilterRow = {category: DashboardFilterCategory, condition: DashboardFilterCondition, value: string | number | Array, key?: string | undefined}; export type DashboardFilterGroup = {operator: DashboardFilterOperator, filters: Array}; export type DashboardFilterSet = {operator: DashboardFilterOperator, groups: Array}; export type DashboardLineExtraQueryParams = {spanType?: ('LLM' | 'AGENT' | 'RETRIEVER' | 'TOOL' | 'CUSTOM') | undefined, metricMetadataKey?: string | undefined, category?: ('SINGLE_TURN' | 'MULTI_TURN' | 'TEST_RUN' | 'TRACE' | 'SPAN' | 'LLM_SPAN' | 'AGENT_SPAN' | 'RETRIEVER_SPAN' | 'TOOL_SPAN' | 'CUSTOM_SPAN' | 'THREAD') | undefined, metricName?: string | undefined, dataType?: ('Traces' | 'Spans' | 'Threads') | undefined, source?: ('User' | 'Reviewer') | undefined} | {[key: string]: string} | null; export type DashboardLineInput = {name: string, color?: DashboardLineColor | undefined, dataModel?: DashboardDataModel | undefined, aggregation?: DashboardAggregation | undefined, filters?: DashboardFilterSet | undefined, extraQueryParams?: DashboardLineExtraQueryParams | undefined}; export type DashboardWidgetInput = {name: string, description?: (string | null) | undefined, type?: DashboardWidgetType | undefined, unit?: DashboardWidgetUnit | undefined, mode?: DashboardWidgetMode | undefined, bucketMode?: DashboardWidgetBucketMode | undefined, dimension?: DashboardDimension | undefined, topK?: DashboardTopK | undefined, startTime?: (string | null) | undefined, endTime?: (string | null) | undefined, layout?: DashboardLayout | undefined, lines?: (Array | null) | undefined}; export type DashboardAggregateGranularity = 'thirty_minutes' | 'hour' | 'day' | 'week' | 'month'; export type WidgetQueryRequest = {widget: DashboardWidgetInput, startTime?: string | undefined, endTime?: string | undefined, granularity?: DashboardAggregateGranularity | undefined}; export type CreateApiV1WidgetsQueryArgs = {body: WidgetQueryRequest}; export type CreateApiV1WidgetsQueryOptions = {baseUrl?: string | undefined, fetch?: typeof fetch | undefined}; export const dashboardWidgetType = z.enum(["LINE", "AREA", "BAR", "STACKED_BAR", "TABLE", "BIG_NUMBER"]); export const dashboardWidgetMode = z.enum(["TIME_SERIES", "DIMENSION_SERIES"]); export const dashboardWidgetUnit = z.enum(["COUNT", "PERCENT", "SCORE", "SECONDS", "USD", "MILLISECONDS"]); export const dashboardSeriesPoint = z.object({x: z.string(), y: z.number().nullable()}); export const dashboardSeries = z.object({key: z.string(), name: z.string(), color: z.string(), lineId: z.string().optional(), points: z.array(dashboardSeriesPoint)}); export const dashboardScalarValue = z.object({key: z.string(), name: z.string(), color: z.string(), lineId: z.string().optional(), value: z.number().nullable()}); export const widgetQueryData = z.object({type: dashboardWidgetType.optional(), mode: dashboardWidgetMode.optional(), kind: z.enum(["TIME_SERIES", "DIMENSION", "BIG_NUMBER", "TABLE"]), unit: dashboardWidgetUnit.optional(), xAxis: z.object({type: z.enum(["time", "category"])}).optional(), series: z.array(dashboardSeries).optional(), values: z.array(dashboardScalarValue).optional(), columns: z.array(z.object({key: z.string(), label: z.string()})).optional(), rows: z.array(z.record(z.string(), z.unknown())).optional()}); export const widgetQueryResponse = z.object({success: z.boolean(), data: widgetQueryData, deprecated: z.boolean()}); export const validateResponse = async ( schema: { readonly '~standard': { readonly validate: (value: unknown) => ({ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }) | Promise<{ readonly value: Output; readonly issues?: undefined } | { readonly issues: ReadonlyArray<{ readonly message: string; readonly path?: ReadonlyArray | undefined }> }> } }, value: unknown ): Promise => { const result = await schema['~standard'].validate(value) if (result.issues !== undefined) { const detail = result.issues .map(({ message, path }) => path?.length ? `${path.map((segment) => String(typeof segment === 'object' ? segment.key : segment)).join('.')}: ${message}` : message ) .join('; ') throw new Error(`Response does not match the schema: ${detail}`, { cause: { issues: result.issues } }) } return result.value }; /** * Query Ad-hoc Widget Data * * POST /v1/widgets/query */ export const createApiV1WidgetsQuery = async (args: CreateApiV1WidgetsQueryArgs, options?: CreateApiV1WidgetsQueryOptions) => { const {baseUrl = "https://api.confident-ai.com", fetch: fetchFn = fetch} = options ?? {} const query = new URLSearchParams() const headers = new Headers() headers.set('content-type', "application/json") headers.set('accept', "application/json") const url = `${baseUrl.replace(/\/+$/, '')}/v1/widgets/query${query.size > 0 ? `?${query}` : ''}` const res = await fetchFn(url, {method: "POST", headers, body: JSON.stringify(args.body)}) if (!res.ok) throw new Error(`Request failed with status ${res.status}`, {cause: {status: res.status, body: await res.text()}}) return validateResponse(widgetQueryResponse, await res.json()) };