/** * KV-based live crawl progress. * * During a crawl, each crawled URL is appended to a KV key so the UI can * poll for a live feed of crawled pages (most recent first). * * The KV entry auto-expires after 30 minutes — it's only needed while * the audit is running. Once finalized, we explicitly delete it. */ import { env } from "cloudflare:workers"; import { z } from "zod"; import { jsonCodec } from "@/shared/json"; const KV_PREFIX = "audit-progress:"; const TTL_SECONDS = 30 * 60; // 30 minutes const MAX_ENTRIES = 300; const crawledUrlEntrySchema = z.object({ url: z.string(), statusCode: z.number(), title: z.string(), /** Unix timestamp ms when this page was crawled */ crawledAt: z.number(), }); type CrawledUrlEntry = z.infer; const crawledEntriesCodec = jsonCodec(z.array(crawledUrlEntrySchema)); function parseCrawledEntries(json: string | null): CrawledUrlEntry[] { if (!json) return []; const parsed = crawledEntriesCodec.safeParse(json); return parsed.success ? parsed.data : []; } function key(auditId: string): string { return `${KV_PREFIX}${auditId}`; } /** * Append multiple crawled URL entries in one KV write. * New entries are prepended and the list is capped. */ async function pushCrawledUrls( auditId: string, nextEntries: CrawledUrlEntry[], ): Promise { if (nextEntries.length === 0) return; const k = key(auditId); const existing = await env.KV.get(k, "text"); const entries = parseCrawledEntries(existing); const merged = [...nextEntries, ...entries].slice(0, MAX_ENTRIES); await env.KV.put(k, JSON.stringify(merged), { expirationTtl: TTL_SECONDS, }); } /** * Read all crawled URL entries for a running audit. * Returns newest-first. */ async function getCrawledUrls(auditId: string): Promise { const data = await env.KV.get(key(auditId), "text"); return parseCrawledEntries(data); } /** * Delete the progress key (called after audit completes). */ async function clear(auditId: string): Promise { await env.KV.delete(key(auditId)); } export const AuditProgressKV = { pushCrawledUrls, getCrawledUrls, clear, } as const;