81 lines
2.1 KiB
TypeScript
81 lines
2.1 KiB
TypeScript
/**
|
|
* KV-based live crawl progress.
|
|
*
|
|
* During a crawl, each crawled URL is appended to a KV key so the UI can
|
|
* poll for a live feed of crawled pages (most recent first).
|
|
*
|
|
* The KV entry auto-expires after 30 minutes — it's only needed while
|
|
* the audit is running. Once finalized, we explicitly delete it.
|
|
*/
|
|
import { env } from "cloudflare:workers";
|
|
import { z } from "zod";
|
|
import { jsonCodec } from "@/shared/json";
|
|
|
|
const KV_PREFIX = "audit-progress:";
|
|
const TTL_SECONDS = 30 * 60; // 30 minutes
|
|
const MAX_ENTRIES = 300;
|
|
|
|
const crawledUrlEntrySchema = z.object({
|
|
url: z.string(),
|
|
statusCode: z.number(),
|
|
title: z.string(),
|
|
/** Unix timestamp ms when this page was crawled */
|
|
crawledAt: z.number(),
|
|
});
|
|
|
|
type CrawledUrlEntry = z.infer<typeof crawledUrlEntrySchema>;
|
|
|
|
const crawledEntriesCodec = jsonCodec(z.array(crawledUrlEntrySchema));
|
|
|
|
function parseCrawledEntries(json: string | null): CrawledUrlEntry[] {
|
|
if (!json) return [];
|
|
const parsed = crawledEntriesCodec.safeParse(json);
|
|
return parsed.success ? parsed.data : [];
|
|
}
|
|
|
|
function key(auditId: string): string {
|
|
return `${KV_PREFIX}${auditId}`;
|
|
}
|
|
|
|
/**
|
|
* Append multiple crawled URL entries in one KV write.
|
|
* New entries are prepended and the list is capped.
|
|
*/
|
|
async function pushCrawledUrls(
|
|
auditId: string,
|
|
nextEntries: CrawledUrlEntry[],
|
|
): Promise<void> {
|
|
if (nextEntries.length === 0) return;
|
|
|
|
const k = key(auditId);
|
|
const existing = await env.KV.get(k, "text");
|
|
const entries = parseCrawledEntries(existing);
|
|
const merged = [...nextEntries, ...entries].slice(0, MAX_ENTRIES);
|
|
|
|
await env.KV.put(k, JSON.stringify(merged), {
|
|
expirationTtl: TTL_SECONDS,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Read all crawled URL entries for a running audit.
|
|
* Returns newest-first.
|
|
*/
|
|
async function getCrawledUrls(auditId: string): Promise<CrawledUrlEntry[]> {
|
|
const data = await env.KV.get(key(auditId), "text");
|
|
return parseCrawledEntries(data);
|
|
}
|
|
|
|
/**
|
|
* Delete the progress key (called after audit completes).
|
|
*/
|
|
async function clear(auditId: string): Promise<void> {
|
|
await env.KV.delete(key(auditId));
|
|
}
|
|
|
|
export const AuditProgressKV = {
|
|
pushCrawledUrls,
|
|
getCrawledUrls,
|
|
clear,
|
|
} as const;
|