From e5e961bf48ecef61b42b480d3e5f04da831520ea Mon Sep 17 00:00:00 2001 From: Ben Senescu <44480372+bensenescu@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:29:42 -0400 Subject: [PATCH] Audit crawl: conservative OOM retries + window carry-over across chunks (#517) * Audit crawl: retry chunks conservatively and stop re-learning the window per chunk Production audits on heavy-page sites (500-700KB/page) died with exceededMemory in the first crawl chunk, and the single step retry re-ran the exact same profile and died again. - claimChunk now reports isRetry (leftover leases from a dead attempt); a retried chunk crawls under RETRY_CRAWL_WINDOW (start 3, max 5, halved byte budget) instead of restarting at window 10. - The adapted window carries across chunks via durable step results, so every ~200 pages no longer re-spikes to the initial window. - First persist sub-batch shrinks to 5 pages so the byte bound sees the site's page weight before a full 25-page batch is in flight. - In-flight HTML budget halved to 8 MiB (16 MiB never constrained the observed ~650KB pages: bound was 25, above the 20 max); growth now requires a full 25-page sample. - Parse-time caps: 1,000 extracted links/images per page so mega-menu and crawler-trap pages can't bloat retained persist batches. * Guard endWindow for instances replaying pre-deploy step results * ci: un-export internal-only interfaces (knip) --- src/server/features/audit/AuditScratchpad.ts | 21 +++-- src/server/lib/audit/crawl-window.test.ts | 28 ++++++- src/server/lib/audit/crawl-window.ts | 79 ++++++++++++++----- src/server/lib/audit/page-analyzer.test.ts | 21 +++++ src/server/lib/audit/page-analyzer.ts | 20 ++++- .../workflows/siteAuditWorkflowCrawl.ts | 55 +++++++++++-- 6 files changed, 186 insertions(+), 38 deletions(-) diff --git a/src/server/features/audit/AuditScratchpad.ts b/src/server/features/audit/AuditScratchpad.ts index ca8c398..59ff6e8 100644 --- a/src/server/features/audit/AuditScratchpad.ts +++ b/src/server/features/audit/AuditScratchpad.ts @@ -26,6 +26,17 @@ export interface ClaimedUrl { inSitemap: boolean; } +interface ClaimedChunk { + urls: ClaimedUrl[]; + /** + * True when a prior attempt of this chunk already claimed work — i.e. the + * caller is a workflow step retry after that attempt died mid-crawl. The + * crawler uses this to retry far more conservatively (the usual cause of a + * dead attempt is exceededMemory). + */ + isRetry: boolean; +} + export interface FrontierStats { /** Pages attempted (crawled or errored) so far. */ attempted: number; @@ -147,12 +158,12 @@ export class AuditScratchpad extends DurableObject { * had leased (link-discovered URLs drain before sitemap-only ones, FIFO * within each class — same ordering as the old in-memory queues). */ - async claimChunk(chunkNo: number, limit: number): Promise { + async claimChunk(chunkNo: number, limit: number): Promise { const existing = this.selectClaimed( `SELECT url, depth, in_sitemap FROM frontier WHERE state = 'leased' AND chunk_no = ?`, chunkNo, ); - if (existing.length > 0) return existing; + if (existing.length > 0) return { urls: existing, isRetry: true }; // A retried step whose earlier attempt already crawled this chunk's // leases must not claim a fresh set under the same chunk number — that // would duplicate work and overshoot the page budget. @@ -164,8 +175,8 @@ export class AuditScratchpad extends DurableObject { chunkNo, ) .one(); - if (done.n > 0) return []; - if (limit <= 0) return []; + if (done.n > 0) return { urls: [], isRetry: true }; + if (limit <= 0) return { urls: [], isRetry: false }; const fresh = this.selectClaimed( `SELECT url, depth, in_sitemap FROM frontier WHERE state = 'pending' @@ -179,7 +190,7 @@ export class AuditScratchpad extends DurableObject { row.url, ); } - return fresh; + return { urls: fresh, isRetry: false }; } /** Persist one crawled sub-batch: completions, mirror rows, links, frontier. */ diff --git a/src/server/lib/audit/crawl-window.test.ts b/src/server/lib/audit/crawl-window.test.ts index 5c3bde7..3cdcf9d 100644 --- a/src/server/lib/audit/crawl-window.test.ts +++ b/src/server/lib/audit/crawl-window.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { adjustCrawlWindow } from "@/server/lib/audit/crawl-window"; +import { + adjustCrawlWindow, + RETRY_CRAWL_WINDOW, +} from "@/server/lib/audit/crawl-window"; import type { CrawledPageResult, PageFetchClass, @@ -79,11 +82,30 @@ describe("adjustCrawlWindow", () => { }); it("caps the window so heavy pages stay inside the byte budget", () => { - // 1 MiB average pages: 16 MiB budget / 1 MiB = window of 16. + // 1 MiB average pages: 8 MiB budget / 1 MiB = window of 8. const recent = Array.from({ length: 25 }, () => page("ok", 300, 1024 * 1024), ); - expect(adjustCrawlWindow(20, recent)).toBe(16); + expect(adjustCrawlWindow(20, recent)).toBe(8); + }); + + it("does not grow on a small sample, but still applies the byte bound", () => { + const fastSmall = Array.from({ length: 5 }, () => page("ok", 400)); + expect(adjustCrawlWindow(10, fastSmall)).toBe(10); + const fastHeavy = Array.from({ length: 5 }, () => + page("ok", 400, 1024 * 1024), + ); + expect(adjustCrawlWindow(10, fastHeavy)).toBe(8); + }); + + it("retry limits keep the window small even on a clean, fast site", () => { + const recent = Array.from({ length: 25 }, () => page("ok", 400)); + expect(adjustCrawlWindow(3, recent, RETRY_CRAWL_WINDOW)).toBe(5); + }); + + it("retry limits shrink below the normal minimum on trouble", () => { + const recent = Array.from({ length: 10 }, () => page("error", 15_000)); + expect(adjustCrawlWindow(3, recent, RETRY_CRAWL_WINDOW)).toBe(2); }); it("keeps the byte bound at the minimum window even for huge pages", () => { diff --git a/src/server/lib/audit/crawl-window.ts b/src/server/lib/audit/crawl-window.ts index 145eb1f..7e2793b 100644 --- a/src/server/lib/audit/crawl-window.ts +++ b/src/server/lib/audit/crawl-window.ts @@ -7,32 +7,69 @@ import type { CrawledPageResult } from "@/server/lib/audit/types"; * slowly (politeness toward struggling or defensive sites) and grows when * the site answers fast. */ -export const INITIAL_CRAWL_WINDOW = 10; -const MIN_WINDOW = 5; -const MAX_WINDOW = 20; -const SLOW_RESPONSE_MS = 10_000; -const FAST_RESPONSE_MS = 1_500; +interface CrawlWindowLimits { + /** Window size a chunk starts with, before any observations. */ + initial: number; + min: number; + max: number; + /** + * Total HTML the in-flight window may buffer at once. Each in-flight page + * holds its body (decoded to a UTF-16 string, roughly doubling it), and the + * workflow shares its isolate's 128 MB memory limit with the rest of the + * worker — production audits died with exceededMemory when a fast site let + * the window grow unchecked. + */ + budgetBytes: number; +} + +export const CRAWL_WINDOW: CrawlWindowLimits = { + initial: 10, + min: 5, + max: 20, + budgetBytes: 8 * 1024 * 1024, +}; /** - * Total HTML the in-flight window may buffer at once. Each in-flight page - * holds its body (decoded to a UTF-16 string, roughly doubling it), and the - * workflow shares its isolate's 128 MB memory limit with the rest of the - * worker — production audits died with exceededMemory when a fast site let - * the window grow unchecked. + * Limits for a chunk whose earlier attempt died mid-crawl — in production + * almost always exceededMemory on a heavy-page site. The retry is the + * chunk's last attempt (retry limit 1), so it must not re-run the exact + * memory profile that just killed the isolate. */ -const IN_FLIGHT_HTML_BUDGET_BYTES = 16 * 1024 * 1024; +export const RETRY_CRAWL_WINDOW: CrawlWindowLimits = { + initial: 3, + min: 2, + max: 5, + budgetBytes: 4 * 1024 * 1024, +}; + +const SLOW_RESPONSE_MS = 10_000; +const FAST_RESPONSE_MS = 1_500; /** Floor for the observed page size so tiny-page sites can't void the bound. */ const MIN_ASSUMED_PAGE_BYTES = 64 * 1024; +/** + * Growth requires a full-size sample. The first persist sub-batch is small + * (so the byte bound reacts to heavy pages early), and a handful of fast + * pages proves too little to widen the window. + */ +const GROWTH_MIN_SAMPLE = 25; + +export function clampCrawlWindow( + size: number, + limits: CrawlWindowLimits, +): number { + return Math.min(Math.max(size, limits.min), limits.max); +} /** * Adapt the window to the last persisted sub-batch. Shrinks on trouble - * (errors, blocks, very slow responses), grows only on a clean and mostly - * fast batch, and is always capped so the batch's average page size times - * the window stays inside the in-flight byte budget. + * (errors, blocks, very slow responses), grows only on a clean, mostly fast, + * full-size batch, and is always capped so the batch's average page size + * times the window stays inside the in-flight byte budget. */ export function adjustCrawlWindow( windowSize: number, recent: CrawledPageResult[], + limits: CrawlWindowLimits = CRAWL_WINDOW, ): number { if (recent.length === 0) return windowSize; const troubled = recent.filter( @@ -42,15 +79,19 @@ export function adjustCrawlWindow( ).length; let next = windowSize; if (troubled * 3 >= recent.length) { - next = Math.max(MIN_WINDOW, Math.floor(windowSize / 2)); + next = Math.max(limits.min, Math.floor(windowSize / 2)); } else { const fast = recent.filter( (page) => page.fetchClass === "ok" && (page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS, ).length; - if (troubled === 0 && fast * 2 >= recent.length) { - next = Math.min(MAX_WINDOW, windowSize + 5); + if ( + troubled === 0 && + fast * 2 >= recent.length && + recent.length >= GROWTH_MIN_SAMPLE + ) { + next = Math.min(limits.max, windowSize + 5); } } @@ -59,8 +100,8 @@ export function adjustCrawlWindow( MIN_ASSUMED_PAGE_BYTES, ); const byteBound = Math.max( - MIN_WINDOW, - Math.floor(IN_FLIGHT_HTML_BUDGET_BYTES / avgPageBytes), + limits.min, + Math.floor(limits.budgetBytes / avgPageBytes), ); return Math.min(next, byteBound); } diff --git a/src/server/lib/audit/page-analyzer.test.ts b/src/server/lib/audit/page-analyzer.test.ts index ccab983..557a536 100644 --- a/src/server/lib/audit/page-analyzer.test.ts +++ b/src/server/lib/audit/page-analyzer.test.ts @@ -206,3 +206,24 @@ describe("analyzeHtml parity with the DOM reference", () => { `); }); }); + +describe("analyzeHtml extraction caps", () => { + it("caps links and images per page", () => { + const links = Array.from( + { length: 1_100 }, + (_, i) => `link ${i}`, + ).join(""); + const images = Array.from( + { length: 1_100 }, + (_, i) => ``, + ).join(""); + const analysis = analyzeHtml( + `${links}${images}`, + "https://example.com/", + 200, + 100, + ); + expect(analysis.links).toHaveLength(1_000); + expect(analysis.images).toHaveLength(1_000); + }); +}); diff --git a/src/server/lib/audit/page-analyzer.ts b/src/server/lib/audit/page-analyzer.ts index 5abc9d3..47db8ce 100644 --- a/src/server/lib/audit/page-analyzer.ts +++ b/src/server/lib/audit/page-analyzer.ts @@ -26,6 +26,15 @@ const HEADING_LEVELS: Record = { h6: 6, }; const MAX_ANCHOR_CHARS = 200; +/** + * Per-page caps on the extracted collections. Crawler-trap and mega-menu + * pages can carry thousands of links/images per page, and crawled pages sit + * in memory in 25-page persist batches — uncapped collections were part of + * the audit engine's exceededMemory profile. Counts derived from these + * arrays saturate at the cap on such pathological pages. + */ +const MAX_EXTRACTED_LINKS = 1_000; +const MAX_EXTRACTED_IMAGES = 1_000; interface OpenAnchor { href: string; @@ -103,6 +112,7 @@ export function analyzeHtml( if (!openAnchor) return; const { href, rel, text } = openAnchor; openAnchor = null; + if (linksByTarget.size >= MAX_EXTRACTED_LINKS) return; const resolved = normalizeUrl(href, pageUrl); if (!resolved || linksByTarget.has(resolved)) return; const anchor = text @@ -148,10 +158,12 @@ export function analyzeHtml( handleLinkTag(attribs); break; case "img": - images.push({ - src: attribs["src"] ?? null, - alt: "alt" in attribs ? attribs["alt"] : null, - }); + if (images.length < MAX_EXTRACTED_IMAGES) { + images.push({ + src: attribs["src"] ?? null, + alt: "alt" in attribs ? attribs["alt"] : null, + }); + } break; case "script": if (attribs["type"] === "application/ld+json") { diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.ts index 5080ce0..b090955 100644 --- a/src/server/workflows/siteAuditWorkflowCrawl.ts +++ b/src/server/workflows/siteAuditWorkflowCrawl.ts @@ -15,7 +15,9 @@ import { import { AuditProgressKV } from "@/server/lib/audit/progress-kv"; import { adjustCrawlWindow, - INITIAL_CRAWL_WINDOW, + clampCrawlWindow, + CRAWL_WINDOW, + RETRY_CRAWL_WINDOW, } from "@/server/lib/audit/crawl-window"; import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers"; import { pgStep } from "@/server/workflows/pgStep"; @@ -34,6 +36,14 @@ const CHUNK_TARGET_PAGES = 200; const CHUNK_SOFT_DEADLINE_MS = 90_000; /** Crawled pages are persisted in sub-batches of this size. */ const PERSIST_BATCH_SIZE = 25; +/** + * The first sub-batch of a chunk is deliberately small: the crawl window + * only adapts when a sub-batch persists, and on a heavy-page site a full + * 25-page batch crawled at the starting window was already enough to + * exceed the isolate's memory. Five pages tell the byte bound what the + * site's pages weigh before the window commits to more. + */ +const FIRST_PERSIST_BATCH_SIZE = 5; /** * Stop launching new fetches while more than this many persist sub-batches * are waiting: persistence is sequential, so when the DB falls behind a fast @@ -91,6 +101,11 @@ export async function runCrawlPhase( let attemptedTotal = 0; let pending = params.seededCount; let zeroProgressChunks = 0; + // The adapted window carries across chunks via durable step results: + // without this every chunk restarted at the initial window and re-learned + // the site's page weight the hard way — on heavy-page sites that meant an + // exceededMemory death every ~200 pages. + let windowHint = CRAWL_WINDOW.initial; while (pending > 0 && attemptedTotal < params.maxPages) { chunkNo += 1; @@ -103,6 +118,7 @@ export async function runCrawlPhase( ...params, chunkNo, attemptedBefore: attemptedTotal, + startWindow: windowHint, }), ); // Apply the chunk's counters even when it did no new work (a retried @@ -110,6 +126,9 @@ export async function runCrawlPhase( // with up-to-date scratchpad totals) — finalize must not see stale ones. attemptedTotal = result.attempted; pending = result.pending; + // `?? initial`: an instance in flight across a deploy replays cached + // step results from before endWindow existed. + windowHint = result.endWindow ?? CRAWL_WINDOW.initial; // One zero-attempt chunk is normal (retry of a completed chunk number); // two in a row means the frontier is unservable — stop with what we // have instead of spinning forever. @@ -122,8 +141,17 @@ export async function runCrawlPhase( } async function runCrawlChunk( - input: CrawlPhaseParams & { chunkNo: number; attemptedBefore: number }, -): Promise<{ attemptedInChunk: number; attempted: number; pending: number }> { + input: CrawlPhaseParams & { + chunkNo: number; + attemptedBefore: number; + startWindow: number; + }, +): Promise<{ + attemptedInChunk: number; + attempted: number; + pending: number; + endWindow: number; +}> { const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } = input; const scratchpad = getAuditScratchpad(auditId); @@ -132,23 +160,34 @@ async function runCrawlChunk( CHUNK_TARGET_PAGES, maxPages - input.attemptedBefore, ); - const claimed = await scratchpad.claimChunk(chunkNo, claimLimit); + const { urls: claimed, isRetry } = await scratchpad.claimChunk( + chunkNo, + claimLimit, + ); if (claimed.length === 0) { const stats = await scratchpad.getStats(); return { attemptedInChunk: 0, attempted: stats.attempted, pending: stats.pending, + endWindow: input.startWindow, }; } const depthByUrl = new Map(claimed.map((entry) => [entry.url, entry.depth])); const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS; - let windowSize = INITIAL_CRAWL_WINDOW; + // A retry means the previous attempt died mid-crawl (in production almost + // always exceededMemory), and it is the chunk's last attempt — so it runs + // under drastically reduced limits instead of the profile that just failed. + const limits = isRetry ? RETRY_CRAWL_WINDOW : CRAWL_WINDOW; + let windowSize = isRetry + ? limits.initial + : clampCrawlWindow(input.startWindow, limits); let nextIndex = 0; let attemptedInChunk = 0; const inFlight = new Set>(); + let persistThreshold = FIRST_PERSIST_BATCH_SIZE; let batch: CrawledPageResult[] = []; // Persistence runs concurrently with fetching (pipelined) but sequentially // with itself, so DB write pressure stays bounded at one batch at a time. @@ -159,7 +198,8 @@ async function runCrawlChunk( if (batch.length === 0) return; const pages = batch; batch = []; - windowSize = adjustCrawlWindow(windowSize, pages); + persistThreshold = PERSIST_BATCH_SIZE; + windowSize = adjustCrawlWindow(windowSize, pages, limits); queuedPersists += 1; persistChain = persistChain .then(() => @@ -184,7 +224,7 @@ async function runCrawlChunk( .then((page) => { attemptedInChunk += 1; batch.push(page); - if (batch.length >= PERSIST_BATCH_SIZE) flush(); + if (batch.length >= persistThreshold) flush(); }) .finally(() => { inFlight.delete(promise); @@ -239,6 +279,7 @@ async function runCrawlChunk( attemptedInChunk, attempted: stats.attempted, pending: stats.pending, + endWindow: windowSize, }; }