diff --git a/src/server/lib/audit/crawl-window.test.ts b/src/server/lib/audit/crawl-window.test.ts index c5c5a39..5c3bde7 100644 --- a/src/server/lib/audit/crawl-window.test.ts +++ b/src/server/lib/audit/crawl-window.test.ts @@ -51,7 +51,7 @@ function page( describe("adjustCrawlWindow", () => { it("keeps the window on an empty batch", () => { - expect(adjustCrawlWindow(25, [])).toBe(25); + expect(adjustCrawlWindow(10, [])).toBe(10); }); it("halves the window when a third of the batch is troubled", () => { @@ -74,27 +74,32 @@ describe("adjustCrawlWindow", () => { it("grows on a clean, fast batch up to the cap", () => { const recent = Array.from({ length: 25 }, () => page("ok", 400)); - expect(adjustCrawlWindow(25, recent)).toBe(30); - expect(adjustCrawlWindow(40, recent)).toBe(40); + expect(adjustCrawlWindow(10, recent)).toBe(15); + expect(adjustCrawlWindow(20, recent)).toBe(20); }); - it("treats heavy pages as trouble even when they respond fast", () => { + it("caps the window so heavy pages stay inside the byte budget", () => { + // 1 MiB average pages: 16 MiB budget / 1 MiB = window of 16. const recent = Array.from({ length: 25 }, () => - page("ok", 300, 2 * 1024 * 1024), + page("ok", 300, 1024 * 1024), ); - expect(adjustCrawlWindow(25, recent)).toBe(12); + expect(adjustCrawlWindow(20, recent)).toBe(16); }); - it("does not grow when some pages are heavy", () => { - const recent = [ - ...Array.from({ length: 4 }, () => page("ok", 300, 1024 * 1024)), - ...Array.from({ length: 21 }, () => page("ok", 300)), - ]; - expect(adjustCrawlWindow(25, recent)).toBe(25); + it("keeps the byte bound at the minimum window even for huge pages", () => { + const recent = Array.from({ length: 25 }, () => + page("ok", 300, 4 * 1024 * 1024), + ); + expect(adjustCrawlWindow(20, recent)).toBe(5); + }); + + it("lets small pages use the full window cap", () => { + const recent = Array.from({ length: 25 }, () => page("ok", 400, 10_000)); + expect(adjustCrawlWindow(20, recent)).toBe(20); }); it("holds steady on a clean but slow batch", () => { const recent = Array.from({ length: 25 }, () => page("ok", 5_000)); - expect(adjustCrawlWindow(25, recent)).toBe(25); + expect(adjustCrawlWindow(15, recent)).toBe(15); }); }); diff --git a/src/server/lib/audit/crawl-window.ts b/src/server/lib/audit/crawl-window.ts index 59fca1e..145eb1f 100644 --- a/src/server/lib/audit/crawl-window.ts +++ b/src/server/lib/audit/crawl-window.ts @@ -7,22 +7,28 @@ import type { CrawledPageResult } from "@/server/lib/audit/types"; * slowly (politeness toward struggling or defensive sites) and grows when * the site answers fast. */ -export const INITIAL_CRAWL_WINDOW = 25; +export const INITIAL_CRAWL_WINDOW = 10; const MIN_WINDOW = 5; -const MAX_WINDOW = 40; +const MAX_WINDOW = 20; const SLOW_RESPONSE_MS = 10_000; const FAST_RESPONSE_MS = 1_500; + /** - * Pages at/above this HTML size count as trouble: each in-flight page - * buffers its body, so a wide window on a heavy-page site is memory - * pressure the response time can't see (it's measured at headers). + * Total HTML the in-flight window may buffer at once. Each in-flight page + * holds its body (decoded to a UTF-16 string, roughly doubling it), and the + * workflow shares its isolate's 128 MB memory limit with the rest of the + * worker — production audits died with exceededMemory when a fast site let + * the window grow unchecked. */ -const HEAVY_PAGE_BYTES = 1024 * 1024; +const IN_FLIGHT_HTML_BUDGET_BYTES = 16 * 1024 * 1024; +/** Floor for the observed page size so tiny-page sites can't void the bound. */ +const MIN_ASSUMED_PAGE_BYTES = 64 * 1024; /** * Adapt the window to the last persisted sub-batch. Shrinks on trouble - * (errors, blocks, very slow responses, heavy bodies), grows only on a - * clean and mostly-fast batch. + * (errors, blocks, very slow responses), grows only on a clean and mostly + * fast batch, and is always capped so the batch's average page size times + * the window stays inside the in-flight byte budget. */ export function adjustCrawlWindow( windowSize: number, @@ -32,19 +38,29 @@ export function adjustCrawlWindow( const troubled = recent.filter( (page) => page.fetchClass !== "ok" || - (page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS || - page.htmlBytes >= HEAVY_PAGE_BYTES, + (page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS, ).length; + let next = windowSize; if (troubled * 3 >= recent.length) { - return Math.max(MIN_WINDOW, Math.floor(windowSize / 2)); + next = Math.max(MIN_WINDOW, Math.floor(windowSize / 2)); + } else { + const fast = recent.filter( + (page) => + page.fetchClass === "ok" && + (page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS, + ).length; + if (troubled === 0 && fast * 2 >= recent.length) { + next = Math.min(MAX_WINDOW, windowSize + 5); + } } - const fast = recent.filter( - (page) => - page.fetchClass === "ok" && - (page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS, - ).length; - if (troubled === 0 && fast * 2 >= recent.length) { - return Math.min(MAX_WINDOW, windowSize + 5); - } - return windowSize; + + const avgPageBytes = Math.max( + recent.reduce((sum, page) => sum + page.htmlBytes, 0) / recent.length, + MIN_ASSUMED_PAGE_BYTES, + ); + const byteBound = Math.max( + MIN_WINDOW, + Math.floor(IN_FLIGHT_HTML_BUDGET_BYTES / avgPageBytes), + ); + return Math.min(next, byteBound); } diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.ts index cf46382..a03463f 100644 --- a/src/server/workflows/site-audit-workflow-helpers.ts +++ b/src/server/workflows/site-audit-workflow-helpers.ts @@ -6,7 +6,7 @@ import { sha256Hex } from "@/server/lib/audit/ids"; import { normalizeUrl } from "@/server/lib/audit/url-utils"; const CRAWL_USER_AGENT = "OpenSEO-Audit/1.0"; -const MAX_HTML_BYTES = 2 * 1024 * 1024; +const MAX_HTML_BYTES = 1024 * 1024; /** * Markers of a bot-mitigation challenge page. We classify these honestly as @@ -104,7 +104,7 @@ export async function crawlPage( const contentType = response.headers.get("content-type") ?? ""; const isHtml = contentType.includes("text/html"); - // Cap what we read: the first 2 MiB still contains the SEO metadata and + // Cap what we read: the first 1 MiB still contains the SEO metadata and // navigation needed by the audit in normal documents. const body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : ""; const fetchClass = classifyFetch( @@ -124,6 +124,9 @@ export async function crawlPage( headerCanonicalUrl, crawlDepth, inSitemap, + // The body was still fetched and buffered; report its size so the + // crawl window's byte budget sees blocked/error pages too. + htmlBytes: body.length, }); } @@ -244,6 +247,7 @@ function emptyPageResult(input: { headerCanonicalUrl: string | null; crawlDepth: number | null; inSitemap: boolean; + htmlBytes?: number; }): CrawledPageResult { return { id: crypto.randomUUID(), @@ -270,7 +274,7 @@ function emptyPageResult(input: { wordCount: 0, contentHash: null, isHtml: false, - htmlBytes: 0, + htmlBytes: input.htmlBytes ?? 0, imagesTotal: 0, imagesMissingAlt: 0, images: [], diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.ts index 0be7859..cba90cc 100644 --- a/src/server/workflows/siteAuditWorkflowCrawl.ts +++ b/src/server/workflows/siteAuditWorkflowCrawl.ts @@ -34,6 +34,13 @@ const CHUNK_TARGET_PAGES = 200; const CHUNK_SOFT_DEADLINE_MS = 90_000; /** Crawled pages are persisted in sub-batches of this size. */ const PERSIST_BATCH_SIZE = 25; +/** + * Stop launching new fetches while more than this many persist sub-batches + * are waiting: persistence is sequential, so when the DB falls behind a fast + * site, unpersisted page results would otherwise pile up in memory without + * bound. + */ +const MAX_QUEUED_PERSIST_BATCHES = 2; /** * Mega-menu/footer-heavy sites can carry 1000+ links per page; cap what we @@ -146,24 +153,30 @@ async function runCrawlChunk( // Persistence runs concurrently with fetching (pipelined) but sequentially // with itself, so DB write pressure stays bounded at one batch at a time. let persistChain: Promise = Promise.resolve(); + let queuedPersists = 0; const flush = () => { if (batch.length === 0) return; const pages = batch; batch = []; windowSize = adjustCrawlWindow(windowSize, pages); - persistChain = persistChain.then(() => - persistCrawledPages({ - auditId, - workflowInstanceId, - origin, - robots, - scratchpad, - pages, - depthByUrl, - maxPages, - }), - ); + queuedPersists += 1; + persistChain = persistChain + .then(() => + persistCrawledPages({ + auditId, + workflowInstanceId, + origin, + robots, + scratchpad, + pages, + depthByUrl, + maxPages, + }), + ) + .finally(() => { + queuedPersists -= 1; + }); }; const launch = (entry: ClaimedUrl) => { @@ -182,14 +195,29 @@ async function runCrawlChunk( while (true) { while ( inFlight.size < windowSize && + queuedPersists <= MAX_QUEUED_PERSIST_BATCHES && nextIndex < claimed.length && Date.now() < deadlineAt ) { launch(claimed[nextIndex]); nextIndex += 1; } - if (inFlight.size === 0) break; - await Promise.race(inFlight); + if (inFlight.size > 0) { + await Promise.race(inFlight); + continue; + } + // Nothing in flight. If launches are only paused by persistence + // backpressure, wait for the queue to drain and resume; otherwise the + // chunk is done (leases exhausted or soft deadline hit). + if ( + queuedPersists > MAX_QUEUED_PERSIST_BATCHES && + nextIndex < claimed.length && + Date.now() < deadlineAt + ) { + await persistChain; + continue; + } + break; } flush(); await persistChain;