Cap site-audit crawl memory: byte-budgeted window, 1 MiB read cap, persist backpressure (#445)

src/server/lib/audit/crawl-window.test.ts

@@ -51,7 +51,7 @@ function page(
 
 describe("adjustCrawlWindow", () => {
   it("keeps the window on an empty batch", () => {
-    expect(adjustCrawlWindow(25, [])).toBe(25);
+    expect(adjustCrawlWindow(10, [])).toBe(10);
   });
 
   it("halves the window when a third of the batch is troubled", () => {
@@ -74,27 +74,32 @@ describe("adjustCrawlWindow", () => {
 
   it("grows on a clean, fast batch up to the cap", () => {
     const recent = Array.from({ length: 25 }, () => page("ok", 400));
-    expect(adjustCrawlWindow(25, recent)).toBe(30);
+    expect(adjustCrawlWindow(10, recent)).toBe(15);
-    expect(adjustCrawlWindow(40, recent)).toBe(40);
+    expect(adjustCrawlWindow(20, recent)).toBe(20);
   });
 
-  it("treats heavy pages as trouble even when they respond fast", () => {
+  it("caps the window so heavy pages stay inside the byte budget", () => {
+    // 1 MiB average pages: 16 MiB budget / 1 MiB = window of 16.
     const recent = Array.from({ length: 25 }, () =>
-      page("ok", 300, 2 * 1024 * 1024),
+      page("ok", 300, 1024 * 1024),
     );
-    expect(adjustCrawlWindow(25, recent)).toBe(12);
+    expect(adjustCrawlWindow(20, recent)).toBe(16);
   });
 
-  it("does not grow when some pages are heavy", () => {
+  it("keeps the byte bound at the minimum window even for huge pages", () => {
-    const recent = [
+    const recent = Array.from({ length: 25 }, () =>
-      ...Array.from({ length: 4 }, () => page("ok", 300, 1024 * 1024)),
+      page("ok", 300, 4 * 1024 * 1024),
-      ...Array.from({ length: 21 }, () => page("ok", 300)),
+    );
-    ];
+    expect(adjustCrawlWindow(20, recent)).toBe(5);
-    expect(adjustCrawlWindow(25, recent)).toBe(25);
+  });
+
+  it("lets small pages use the full window cap", () => {
+    const recent = Array.from({ length: 25 }, () => page("ok", 400, 10_000));
+    expect(adjustCrawlWindow(20, recent)).toBe(20);
   });
 
   it("holds steady on a clean but slow batch", () => {
     const recent = Array.from({ length: 25 }, () => page("ok", 5_000));
-    expect(adjustCrawlWindow(25, recent)).toBe(25);
+    expect(adjustCrawlWindow(15, recent)).toBe(15);
   });
 });

src/server/lib/audit/crawl-window.ts

@@ -7,22 +7,28 @@ import type { CrawledPageResult } from "@/server/lib/audit/types";
  * slowly (politeness toward struggling or defensive sites) and grows when
  * the site answers fast.
  */
-export const INITIAL_CRAWL_WINDOW = 25;
+export const INITIAL_CRAWL_WINDOW = 10;
 const MIN_WINDOW = 5;
-const MAX_WINDOW = 40;
+const MAX_WINDOW = 20;
 const SLOW_RESPONSE_MS = 10_000;
 const FAST_RESPONSE_MS = 1_500;
+
 /**
- * Pages at/above this HTML size count as trouble: each in-flight page
+ * Total HTML the in-flight window may buffer at once. Each in-flight page
- * buffers its body, so a wide window on a heavy-page site is memory
+ * holds its body (decoded to a UTF-16 string, roughly doubling it), and the
- * pressure the response time can't see (it's measured at headers).
+ * workflow shares its isolate's 128 MB memory limit with the rest of the
+ * worker — production audits died with exceededMemory when a fast site let
+ * the window grow unchecked.
  */
-const HEAVY_PAGE_BYTES = 1024 * 1024;
+const IN_FLIGHT_HTML_BUDGET_BYTES = 16 * 1024 * 1024;
+/** Floor for the observed page size so tiny-page sites can't void the bound. */
+const MIN_ASSUMED_PAGE_BYTES = 64 * 1024;
 
 /**
  * Adapt the window to the last persisted sub-batch. Shrinks on trouble
- * (errors, blocks, very slow responses, heavy bodies), grows only on a
+ * (errors, blocks, very slow responses), grows only on a clean and mostly
- * clean and mostly-fast batch.
+ * fast batch, and is always capped so the batch's average page size times
+ * the window stays inside the in-flight byte budget.
  */
 export function adjustCrawlWindow(
   windowSize: number,
@@ -32,19 +38,29 @@ export function adjustCrawlWindow(
   const troubled = recent.filter(
     (page) =>
       page.fetchClass !== "ok" ||
-      (page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS ||
+      (page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS,
-      page.htmlBytes >= HEAVY_PAGE_BYTES,
   ).length;
+  let next = windowSize;
   if (troubled * 3 >= recent.length) {
-    return Math.max(MIN_WINDOW, Math.floor(windowSize / 2));
+    next = Math.max(MIN_WINDOW, Math.floor(windowSize / 2));
-  }
+  } else {
     const fast = recent.filter(
       (page) =>
         page.fetchClass === "ok" &&
         (page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS,
     ).length;
     if (troubled === 0 && fast * 2 >= recent.length) {
-    return Math.min(MAX_WINDOW, windowSize + 5);
+      next = Math.min(MAX_WINDOW, windowSize + 5);
     }
-  return windowSize;
+  }
+
+  const avgPageBytes = Math.max(
+    recent.reduce((sum, page) => sum + page.htmlBytes, 0) / recent.length,
+    MIN_ASSUMED_PAGE_BYTES,
+  );
+  const byteBound = Math.max(
+    MIN_WINDOW,
+    Math.floor(IN_FLIGHT_HTML_BUDGET_BYTES / avgPageBytes),
+  );
+  return Math.min(next, byteBound);
 }

src/server/workflows/site-audit-workflow-helpers.ts

@@ -6,7 +6,7 @@ import { sha256Hex } from "@/server/lib/audit/ids";
 import { normalizeUrl } from "@/server/lib/audit/url-utils";
 
 const CRAWL_USER_AGENT = "OpenSEO-Audit/1.0";
-const MAX_HTML_BYTES = 2 * 1024 * 1024;
+const MAX_HTML_BYTES = 1024 * 1024;
 
 /**
  * Markers of a bot-mitigation challenge page. We classify these honestly as
@@ -104,7 +104,7 @@ export async function crawlPage(
 
     const contentType = response.headers.get("content-type") ?? "";
     const isHtml = contentType.includes("text/html");
-    // Cap what we read: the first 2 MiB still contains the SEO metadata and
+    // Cap what we read: the first 1 MiB still contains the SEO metadata and
     // navigation needed by the audit in normal documents.
     const body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";
     const fetchClass = classifyFetch(
@@ -124,6 +124,9 @@ export async function crawlPage(
         headerCanonicalUrl,
         crawlDepth,
         inSitemap,
+        // The body was still fetched and buffered; report its size so the
+        // crawl window's byte budget sees blocked/error pages too.
+        htmlBytes: body.length,
       });
     }
 
@@ -244,6 +247,7 @@ function emptyPageResult(input: {
   headerCanonicalUrl: string | null;
   crawlDepth: number | null;
   inSitemap: boolean;
+  htmlBytes?: number;
 }): CrawledPageResult {
   return {
     id: crypto.randomUUID(),
@@ -270,7 +274,7 @@ function emptyPageResult(input: {
     wordCount: 0,
     contentHash: null,
     isHtml: false,
-    htmlBytes: 0,
+    htmlBytes: input.htmlBytes ?? 0,
     imagesTotal: 0,
     imagesMissingAlt: 0,
     images: [],

src/server/workflows/siteAuditWorkflowCrawl.ts

@@ -34,6 +34,13 @@ const CHUNK_TARGET_PAGES = 200;
 const CHUNK_SOFT_DEADLINE_MS = 90_000;
 /** Crawled pages are persisted in sub-batches of this size. */
 const PERSIST_BATCH_SIZE = 25;
+/**
+ * Stop launching new fetches while more than this many persist sub-batches
+ * are waiting: persistence is sequential, so when the DB falls behind a fast
+ * site, unpersisted page results would otherwise pile up in memory without
+ * bound.
+ */
+const MAX_QUEUED_PERSIST_BATCHES = 2;
 
 /**
  * Mega-menu/footer-heavy sites can carry 1000+ links per page; cap what we
@@ -146,13 +153,16 @@ async function runCrawlChunk(
   // Persistence runs concurrently with fetching (pipelined) but sequentially
   // with itself, so DB write pressure stays bounded at one batch at a time.
   let persistChain: Promise<unknown> = Promise.resolve();
+  let queuedPersists = 0;
 
   const flush = () => {
     if (batch.length === 0) return;
     const pages = batch;
     batch = [];
     windowSize = adjustCrawlWindow(windowSize, pages);
-    persistChain = persistChain.then(() =>
+    queuedPersists += 1;
+    persistChain = persistChain
+      .then(() =>
         persistCrawledPages({
           auditId,
           workflowInstanceId,
@@ -163,7 +173,10 @@ async function runCrawlChunk(
           depthByUrl,
           maxPages,
         }),
-    );
+      )
+      .finally(() => {
+        queuedPersists -= 1;
+      });
   };
 
   const launch = (entry: ClaimedUrl) => {
@@ -182,14 +195,29 @@ async function runCrawlChunk(
   while (true) {
     while (
       inFlight.size < windowSize &&
+      queuedPersists <= MAX_QUEUED_PERSIST_BATCHES &&
       nextIndex < claimed.length &&
       Date.now() < deadlineAt
     ) {
       launch(claimed[nextIndex]);
       nextIndex += 1;
     }
-    if (inFlight.size === 0) break;
+    if (inFlight.size > 0) {
       await Promise.race(inFlight);
+      continue;
+    }
+    // Nothing in flight. If launches are only paused by persistence
+    // backpressure, wait for the queue to drain and resume; otherwise the
+    // chunk is done (leases exhausted or soft deadline hit).
+    if (
+      queuedPersists > MAX_QUEUED_PERSIST_BATCHES &&
+      nextIndex < claimed.length &&
+      Date.now() < deadlineAt
+    ) {
+      await persistChain;
+      continue;
+    }
+    break;
   }
   flush();
   await persistChain;