Cap site-audit crawl memory: byte-budgeted window, 1 MiB read cap, persist backpressure (#445)

src/server/lib/audit/crawl-window.test.ts

@@ -51,7 +51,7 @@ function page(
 
 describe("adjustCrawlWindow", () => {
   it("keeps the window on an empty batch", () => {
-    expect(adjustCrawlWindow(25, [])).toBe(25);
+    expect(adjustCrawlWindow(10, [])).toBe(10);
   });
 
   it("halves the window when a third of the batch is troubled", () => {
@@ -74,27 +74,32 @@ describe("adjustCrawlWindow", () => {
 
   it("grows on a clean, fast batch up to the cap", () => {
     const recent = Array.from({ length: 25 }, () => page("ok", 400));
-    expect(adjustCrawlWindow(25, recent)).toBe(30);
-    expect(adjustCrawlWindow(40, recent)).toBe(40);
+    expect(adjustCrawlWindow(10, recent)).toBe(15);
+    expect(adjustCrawlWindow(20, recent)).toBe(20);
   });
 
-  it("treats heavy pages as trouble even when they respond fast", () => {
+  it("caps the window so heavy pages stay inside the byte budget", () => {
+    // 1 MiB average pages: 16 MiB budget / 1 MiB = window of 16.
     const recent = Array.from({ length: 25 }, () =>
-      page("ok", 300, 2 * 1024 * 1024),
+      page("ok", 300, 1024 * 1024),
     );
-    expect(adjustCrawlWindow(25, recent)).toBe(12);
+    expect(adjustCrawlWindow(20, recent)).toBe(16);
   });
 
-  it("does not grow when some pages are heavy", () => {
-    const recent = [
-      ...Array.from({ length: 4 }, () => page("ok", 300, 1024 * 1024)),
-      ...Array.from({ length: 21 }, () => page("ok", 300)),
-    ];
-    expect(adjustCrawlWindow(25, recent)).toBe(25);
+  it("keeps the byte bound at the minimum window even for huge pages", () => {
+    const recent = Array.from({ length: 25 }, () =>
+      page("ok", 300, 4 * 1024 * 1024),
+    );
+    expect(adjustCrawlWindow(20, recent)).toBe(5);
+  });
+
+  it("lets small pages use the full window cap", () => {
+    const recent = Array.from({ length: 25 }, () => page("ok", 400, 10_000));
+    expect(adjustCrawlWindow(20, recent)).toBe(20);
   });
 
   it("holds steady on a clean but slow batch", () => {
     const recent = Array.from({ length: 25 }, () => page("ok", 5_000));
-    expect(adjustCrawlWindow(25, recent)).toBe(25);
+    expect(adjustCrawlWindow(15, recent)).toBe(15);
   });
 });

src/server/lib/audit/crawl-window.ts

@@ -7,22 +7,28 @@ import type { CrawledPageResult } from "@/server/lib/audit/types";
  * slowly (politeness toward struggling or defensive sites) and grows when
  * the site answers fast.
  */
-export const INITIAL_CRAWL_WINDOW = 25;
+export const INITIAL_CRAWL_WINDOW = 10;
 const MIN_WINDOW = 5;
-const MAX_WINDOW = 40;
+const MAX_WINDOW = 20;
 const SLOW_RESPONSE_MS = 10_000;
 const FAST_RESPONSE_MS = 1_500;
+
 /**
- * Pages at/above this HTML size count as trouble: each in-flight page
- * buffers its body, so a wide window on a heavy-page site is memory
- * pressure the response time can't see (it's measured at headers).
+ * Total HTML the in-flight window may buffer at once. Each in-flight page
+ * holds its body (decoded to a UTF-16 string, roughly doubling it), and the
+ * workflow shares its isolate's 128 MB memory limit with the rest of the
+ * worker — production audits died with exceededMemory when a fast site let
+ * the window grow unchecked.
  */
-const HEAVY_PAGE_BYTES = 1024 * 1024;
+const IN_FLIGHT_HTML_BUDGET_BYTES = 16 * 1024 * 1024;
+/** Floor for the observed page size so tiny-page sites can't void the bound. */
+const MIN_ASSUMED_PAGE_BYTES = 64 * 1024;
 
 /**
  * Adapt the window to the last persisted sub-batch. Shrinks on trouble
- * (errors, blocks, very slow responses, heavy bodies), grows only on a
- * clean and mostly-fast batch.
+ * (errors, blocks, very slow responses), grows only on a clean and mostly
+ * fast batch, and is always capped so the batch's average page size times
+ * the window stays inside the in-flight byte budget.
  */
 export function adjustCrawlWindow(
   windowSize: number,
@@ -32,19 +38,29 @@ export function adjustCrawlWindow(
   const troubled = recent.filter(
     (page) =>
       page.fetchClass !== "ok" ||
-      (page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS ||
-      page.htmlBytes >= HEAVY_PAGE_BYTES,
+      (page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS,
   ).length;
+  let next = windowSize;
   if (troubled * 3 >= recent.length) {
-    return Math.max(MIN_WINDOW, Math.floor(windowSize / 2));
+    next = Math.max(MIN_WINDOW, Math.floor(windowSize / 2));
+  } else {
+    const fast = recent.filter(
+      (page) =>
+        page.fetchClass === "ok" &&
+        (page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS,
+    ).length;
+    if (troubled === 0 && fast * 2 >= recent.length) {
+      next = Math.min(MAX_WINDOW, windowSize + 5);
+    }
   }
-  const fast = recent.filter(
-    (page) =>
-      page.fetchClass === "ok" &&
-      (page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS,
-  ).length;
-  if (troubled === 0 && fast * 2 >= recent.length) {
-    return Math.min(MAX_WINDOW, windowSize + 5);
-  }
-  return windowSize;
+
+  const avgPageBytes = Math.max(
+    recent.reduce((sum, page) => sum + page.htmlBytes, 0) / recent.length,
+    MIN_ASSUMED_PAGE_BYTES,
+  );
+  const byteBound = Math.max(
+    MIN_WINDOW,
+    Math.floor(IN_FLIGHT_HTML_BUDGET_BYTES / avgPageBytes),
+  );
+  return Math.min(next, byteBound);
 }

src/server/workflows/site-audit-workflow-helpers.ts

@@ -6,7 +6,7 @@ import { sha256Hex } from "@/server/lib/audit/ids";
 import { normalizeUrl } from "@/server/lib/audit/url-utils";
 
 const CRAWL_USER_AGENT = "OpenSEO-Audit/1.0";
-const MAX_HTML_BYTES = 2 * 1024 * 1024;
+const MAX_HTML_BYTES = 1024 * 1024;
 
 /**
  * Markers of a bot-mitigation challenge page. We classify these honestly as
@@ -104,7 +104,7 @@ export async function crawlPage(
 
     const contentType = response.headers.get("content-type") ?? "";
     const isHtml = contentType.includes("text/html");
-    // Cap what we read: the first 2 MiB still contains the SEO metadata and
+    // Cap what we read: the first 1 MiB still contains the SEO metadata and
     // navigation needed by the audit in normal documents.
     const body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";
     const fetchClass = classifyFetch(
@@ -124,6 +124,9 @@ export async function crawlPage(
         headerCanonicalUrl,
         crawlDepth,
         inSitemap,
+        // The body was still fetched and buffered; report its size so the
+        // crawl window's byte budget sees blocked/error pages too.
+        htmlBytes: body.length,
       });
     }
 
@@ -244,6 +247,7 @@ function emptyPageResult(input: {
   headerCanonicalUrl: string | null;
   crawlDepth: number | null;
   inSitemap: boolean;
+  htmlBytes?: number;
 }): CrawledPageResult {
   return {
     id: crypto.randomUUID(),
@@ -270,7 +274,7 @@ function emptyPageResult(input: {
     wordCount: 0,
     contentHash: null,
     isHtml: false,
-    htmlBytes: 0,
+    htmlBytes: input.htmlBytes ?? 0,
     imagesTotal: 0,
     imagesMissingAlt: 0,
     images: [],

src/server/workflows/siteAuditWorkflowCrawl.ts

@@ -34,6 +34,13 @@ const CHUNK_TARGET_PAGES = 200;
 const CHUNK_SOFT_DEADLINE_MS = 90_000;
 /** Crawled pages are persisted in sub-batches of this size. */
 const PERSIST_BATCH_SIZE = 25;
+/**
+ * Stop launching new fetches while more than this many persist sub-batches
+ * are waiting: persistence is sequential, so when the DB falls behind a fast
+ * site, unpersisted page results would otherwise pile up in memory without
+ * bound.
+ */
+const MAX_QUEUED_PERSIST_BATCHES = 2;
 
 /**
  * Mega-menu/footer-heavy sites can carry 1000+ links per page; cap what we
@@ -146,24 +153,30 @@ async function runCrawlChunk(
   // Persistence runs concurrently with fetching (pipelined) but sequentially
   // with itself, so DB write pressure stays bounded at one batch at a time.
   let persistChain: Promise<unknown> = Promise.resolve();
+  let queuedPersists = 0;
 
   const flush = () => {
     if (batch.length === 0) return;
     const pages = batch;
     batch = [];
     windowSize = adjustCrawlWindow(windowSize, pages);
-    persistChain = persistChain.then(() =>
-      persistCrawledPages({
-        auditId,
-        workflowInstanceId,
-        origin,
-        robots,
-        scratchpad,
-        pages,
-        depthByUrl,
-        maxPages,
-      }),
-    );
+    queuedPersists += 1;
+    persistChain = persistChain
+      .then(() =>
+        persistCrawledPages({
+          auditId,
+          workflowInstanceId,
+          origin,
+          robots,
+          scratchpad,
+          pages,
+          depthByUrl,
+          maxPages,
+        }),
+      )
+      .finally(() => {
+        queuedPersists -= 1;
+      });
   };
 
   const launch = (entry: ClaimedUrl) => {
@@ -182,14 +195,29 @@ async function runCrawlChunk(
   while (true) {
     while (
       inFlight.size < windowSize &&
+      queuedPersists <= MAX_QUEUED_PERSIST_BATCHES &&
       nextIndex < claimed.length &&
       Date.now() < deadlineAt
     ) {
       launch(claimed[nextIndex]);
       nextIndex += 1;
     }
-    if (inFlight.size === 0) break;
-    await Promise.race(inFlight);
+    if (inFlight.size > 0) {
+      await Promise.race(inFlight);
+      continue;
+    }
+    // Nothing in flight. If launches are only paused by persistence
+    // backpressure, wait for the queue to drain and resume; otherwise the
+    // chunk is done (leases exhausted or soft deadline hit).
+    if (
+      queuedPersists > MAX_QUEUED_PERSIST_BATCHES &&
+      nextIndex < claimed.length &&
+      Date.now() < deadlineAt
+    ) {
+      await persistChain;
+      continue;
+    }
+    break;
   }
   flush();
   await persistChain;