From 6ad65f59e1131f703eda3024e2a70555f55ed37b Mon Sep 17 00:00:00 2001
From: Ben Senescu <44480372+bensenescu@users.noreply.github.com>
Date: Sat, 18 Jul 2026 23:49:30 -0400
Subject: [PATCH] Bound audit crawl resource usage (#401)

Cap crawled HTML bodies at 2 MiB before Cheerio parsing, bulk-insert
audit links on Postgres instead of per-row Hyperdrive round trips, and
raise the Worker CPU ceiling to the 5-minute paid max for link-heavy
crawl steps.
---
 alchemy.run.ts                                |  4 ++
 .../audit/repositories/AuditRepository.ts     | 19 ++++++++--
 .../workflows/site-audit-workflow-helpers.ts  | 38 ++++++++++++++++++-
 3 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/alchemy.run.ts b/alchemy.run.ts
index 2e3c534..d95a6e1 100644
--- a/alchemy.run.ts
+++ b/alchemy.run.ts
@@ -247,6 +247,10 @@ export default Alchemy.Stack(
         date: wrangler.compatibility_date,
         flags: wrangler.compatibility_flags,
       },
+      // Site audits parse and persist batches of HTML inside Workflow steps.
+      // Paid Workers permit up to five minutes; keep headroom for unusually
+      // link-heavy sites after bounding page bodies and bulk-writing links.
+      limits: { cpuMs: 300_000 },
       observability: {
         enabled: wrangler.observability?.enabled ?? true,
         traces: { enabled: wrangler.observability?.traces?.enabled ?? false },
diff --git a/src/server/features/audit/repositories/AuditRepository.ts b/src/server/features/audit/repositories/AuditRepository.ts
index 9fa0ad0..28c2767 100644
--- a/src/server/features/audit/repositories/AuditRepository.ts
+++ b/src/server/features/audit/repositories/AuditRepository.ts
@@ -12,6 +12,7 @@ import {
   auditLinks,
   auditPages,
 } from "@/db/schema";
+import { getDatabaseProvider } from "@/db/provider";
 import { executeInBatches } from "@/db/runBatch";
 import { AUDIT_ISSUE_TYPES } from "@/shared/audit-issues";
 import { deterministicAuditRowId } from "@/server/lib/audit/ids";
@@ -28,6 +29,7 @@ import type {
 // checks. Mega-menu/footer-heavy sites can carry 1000+ links per page; cap
 // what we store so a 10k-page crawl can't write tens of millions of link rows.
 const MAX_STORED_LINKS_PER_PAGE = 500;
+const POSTGRES_LINK_INSERT_SIZE = 500;
 
 async function createAudit(data: {
   id: string;
@@ -207,9 +209,20 @@ async function insertCrawledBatch(
         })),
     ),
   );
-  await executeInBatches(linkRows, (tx, row) =>
-    tx.insert(auditLinks).values(row).onConflictDoNothing(),
-  );
+  if (getDatabaseProvider() === "postgres") {
+    // A Postgres transaction executes runBatch statements sequentially. Bulk
+    // values avoid thousands of Hyperdrive round trips on link-heavy pages.
+    for (let i = 0; i < linkRows.length; i += POSTGRES_LINK_INSERT_SIZE) {
+      await db
+        .insert(auditLinks)
+        .values(linkRows.slice(i, i + POSTGRES_LINK_INSERT_SIZE))
+        .onConflictDoNothing();
+    }
+  } else {
+    await executeInBatches(linkRows, (tx, row) =>
+      tx.insert(auditLinks).values(row).onConflictDoNothing(),
+    );
+  }
 
   await insertIssues(auditId, issues);
 }
diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.ts
index 910cd3d..ff08033 100644
--- a/src/server/workflows/site-audit-workflow-helpers.ts
+++ b/src/server/workflows/site-audit-workflow-helpers.ts
@@ -6,6 +6,7 @@ import { sha256Hex } from "@/server/lib/audit/ids";
 import { normalizeUrl } from "@/server/lib/audit/url-utils";
 
 const CRAWL_USER_AGENT = "OpenSEO-Audit/1.0";
+const MAX_HTML_BYTES = 2 * 1024 * 1024;
 
 /**
  * Markers of a bot-mitigation challenge page. We classify these honestly as
@@ -103,7 +104,10 @@ export async function crawlPage(
 
     const contentType = response.headers.get("content-type") ?? "";
     const isHtml = contentType.includes("text/html");
-    const body = isHtml ? await response.text() : "";
+    // Large pages make Cheerio disproportionately expensive and can exhaust a
+    // crawl step's CPU or isolate memory. The first 2 MiB still contains the
+    // SEO metadata and navigation needed by the audit in normal documents.
+    const body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";
     const fetchClass = classifyFetch(
       statusCode,
       response.headers,
@@ -198,6 +202,38 @@ export async function crawlPage(
   }
 }
 
+async function readTextUpTo(response: Response, maxBytes: number) {
+  if (!response.body) return "";
+
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  const parts: string[] = [];
+  let bytesRead = 0;
+
+  try {
+    while (bytesRead < maxBytes) {
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      const remaining = maxBytes - bytesRead;
+      const chunk =
+        value.byteLength > remaining ? value.subarray(0, remaining) : value;
+      bytesRead += chunk.byteLength;
+      parts.push(decoder.decode(chunk, { stream: true }));
+
+      if (bytesRead >= maxBytes) {
+        await reader.cancel();
+        break;
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  parts.push(decoder.decode());
+  return parts.join("");
+}
+
 function emptyPageResult(input: {
   url: string;
   statusCode: number;