Audit crawler: send a browser navigation header set (tier-1 anti-bot)

The crawler identified as `OpenSEO-Audit/1.0` with almost no headers, which
naive bot filters and security plugins block outright.

- New crawl-request.ts: AUDIT_USER_AGENT (a current Chrome string),
  buildAuditHeaders() (Accept, Accept-Language, Sec-Fetch-*, Sec-Ch-Ua,
  Upgrade-Insecure-Requests), and fetchForAudit() — fetch + those headers +
  one retry on a transient 429/503.
- Wired into the page crawl (site-audit-workflow-helpers), robots.txt +
  sitemap discovery, and start-URL redirect probing.

Gets past the naive tier; still reported as "blocked" for JS/TLS challenges
(Cloudflare Managed Challenge, DataDome) — those need a real browser. Doc note
points operators at WAF IP/UA allowlisting for their own sites.

No env dependency (keeps the audit lib importable without a cloudflare:workers
mock). tsc / oxlint / knip clean; new crawl-request.test.ts (5); suite
otherwise unchanged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

docs/SELF_HOSTING_TEAM_MODE.md

@@ -92,6 +92,10 @@ app keeps working; the Activity tab just shows nothing.
 
 - Password reset by email is not available in `team` mode. The owner/admins
   reset passwords from Settings → Users.
+- Site audit sends a Chrome-like User-Agent and browser headers, which gets
+  past naive bot filters but not JS/TLS challenges (Cloudflare Managed
+  Challenge, DataDome). For those, allowlist the server's IP or the crawler's
+  User-Agent in the target site's WAF.
 - The MCP server and its OAuth flow are hosted-only for now; `team` deployments
   serve the app UI only.
 - Rolling back: set `AUTH_MODE=local_noauth`, rebuild, restart. Existing users

src/server/lib/audit/crawl-request.test.ts

@@ -0,0 +1,82 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  AUDIT_USER_AGENT,
+  buildAuditHeaders,
+  fetchForAudit,
+} from "./crawl-request";
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+  vi.useRealTimers();
+});
+
+describe("buildAuditHeaders", () => {
+  it("sends a browser navigation header set", () => {
+    const headers = buildAuditHeaders();
+    expect(headers["User-Agent"]).toBe(AUDIT_USER_AGENT);
+    expect(AUDIT_USER_AGENT).toMatch(/Chrome\/\d+\.0\.0\.0 Safari/);
+    expect(headers["Accept-Language"]).toContain("en");
+    expect(headers["Sec-Fetch-Mode"]).toBe("navigate");
+    expect(headers["Upgrade-Insecure-Requests"]).toBe("1");
+  });
+
+  it("lets callers override individual entries", () => {
+    expect(buildAuditHeaders({ Accept: "text/plain" }).Accept).toBe(
+      "text/plain",
+    );
+  });
+});
+
+describe("fetchForAudit", () => {
+  it("returns the first response when it is not a transient block", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValue(new Response("ok", { status: 200 }));
+    vi.stubGlobal("fetch", fetchMock);
+
+    const res = await fetchForAudit("https://example.com");
+    expect(res.status).toBe(200);
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it("applies the browser headers", async () => {
+    let seen: HeadersInit | undefined;
+    vi.stubGlobal(
+      "fetch",
+      vi.fn((_url: string, init?: RequestInit) => {
+        seen = init?.headers;
+        return Promise.resolve(new Response("ok", { status: 200 }));
+      }),
+    );
+
+    await fetchForAudit("https://example.com");
+    expect(seen).toMatchObject({ "User-Agent": AUDIT_USER_AGENT });
+  });
+
+  it("retries once on 503 and returns the retry response", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(new Response("busy", { status: 503 }))
+      .mockResolvedValueOnce(new Response("ok", { status: 200 }));
+    vi.stubGlobal("fetch", fetchMock);
+    vi.useFakeTimers();
+
+    const promise = fetchForAudit("https://example.com");
+    await vi.runAllTimersAsync();
+    const res = await promise;
+
+    expect(res.status).toBe(200);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
+  it("does not retry a hard 403", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValue(new Response("no", { status: 403 }));
+    vi.stubGlobal("fetch", fetchMock);
+
+    const res = await fetchForAudit("https://example.com");
+    expect(res.status).toBe(403);
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+});

src/server/lib/audit/crawl-request.ts

@@ -0,0 +1,48 @@
+// Tier-1 anti-bot handling for the site-audit crawler: present each request as
+// an ordinary top-level Chrome navigation. This gets past naive user-agent
+// blocklists, basic WAF rules, and security plugins. It does NOT defeat JS or
+// TLS-fingerprint challenges (Cloudflare Managed Challenge, DataDome, Akamai) —
+// those need a real browser and are still reported as "blocked".
+
+const CHROME_MAJOR = "141";
+
+export const AUDIT_USER_AGENT =
+  `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ` +
+  `(KHTML, like Gecko) Chrome/${CHROME_MAJOR}.0.0.0 Safari/537.36`;
+
+export function buildAuditHeaders(
+  overrides: Record<string, string> = {},
+): Record<string, string> {
+  return {
+    "User-Agent": AUDIT_USER_AGENT,
+    Accept:
+      "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Upgrade-Insecure-Requests": "1",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Sec-Ch-Ua": `"Chromium";v="${CHROME_MAJOR}", "Google Chrome";v="${CHROME_MAJOR}", "Not?A_Brand";v="24"`,
+    "Sec-Ch-Ua-Mobile": "?0",
+    "Sec-Ch-Ua-Platform": '"Windows"',
+    ...overrides,
+  };
+}
+
+const TRANSIENT_BLOCK_STATUS = new Set([429, 503]);
+
+// fetch() with browser headers and a single retry on a transient throttle /
+// challenge response. A hard block (403, persistent 429) will not clear, so the
+// retry is capped tight — ~1s of added latency at most.
+export async function fetchForAudit(
+  url: string,
+  init: RequestInit & { headers?: Record<string, string> } = {},
+): Promise<Response> {
+  const headers = buildAuditHeaders(init.headers);
+  const first = await fetch(url, { ...init, headers });
+  if (!TRANSIENT_BLOCK_STATUS.has(first.status)) return first;
+
+  await new Promise((resolve) => setTimeout(resolve, 1_000));
+  return fetch(url, { ...init, headers });
+}

src/server/lib/audit/discovery.ts

@@ -3,6 +3,7 @@
  */
 import robotsParser from "robots-parser";
 import { XMLParser } from "fast-xml-parser";
+import { buildAuditHeaders } from "./crawl-request";
 import { isSameOrigin, normalizeUrl } from "./url-utils";
 
 const SITEMAP_FETCH_TIMEOUT_MS = 15_000;
@@ -40,7 +41,7 @@ export interface RobotsResult {
 async function fetchRobotsTxtText(origin: string): Promise<string | null> {
   try {
     const response = await fetch(`${origin}/robots.txt`, {
-      headers: { "User-Agent": "OpenSEO-Audit/1.0" },
+      headers: buildAuditHeaders({ Accept: "text/plain,*/*;q=0.8" }),
       signal: AbortSignal.timeout(10_000),
     });
 
@@ -173,7 +174,9 @@ async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{
   for (let attempt = 0; attempt <= SITEMAP_RETRIES; attempt++) {
     try {
       const response = await fetch(normalizedSitemapUrl, {
-        headers: { "User-Agent": "OpenSEO-Audit/1.0" },
+        headers: buildAuditHeaders({
+          Accept: "application/xml,text/xml;q=0.9,*/*;q=0.8",
+        }),
         signal: AbortSignal.timeout(SITEMAP_FETCH_TIMEOUT_MS),
       });
 

src/server/lib/audit/url-policy.ts

@@ -1,4 +1,5 @@
 import { AppError } from "@/server/lib/errors";
+import { buildAuditHeaders } from "./crawl-request";
 
 const BLOCKED_HOSTS = new Set([
   "localhost",
@@ -263,7 +264,7 @@ export async function resolveStartUrlRedirects(
       response = await fetch(current, {
         method: "HEAD",
         redirect: "manual",
-        headers: { "User-Agent": "OpenSEO-Audit/1.0" },
+        headers: buildAuditHeaders(),
         signal: AbortSignal.timeout(START_URL_PROBE_TIMEOUT_MS),
       });
     } catch {

src/server/workflows/site-audit-workflow-helpers.ts

@@ -2,10 +2,10 @@ import type {
   CrawledPageResult,
   PageFetchClass,
 } from "@/server/lib/audit/types";
+import { fetchForAudit } from "@/server/lib/audit/crawl-request";
 import { sha256Hex } from "@/server/lib/audit/ids";
 import { normalizeUrl } from "@/server/lib/audit/url-utils";
 
-const CRAWL_USER_AGENT = "OpenSEO-Audit/1.0";
 const MAX_HTML_BYTES = 1024 * 1024;
 
 /**
@@ -69,11 +69,7 @@ export async function crawlPage(
     // /docs/) need no special handling: normalizeUrl preserves trailing
     // slashes, so /docs and /docs/ are distinct URLs and the redirect resolves
     // to its canonical target instead of cycling back to its own source.
-    const response = await fetch(url, {
+    const response = await fetchForAudit(url, {
-      headers: {
-        "User-Agent": CRAWL_USER_AGENT,
-        Accept: "text/html,application/xhtml+xml",
-      },
       redirect: "manual",
       signal: AbortSignal.timeout(15_000),
     });