diff --git a/docs/SELF_HOSTING_TEAM_MODE.md b/docs/SELF_HOSTING_TEAM_MODE.md index cd908e5..56e684e 100644 --- a/docs/SELF_HOSTING_TEAM_MODE.md +++ b/docs/SELF_HOSTING_TEAM_MODE.md @@ -92,6 +92,10 @@ app keeps working; the Activity tab just shows nothing. - Password reset by email is not available in `team` mode. The owner/admins reset passwords from Settings → Users. +- Site audit sends a Chrome-like User-Agent and browser headers, which gets + past naive bot filters but not JS/TLS challenges (Cloudflare Managed + Challenge, DataDome). For those, allowlist the server's IP or the crawler's + User-Agent in the target site's WAF. - The MCP server and its OAuth flow are hosted-only for now; `team` deployments serve the app UI only. - Rolling back: set `AUTH_MODE=local_noauth`, rebuild, restart. Existing users diff --git a/src/server/lib/audit/crawl-request.test.ts b/src/server/lib/audit/crawl-request.test.ts new file mode 100644 index 0000000..aefce35 --- /dev/null +++ b/src/server/lib/audit/crawl-request.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + AUDIT_USER_AGENT, + buildAuditHeaders, + fetchForAudit, +} from "./crawl-request"; + +afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); +}); + +describe("buildAuditHeaders", () => { + it("sends a browser navigation header set", () => { + const headers = buildAuditHeaders(); + expect(headers["User-Agent"]).toBe(AUDIT_USER_AGENT); + expect(AUDIT_USER_AGENT).toMatch(/Chrome\/\d+\.0\.0\.0 Safari/); + expect(headers["Accept-Language"]).toContain("en"); + expect(headers["Sec-Fetch-Mode"]).toBe("navigate"); + expect(headers["Upgrade-Insecure-Requests"]).toBe("1"); + }); + + it("lets callers override individual entries", () => { + expect(buildAuditHeaders({ Accept: "text/plain" }).Accept).toBe( + "text/plain", + ); + }); +}); + +describe("fetchForAudit", () => { + it("returns the first response when it is not a transient block", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(new Response("ok", { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + + const res = await fetchForAudit("https://example.com"); + expect(res.status).toBe(200); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("applies the browser headers", async () => { + let seen: HeadersInit | undefined; + vi.stubGlobal( + "fetch", + vi.fn((_url: string, init?: RequestInit) => { + seen = init?.headers; + return Promise.resolve(new Response("ok", { status: 200 })); + }), + ); + + await fetchForAudit("https://example.com"); + expect(seen).toMatchObject({ "User-Agent": AUDIT_USER_AGENT }); + }); + + it("retries once on 503 and returns the retry response", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response("busy", { status: 503 })) + .mockResolvedValueOnce(new Response("ok", { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + vi.useFakeTimers(); + + const promise = fetchForAudit("https://example.com"); + await vi.runAllTimersAsync(); + const res = await promise; + + expect(res.status).toBe(200); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("does not retry a hard 403", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(new Response("no", { status: 403 })); + vi.stubGlobal("fetch", fetchMock); + + const res = await fetchForAudit("https://example.com"); + expect(res.status).toBe(403); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/server/lib/audit/crawl-request.ts b/src/server/lib/audit/crawl-request.ts new file mode 100644 index 0000000..a1eb34a --- /dev/null +++ b/src/server/lib/audit/crawl-request.ts @@ -0,0 +1,48 @@ +// Tier-1 anti-bot handling for the site-audit crawler: present each request as +// an ordinary top-level Chrome navigation. This gets past naive user-agent +// blocklists, basic WAF rules, and security plugins. It does NOT defeat JS or +// TLS-fingerprint challenges (Cloudflare Managed Challenge, DataDome, Akamai) — +// those need a real browser and are still reported as "blocked". + +const CHROME_MAJOR = "141"; + +export const AUDIT_USER_AGENT = + `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ` + + `(KHTML, like Gecko) Chrome/${CHROME_MAJOR}.0.0.0 Safari/537.36`; + +export function buildAuditHeaders( + overrides: Record = {}, +): Record { + return { + "User-Agent": AUDIT_USER_AGENT, + Accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Sec-Ch-Ua": `"Chromium";v="${CHROME_MAJOR}", "Google Chrome";v="${CHROME_MAJOR}", "Not?A_Brand";v="24"`, + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": '"Windows"', + ...overrides, + }; +} + +const TRANSIENT_BLOCK_STATUS = new Set([429, 503]); + +// fetch() with browser headers and a single retry on a transient throttle / +// challenge response. A hard block (403, persistent 429) will not clear, so the +// retry is capped tight — ~1s of added latency at most. +export async function fetchForAudit( + url: string, + init: RequestInit & { headers?: Record } = {}, +): Promise { + const headers = buildAuditHeaders(init.headers); + const first = await fetch(url, { ...init, headers }); + if (!TRANSIENT_BLOCK_STATUS.has(first.status)) return first; + + await new Promise((resolve) => setTimeout(resolve, 1_000)); + return fetch(url, { ...init, headers }); +} diff --git a/src/server/lib/audit/discovery.ts b/src/server/lib/audit/discovery.ts index 1113f1a..b47dadb 100644 --- a/src/server/lib/audit/discovery.ts +++ b/src/server/lib/audit/discovery.ts @@ -3,6 +3,7 @@ */ import robotsParser from "robots-parser"; import { XMLParser } from "fast-xml-parser"; +import { buildAuditHeaders } from "./crawl-request"; import { isSameOrigin, normalizeUrl } from "./url-utils"; const SITEMAP_FETCH_TIMEOUT_MS = 15_000; @@ -40,7 +41,7 @@ export interface RobotsResult { async function fetchRobotsTxtText(origin: string): Promise { try { const response = await fetch(`${origin}/robots.txt`, { - headers: { "User-Agent": "OpenSEO-Audit/1.0" }, + headers: buildAuditHeaders({ Accept: "text/plain,*/*;q=0.8" }), signal: AbortSignal.timeout(10_000), }); @@ -173,7 +174,9 @@ async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{ for (let attempt = 0; attempt <= SITEMAP_RETRIES; attempt++) { try { const response = await fetch(normalizedSitemapUrl, { - headers: { "User-Agent": "OpenSEO-Audit/1.0" }, + headers: buildAuditHeaders({ + Accept: "application/xml,text/xml;q=0.9,*/*;q=0.8", + }), signal: AbortSignal.timeout(SITEMAP_FETCH_TIMEOUT_MS), }); diff --git a/src/server/lib/audit/url-policy.ts b/src/server/lib/audit/url-policy.ts index c2fb5d3..d1c0fc1 100644 --- a/src/server/lib/audit/url-policy.ts +++ b/src/server/lib/audit/url-policy.ts @@ -1,4 +1,5 @@ import { AppError } from "@/server/lib/errors"; +import { buildAuditHeaders } from "./crawl-request"; const BLOCKED_HOSTS = new Set([ "localhost", @@ -263,7 +264,7 @@ export async function resolveStartUrlRedirects( response = await fetch(current, { method: "HEAD", redirect: "manual", - headers: { "User-Agent": "OpenSEO-Audit/1.0" }, + headers: buildAuditHeaders(), signal: AbortSignal.timeout(START_URL_PROBE_TIMEOUT_MS), }); } catch { diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.ts index a03463f..5dcb79e 100644 --- a/src/server/workflows/site-audit-workflow-helpers.ts +++ b/src/server/workflows/site-audit-workflow-helpers.ts @@ -2,10 +2,10 @@ import type { CrawledPageResult, PageFetchClass, } from "@/server/lib/audit/types"; +import { fetchForAudit } from "@/server/lib/audit/crawl-request"; import { sha256Hex } from "@/server/lib/audit/ids"; import { normalizeUrl } from "@/server/lib/audit/url-utils"; -const CRAWL_USER_AGENT = "OpenSEO-Audit/1.0"; const MAX_HTML_BYTES = 1024 * 1024; /** @@ -69,11 +69,7 @@ export async function crawlPage( // /docs/) need no special handling: normalizeUrl preserves trailing // slashes, so /docs and /docs/ are distinct URLs and the redirect resolves // to its canonical target instead of cycling back to its own source. - const response = await fetch(url, { - headers: { - "User-Agent": CRAWL_USER_AGENT, - Accept: "text/html,application/xhtml+xml", - }, + const response = await fetchForAudit(url, { redirect: "manual", signal: AbortSignal.timeout(15_000), });