17dfca406f
The crawler identified as `OpenSEO-Audit/1.0` with almost no headers, which naive bot filters and security plugins block outright. - New crawl-request.ts: AUDIT_USER_AGENT (a current Chrome string), buildAuditHeaders() (Accept, Accept-Language, Sec-Fetch-*, Sec-Ch-Ua, Upgrade-Insecure-Requests), and fetchForAudit() — fetch + those headers + one retry on a transient 429/503. - Wired into the page crawl (site-audit-workflow-helpers), robots.txt + sitemap discovery, and start-URL redirect probing. Gets past the naive tier; still reported as "blocked" for JS/TLS challenges (Cloudflare Managed Challenge, DataDome) — those need a real browser. Doc note points operators at WAF IP/UA allowlisting for their own sites. No env dependency (keeps the audit lib importable without a cloudflare:workers mock). tsc / oxlint / knip clean; new crawl-request.test.ts (5); suite otherwise unchanged. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
83 lines
2.5 KiB
TypeScript
83 lines
2.5 KiB
TypeScript
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import {
|
|
AUDIT_USER_AGENT,
|
|
buildAuditHeaders,
|
|
fetchForAudit,
|
|
} from "./crawl-request";
|
|
|
|
afterEach(() => {
|
|
vi.unstubAllGlobals();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
describe("buildAuditHeaders", () => {
|
|
it("sends a browser navigation header set", () => {
|
|
const headers = buildAuditHeaders();
|
|
expect(headers["User-Agent"]).toBe(AUDIT_USER_AGENT);
|
|
expect(AUDIT_USER_AGENT).toMatch(/Chrome\/\d+\.0\.0\.0 Safari/);
|
|
expect(headers["Accept-Language"]).toContain("en");
|
|
expect(headers["Sec-Fetch-Mode"]).toBe("navigate");
|
|
expect(headers["Upgrade-Insecure-Requests"]).toBe("1");
|
|
});
|
|
|
|
it("lets callers override individual entries", () => {
|
|
expect(buildAuditHeaders({ Accept: "text/plain" }).Accept).toBe(
|
|
"text/plain",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("fetchForAudit", () => {
|
|
it("returns the first response when it is not a transient block", async () => {
|
|
const fetchMock = vi
|
|
.fn()
|
|
.mockResolvedValue(new Response("ok", { status: 200 }));
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
const res = await fetchForAudit("https://example.com");
|
|
expect(res.status).toBe(200);
|
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("applies the browser headers", async () => {
|
|
let seen: HeadersInit | undefined;
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn((_url: string, init?: RequestInit) => {
|
|
seen = init?.headers;
|
|
return Promise.resolve(new Response("ok", { status: 200 }));
|
|
}),
|
|
);
|
|
|
|
await fetchForAudit("https://example.com");
|
|
expect(seen).toMatchObject({ "User-Agent": AUDIT_USER_AGENT });
|
|
});
|
|
|
|
it("retries once on 503 and returns the retry response", async () => {
|
|
const fetchMock = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(new Response("busy", { status: 503 }))
|
|
.mockResolvedValueOnce(new Response("ok", { status: 200 }));
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
vi.useFakeTimers();
|
|
|
|
const promise = fetchForAudit("https://example.com");
|
|
await vi.runAllTimersAsync();
|
|
const res = await promise;
|
|
|
|
expect(res.status).toBe(200);
|
|
expect(fetchMock).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("does not retry a hard 403", async () => {
|
|
const fetchMock = vi
|
|
.fn()
|
|
.mockResolvedValue(new Response("no", { status: 403 }));
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
const res = await fetchForAudit("https://example.com");
|
|
expect(res.status).toBe(403);
|
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|