Site Audit - Improve reliability and performance with Durable Objects
This commit is contained in:
Ben Senescu
GitHub
parent
9d19e43990
commit
1e8a924c4c
@ -20,3 +20,15 @@ data, or sensitive paths.
|
||||
## Resolved
|
||||
|
||||
Move fixed entries here, mark them checked, and append the resolving date or commit.
|
||||
|
||||
## badseo harness vs `wrangler dev`: sitemap emits badseo.dev locs locally
|
||||
|
||||
`badseo/scripts/run-audit.ts` against a local `wrangler dev --port 8787` fails 4
|
||||
sitemap-dependent checks (orphan page, 500, 403, duplicate-content) with
|
||||
NOT CRAWLED: wrangler dev adopts the `badseo.dev` custom-domain route as the
|
||||
host the worker sees, so `/sitemap.xml` emits `http://badseo.dev/...` locs that
|
||||
the crawler's same-origin filter drops. Run it as
|
||||
`wrangler dev --port 8787 --local-upstream "localhost:8787"` (after
|
||||
`vite build`). Also: `pnpm --filter badseo audit` fails with
|
||||
"Unknown option: 'recursive'" from the repo root — badseo is its own pnpm
|
||||
workspace, not a root workspace member; use `npx tsx badseo/scripts/run-audit.ts`.
|
||||
|
||||
@ -400,8 +400,10 @@ export default Alchemy.Stack(
|
||||
// Prod-only: pooled Postgres via the existing Hyperdrive config.
|
||||
...(prod ? { HYPERDRIVE: makeHyperdrive() } : {}),
|
||||
|
||||
// Durable Objects (Agents SDK chat agents). Alchemy backs new DO
|
||||
// classes with SQLite storage, which both require.
|
||||
// Durable Objects (chat agents + the per-audit crawl scratchpad).
|
||||
// Alchemy backs new DO classes with SQLite storage, which all of
|
||||
// them require; the `migrations` array in wrangler.jsonc only
|
||||
// applies to the wrangler/workerd surfaces (local dev, Docker).
|
||||
...Object.fromEntries(
|
||||
wrangler.durable_objects.bindings.map((binding) => [
|
||||
binding.name,
|
||||
|
||||
@ -46,7 +46,7 @@ a:focus-visible, button:focus-visible { outline: 3px solid #ff6a57; outline-offs
|
||||
.home-copy > p { max-width: 860px; margin: 42px 0 0; font-family: var(--serif); font-size: clamp(28px, 2.2vw, 32px); line-height: 1.08; }
|
||||
.home-issues { scroll-margin-top: 24px; border-top: 3px solid var(--navy); }
|
||||
.home-issue-group { margin: 22px 0 54px; }
|
||||
.home-issue-group h3 { margin: 0; padding: 18px 0 14px; border-bottom: 1px solid var(--line); font-family: var(--condensed); font-size: 30px; line-height: 1; }
|
||||
.home-issue-group h2 { margin: 0; padding: 18px 0 14px; border-bottom: 1px solid var(--line); font-family: var(--condensed); font-size: 30px; line-height: 1; }
|
||||
.home-case-head, .home-case-row { display: grid; grid-template-columns: minmax(220px, .72fr) minmax(320px, 1.28fr); gap: clamp(30px, 6vw, 90px); }
|
||||
.home-case-head { padding: 11px 0; border-bottom: 1px solid var(--line-soft); color: var(--muted); font-family: var(--mono); font-size: 8px; }
|
||||
.home-case-row { min-height: 72px; align-items: center; padding: 16px 0; border-bottom: 1px solid var(--line-soft); text-decoration: none; }
|
||||
|
||||
@ -75,7 +75,12 @@ export interface FileRoutesById {
|
||||
export interface FileRouteTypes {
|
||||
fileRoutesByFullPath: FileRoutesByFullPath
|
||||
fullPaths:
|
||||
'/' | '/$' | '/catalog' | '/privacy' | '/robots.txt' | '/sitemap.xml'
|
||||
| '/'
|
||||
| '/$'
|
||||
| '/catalog'
|
||||
| '/privacy'
|
||||
| '/robots.txt'
|
||||
| '/sitemap.xml'
|
||||
fileRoutesByTo: FileRoutesByTo
|
||||
to: '/' | '/$' | '/catalog' | '/privacy' | '/robots.txt' | '/sitemap.xml'
|
||||
id:
|
||||
|
||||
@ -41,7 +41,7 @@ function HomePage() {
|
||||
<div className="home-issues" id="issues">
|
||||
{homepageCategories.map((category) => (
|
||||
<section className="home-issue-group" key={category}>
|
||||
<h3>{category}</h3>
|
||||
<h2>{category}</h2>
|
||||
<div className="home-case-head" aria-hidden="true">
|
||||
<span>Issue</span>
|
||||
<span>What it demonstrates</span>
|
||||
|
||||
3
drizzle-pg/0014_solid_centennial.sql
Normal file
3
drizzle-pg/0014_solid_centennial.sql
Normal file
@ -0,0 +1,3 @@
|
||||
ALTER TABLE "audits" ADD COLUMN "error_code" text;--> statement-breakpoint
|
||||
ALTER TABLE "audits" ADD COLUMN "error_detail" text;--> statement-breakpoint
|
||||
ALTER TABLE "audits" ADD COLUMN "failed_phase" text;
|
||||
3732
drizzle-pg/meta/0014_snapshot.json
Normal file
3732
drizzle-pg/meta/0014_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -99,6 +99,13 @@
|
||||
"when": 1784426969390,
|
||||
"tag": "0013_sleepy_black_tarantula",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 14,
|
||||
"version": "7",
|
||||
"when": 1784588248793,
|
||||
"tag": "0014_solid_centennial",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
3
drizzle/0037_small_caretaker.sql
Normal file
3
drizzle/0037_small_caretaker.sql
Normal file
@ -0,0 +1,3 @@
|
||||
ALTER TABLE `audits` ADD `error_code` text;--> statement-breakpoint
|
||||
ALTER TABLE `audits` ADD `error_detail` text;--> statement-breakpoint
|
||||
ALTER TABLE `audits` ADD `failed_phase` text;
|
||||
3401
drizzle/meta/0037_snapshot.json
Normal file
3401
drizzle/meta/0037_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -260,6 +260,13 @@
|
||||
"when": 1784426967421,
|
||||
"tag": "0036_curvy_silk_fever",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 37,
|
||||
"version": "6",
|
||||
"when": 1784588247803,
|
||||
"tag": "0037_small_caretaker",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -91,12 +91,12 @@
|
||||
"ai": "^6.0.199",
|
||||
"autumn-js": "^1.2.33",
|
||||
"better-auth": "^1.6.22",
|
||||
"cheerio": "^1.2.0",
|
||||
"cloudflare": "^5.2.0",
|
||||
"daisyui": "^5.5.5",
|
||||
"dataforseo-client": "^2.0.19",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"fast-xml-parser": "^5.4.1",
|
||||
"htmlparser2": "^10.1.0",
|
||||
"jose": "^6.0.12",
|
||||
"lucide-react": "^0.542.0",
|
||||
"papaparse": "^5.5.3",
|
||||
@ -132,6 +132,7 @@
|
||||
"@vitejs/plugin-react": "^4.6.0",
|
||||
"alchemy": "2.0.0-beta.61",
|
||||
"chalk": "^5.6.2",
|
||||
"cheerio": "^1.2.0",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
"effect": "4.0.0-beta.93",
|
||||
"knip": "^5.88.1",
|
||||
|
||||
9
pnpm-lock.yaml
generated
9
pnpm-lock.yaml
generated
@ -75,9 +75,6 @@ importers:
|
||||
better-auth:
|
||||
specifier: ^1.6.22
|
||||
version: 1.6.22(@cloudflare/workers-types@4.20260611.1)(@opentelemetry/api@1.9.1)(@tanstack/react-start@1.168.26(esbuild@0.28.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(rolldown@1.0.1)(rollup@4.59.0)(vite@7.3.6(@types/node@22.19.11)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)))(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@4.20260611.1)(@libsql/client@0.15.15)(@opentelemetry/api@1.9.1)(kysely@0.29.2)(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(postgres@3.4.9)(sql.js@1.14.1))(mongodb@7.2.0(@aws-sdk/credential-providers@3.1080.0)(@mongodb-js/zstd@7.0.0))(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.6(@types/debug@4.1.13)(@types/node@22.19.11)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0))
|
||||
cheerio:
|
||||
specifier: ^1.2.0
|
||||
version: 1.2.0
|
||||
cloudflare:
|
||||
specifier: ^5.2.0
|
||||
version: 5.2.0
|
||||
@ -93,6 +90,9 @@ importers:
|
||||
fast-xml-parser:
|
||||
specifier: ^5.4.1
|
||||
version: 5.8.0
|
||||
htmlparser2:
|
||||
specifier: ^10.1.0
|
||||
version: 10.1.0
|
||||
jose:
|
||||
specifier: ^6.0.12
|
||||
version: 6.1.3
|
||||
@ -193,6 +193,9 @@ importers:
|
||||
chalk:
|
||||
specifier: ^5.6.2
|
||||
version: 5.6.2
|
||||
cheerio:
|
||||
specifier: ^1.2.0
|
||||
version: 1.2.0
|
||||
drizzle-kit:
|
||||
specifier: ^0.31.10
|
||||
version: 0.31.10
|
||||
|
||||
125
specs/0009-site-audit-crawl-architecture.md
Normal file
125
specs/0009-site-audit-crawl-architecture.md
Normal file
@ -0,0 +1,125 @@
|
||||
# Site audit crawl architecture
|
||||
|
||||
## Status
|
||||
|
||||
Accepted
|
||||
|
||||
## Context
|
||||
|
||||
The site audit runs as a Cloudflare Workflow. The original design kept all
|
||||
crawl state inside the workflow run: the URL frontier and page summaries lived
|
||||
in workflow memory and step returns, pages were parsed with a full DOM parser
|
||||
(cheerio), and link edges were bulk-inserted into the app database during the
|
||||
crawl. On large or slow sites this collided with platform limits:
|
||||
|
||||
- **Isolate memory.** DOM parsing expands a page several times over, multiplied
|
||||
by concurrent parses, plus a whole-crawl summaries array that was rebuilt on
|
||||
every workflow replay.
|
||||
- **Step-output cap (~1 MiB).** Sitemap seed lists flowed through step returns
|
||||
and could exceed it outright.
|
||||
- **Step timeout × retry.** The default 10-minute step timeout with default
|
||||
retries meant a deterministically failing step burned about an hour before
|
||||
the audit died.
|
||||
- **Batch head-of-line blocking.** Fixed-size crawl batches waited on their
|
||||
slowest fetch; one tarpit URL stalled a whole batch.
|
||||
- **Invisible failure.** Workflow-level kills (OOM, CPU) skip the error
|
||||
handler, so audit rows could stay `running` forever with no recorded reason,
|
||||
and the UI refused to show the pages that had already been crawled and
|
||||
persisted.
|
||||
|
||||
## Decision
|
||||
|
||||
Split the audit into a control plane and a data plane.
|
||||
|
||||
### Orchestration stays in Workflows; crawl state moves to a Durable Object
|
||||
|
||||
`SiteAuditWorkflow` remains the orchestrator (phase ordering, durable steps,
|
||||
retries). All transient crawl state lives in `AuditScratchpad`, a SQLite-backed
|
||||
Durable Object, one instance per audit (`idFromName(auditId)`):
|
||||
|
||||
- **frontier** — URL queue and dedup set (URL primary key), with
|
||||
`pending / leased / crawled` states keyed by chunk number;
|
||||
- **links** — internal link edges, primary-keyed for idempotent re-insert;
|
||||
- **page_mirror** — the few columns the finalize link checks need.
|
||||
|
||||
The app database (D1/Postgres) keeps only what the product reads: `audits`
|
||||
(plus failure columns), `audit_pages`, `audit_issues`,
|
||||
`audit_lighthouse_results`. Link edges are never persisted to the app DB; the
|
||||
cross-page link checks (broken internal links, orphan pages) run as SQL inside
|
||||
the DO at finalize.
|
||||
|
||||
Rejected alternatives: Cloudflare Queues (no dedup, no counters, no completion
|
||||
signal — a frontier is mostly those things), a Postgres frontier table (chatty
|
||||
hot-path writes, dual-dialect surface), R2 spill files (not queryable).
|
||||
|
||||
### Chunked rolling crawl
|
||||
|
||||
The crawl phase is a loop of `crawl-chunk-N` steps. Each chunk leases up to
|
||||
~200 URLs from the DO and crawls them with a rolling concurrency window: the
|
||||
moment a fetch settles, the next URL launches. The window self-adjusts between
|
||||
5 and 40 based on recent fetch health (errors, blocked fetches, slow responses,
|
||||
oversized bodies shrink it; clean fast batches grow it). A soft deadline ends
|
||||
the chunk early and releases unfetched leases. Persistence is pipelined with
|
||||
fetching but serialized with itself: page rows and per-page issues go to the
|
||||
app DB, link edges and frontier updates go to the DO, and progress counters
|
||||
update per sub-batch. Step returns carry only counters, so no step output
|
||||
scales with site size.
|
||||
|
||||
Everything is idempotent under step retries: chunk-keyed leases (a retried
|
||||
step re-receives exactly the URLs its failed attempt held, and refuses a fresh
|
||||
claim for an already-crawled chunk number), deterministic page-row ids, and
|
||||
insert-or-ignore/replace writes on stable keys.
|
||||
|
||||
### Streaming HTML parsing
|
||||
|
||||
Pages are parsed with htmlparser2's streaming tokenizer — no DOM is built, so
|
||||
per-page memory is constant. cheerio (which uses the same tokenizer
|
||||
internally) remains only as a test reference: the parser test suite asserts
|
||||
extraction parity against it, and the badseo fixture harness asserts identical
|
||||
issue output end to end.
|
||||
|
||||
### Failure handling and graceful degradation
|
||||
|
||||
- `audits` carries `error_code` (a closed vocabulary mapped from real platform
|
||||
error strings — `step_timeout`, `oom`, `cpu_limit`, `db_error`,
|
||||
`step_output_too_large`, `workflow_internal`, `instance_lost`, `unknown`),
|
||||
`error_detail`, and `failed_phase`.
|
||||
- A watchdog on the existing `*/15` cron reconciles audit rows stuck in
|
||||
`running` against the Workflows API; the audit status read path does the
|
||||
same lazily. Only confirmed instance-not-found errors (after a grace period)
|
||||
count as a lost instance, so transient API failures never fail a live audit.
|
||||
- A failed audit shows everything crawled before the failure ("stopped early
|
||||
after N pages") instead of hiding results.
|
||||
- The start URL follows redirects (with per-hop SSRF revalidation) before the
|
||||
crawl anchors its origin, so apex→www and cross-TLD redirects don't dead-end
|
||||
the crawl after one page.
|
||||
|
||||
### DO lifecycle and platform-limit guards
|
||||
|
||||
- `destroy()` (delete alarm, then all storage) runs on audit success and on
|
||||
audit deletion. Every DO construction schedules a 7-day self-cleanup alarm,
|
||||
so any instantiation — including a write racing in after destroy, or a
|
||||
workflow that dies before seeding — is eventually wiped. Failed audits keep
|
||||
their scratchpad for those 7 days as the resume/debug artifact.
|
||||
- Guards where the platform has hard edges: link writes stop at a storage
|
||||
budget (below the free-plan per-object SQLite cap; orphan detection is
|
||||
skipped when the link graph was truncated), discovered-URL batches are
|
||||
capped below the serialized-RPC limit, and sitemap documents are read up to
|
||||
a byte cap and skipped whole beyond it.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Workflow heap and step state are O(one chunk) regardless of site size; the
|
||||
OOM/step-output failure classes are gone by construction.
|
||||
- Hostile or slow sites degrade to a slower window and finish (or fail with a
|
||||
classified reason and visible partials) instead of burning an hour of
|
||||
retries.
|
||||
- Every failure is aggregable by `error_code` in plain SQL, and zombie
|
||||
`running` rows self-heal.
|
||||
- Crawl-state code is provider-independent (DO SQLite exists in workerd), so
|
||||
self-host does not add a dialect surface for the frontier or links.
|
||||
- The scratchpad is opaque from outside the DO; the 7-day retention of failed
|
||||
audits' state is the debugging window.
|
||||
- Resume-from-frontier ("retry finishes the last N pages instead of
|
||||
recrawling") is enabled by the retained frontier but intentionally not
|
||||
built yet.
|
||||
@ -1,6 +1,6 @@
|
||||
import { AlertCircle, CheckCircle, Loader2 } from "lucide-react";
|
||||
|
||||
export const SUPPORT_URL = "https://everyapp.dev/support";
|
||||
export const SUPPORT_EMAIL = "ben@openseo.so";
|
||||
|
||||
export function extractPathname(url: string): string {
|
||||
try {
|
||||
|
||||
@ -37,6 +37,13 @@ export const audits = sqliteTable(
|
||||
lighthouseCompleted: integer("lighthouse_completed").notNull().default(0),
|
||||
lighthouseFailed: integer("lighthouse_failed").notNull().default(0),
|
||||
currentPhase: text("current_phase").default("discovery"),
|
||||
// Failure diagnostics; null unless status = "failed". errorCode is a
|
||||
// closed vocabulary (see classifyAuditError) so failures are aggregable;
|
||||
// errorDetail is the raw message, truncated. failedPhase records which
|
||||
// currentPhase the audit was in when it died.
|
||||
errorCode: text("error_code"),
|
||||
errorDetail: text("error_detail"),
|
||||
failedPhase: text("failed_phase"),
|
||||
startedAt: text("started_at")
|
||||
.notNull()
|
||||
.default(sql`(current_timestamp)`),
|
||||
|
||||
@ -44,6 +44,13 @@ export const audits = pgTable(
|
||||
lighthouseCompleted: integer("lighthouse_completed").notNull().default(0),
|
||||
lighthouseFailed: integer("lighthouse_failed").notNull().default(0),
|
||||
currentPhase: text("current_phase").default("discovery"),
|
||||
// Failure diagnostics; null unless status = "failed". errorCode is a
|
||||
// closed vocabulary (see classifyAuditError) so failures are aggregable;
|
||||
// errorDetail is the raw message, truncated. failedPhase records which
|
||||
// currentPhase the audit was in when it died.
|
||||
errorCode: text("error_code"),
|
||||
errorDetail: text("error_detail"),
|
||||
failedPhase: text("failed_phase"),
|
||||
startedAt: timestampColumn("started_at").notNull().default(isoNow),
|
||||
completedAt: timestampColumn("completed_at"),
|
||||
},
|
||||
|
||||
@ -5,6 +5,7 @@ import { getTableConfig as getSqliteTableConfig } from "drizzle-orm/sqlite-core"
|
||||
import { getTableConfig as getPgTableConfig } from "drizzle-orm/pg-core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import * as sqliteApp from "./app.schema";
|
||||
import * as sqliteAudit from "./audit.schema";
|
||||
import * as sqliteSam from "./sam.schema";
|
||||
import * as sqliteAuth from "./better-auth-schema";
|
||||
import * as sqliteBilling from "./billing.schema";
|
||||
@ -12,6 +13,7 @@ import * as sqliteGsc from "./gsc.schema";
|
||||
import * as sqliteReddit from "./reddit-attribution.schema";
|
||||
import * as sqliteTelemetry from "./telemetry.schema";
|
||||
import * as pgApp from "./pg/app.schema";
|
||||
import * as pgAudit from "./pg/audit.schema";
|
||||
import * as pgSam from "./pg/sam.schema";
|
||||
import * as pgAuth from "./pg/better-auth-schema";
|
||||
import * as pgBilling from "./pg/billing.schema";
|
||||
@ -135,6 +137,7 @@ function foreignKeys(table: Table, dialect: Dialect): string[] {
|
||||
|
||||
const sqliteAppTables = tablesFrom(
|
||||
sqliteApp,
|
||||
sqliteAudit,
|
||||
sqliteSam,
|
||||
sqliteBilling,
|
||||
sqliteGsc,
|
||||
@ -143,6 +146,7 @@ const sqliteAppTables = tablesFrom(
|
||||
);
|
||||
const pgAppTables = tablesFrom(
|
||||
pgApp,
|
||||
pgAudit,
|
||||
pgSam,
|
||||
pgBilling,
|
||||
pgGsc,
|
||||
|
||||
4
src/env.d.ts
vendored
4
src/env.d.ts
vendored
@ -12,6 +12,10 @@ declare namespace Cloudflare {
|
||||
// Durable Object backing the SAM in-app agent (see wrangler.jsonc).
|
||||
SAM_CHAT: DurableObjectNamespace;
|
||||
|
||||
// Durable Object holding per-audit crawl scratch state (frontier, link
|
||||
// edges, page mirror). Untyped here; getAuditScratchpad narrows the stub.
|
||||
AUDIT_SCRATCHPAD: DurableObjectNamespace;
|
||||
|
||||
AUTH_MODE?: "cloudflare_access" | "local_noauth" | "hosted";
|
||||
BYPASS_EMAIL_VERIFICATION?: string;
|
||||
TEAM_DOMAIN?: string;
|
||||
|
||||
@ -16,7 +16,7 @@ import {
|
||||
formatStartedAt,
|
||||
HttpStatusBadge,
|
||||
StatusBadge,
|
||||
SUPPORT_URL,
|
||||
SUPPORT_EMAIL,
|
||||
} from "@/client/features/audit/shared";
|
||||
|
||||
export const Route = createFileRoute<"/_project/p/$projectId/audit/">(
|
||||
@ -87,10 +87,13 @@ function AuditDetail({
|
||||
const isFailed = statusQuery.data?.status === "failed";
|
||||
const isRunning = statusQuery.data?.status === "running";
|
||||
|
||||
// Failed audits keep whatever pages were crawled before the failure
|
||||
// (persistence is per-batch), so fetch results for them too and show the
|
||||
// partial crawl instead of a dead end.
|
||||
const resultsQuery = useQuery({
|
||||
queryKey: ["audit-results", projectId, auditId],
|
||||
queryFn: () => getAuditResults({ data: { projectId, auditId } }),
|
||||
enabled: isComplete,
|
||||
enabled: isComplete || isFailed,
|
||||
});
|
||||
|
||||
if (statusQuery.isLoading) {
|
||||
@ -118,8 +121,15 @@ function AuditDetail({
|
||||
}
|
||||
|
||||
const status = statusQuery.data;
|
||||
const partialPageCount = isFailed
|
||||
? (resultsQuery.data?.pages.length ?? 0)
|
||||
: 0;
|
||||
const failedWithResults = isFailed && partialPageCount > 0;
|
||||
// Wait for the results fetch before choosing between the "partial results"
|
||||
// banner and the zero-page support CTA, so the CTA doesn't flash first.
|
||||
const showSupportCta =
|
||||
isFailed || (isComplete && status && status.pagesCrawled <= 1);
|
||||
(isFailed && resultsQuery.isSuccess && !failedWithResults) ||
|
||||
(isComplete && status && status.pagesCrawled <= 1);
|
||||
|
||||
return (
|
||||
<div className="px-4 py-4 md:px-6 md:py-6 pb-24 md:pb-8 overflow-auto">
|
||||
@ -161,15 +171,12 @@ function AuditDetail({
|
||||
Site audit couldn't fully crawl this website.
|
||||
</p>
|
||||
<p>
|
||||
This is often caused by anti-bot or firewall settings. Reach out
|
||||
at{" "}
|
||||
This is often caused by anti-bot or firewall settings. Email{" "}
|
||||
<a
|
||||
className="link link-primary"
|
||||
href={SUPPORT_URL}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
href={`mailto:${SUPPORT_EMAIL}`}
|
||||
>
|
||||
everyapp.dev/support
|
||||
{SUPPORT_EMAIL}
|
||||
</a>{" "}
|
||||
and we'll help configure auditing for your site.
|
||||
</p>
|
||||
@ -177,7 +184,30 @@ function AuditDetail({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isComplete && resultsQuery.data && (
|
||||
{failedWithResults && (
|
||||
<div className="alert alert-warning">
|
||||
<AlertCircle className="size-5" />
|
||||
<div className="space-y-1">
|
||||
<p className="font-medium">
|
||||
This audit stopped early after {partialPageCount} page
|
||||
{partialPageCount === 1 ? "" : "s"}.
|
||||
</p>
|
||||
<p>
|
||||
The results below cover everything crawled before it stopped.
|
||||
Run a new audit to try again, or email{" "}
|
||||
<a
|
||||
className="link link-primary"
|
||||
href={`mailto:${SUPPORT_EMAIL}`}
|
||||
>
|
||||
{SUPPORT_EMAIL}
|
||||
</a>{" "}
|
||||
if this keeps happening.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(isComplete || failedWithResults) && resultsQuery.data && (
|
||||
<ResultsView
|
||||
projectId={projectId}
|
||||
data={resultsQuery.data}
|
||||
|
||||
@ -7,6 +7,7 @@ import { resolveUserContextFromHeaders } from "@/middleware/ensure-user/resolve"
|
||||
import { ProjectRepository } from "@/server/features/projects/repositories/ProjectRepository";
|
||||
import { SamSessionRepository } from "@/server/features/sam/SamSessionRepository";
|
||||
import { runScheduledRankChecks } from "@/server/features/rank-tracking/services/scheduledRankChecks";
|
||||
import { reconcileStaleAudits } from "@/server/features/audit/services/auditReconciler";
|
||||
import { getOrCreateOrganizationCustomer } from "@/server/billing/subscription";
|
||||
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
|
||||
import { getAuthMode, isHostedAuthMode } from "@/lib/auth-mode";
|
||||
@ -177,6 +178,8 @@ export { RankCheckWorkflow } from "./server/workflows/RankCheckWorkflow";
|
||||
export { OnboardingChatAgent } from "./server/features/onboarding/OnboardingChatAgent";
|
||||
// Durable Object class for the SAM in-app agent (Agents SDK).
|
||||
export { SamChatAgent } from "./server/features/sam/SamChatAgent";
|
||||
// Durable Object class for the per-audit crawl scratchpad.
|
||||
export { AuditScratchpad } from "./server/features/audit/AuditScratchpad";
|
||||
|
||||
export default {
|
||||
fetch,
|
||||
@ -187,5 +190,8 @@ export default {
|
||||
) {
|
||||
// Scope a per-request Postgres client for the cron run (no-op in D1 mode).
|
||||
await withPgClient(() => runScheduledRankChecks(env));
|
||||
// Watchdog: reconcile audits stuck in "running" whose workflow died
|
||||
// without reaching mark-failed (OOM/CPU kills, expired instances).
|
||||
await withPgClient(() => reconcileStaleAudits());
|
||||
},
|
||||
};
|
||||
|
||||
372
src/server/features/audit/AuditScratchpad.ts
Normal file
372
src/server/features/audit/AuditScratchpad.ts
Normal file
@ -0,0 +1,372 @@
|
||||
/**
|
||||
* Per-audit crawl scratchpad — a SQLite-backed Durable Object.
|
||||
*
|
||||
* Holds all transient crawl state for one audit (id = auditId): the frontier
|
||||
* (URL queue + seen-set), the internal-link edges, and a slim mirror of page
|
||||
* rows. This keeps chatty crawl-loop writes off Postgres, removes the ~1MiB
|
||||
* Workflow step-output limits (nothing large flows through step returns
|
||||
* anymore), and gives the finalize link checks a local SQL database.
|
||||
*
|
||||
* Lifecycle: seeded by the discover-urls step, read/written by every crawl
|
||||
* chunk, queried once at finalize, then destroyed on success. A self-cleanup
|
||||
* alarm set on every construction guarantees any instantiation — including a
|
||||
* write racing in after destroy() — is wiped after 7 days (failed audits'
|
||||
* state doubles as the resume/debug artifact until then).
|
||||
*
|
||||
* All methods are synchronous inside (SQLite in DOs is sync), so each RPC is
|
||||
* effectively atomic. Writes are idempotent: the workflow retries steps, so
|
||||
* every insert is OR IGNORE / OR REPLACE on a stable key.
|
||||
*/
|
||||
import { DurableObject, env } from "cloudflare:workers";
|
||||
|
||||
export interface ClaimedUrl {
|
||||
url: string;
|
||||
/** Clicks from the start URL; null when only reachable via sitemap. */
|
||||
depth: number | null;
|
||||
inSitemap: boolean;
|
||||
}
|
||||
|
||||
export interface FrontierStats {
|
||||
/** Pages attempted (crawled or errored) so far. */
|
||||
attempted: number;
|
||||
/** URLs still waiting to be crawled. */
|
||||
pending: number;
|
||||
/** Every URL ever enqueued (attempted + pending + leased). */
|
||||
seen: number;
|
||||
}
|
||||
|
||||
interface ScratchpadPageRow {
|
||||
pageId: string;
|
||||
url: string;
|
||||
statusCode: number;
|
||||
fetchClass: string;
|
||||
redirectUrl: string | null;
|
||||
}
|
||||
|
||||
export interface ScratchpadLinkRow {
|
||||
sourcePageId: string;
|
||||
sourceUrl: string;
|
||||
targetUrl: string;
|
||||
anchor: string | null;
|
||||
isNofollow: boolean;
|
||||
}
|
||||
|
||||
interface RecordBatchInput {
|
||||
/** URLs whose crawl attempt finished (successfully or not). */
|
||||
crawledUrls: string[];
|
||||
pages: ScratchpadPageRow[];
|
||||
links: ScratchpadLinkRow[];
|
||||
/** Newly discovered same-origin URLs to enqueue (already policy-filtered). */
|
||||
discovered: Array<{ url: string; depth: number | null }>;
|
||||
}
|
||||
|
||||
interface BrokenLinkRow {
|
||||
sourcePageId: string;
|
||||
sourceUrl: string;
|
||||
targetUrl: string;
|
||||
targetStatus: number;
|
||||
}
|
||||
|
||||
interface OrphanPageRow {
|
||||
pageId: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
const BROKEN_LINK_ISSUE_CAP = 2_000;
|
||||
const CLEANUP_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
/**
|
||||
* Stop storing link edges once the database reaches this size. Link rows are
|
||||
* the only unbounded-per-page data; a pathological link-dense site could
|
||||
* otherwise hit the platform's per-object SQLite cap (1 GB on the free plan
|
||||
* self-hosters may run on) and fail the crawl with SQLITE_FULL. Past the
|
||||
* budget the crawl continues — the audit just loses link-graph issues.
|
||||
*/
|
||||
const LINK_STORAGE_BUDGET_BYTES = 500 * 1024 * 1024;
|
||||
|
||||
export class AuditScratchpad extends DurableObject {
|
||||
constructor(ctx: DurableObjectState, workerEnv: Env) {
|
||||
super(ctx, workerEnv);
|
||||
this.ctx.storage.sql.exec(`
|
||||
CREATE TABLE IF NOT EXISTS frontier (
|
||||
url TEXT PRIMARY KEY,
|
||||
depth INTEGER,
|
||||
source TEXT NOT NULL,
|
||||
in_sitemap INTEGER NOT NULL DEFAULT 0,
|
||||
state TEXT NOT NULL DEFAULT 'pending',
|
||||
chunk_no INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS frontier_claim_idx ON frontier (state, source);
|
||||
CREATE TABLE IF NOT EXISTS links (
|
||||
source_page_id TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL,
|
||||
target_url TEXT NOT NULL,
|
||||
anchor TEXT,
|
||||
is_nofollow INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (source_page_id, target_url)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS links_target_idx ON links (target_url);
|
||||
CREATE TABLE IF NOT EXISTS page_mirror (
|
||||
page_id TEXT PRIMARY KEY,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
status_code INTEGER,
|
||||
fetch_class TEXT NOT NULL,
|
||||
redirect_url TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS page_mirror_redirect_idx ON page_mirror (redirect_url);
|
||||
`);
|
||||
// Guarantee the cleanup alarm on EVERY instantiation, not just at seed:
|
||||
// any RPC (even one racing in right after destroy()) re-creates the
|
||||
// tables above, and without an alarm that storage would leak forever.
|
||||
// blockConcurrencyWhile gates RPC delivery on its own; the returned
|
||||
// promise doesn't need observing (constructors can't await).
|
||||
void this.ctx.blockConcurrencyWhile(() => this.ensureCleanupAlarm());
|
||||
}
|
||||
|
||||
async seedStart(url: string): Promise<void> {
|
||||
this.ctx.storage.sql.exec(
|
||||
`INSERT OR IGNORE INTO frontier (url, depth, source, in_sitemap) VALUES (?, 0, 'link', 0)`,
|
||||
url,
|
||||
);
|
||||
}
|
||||
|
||||
async seedSitemapUrls(urls: string[]): Promise<void> {
|
||||
for (const url of urls) {
|
||||
// Upsert so a URL that already exists (e.g. the start URL) still gets
|
||||
// its in-sitemap flag; queue position and depth stay as first seen.
|
||||
this.ctx.storage.sql.exec(
|
||||
`INSERT INTO frontier (url, depth, source, in_sitemap) VALUES (?, NULL, 'sitemap', 1)
|
||||
ON CONFLICT(url) DO UPDATE SET in_sitemap = 1`,
|
||||
url,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lease the next batch of pending URLs for one crawl chunk. Idempotent per
|
||||
* chunkNo: a retried step gets back exactly the URLs its failed attempt
|
||||
* had leased (link-discovered URLs drain before sitemap-only ones, FIFO
|
||||
* within each class — same ordering as the old in-memory queues).
|
||||
*/
|
||||
async claimChunk(chunkNo: number, limit: number): Promise<ClaimedUrl[]> {
|
||||
const existing = this.selectClaimed(
|
||||
`SELECT url, depth, in_sitemap FROM frontier WHERE state = 'leased' AND chunk_no = ?`,
|
||||
chunkNo,
|
||||
);
|
||||
if (existing.length > 0) return existing;
|
||||
// A retried step whose earlier attempt already crawled this chunk's
|
||||
// leases must not claim a fresh set under the same chunk number — that
|
||||
// would duplicate work and overshoot the page budget.
|
||||
const done = this.ctx.storage.sql
|
||||
.exec<{
|
||||
n: number;
|
||||
}>(
|
||||
`SELECT COUNT(*) AS n FROM frontier WHERE chunk_no = ? AND state = 'crawled'`,
|
||||
chunkNo,
|
||||
)
|
||||
.one();
|
||||
if (done.n > 0) return [];
|
||||
if (limit <= 0) return [];
|
||||
|
||||
const fresh = this.selectClaimed(
|
||||
`SELECT url, depth, in_sitemap FROM frontier WHERE state = 'pending'
|
||||
ORDER BY CASE source WHEN 'link' THEN 0 ELSE 1 END, rowid LIMIT ?`,
|
||||
limit,
|
||||
);
|
||||
for (const row of fresh) {
|
||||
this.ctx.storage.sql.exec(
|
||||
`UPDATE frontier SET state = 'leased', chunk_no = ? WHERE url = ?`,
|
||||
chunkNo,
|
||||
row.url,
|
||||
);
|
||||
}
|
||||
return fresh;
|
||||
}
|
||||
|
||||
/** Persist one crawled sub-batch: completions, mirror rows, links, frontier. */
|
||||
async recordBatch(input: RecordBatchInput): Promise<FrontierStats> {
|
||||
for (const url of input.crawledUrls) {
|
||||
this.ctx.storage.sql.exec(
|
||||
`UPDATE frontier SET state = 'crawled' WHERE url = ?`,
|
||||
url,
|
||||
);
|
||||
}
|
||||
for (const page of input.pages) {
|
||||
this.ctx.storage.sql.exec(
|
||||
`INSERT OR REPLACE INTO page_mirror (page_id, url, status_code, fetch_class, redirect_url)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
page.pageId,
|
||||
page.url,
|
||||
page.statusCode,
|
||||
page.fetchClass,
|
||||
page.redirectUrl,
|
||||
);
|
||||
}
|
||||
// Database size only ever grows (deletes happen solely via destroy), so
|
||||
// once the budget trips it stays tripped — runFinalizeChecks uses the
|
||||
// same comparison to know the link graph is incomplete.
|
||||
if (this.ctx.storage.sql.databaseSize < LINK_STORAGE_BUDGET_BYTES) {
|
||||
for (const link of input.links) {
|
||||
this.ctx.storage.sql.exec(
|
||||
`INSERT OR IGNORE INTO links (source_page_id, source_url, target_url, anchor, is_nofollow)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
link.sourcePageId,
|
||||
link.sourceUrl,
|
||||
link.targetUrl,
|
||||
link.anchor,
|
||||
link.isNofollow ? 1 : 0,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const found of input.discovered) {
|
||||
// OR IGNORE: already-seen URLs (crawled, leased, or pending) keep
|
||||
// their existing row — this is the global dedup.
|
||||
this.ctx.storage.sql.exec(
|
||||
`INSERT OR IGNORE INTO frontier (url, depth, source, in_sitemap) VALUES (?, ?, 'link', 0)`,
|
||||
found.url,
|
||||
found.depth,
|
||||
);
|
||||
}
|
||||
return this.stats();
|
||||
}
|
||||
|
||||
/** Return unattempted leases to the queue (chunk soft-deadline hit). */
|
||||
async releaseUrls(urls: string[]): Promise<void> {
|
||||
for (const url of urls) {
|
||||
this.ctx.storage.sql.exec(
|
||||
`UPDATE frontier SET state = 'pending', chunk_no = NULL WHERE url = ? AND state = 'leased'`,
|
||||
url,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async getStats(): Promise<FrontierStats> {
|
||||
return this.stats();
|
||||
}
|
||||
|
||||
/**
|
||||
* The two finalize checks that need link edges, as local SQL. Mirrors the
|
||||
* former Postgres implementations in multipage.ts exactly.
|
||||
*/
|
||||
async runFinalizeChecks(input: {
|
||||
startUrl: string;
|
||||
crawlCompleted: boolean;
|
||||
}): Promise<{ brokenLinks: BrokenLinkRow[]; orphanPages: OrphanPageRow[] }> {
|
||||
// Only flag targets we actually crawled and saw fail — never inferred
|
||||
// from absence. Blocked targets (WAF challenges) are excluded: a 403
|
||||
// from bot protection is not evidence of a broken link.
|
||||
const brokenLinks = this.ctx.storage.sql
|
||||
.exec<{
|
||||
source_page_id: string;
|
||||
source_url: string;
|
||||
target_url: string;
|
||||
target_status: number;
|
||||
}>(
|
||||
`SELECT l.source_page_id, l.source_url, l.target_url, m.status_code AS target_status
|
||||
FROM links l JOIN page_mirror m ON m.url = l.target_url
|
||||
WHERE m.status_code >= 400 AND m.fetch_class = 'ok'
|
||||
ORDER BY l.source_page_id, l.target_url
|
||||
LIMIT ?`,
|
||||
BROKEN_LINK_ISSUE_CAP,
|
||||
)
|
||||
.toArray()
|
||||
.map((row) => ({
|
||||
sourcePageId: row.source_page_id,
|
||||
sourceUrl: row.source_url,
|
||||
targetUrl: row.target_url,
|
||||
targetStatus: row.target_status,
|
||||
}));
|
||||
|
||||
// A live 2xx page is an orphan when no OTHER crawled page links to it
|
||||
// and nothing redirects to it. Only meaningful on a completed crawl with
|
||||
// a complete link graph — if the storage budget truncated link writes,
|
||||
// "no inbound edge" is missing data, not evidence of orphanhood.
|
||||
const linkGraphComplete =
|
||||
this.ctx.storage.sql.databaseSize < LINK_STORAGE_BUDGET_BYTES;
|
||||
const orphanPages =
|
||||
input.crawlCompleted && linkGraphComplete
|
||||
? this.ctx.storage.sql
|
||||
.exec<{ page_id: string; url: string }>(
|
||||
`SELECT m.page_id, m.url FROM page_mirror m
|
||||
WHERE m.url != ?
|
||||
AND m.fetch_class = 'ok'
|
||||
AND m.status_code >= 200 AND m.status_code < 300
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM links l
|
||||
WHERE l.target_url = m.url AND l.source_page_id != m.page_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM page_mirror r WHERE r.redirect_url = m.url
|
||||
)`,
|
||||
input.startUrl,
|
||||
)
|
||||
.toArray()
|
||||
.map((row) => ({ pageId: row.page_id, url: row.url }))
|
||||
: [];
|
||||
|
||||
return { brokenLinks, orphanPages };
|
||||
}
|
||||
|
||||
/** Wipe all state (success path, or explicit audit deletion). */
|
||||
async destroy(): Promise<void> {
|
||||
await this.ctx.storage.deleteAlarm();
|
||||
await this.ctx.storage.deleteAll();
|
||||
}
|
||||
|
||||
/**
|
||||
* Self-cleanup for audits whose workflow died without reaching finalize.
|
||||
* Full destroy(), not just deleteAll(): under our compatibility date,
|
||||
* deleteAll() does not clear alarm state, and a leftover alarm would keep
|
||||
* the object (and its billing) alive.
|
||||
*/
|
||||
async alarm(): Promise<void> {
|
||||
await this.destroy();
|
||||
}
|
||||
|
||||
private async ensureCleanupAlarm(): Promise<void> {
|
||||
const existing = await this.ctx.storage.getAlarm();
|
||||
if (existing === null) {
|
||||
await this.ctx.storage.setAlarm(Date.now() + CLEANUP_AFTER_MS);
|
||||
}
|
||||
}
|
||||
|
||||
private selectClaimed(query: string, param: number): ClaimedUrl[] {
|
||||
return this.ctx.storage.sql
|
||||
.exec<{ url: string; depth: number | null; in_sitemap: number }>(
|
||||
query,
|
||||
param,
|
||||
)
|
||||
.toArray()
|
||||
.map((row) => ({
|
||||
url: row.url,
|
||||
depth: row.depth,
|
||||
inSitemap: row.in_sitemap === 1,
|
||||
}));
|
||||
}
|
||||
|
||||
private stats(): FrontierStats {
|
||||
const row = this.ctx.storage.sql
|
||||
.exec<{ attempted: number; pending: number; seen: number }>(
|
||||
`SELECT
|
||||
COUNT(*) FILTER (WHERE state = 'crawled') AS attempted,
|
||||
COUNT(*) FILTER (WHERE state = 'pending') AS pending,
|
||||
COUNT(*) AS seen
|
||||
FROM frontier`,
|
||||
)
|
||||
.one();
|
||||
return {
|
||||
attempted: row.attempted,
|
||||
pending: row.pending,
|
||||
seen: row.seen,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Stub for the audit's scratchpad DO (one instance per audit id). */
|
||||
export function getAuditScratchpad(auditId: string) {
|
||||
// env.d.ts declares the binding untyped (ambient contexts can't import the
|
||||
// class); narrow here so callers get typed RPC methods.
|
||||
const namespace =
|
||||
// oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the binding is declared as this class in wrangler.jsonc
|
||||
env.AUDIT_SCRATCHPAD as unknown as DurableObjectNamespace<AuditScratchpad>;
|
||||
return namespace.get(namespace.idFromName(auditId));
|
||||
}
|
||||
@ -1,18 +1,17 @@
|
||||
/**
|
||||
* Data access layer for site audit tables.
|
||||
* Provider-aware (D1 or Postgres) via the `@/db` handle. Covers audits,
|
||||
* audit_pages, audit_links, audit_issues, and stored Lighthouse results.
|
||||
* audit_pages, audit_issues, and stored Lighthouse results. Link edges live
|
||||
* in the per-audit scratchpad Durable Object, not here.
|
||||
*/
|
||||
import { and, desc, eq } from "drizzle-orm";
|
||||
import { and, count, desc, eq } from "drizzle-orm";
|
||||
import { db } from "@/db";
|
||||
import {
|
||||
audits,
|
||||
auditIssues,
|
||||
auditLighthouseResults,
|
||||
auditLinks,
|
||||
auditPages,
|
||||
} from "@/db/schema";
|
||||
import { getDatabaseProvider } from "@/db/provider";
|
||||
import { executeInBatches } from "@/db/runBatch";
|
||||
import { AUDIT_ISSUE_TYPES } from "@/shared/audit-issues";
|
||||
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
|
||||
@ -23,14 +22,6 @@ import type {
|
||||
LighthouseResult,
|
||||
} from "@/server/lib/audit/types";
|
||||
|
||||
// Only internal links are stored: both consumers (broken-internal-link and
|
||||
// orphan checks) filter on isInternal, and per-page external counts already
|
||||
// live on audit_pages. External rows come back when P1 adds external-link
|
||||
// checks. Mega-menu/footer-heavy sites can carry 1000+ links per page; cap
|
||||
// what we store so a 10k-page crawl can't write tens of millions of link rows.
|
||||
const MAX_STORED_LINKS_PER_PAGE = 500;
|
||||
const POSTGRES_LINK_INSERT_SIZE = 500;
|
||||
|
||||
async function createAudit(data: {
|
||||
id: string;
|
||||
projectId: string;
|
||||
@ -102,7 +93,15 @@ async function completeAudit(
|
||||
);
|
||||
}
|
||||
|
||||
async function failAudit(auditId: string, workflowInstanceId: string) {
|
||||
async function failAudit(
|
||||
auditId: string,
|
||||
workflowInstanceId: string,
|
||||
errorInfo?: {
|
||||
errorCode: string;
|
||||
errorDetail: string;
|
||||
failedPhase: string | null;
|
||||
},
|
||||
) {
|
||||
// Only a running audit can transition to failed: the getStatus reconciler
|
||||
// races the workflow's own finalize, and without this guard it could flip
|
||||
// a just-completed audit to failed.
|
||||
@ -112,6 +111,11 @@ async function failAudit(auditId: string, workflowInstanceId: string) {
|
||||
status: "failed",
|
||||
completedAt: new Date().toISOString(),
|
||||
currentPhase: "failed",
|
||||
...(errorInfo && {
|
||||
errorCode: errorInfo.errorCode,
|
||||
errorDetail: errorInfo.errorDetail,
|
||||
failedPhase: errorInfo.failedPhase,
|
||||
}),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
@ -135,15 +139,15 @@ async function getAuditForWorkflow(
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist one crawl batch (pages + link edges + per-page issues).
|
||||
* Called inside the crawl-batch Workflow step so results land in D1
|
||||
* incrementally instead of accumulating in memory until finalize.
|
||||
* Persist one crawled sub-batch (pages + per-page issues). Called inside the
|
||||
* crawl-chunk Workflow step so results land in the app DB incrementally
|
||||
* instead of accumulating in memory until finalize. Link edges go to the
|
||||
* audit's scratchpad DO, not here.
|
||||
*
|
||||
* Idempotent on step retry: callers assign deterministic page ids
|
||||
* (deterministicAuditRowId) and link/issue ids are derived from stable
|
||||
* content. Page rows upsert (a retried fetch may legitimately differ — last
|
||||
* attempt wins, matching what the step returns); links and issues are
|
||||
* insert-or-ignore.
|
||||
* (deterministicAuditRowId) and issue ids are derived from stable content.
|
||||
* Page rows upsert (a retried fetch may legitimately differ — last attempt
|
||||
* wins); issues are insert-or-ignore.
|
||||
*/
|
||||
async function insertCrawledBatch(
|
||||
auditId: string,
|
||||
@ -192,38 +196,6 @@ async function insertCrawledBatch(
|
||||
.onConflictDoUpdate({ target: auditPages.id, set: dataColumns });
|
||||
});
|
||||
|
||||
const linkRows = await Promise.all(
|
||||
pages.flatMap((page) =>
|
||||
page.links
|
||||
.filter((link) => link.isInternal)
|
||||
.slice(0, MAX_STORED_LINKS_PER_PAGE)
|
||||
.map(async (link) => ({
|
||||
id: await deterministicAuditRowId(auditId, page.url, link.targetUrl),
|
||||
auditId,
|
||||
sourcePageId: page.id,
|
||||
sourceUrl: page.url,
|
||||
targetUrl: link.targetUrl,
|
||||
anchor: link.anchor,
|
||||
isInternal: link.isInternal,
|
||||
isNofollow: link.isNofollow,
|
||||
})),
|
||||
),
|
||||
);
|
||||
if (getDatabaseProvider() === "postgres") {
|
||||
// A Postgres transaction executes runBatch statements sequentially. Bulk
|
||||
// values avoid thousands of Hyperdrive round trips on link-heavy pages.
|
||||
for (let i = 0; i < linkRows.length; i += POSTGRES_LINK_INSERT_SIZE) {
|
||||
await db
|
||||
.insert(auditLinks)
|
||||
.values(linkRows.slice(i, i + POSTGRES_LINK_INSERT_SIZE))
|
||||
.onConflictDoNothing();
|
||||
}
|
||||
} else {
|
||||
await executeInBatches(linkRows, (tx, row) =>
|
||||
tx.insert(auditLinks).values(row).onConflictDoNothing(),
|
||||
);
|
||||
}
|
||||
|
||||
await insertIssues(auditId, issues);
|
||||
}
|
||||
|
||||
@ -336,6 +308,19 @@ async function getPagesForAudit(auditId: string) {
|
||||
.where(eq(auditPages.auditId, auditId));
|
||||
}
|
||||
|
||||
async function countBlockedPages(auditId: string): Promise<number> {
|
||||
const rows = await db
|
||||
.select({ blocked: count() })
|
||||
.from(auditPages)
|
||||
.where(
|
||||
and(
|
||||
eq(auditPages.auditId, auditId),
|
||||
eq(auditPages.fetchClass, "blocked"),
|
||||
),
|
||||
);
|
||||
return rows[0]?.blocked ?? 0;
|
||||
}
|
||||
|
||||
async function hasPagesForAudit(auditId: string): Promise<boolean> {
|
||||
const rows = await db
|
||||
.select({ id: auditPages.id })
|
||||
@ -449,6 +434,7 @@ export const AuditRepository = {
|
||||
getLatestAuditForProject,
|
||||
getIssuesForAudit,
|
||||
getPagesForAudit,
|
||||
countBlockedPages,
|
||||
hasPagesForAudit,
|
||||
getAuditsByProject,
|
||||
getAuditUsageForUser,
|
||||
|
||||
@ -5,6 +5,7 @@ import {
|
||||
type BillingCustomerContext,
|
||||
} from "@/server/billing/subscription";
|
||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||
import { getAuditScratchpad } from "@/server/features/audit/AuditScratchpad";
|
||||
import {
|
||||
AUDIT_LIMITS,
|
||||
clampAuditMaxPages,
|
||||
@ -18,7 +19,11 @@ import {
|
||||
type AuditConfig,
|
||||
type LighthouseStrategy,
|
||||
} from "@/server/lib/audit/types";
|
||||
import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
|
||||
import {
|
||||
normalizeAndValidateStartUrl,
|
||||
resolveStartUrlRedirects,
|
||||
} from "@/server/lib/audit/url-policy";
|
||||
import { reconcileRunningAudit } from "@/server/features/audit/services/auditReconciler";
|
||||
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
|
||||
|
||||
// Plan-tier limits are the abuse bound in hosted mode: free accounts get one
|
||||
@ -61,7 +66,12 @@ async function startAudit(input: {
|
||||
|
||||
const auditId = crypto.randomUUID();
|
||||
const config: AuditConfig = { maxPages, lighthouseStrategy };
|
||||
const startUrl = await normalizeAndValidateStartUrl(input.startUrl);
|
||||
// Anchor the audit to the site's real origin: a start domain that 301s
|
||||
// elsewhere (…net -> …com, apex -> www) would otherwise dead-end after
|
||||
// one page at the same-origin crawl boundary.
|
||||
const startUrl = await resolveStartUrlRedirects(
|
||||
await normalizeAndValidateStartUrl(input.startUrl),
|
||||
);
|
||||
|
||||
await AuditRepository.createAudit({
|
||||
id: auditId,
|
||||
@ -125,22 +135,13 @@ async function getStatus(auditId: string, projectId: string) {
|
||||
throw new AppError("NOT_FOUND", "Audit not found in this project.");
|
||||
|
||||
// Self-heal audits whose workflow died without reaching the mark-failed
|
||||
// step (instance terminated, mark-failed itself failed, deploys, ...).
|
||||
// step (instance terminated/errored, instance expired from retention, ...).
|
||||
// Without this they stay "running" forever and hold capacity.
|
||||
if (audit.status === "running" && audit.workflowInstanceId) {
|
||||
try {
|
||||
const instance = await env.SITE_AUDIT_WORKFLOW.get(
|
||||
audit.workflowInstanceId,
|
||||
);
|
||||
const { status } = await instance.status();
|
||||
if (status === "errored" || status === "terminated") {
|
||||
await AuditRepository.failAudit(audit.id, audit.workflowInstanceId);
|
||||
if (audit.status === "running") {
|
||||
const reconciled = await reconcileRunningAudit(audit);
|
||||
if (reconciled) {
|
||||
audit =
|
||||
(await AuditRepository.getAuditForProject(auditId, projectId)) ??
|
||||
audit;
|
||||
}
|
||||
} catch {
|
||||
// Instance not found or status unavailable — leave the audit as-is.
|
||||
(await AuditRepository.getAuditForProject(auditId, projectId)) ?? audit;
|
||||
}
|
||||
}
|
||||
|
||||
@ -154,6 +155,7 @@ async function getStatus(auditId: string, projectId: string) {
|
||||
lighthouseCompleted: audit.lighthouseCompleted,
|
||||
lighthouseFailed: audit.lighthouseFailed,
|
||||
currentPhase: audit.currentPhase,
|
||||
errorCode: audit.errorCode,
|
||||
startedAt: audit.startedAt,
|
||||
completedAt: audit.completedAt,
|
||||
};
|
||||
@ -257,6 +259,13 @@ async function remove(auditId: string, projectId: string) {
|
||||
}
|
||||
|
||||
await AuditRepository.deleteAuditForProject(auditId, projectId);
|
||||
// Best-effort: drop the crawl scratchpad DO with the audit. A missed
|
||||
// destroy self-cleans via the DO's 7-day alarm.
|
||||
try {
|
||||
await getAuditScratchpad(auditId).destroy();
|
||||
} catch (error) {
|
||||
console.warn(`Failed to destroy audit scratchpad ${auditId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
export const AuditService = {
|
||||
|
||||
176
src/server/features/audit/services/auditReconciler.ts
Normal file
176
src/server/features/audit/services/auditReconciler.ts
Normal file
@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Reconciles audits stuck in "running" with the actual state of their
|
||||
* Cloudflare Workflow instance.
|
||||
*
|
||||
* A workflow killed by the platform (OOM, CPU limit, deploy reset) never
|
||||
* reaches its own mark-failed step, and an instance can expire from Workflows
|
||||
* retention entirely. Two callers close that gap:
|
||||
* - AuditService.getStatus (lazy: whenever the UI polls a running audit)
|
||||
* - the scheduled watchdog (reconcileStaleAudits, every cron tick) — so
|
||||
* zombie rows die even if nobody ever reopens the page, and the instance
|
||||
* error is copied into the row before Workflows retention deletes it.
|
||||
*/
|
||||
import { env } from "cloudflare:workers";
|
||||
import { and, eq, lt } from "drizzle-orm";
|
||||
import { db } from "@/db";
|
||||
import { audits, projects } from "@/db/schema";
|
||||
import { getDatabaseProvider } from "@/db/provider";
|
||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||
import {
|
||||
classifyAuditError,
|
||||
type AuditErrorInfo,
|
||||
} from "@/server/lib/audit/audit-errors";
|
||||
import { captureServerEvent } from "@/server/lib/posthog";
|
||||
|
||||
/**
|
||||
* Don't declare an instance "lost" until the audit is comfortably older than
|
||||
* any legitimate create/start delay: startAudit inserts the row before
|
||||
* creating the workflow, so a brand-new audit can briefly have no instance.
|
||||
*/
|
||||
const INSTANCE_LOST_GRACE_MS = 10 * 60 * 1000;
|
||||
|
||||
/** Audits still "running" after this long get reconciled by the watchdog. */
|
||||
const STALE_RUNNING_AFTER_MS = 15 * 60 * 1000;
|
||||
|
||||
const WATCHDOG_BATCH_LIMIT = 100;
|
||||
|
||||
type RunningAudit = {
|
||||
id: string;
|
||||
workflowInstanceId: string | null;
|
||||
startedAt: string;
|
||||
currentPhase: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* If the audit's workflow instance is dead (errored/terminated) or gone,
|
||||
* flip the row to failed with a classified error. Returns the error info
|
||||
* when the row was flipped, null when the audit is genuinely still running.
|
||||
*/
|
||||
export async function reconcileRunningAudit(
|
||||
audit: RunningAudit,
|
||||
): Promise<AuditErrorInfo | null> {
|
||||
if (!audit.workflowInstanceId) return null;
|
||||
|
||||
let errorInfo: AuditErrorInfo | null = null;
|
||||
try {
|
||||
const instance = await env.SITE_AUDIT_WORKFLOW.get(
|
||||
audit.workflowInstanceId,
|
||||
);
|
||||
const status = await instance.status();
|
||||
if (status.status === "errored" || status.status === "terminated") {
|
||||
errorInfo = status.error
|
||||
? classifyAuditError(
|
||||
typeof status.error === "string"
|
||||
? status.error
|
||||
: (status.error.message ?? JSON.stringify(status.error)),
|
||||
)
|
||||
: {
|
||||
errorCode: "unknown",
|
||||
errorDetail: `Workflow instance ${status.status}`,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
// Instance not found (never created, or expired from Workflows
|
||||
// retention). Past the grace window that means the audit can never
|
||||
// finish — without this branch such rows stay "running" forever.
|
||||
// Transient status/control-plane errors must NOT fail a live audit,
|
||||
// so only a confirmed not-found counts.
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!/not[ _]?found/i.test(message)) return null;
|
||||
if (!isOlderThan(audit.startedAt, INSTANCE_LOST_GRACE_MS)) return null;
|
||||
errorInfo = {
|
||||
errorCode: "instance_lost",
|
||||
errorDetail: "Workflow instance not found",
|
||||
};
|
||||
}
|
||||
if (!errorInfo) return null;
|
||||
|
||||
await AuditRepository.failAudit(audit.id, audit.workflowInstanceId, {
|
||||
...errorInfo,
|
||||
failedPhase: audit.currentPhase,
|
||||
});
|
||||
return errorInfo;
|
||||
}
|
||||
|
||||
/** Cron watchdog: sweep stale running audits and reconcile each. */
|
||||
export async function reconcileStaleAudits() {
|
||||
const cutoff = new Date(Date.now() - STALE_RUNNING_AFTER_MS);
|
||||
const stale = await getStaleRunningAudits(cutoff, WATCHDOG_BATCH_LIMIT);
|
||||
|
||||
for (const audit of stale) {
|
||||
try {
|
||||
const errorInfo = await reconcileRunningAudit(audit);
|
||||
if (!errorInfo) continue;
|
||||
|
||||
console.log(
|
||||
`Audit watchdog: marked ${audit.id} failed (${errorInfo.errorCode}, phase=${audit.currentPhase})`,
|
||||
);
|
||||
const project = await db.query.projects.findFirst({
|
||||
where: eq(projects.id, audit.projectId),
|
||||
});
|
||||
if (!project) continue;
|
||||
await captureServerEvent({
|
||||
distinctId: audit.startedByUserId,
|
||||
event: "site_audit:complete",
|
||||
organizationId: project.organizationId,
|
||||
properties: {
|
||||
project_id: audit.projectId,
|
||||
status: "failed",
|
||||
reconciled_by: "watchdog",
|
||||
error_code: errorInfo.errorCode,
|
||||
pages_crawled: audit.pagesCrawled,
|
||||
pages_total: audit.pagesTotal,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
// One unreconcilable audit must not stop the sweep.
|
||||
console.error(`Audit watchdog: failed to reconcile ${audit.id}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Running audits started before the cutoff — watchdog candidates. Timestamps
|
||||
* are stored as text, so this compares lexicographically: PG rows use ISO
|
||||
* with a "T", D1-default rows use "YYYY-MM-DD HH:MM:SS" (space < "T", so a
|
||||
* same-day ISO cutoff would misorder against D1 rows without the reformat).
|
||||
*/
|
||||
async function getStaleRunningAudits(cutoff: Date, limit: number) {
|
||||
const iso = cutoff.toISOString();
|
||||
const startedBefore =
|
||||
getDatabaseProvider() === "postgres"
|
||||
? iso
|
||||
: iso.replace("T", " ").slice(0, 19);
|
||||
// Oldest first: genuinely dead audits age past the cutoff and stay there,
|
||||
// while long-but-live crawls are the newest of the stale set — without a
|
||||
// deterministic order they could occupy the whole batch every sweep and
|
||||
// starve real zombies.
|
||||
return db.query.audits.findMany({
|
||||
where: and(
|
||||
eq(audits.status, "running"),
|
||||
lt(audits.startedAt, startedBefore),
|
||||
),
|
||||
orderBy: audits.startedAt,
|
||||
columns: {
|
||||
id: true,
|
||||
workflowInstanceId: true,
|
||||
projectId: true,
|
||||
startedByUserId: true,
|
||||
startedAt: true,
|
||||
currentPhase: true,
|
||||
pagesCrawled: true,
|
||||
pagesTotal: true,
|
||||
},
|
||||
limit,
|
||||
});
|
||||
}
|
||||
|
||||
function isOlderThan(startedAt: string, ageMs: number): boolean {
|
||||
// D1-default timestamps ("YYYY-MM-DD HH:MM:SS") lack the T/Z; normalize so
|
||||
// Date.parse reads them as UTC, matching the PG ISO format.
|
||||
const parsed = Date.parse(
|
||||
startedAt.includes("T") ? startedAt : `${startedAt.replace(" ", "T")}Z`,
|
||||
);
|
||||
if (Number.isNaN(parsed)) return true;
|
||||
return parsed < Date.now() - ageMs;
|
||||
}
|
||||
79
src/server/lib/audit/audit-errors.ts
Normal file
79
src/server/lib/audit/audit-errors.ts
Normal file
@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Classification of audit failures into a closed error-code vocabulary.
|
||||
*
|
||||
* Codes are written to audits.error_code by the workflow's mark-failed step
|
||||
* and by the stale-audit reconciler (which reads the message off the dead
|
||||
* Workflows instance). Keeping the vocabulary closed makes failures
|
||||
* aggregable in SQL and lets the UI map codes to friendly copy without
|
||||
* leaking raw infrastructure errors to users.
|
||||
*/
|
||||
|
||||
const AUDIT_ERROR_CODES = [
|
||||
// A workflow step exceeded its timeout (typically a site that hangs
|
||||
// connections or a stalled DB write).
|
||||
"step_timeout",
|
||||
// The Worker isolate was killed for exceeding its 128MB memory limit.
|
||||
"oom",
|
||||
// The Worker isolate was killed for exceeding its CPU time limit.
|
||||
"cpu_limit",
|
||||
// A database query failed permanently.
|
||||
"db_error",
|
||||
// A workflow step tried to persist more than the ~1MiB durable-state cap.
|
||||
"step_output_too_large",
|
||||
// Cloudflare Workflows internal error (not caused by our code).
|
||||
"workflow_internal",
|
||||
// The workflow instance no longer exists (expired from retention or never
|
||||
// created) while the audit row still said "running".
|
||||
"instance_lost",
|
||||
// Anything we could not classify.
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
export type AuditErrorCode = (typeof AUDIT_ERROR_CODES)[number];
|
||||
|
||||
export interface AuditErrorInfo {
|
||||
errorCode: AuditErrorCode;
|
||||
errorDetail: string;
|
||||
}
|
||||
|
||||
const ERROR_DETAIL_MAX_CHARS = 500;
|
||||
|
||||
/**
|
||||
* Classify an error (thrown in the workflow, or read back from a dead
|
||||
* Workflows instance) into an AuditErrorCode. Matches the exact failure
|
||||
* messages observed in production instances.
|
||||
*/
|
||||
export function classifyAuditError(error: unknown): AuditErrorInfo {
|
||||
const message =
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: typeof error === "string"
|
||||
? error
|
||||
: String(error);
|
||||
|
||||
return {
|
||||
errorCode: classifyMessage(message),
|
||||
errorDetail: message.slice(0, ERROR_DETAIL_MAX_CHARS),
|
||||
};
|
||||
}
|
||||
|
||||
function classifyMessage(message: string): AuditErrorCode {
|
||||
if (message.includes("exceeded memory limit")) return "oom";
|
||||
if (message.includes("exceeded CPU time limit")) return "cpu_limit";
|
||||
if (
|
||||
message.includes("WorkflowTimeoutError") ||
|
||||
message.includes("Execution timed out")
|
||||
) {
|
||||
return "step_timeout";
|
||||
}
|
||||
if (message.includes("output is too large")) return "step_output_too_large";
|
||||
if (message.includes("WorkflowInternalError")) return "workflow_internal";
|
||||
if (
|
||||
message.startsWith("Failed query:") ||
|
||||
message.includes("D1_ERROR") ||
|
||||
message.includes("Postgres database accessed outside a request scope")
|
||||
) {
|
||||
return "db_error";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
100
src/server/lib/audit/crawl-window.test.ts
Normal file
100
src/server/lib/audit/crawl-window.test.ts
Normal file
@ -0,0 +1,100 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { adjustCrawlWindow } from "@/server/lib/audit/crawl-window";
|
||||
import type {
|
||||
CrawledPageResult,
|
||||
PageFetchClass,
|
||||
} from "@/server/lib/audit/types";
|
||||
|
||||
function page(
|
||||
fetchClass: PageFetchClass,
|
||||
responseTimeMs: number,
|
||||
htmlBytes = 10_000,
|
||||
): CrawledPageResult {
|
||||
return {
|
||||
id: "",
|
||||
url: "https://example.com/",
|
||||
statusCode: fetchClass === "ok" ? 200 : 0,
|
||||
fetchClass,
|
||||
redirectUrl: null,
|
||||
title: "",
|
||||
metaDescription: "",
|
||||
canonicalUrl: null,
|
||||
robotsMeta: null,
|
||||
xRobotsTag: null,
|
||||
headerCanonicalUrl: null,
|
||||
ogTitle: null,
|
||||
ogDescription: null,
|
||||
ogImage: null,
|
||||
h1Count: 0,
|
||||
h2Count: 0,
|
||||
h3Count: 0,
|
||||
h4Count: 0,
|
||||
h5Count: 0,
|
||||
h6Count: 0,
|
||||
headingOrder: [],
|
||||
wordCount: 0,
|
||||
contentHash: null,
|
||||
isHtml: true,
|
||||
htmlBytes,
|
||||
imagesTotal: 0,
|
||||
imagesMissingAlt: 0,
|
||||
images: [],
|
||||
links: [],
|
||||
hasStructuredData: false,
|
||||
hreflangTags: [],
|
||||
isIndexable: true,
|
||||
responseTimeMs,
|
||||
crawlDepth: 0,
|
||||
inSitemap: false,
|
||||
};
|
||||
}
|
||||
|
||||
describe("adjustCrawlWindow", () => {
|
||||
it("keeps the window on an empty batch", () => {
|
||||
expect(adjustCrawlWindow(25, [])).toBe(25);
|
||||
});
|
||||
|
||||
it("halves the window when a third of the batch is troubled", () => {
|
||||
const recent = [
|
||||
...Array.from({ length: 9 }, () => page("error", 15_000)),
|
||||
...Array.from({ length: 16 }, () => page("ok", 500)),
|
||||
];
|
||||
expect(adjustCrawlWindow(25, recent)).toBe(12);
|
||||
});
|
||||
|
||||
it("treats blocked fetches as trouble", () => {
|
||||
const recent = Array.from({ length: 10 }, () => page("blocked", 300));
|
||||
expect(adjustCrawlWindow(20, recent)).toBe(10);
|
||||
});
|
||||
|
||||
it("never shrinks below the minimum", () => {
|
||||
const recent = Array.from({ length: 10 }, () => page("error", 15_000));
|
||||
expect(adjustCrawlWindow(6, recent)).toBe(5);
|
||||
});
|
||||
|
||||
it("grows on a clean, fast batch up to the cap", () => {
|
||||
const recent = Array.from({ length: 25 }, () => page("ok", 400));
|
||||
expect(adjustCrawlWindow(25, recent)).toBe(30);
|
||||
expect(adjustCrawlWindow(40, recent)).toBe(40);
|
||||
});
|
||||
|
||||
it("treats heavy pages as trouble even when they respond fast", () => {
|
||||
const recent = Array.from({ length: 25 }, () =>
|
||||
page("ok", 300, 2 * 1024 * 1024),
|
||||
);
|
||||
expect(adjustCrawlWindow(25, recent)).toBe(12);
|
||||
});
|
||||
|
||||
it("does not grow when some pages are heavy", () => {
|
||||
const recent = [
|
||||
...Array.from({ length: 4 }, () => page("ok", 300, 1024 * 1024)),
|
||||
...Array.from({ length: 21 }, () => page("ok", 300)),
|
||||
];
|
||||
expect(adjustCrawlWindow(25, recent)).toBe(25);
|
||||
});
|
||||
|
||||
it("holds steady on a clean but slow batch", () => {
|
||||
const recent = Array.from({ length: 25 }, () => page("ok", 5_000));
|
||||
expect(adjustCrawlWindow(25, recent)).toBe(25);
|
||||
});
|
||||
});
|
||||
50
src/server/lib/audit/crawl-window.ts
Normal file
50
src/server/lib/audit/crawl-window.ts
Normal file
@ -0,0 +1,50 @@
|
||||
import type { CrawledPageResult } from "@/server/lib/audit/types";
|
||||
|
||||
/**
|
||||
* Rolling fetch-concurrency window for the crawl. Unlike fixed batches, a
|
||||
* slow page only occupies one slot instead of stalling a whole batch. The
|
||||
* window adapts to the site: it shrinks when fetches error/block/crawl
|
||||
* slowly (politeness toward struggling or defensive sites) and grows when
|
||||
* the site answers fast.
|
||||
*/
|
||||
export const INITIAL_CRAWL_WINDOW = 25;
|
||||
const MIN_WINDOW = 5;
|
||||
const MAX_WINDOW = 40;
|
||||
const SLOW_RESPONSE_MS = 10_000;
|
||||
const FAST_RESPONSE_MS = 1_500;
|
||||
/**
|
||||
* Pages at/above this HTML size count as trouble: each in-flight page
|
||||
* buffers its body, so a wide window on a heavy-page site is memory
|
||||
* pressure the response time can't see (it's measured at headers).
|
||||
*/
|
||||
const HEAVY_PAGE_BYTES = 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Adapt the window to the last persisted sub-batch. Shrinks on trouble
|
||||
* (errors, blocks, very slow responses, heavy bodies), grows only on a
|
||||
* clean and mostly-fast batch.
|
||||
*/
|
||||
export function adjustCrawlWindow(
|
||||
windowSize: number,
|
||||
recent: CrawledPageResult[],
|
||||
): number {
|
||||
if (recent.length === 0) return windowSize;
|
||||
const troubled = recent.filter(
|
||||
(page) =>
|
||||
page.fetchClass !== "ok" ||
|
||||
(page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS ||
|
||||
page.htmlBytes >= HEAVY_PAGE_BYTES,
|
||||
).length;
|
||||
if (troubled * 3 >= recent.length) {
|
||||
return Math.max(MIN_WINDOW, Math.floor(windowSize / 2));
|
||||
}
|
||||
const fast = recent.filter(
|
||||
(page) =>
|
||||
page.fetchClass === "ok" &&
|
||||
(page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS,
|
||||
).length;
|
||||
if (troubled === 0 && fast * 2 >= recent.length) {
|
||||
return Math.min(MAX_WINDOW, windowSize + 5);
|
||||
}
|
||||
return windowSize;
|
||||
}
|
||||
@ -6,10 +6,21 @@ import { XMLParser } from "fast-xml-parser";
|
||||
import { isSameOrigin, normalizeUrl } from "./url-utils";
|
||||
|
||||
const SITEMAP_FETCH_TIMEOUT_MS = 15_000;
|
||||
// robots.txt is checkpointed as durable Workflow step state (~1MiB cap, shared
|
||||
// with the rest of the step's return). RFC 9309 requires parsers to handle at
|
||||
// least 500 KiB and permits ignoring anything beyond it — Google does exactly
|
||||
// that — so this cap matches standard crawler behavior while keeping a
|
||||
// misbehaving server (e.g. HTML at /robots.txt) from blowing the step limit.
|
||||
const MAX_ROBOTS_TXT_BYTES = 500 * 1024;
|
||||
const MAX_SITEMAP_DEPTH = 3;
|
||||
const MAX_SITEMAP_DOCS = 300;
|
||||
const SITEMAP_CONCURRENCY = 5;
|
||||
const SITEMAP_RETRIES = 1;
|
||||
// Sitemap shards can legally reach 50 MB and SITEMAP_CONCURRENCY of them are
|
||||
// read at once, so unbounded reads can exhaust Worker memory. Oversized
|
||||
// shards are skipped whole — truncated XML would not parse anyway, and real
|
||||
// generators shard far below this.
|
||||
const MAX_SITEMAP_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
const xmlParser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
@ -34,7 +45,7 @@ async function fetchRobotsTxtText(origin: string): Promise<string | null> {
|
||||
});
|
||||
|
||||
if (!response.ok) return null;
|
||||
return await response.text();
|
||||
return (await response.text()).slice(0, MAX_ROBOTS_TXT_BYTES);
|
||||
} catch (error) {
|
||||
console.warn("Failed to fetch robots.txt:", error);
|
||||
return null;
|
||||
@ -119,6 +130,34 @@ function isTimeoutError(error: unknown): boolean {
|
||||
return "name" in error && error.name === "TimeoutError";
|
||||
}
|
||||
|
||||
/** Read a response body up to maxBytes; null when the body exceeds it. */
|
||||
async function readBodyCapped(
|
||||
response: Response,
|
||||
maxBytes: number,
|
||||
): Promise<string | null> {
|
||||
if (!response.body) return "";
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
total += value.byteLength;
|
||||
if (total > maxBytes) {
|
||||
await reader.cancel();
|
||||
return null;
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
const joined = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
joined.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return new TextDecoder().decode(joined);
|
||||
}
|
||||
|
||||
async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{
|
||||
nestedSitemaps: string[];
|
||||
pageUrls: string[];
|
||||
@ -147,8 +186,11 @@ async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{
|
||||
return { nestedSitemaps: [], pageUrls: [], timedOut: false };
|
||||
}
|
||||
|
||||
const body = await response.text();
|
||||
if (!isProbablySitemapXml(response.headers.get("content-type"), body)) {
|
||||
const body = await readBodyCapped(response, MAX_SITEMAP_BYTES);
|
||||
if (
|
||||
body === null ||
|
||||
!isProbablySitemapXml(response.headers.get("content-type"), body)
|
||||
) {
|
||||
return { nestedSitemaps: [], pageUrls: [], timedOut: false };
|
||||
}
|
||||
|
||||
|
||||
@ -1,15 +1,15 @@
|
||||
/**
|
||||
* Cross-page (multipage) issue checks.
|
||||
* Cross-page (multipage) issue checks over the app DB's page rows:
|
||||
* duplicates and redirect chains/loops. Pure set-queries over crawl data —
|
||||
* no fetching, no DOM.
|
||||
*
|
||||
* These run once after the crawl, against the rows the crawl wrote to D1:
|
||||
* duplicates, broken internal links, redirect chains/loops, orphan pages.
|
||||
* Pure set-queries over crawl data — no fetching, no DOM.
|
||||
* The two link-edge checks (broken-internal-link, orphan-page) live in the
|
||||
* audit's scratchpad Durable Object (AuditScratchpad.runFinalizeChecks),
|
||||
* next to the link edges themselves — link rows never touch the app DB.
|
||||
*/
|
||||
import { and, eq, gte, lt, ne, notExists, sql } from "drizzle-orm";
|
||||
import { alias } from "drizzle-orm/sqlite-core";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "@/db";
|
||||
import { auditLinks, auditPages } from "@/db/schema";
|
||||
import { normalizeUrl } from "@/server/lib/audit/url-utils";
|
||||
import { auditPages } from "@/db/schema";
|
||||
import {
|
||||
findDuplicates,
|
||||
findRedirectChainsAndLoops,
|
||||
@ -17,13 +17,8 @@ import {
|
||||
} from "@/server/lib/audit/issues/multipage-checks";
|
||||
import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters";
|
||||
|
||||
const BROKEN_LINK_ISSUE_CAP = 2_000;
|
||||
|
||||
export async function runMultipageChecks(input: {
|
||||
auditId: string;
|
||||
startUrl: string;
|
||||
/** Orphan detection only makes sense when the crawl wasn't truncated. */
|
||||
crawlCompleted: boolean;
|
||||
}): Promise<DetectedIssue[]> {
|
||||
const pages: SlimPage[] = await db
|
||||
.select({
|
||||
@ -43,112 +38,5 @@ export async function runMultipageChecks(input: {
|
||||
.from(auditPages)
|
||||
.where(eq(auditPages.auditId, input.auditId));
|
||||
|
||||
const issues: DetectedIssue[] = [
|
||||
...findDuplicates(pages),
|
||||
...findRedirectChainsAndLoops(pages),
|
||||
...(await findBrokenInternalLinks(input.auditId)),
|
||||
];
|
||||
|
||||
if (input.crawlCompleted) {
|
||||
// Page rows store normalized URLs; normalize the start URL the same way
|
||||
// so the orphan exclusion matches.
|
||||
const normalizedStart = normalizeUrl(input.startUrl) ?? input.startUrl;
|
||||
issues.push(...(await findOrphanPages(input.auditId, normalizedStart)));
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function findBrokenInternalLinks(
|
||||
auditId: string,
|
||||
): Promise<DetectedIssue[]> {
|
||||
// Only flag targets we actually crawled and saw fail — never inferred from
|
||||
// absence. Blocked targets (WAF challenges) are excluded: a 403 from bot
|
||||
// protection is not evidence of a broken link.
|
||||
const rows = await db
|
||||
.select({
|
||||
sourcePageId: auditLinks.sourcePageId,
|
||||
sourceUrl: auditLinks.sourceUrl,
|
||||
targetUrl: auditLinks.targetUrl,
|
||||
targetStatus: auditPages.statusCode,
|
||||
})
|
||||
.from(auditLinks)
|
||||
.innerJoin(
|
||||
auditPages,
|
||||
and(
|
||||
eq(auditPages.auditId, auditLinks.auditId),
|
||||
eq(auditPages.url, auditLinks.targetUrl),
|
||||
),
|
||||
)
|
||||
.where(
|
||||
and(
|
||||
eq(auditLinks.auditId, auditId),
|
||||
eq(auditLinks.isInternal, true),
|
||||
gte(auditPages.statusCode, 400),
|
||||
eq(auditPages.fetchClass, "ok"),
|
||||
),
|
||||
)
|
||||
.limit(BROKEN_LINK_ISSUE_CAP);
|
||||
|
||||
return rows.map((row) => ({
|
||||
issueType: "broken-internal-link" as const,
|
||||
pageId: row.sourcePageId,
|
||||
pageUrl: row.sourceUrl,
|
||||
dedupeKey: row.targetUrl,
|
||||
details: { targetUrl: row.targetUrl, targetStatus: row.targetStatus },
|
||||
}));
|
||||
}
|
||||
|
||||
async function findOrphanPages(
|
||||
auditId: string,
|
||||
startUrl: string,
|
||||
): Promise<DetectedIssue[]> {
|
||||
// A live 2xx page is an orphan when no OTHER crawled page links to it and
|
||||
// nothing redirects to it. Only meaningful on a completed crawl: on a
|
||||
// truncated one, "no observed inlinks" is true of nearly everything.
|
||||
// Error/redirect rows aren't orphans — they already get their own issues.
|
||||
const inlink = db
|
||||
.select({ one: sql`1` })
|
||||
.from(auditLinks)
|
||||
.where(
|
||||
and(
|
||||
eq(auditLinks.auditId, auditId),
|
||||
eq(auditLinks.isInternal, true),
|
||||
eq(auditLinks.targetUrl, auditPages.url),
|
||||
// Self-links (breadcrumbs, anchors) don't make a page reachable.
|
||||
ne(auditLinks.sourcePageId, auditPages.id),
|
||||
),
|
||||
);
|
||||
|
||||
const redirectSourcePages = alias(auditPages, "redirect_sources");
|
||||
const redirectSources = db
|
||||
.select({ one: sql`1` })
|
||||
.from(redirectSourcePages)
|
||||
.where(
|
||||
and(
|
||||
eq(redirectSourcePages.auditId, auditId),
|
||||
eq(redirectSourcePages.redirectUrl, auditPages.url),
|
||||
),
|
||||
);
|
||||
|
||||
const rows = await db
|
||||
.select({ id: auditPages.id, url: auditPages.url })
|
||||
.from(auditPages)
|
||||
.where(
|
||||
and(
|
||||
eq(auditPages.auditId, auditId),
|
||||
ne(auditPages.url, startUrl),
|
||||
eq(auditPages.fetchClass, "ok"),
|
||||
gte(auditPages.statusCode, 200),
|
||||
lt(auditPages.statusCode, 300),
|
||||
notExists(inlink),
|
||||
notExists(redirectSources),
|
||||
),
|
||||
);
|
||||
|
||||
return rows.map((row) => ({
|
||||
issueType: "orphan-page" as const,
|
||||
pageId: row.id,
|
||||
pageUrl: row.url,
|
||||
}));
|
||||
return [...findDuplicates(pages), ...findRedirectChainsAndLoops(pages)];
|
||||
}
|
||||
|
||||
@ -41,6 +41,7 @@ function makePage(overrides: Partial<CrawledPageResult>): CrawledPageResult {
|
||||
wordCount: 500,
|
||||
contentHash: "abc123",
|
||||
isHtml: true,
|
||||
htmlBytes: 10_000,
|
||||
imagesTotal: 0,
|
||||
imagesMissingAlt: 0,
|
||||
images: [],
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
* Per-page issue reporters.
|
||||
*
|
||||
* Each reporter is a pure function over a single crawled page record —
|
||||
* DOM-free by design (cheerio runs once in crawlPage), so the engine works
|
||||
* DOM-free by design (HTML parsing runs once in crawlPage), so the engine works
|
||||
* over any crawl source that can produce a CrawledPageResult.
|
||||
*
|
||||
* Cross-page checks (duplicates, broken links, orphans, redirect chains)
|
||||
|
||||
208
src/server/lib/audit/page-analyzer.test.ts
Normal file
208
src/server/lib/audit/page-analyzer.test.ts
Normal file
@ -0,0 +1,208 @@
|
||||
/**
|
||||
* Parity tests for the streaming (htmlparser2) page analyzer against a
|
||||
* cheerio/DOM reference implementation — the exact logic the analyzer
|
||||
* replaced. Cheerio stays as a devDependency for this test only.
|
||||
*/
|
||||
import * as cheerio from "cheerio";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { analyzeHtml } from "@/server/lib/audit/page-analyzer";
|
||||
import { normalizeUrl, isSameOrigin } from "@/server/lib/audit/url-utils";
|
||||
import type { PageAnalysis, PageLink } from "@/server/lib/audit/types";
|
||||
|
||||
/** The previous cheerio implementation, verbatim (minus passthrough fields). */
|
||||
function analyzeHtmlWithCheerio(html: string, pageUrl: string): PageAnalysis {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const title = $("title").first().text().trim();
|
||||
const metaDescription =
|
||||
$('meta[name="description"]').first().attr("content")?.trim() ?? "";
|
||||
const canonical = $('link[rel="canonical"]').first().attr("href") ?? null;
|
||||
const robotsMeta = $('meta[name="robots"]').first().attr("content") ?? null;
|
||||
const ogTitle =
|
||||
$('meta[property="og:title"]').first().attr("content") ?? null;
|
||||
const ogDescription =
|
||||
$('meta[property="og:description"]').first().attr("content") ?? null;
|
||||
const ogImage =
|
||||
$('meta[property="og:image"]').first().attr("content") ?? null;
|
||||
|
||||
const h1s: string[] = [];
|
||||
$("h1").each((_, el) => {
|
||||
h1s.push($(el).text().trim());
|
||||
});
|
||||
|
||||
const headingOrder: number[] = [];
|
||||
$("h1, h2, h3, h4, h5, h6").each((_, el) => {
|
||||
const tag =
|
||||
"tagName" in el && typeof el.tagName === "string"
|
||||
? el.tagName.toLowerCase()
|
||||
: null;
|
||||
if (tag) {
|
||||
const level = parseInt(tag.charAt(1), 10);
|
||||
if (!isNaN(level)) headingOrder.push(level);
|
||||
}
|
||||
});
|
||||
|
||||
const bodyClone = $("body").clone();
|
||||
bodyClone.find("script, style, noscript, svg").remove();
|
||||
const bodyText = bodyClone.text().replace(/\s+/g, " ").trim();
|
||||
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
||||
|
||||
const images: Array<{ src: string | null; alt: string | null }> = [];
|
||||
$("img").each((_, el) => {
|
||||
images.push({
|
||||
src: $(el).attr("src") ?? null,
|
||||
alt: $(el).attr("alt") ?? null,
|
||||
});
|
||||
});
|
||||
|
||||
const linksByTarget = new Map<string, PageLink>();
|
||||
$("a[href]").each((_, el) => {
|
||||
const href = $(el).attr("href");
|
||||
if (!href) return;
|
||||
if (/^(javascript:|mailto:|tel:|#)/.test(href)) return;
|
||||
const resolved = normalizeUrl(href, pageUrl);
|
||||
if (!resolved) return;
|
||||
if (linksByTarget.has(resolved)) return;
|
||||
const anchor = $(el).text().replace(/\s+/g, " ").trim().slice(0, 200);
|
||||
const rel = $(el).attr("rel")?.toLowerCase() ?? "";
|
||||
linksByTarget.set(resolved, {
|
||||
targetUrl: resolved,
|
||||
anchor: anchor || null,
|
||||
isInternal: isSameOrigin(resolved, pageUrl),
|
||||
isNofollow: rel.split(/\s+/).includes("nofollow"),
|
||||
});
|
||||
});
|
||||
|
||||
let hasStructuredData = false;
|
||||
$('script[type="application/ld+json"]').each(() => {
|
||||
hasStructuredData = true;
|
||||
});
|
||||
|
||||
const hreflangTags: string[] = [];
|
||||
$('link[rel="alternate"][hreflang]').each((_, el) => {
|
||||
const hreflang = $(el).attr("hreflang");
|
||||
if (hreflang) hreflangTags.push(hreflang);
|
||||
});
|
||||
|
||||
return {
|
||||
url: pageUrl,
|
||||
statusCode: 200,
|
||||
redirectUrl: null,
|
||||
responseTimeMs: 0,
|
||||
title,
|
||||
metaDescription,
|
||||
canonical,
|
||||
robotsMeta,
|
||||
ogTitle,
|
||||
ogDescription,
|
||||
ogImage,
|
||||
h1s,
|
||||
headingOrder,
|
||||
wordCount,
|
||||
bodyText,
|
||||
images,
|
||||
links: Array.from(linksByTarget.values()),
|
||||
hasStructuredData,
|
||||
hreflangTags,
|
||||
};
|
||||
}
|
||||
|
||||
const PAGE_URL = "https://example.com/blog/post";
|
||||
|
||||
function expectParity(html: string) {
|
||||
const streamed = analyzeHtml(html, PAGE_URL, 200, 0);
|
||||
const reference = analyzeHtmlWithCheerio(html, PAGE_URL);
|
||||
expect(streamed).toEqual(reference);
|
||||
}
|
||||
|
||||
describe("analyzeHtml parity with the DOM reference", () => {
|
||||
it("matches on a full, well-formed document", () => {
|
||||
expectParity(`<!DOCTYPE html>
|
||||
<html><head>
|
||||
<title> The Title </title>
|
||||
<meta name="description" content=" A description. ">
|
||||
<meta name="robots" content="index, follow">
|
||||
<meta property="og:title" content="OG Title">
|
||||
<meta property="og:description" content="OG Desc">
|
||||
<meta property="og:image" content="/og.png">
|
||||
<link rel="canonical" href="https://example.com/blog/post">
|
||||
<link rel="alternate" hreflang="en" href="/en">
|
||||
<link rel="alternate" hreflang="de" href="/de">
|
||||
<script type="application/ld+json">{"@type":"Article"}</script>
|
||||
</head><body>
|
||||
<h1>Main <em>Heading</em></h1>
|
||||
<h2>Sub</h2><h3>Deep</h3><h2>Sub 2</h2>
|
||||
<p>Some visible body text with <strong>bold words</strong> here.</p>
|
||||
<script>ignored();</script>
|
||||
<style>.x{}</style>
|
||||
<svg><title>icon</title><text>svg text</text></svg>
|
||||
<noscript><img src="/ns.png"><p>noscript text</p></noscript>
|
||||
<img src="/a.png" alt="A picture">
|
||||
<img src="/b.png" alt="">
|
||||
<img src="/c.png">
|
||||
<img alt="no src">
|
||||
<a href="/relative">Relative <span>link</span></a>
|
||||
<a href="https://example.com/relative">Duplicate target</a>
|
||||
<a href="https://other.example/x" rel="NoFollow sponsored">External</a>
|
||||
<a href="mailto:x@example.com">Mail</a>
|
||||
<a href="#frag">Fragment</a>
|
||||
<a href="javascript:void(0)">JS</a>
|
||||
<a href="/empty-anchor"><img src="/img-link.png" alt=""></a>
|
||||
</body></html>`);
|
||||
});
|
||||
|
||||
it("matches on documents with no head, body, or title", () => {
|
||||
expectParity(
|
||||
`<h1>Bare fragment</h1><p>Just text and a <a href="/x">link</a>.</p>`,
|
||||
);
|
||||
});
|
||||
|
||||
it("matches on an empty document", () => {
|
||||
expectParity("");
|
||||
});
|
||||
|
||||
it("matches on a document with only a head", () => {
|
||||
expectParity(
|
||||
`<html><head><title>Head only</title><meta name="description" content="d"></head></html>`,
|
||||
);
|
||||
});
|
||||
|
||||
it("matches with duplicate metas and titles (first wins)", () => {
|
||||
expectParity(`<html><head>
|
||||
<title>First</title><title>Second</title>
|
||||
<meta name="description" content="first desc">
|
||||
<meta name="description" content="second desc">
|
||||
<link rel="canonical" href="/first"><link rel="canonical" href="/second">
|
||||
</head><body><p>text</p></body></html>`);
|
||||
});
|
||||
|
||||
it("matches on unclosed and misnested tags", () => {
|
||||
expectParity(`<html><body>
|
||||
<h1>Unclosed heading
|
||||
<p>Paragraph <b>bold <i>both</b> italic?</i>
|
||||
<a href="/one">first <a href="/two">second</a>
|
||||
<div>trailing text`);
|
||||
});
|
||||
|
||||
it("matches on entity-heavy content", () => {
|
||||
expectParity(`<html><head><title>A & B <C></title></head>
|
||||
<body><h1>Café "menu"</h1>
|
||||
<p>1 < 2 && 3 > 2</p>
|
||||
<a href="/x?a=1&b=2">Query & anchor</a></body></html>`);
|
||||
});
|
||||
|
||||
it("matches heading order across nesting", () => {
|
||||
expectParity(`<body><h3>three</h3><div><h1>one</h1><section><h2>two</h2>
|
||||
<h6>six</h6></section></div><h4>four</h4></body>`);
|
||||
});
|
||||
|
||||
it("matches word counts with whitespace-heavy markup", () => {
|
||||
expectParity(`<body>
|
||||
<p>
|
||||
one
|
||||
two three
|
||||
</p>
|
||||
<ul><li>four</li><li>five</li></ul>
|
||||
</body>`);
|
||||
});
|
||||
});
|
||||
@ -1,14 +1,38 @@
|
||||
/**
|
||||
* HTML page analyzer using cheerio.
|
||||
* HTML page analyzer using htmlparser2's streaming tokenizer.
|
||||
*
|
||||
* Extracts SEO-relevant data from a page's HTML:
|
||||
* title, meta description, headings, images, links, canonical, OG tags,
|
||||
* structured data, robots meta, word count, hreflang.
|
||||
* Extracts SEO-relevant data from a page's HTML: title, meta description,
|
||||
* headings, images, links, canonical, OG tags, structured data, robots meta,
|
||||
* word count, hreflang.
|
||||
*
|
||||
* Deliberately NOT a DOM parser: the previous cheerio implementation built a
|
||||
* full DOM (~5-10x the HTML's size) per page, and with 25 concurrent parses
|
||||
* on a 128MB isolate that was the audit engine's dominant OOM cause. The
|
||||
* tokenizer keeps only the accumulated text and extracted fields in memory.
|
||||
*/
|
||||
import * as cheerio from "cheerio";
|
||||
import { Parser } from "htmlparser2";
|
||||
import { normalizeUrl, isSameOrigin } from "./url-utils";
|
||||
import type { PageAnalysis, PageLink } from "./types";
|
||||
|
||||
const SKIPPED_LINK_PROTOCOLS = /^(javascript:|mailto:|tel:|#)/;
|
||||
/** Subtrees whose text is not visible content. */
|
||||
const NON_CONTENT_TAGS = new Set(["script", "style", "noscript", "svg"]);
|
||||
const HEADING_LEVELS: Record<string, number> = {
|
||||
h1: 1,
|
||||
h2: 2,
|
||||
h3: 3,
|
||||
h4: 4,
|
||||
h5: 5,
|
||||
h6: 6,
|
||||
};
|
||||
const MAX_ANCHOR_CHARS = 200;
|
||||
|
||||
interface OpenAnchor {
|
||||
href: string;
|
||||
rel: string;
|
||||
text: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze an HTML string and extract all SEO-relevant data.
|
||||
*/
|
||||
@ -19,102 +43,195 @@ export function analyzeHtml(
|
||||
responseTimeMs: number,
|
||||
redirectUrl: string | null = null,
|
||||
): PageAnalysis {
|
||||
const $ = cheerio.load(html);
|
||||
let title: string | null = null;
|
||||
let titleDepth = 0;
|
||||
let titleDone = false;
|
||||
// parse5 (the old DOM path) treats <noscript> content as raw text when
|
||||
// scripting is enabled; skip element extraction inside it to match.
|
||||
let noscriptDepth = 0;
|
||||
let metaDescription: string | null = null;
|
||||
let canonical: string | null = null;
|
||||
let robotsMeta: string | null = null;
|
||||
let ogTitle: string | null = null;
|
||||
let ogDescription: string | null = null;
|
||||
let ogImage: string | null = null;
|
||||
let hasStructuredData = false;
|
||||
const hreflangTags: string[] = [];
|
||||
|
||||
const title = $("title").first().text().trim();
|
||||
|
||||
const metaDescription =
|
||||
$('meta[name="description"]').first().attr("content")?.trim() ?? "";
|
||||
|
||||
const canonical = $('link[rel="canonical"]').first().attr("href") ?? null;
|
||||
|
||||
const robotsMeta = $('meta[name="robots"]').first().attr("content") ?? null;
|
||||
|
||||
// --- Open Graph ---
|
||||
const ogTitle =
|
||||
$('meta[property="og:title"]').first().attr("content") ?? null;
|
||||
const ogDescription =
|
||||
$('meta[property="og:description"]').first().attr("content") ?? null;
|
||||
const ogImage =
|
||||
$('meta[property="og:image"]').first().attr("content") ?? null;
|
||||
|
||||
// --- Headings ---
|
||||
const h1s: string[] = [];
|
||||
$("h1").each((_, el) => {
|
||||
h1s.push($(el).text().trim());
|
||||
});
|
||||
|
||||
const headingOrder: number[] = [];
|
||||
$("h1, h2, h3, h4, h5, h6").each((_, el) => {
|
||||
const tag =
|
||||
"tagName" in el && typeof el.tagName === "string"
|
||||
? el.tagName.toLowerCase()
|
||||
: null;
|
||||
if (tag) {
|
||||
const level = parseInt(tag.charAt(1), 10);
|
||||
if (!isNaN(level)) headingOrder.push(level);
|
||||
}
|
||||
});
|
||||
|
||||
// --- Word count (visible text in body) ---
|
||||
// Remove script/style/noscript tags, then count words in remaining text
|
||||
const bodyClone = $("body").clone();
|
||||
bodyClone.find("script, style, noscript, svg").remove();
|
||||
const bodyText = bodyClone.text().replace(/\s+/g, " ").trim();
|
||||
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
||||
let openH1: string[] | null = null;
|
||||
|
||||
const images: Array<{ src: string | null; alt: string | null }> = [];
|
||||
$("img").each((_, el) => {
|
||||
images.push({
|
||||
src: $(el).attr("src") ?? null,
|
||||
alt: $(el).attr("alt") ?? null,
|
||||
});
|
||||
});
|
||||
|
||||
// --- Links (deduped by target URL; first anchor wins) ---
|
||||
const linksByTarget = new Map<string, PageLink>();
|
||||
let openAnchor: OpenAnchor | null = null;
|
||||
|
||||
$("a[href]").each((_, el) => {
|
||||
const href = $(el).attr("href");
|
||||
if (!href) return;
|
||||
// Visible text: prefer text inside an explicit <body>; when the document
|
||||
// never opens one (fragments), fall back to all non-head text. Both
|
||||
// exclude NON_CONTENT_TAGS subtrees.
|
||||
let suppressDepth = 0;
|
||||
let bodyDepth = 0;
|
||||
let headDepth = 0;
|
||||
let sawBody = false;
|
||||
const bodyParts: string[] = [];
|
||||
const fallbackParts: string[] = [];
|
||||
|
||||
// Skip javascript:, mailto:, tel:, #anchors
|
||||
if (/^(javascript:|mailto:|tel:|#)/.test(href)) return;
|
||||
const handleMetaTag = (attribs: Record<string, string>) => {
|
||||
const content = attribs["content"];
|
||||
if (attribs["name"] === "description") {
|
||||
metaDescription ??= content?.trim() ?? "";
|
||||
} else if (attribs["name"] === "robots") {
|
||||
robotsMeta ??= content ?? null;
|
||||
} else if (attribs["property"] === "og:title") {
|
||||
ogTitle ??= content ?? null;
|
||||
} else if (attribs["property"] === "og:description") {
|
||||
ogDescription ??= content ?? null;
|
||||
} else if (attribs["property"] === "og:image") {
|
||||
ogImage ??= content ?? null;
|
||||
}
|
||||
};
|
||||
|
||||
const handleLinkTag = (attribs: Record<string, string>) => {
|
||||
if (attribs["rel"] === "canonical") {
|
||||
canonical ??= attribs["href"] ?? null;
|
||||
} else if (attribs["rel"] === "alternate" && attribs["hreflang"]) {
|
||||
hreflangTags.push(attribs["hreflang"]);
|
||||
}
|
||||
};
|
||||
|
||||
const closeAnchor = () => {
|
||||
if (!openAnchor) return;
|
||||
const { href, rel, text } = openAnchor;
|
||||
openAnchor = null;
|
||||
const resolved = normalizeUrl(href, pageUrl);
|
||||
if (!resolved) return;
|
||||
if (linksByTarget.has(resolved)) return;
|
||||
|
||||
const anchor = $(el).text().replace(/\s+/g, " ").trim().slice(0, 200);
|
||||
const rel = $(el).attr("rel")?.toLowerCase() ?? "";
|
||||
if (!resolved || linksByTarget.has(resolved)) return;
|
||||
const anchor = text
|
||||
.join("")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, MAX_ANCHOR_CHARS);
|
||||
linksByTarget.set(resolved, {
|
||||
targetUrl: resolved,
|
||||
anchor: anchor || null,
|
||||
isInternal: isSameOrigin(resolved, pageUrl),
|
||||
isNofollow: rel.split(/\s+/).includes("nofollow"),
|
||||
});
|
||||
});
|
||||
const links = Array.from(linksByTarget.values());
|
||||
};
|
||||
|
||||
// --- Structured data (JSON-LD) ---
|
||||
let hasStructuredData = false;
|
||||
$('script[type="application/ld+json"]').each(() => {
|
||||
const parser = new Parser(
|
||||
{
|
||||
onopentag(name, attribs) {
|
||||
if (NON_CONTENT_TAGS.has(name)) {
|
||||
suppressDepth += 1;
|
||||
}
|
||||
if (name === "noscript") noscriptDepth += 1;
|
||||
if (noscriptDepth > 0) return;
|
||||
switch (name) {
|
||||
case "title":
|
||||
// Ignore <title> inside <svg> — only the document title counts.
|
||||
if (!titleDone && suppressDepth === 0) {
|
||||
titleDepth += 1;
|
||||
if (title === null) title = "";
|
||||
}
|
||||
break;
|
||||
case "head":
|
||||
headDepth += 1;
|
||||
break;
|
||||
case "body":
|
||||
bodyDepth += 1;
|
||||
sawBody = true;
|
||||
break;
|
||||
case "meta":
|
||||
handleMetaTag(attribs);
|
||||
break;
|
||||
case "link":
|
||||
handleLinkTag(attribs);
|
||||
break;
|
||||
case "img":
|
||||
images.push({
|
||||
src: attribs["src"] ?? null,
|
||||
alt: "alt" in attribs ? attribs["alt"] : null,
|
||||
});
|
||||
break;
|
||||
case "script":
|
||||
if (attribs["type"] === "application/ld+json") {
|
||||
hasStructuredData = true;
|
||||
});
|
||||
}
|
||||
break;
|
||||
case "a": {
|
||||
// HTML forbids nested <a>; browsers implicitly close the open
|
||||
// one, and the tokenizer has no tree correction, so mirror that.
|
||||
closeAnchor();
|
||||
const href = attribs["href"];
|
||||
if (href && !SKIPPED_LINK_PROTOCOLS.test(href)) {
|
||||
openAnchor = {
|
||||
href,
|
||||
rel: attribs["rel"]?.toLowerCase() ?? "",
|
||||
text: [],
|
||||
};
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
const headingLevel = HEADING_LEVELS[name];
|
||||
if (headingLevel !== undefined) {
|
||||
headingOrder.push(headingLevel);
|
||||
if (headingLevel === 1 && openH1 === null) openH1 = [];
|
||||
}
|
||||
},
|
||||
ontext(text) {
|
||||
if (suppressDepth > 0) return;
|
||||
if (titleDepth > 0) {
|
||||
if (title !== null) title += text;
|
||||
return;
|
||||
}
|
||||
if (openH1) openH1.push(text);
|
||||
if (openAnchor) openAnchor.text.push(text);
|
||||
if (bodyDepth > 0) {
|
||||
bodyParts.push(text);
|
||||
} else if (headDepth === 0) {
|
||||
fallbackParts.push(text);
|
||||
}
|
||||
},
|
||||
onclosetag(name) {
|
||||
if (NON_CONTENT_TAGS.has(name) && suppressDepth > 0) {
|
||||
suppressDepth -= 1;
|
||||
}
|
||||
if (name === "noscript" && noscriptDepth > 0) {
|
||||
noscriptDepth -= 1;
|
||||
return;
|
||||
}
|
||||
if (noscriptDepth > 0) return;
|
||||
if (name === "title" && titleDepth > 0) {
|
||||
titleDepth -= 1;
|
||||
if (titleDepth === 0) titleDone = true;
|
||||
}
|
||||
if (name === "head" && headDepth > 0) headDepth -= 1;
|
||||
if (name === "body" && bodyDepth > 0) bodyDepth -= 1;
|
||||
if (name === "a") closeAnchor();
|
||||
if (name === "h1" && openH1) {
|
||||
h1s.push(openH1.join("").trim());
|
||||
openH1 = null;
|
||||
}
|
||||
},
|
||||
},
|
||||
// Defaults (non-XML mode): lowercased tag/attribute names, decoded
|
||||
// entities — matching what the DOM-based implementation saw.
|
||||
);
|
||||
parser.write(html);
|
||||
parser.end();
|
||||
|
||||
const hreflangTags: string[] = [];
|
||||
$('link[rel="alternate"][hreflang]').each((_, el) => {
|
||||
const hreflang = $(el).attr("hreflang");
|
||||
if (hreflang) hreflangTags.push(hreflang);
|
||||
});
|
||||
const rawText = (sawBody ? bodyParts : fallbackParts).join("");
|
||||
const bodyText = rawText.replace(/\s+/g, " ").trim();
|
||||
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
||||
|
||||
return {
|
||||
url: pageUrl,
|
||||
statusCode,
|
||||
redirectUrl,
|
||||
responseTimeMs,
|
||||
title,
|
||||
metaDescription,
|
||||
title: (title ?? "").trim(),
|
||||
metaDescription: metaDescription ?? "",
|
||||
canonical,
|
||||
robotsMeta,
|
||||
ogTitle,
|
||||
@ -125,7 +242,7 @@ export function analyzeHtml(
|
||||
wordCount,
|
||||
bodyText,
|
||||
images,
|
||||
links,
|
||||
links: Array.from(linksByTarget.values()),
|
||||
hasStructuredData,
|
||||
hreflangTags,
|
||||
};
|
||||
|
||||
@ -50,7 +50,7 @@ export interface PageLink {
|
||||
isNofollow: boolean;
|
||||
}
|
||||
|
||||
/** Data extracted from a single page via cheerio. */
|
||||
/** Data extracted from a single page's HTML. */
|
||||
export interface PageAnalysis {
|
||||
url: string;
|
||||
statusCode: number;
|
||||
@ -106,8 +106,9 @@ export interface LighthouseResult {
|
||||
}
|
||||
|
||||
/**
|
||||
* Full result of crawling one page. Persisted to D1 inside the crawl-batch
|
||||
* step; never accumulated in memory or returned as durable step state.
|
||||
* Full result of crawling one page. Persisted to the app DB inside the
|
||||
* crawl-chunk step; never accumulated in memory or returned as durable
|
||||
* step state.
|
||||
*/
|
||||
export interface CrawledPageResult {
|
||||
id: string;
|
||||
@ -139,6 +140,12 @@ export interface CrawledPageResult {
|
||||
* checked; a PDF must not). Transient — not persisted.
|
||||
*/
|
||||
isHtml: boolean;
|
||||
/**
|
||||
* HTML size read for this page (approximate; capped at MAX_HTML_BYTES).
|
||||
* Transient — feeds the crawl window's memory-pressure signal, since
|
||||
* response time is measured at headers and says nothing about body size.
|
||||
*/
|
||||
htmlBytes: number;
|
||||
imagesTotal: number;
|
||||
imagesMissingAlt: number;
|
||||
images: Array<{ src: string | null; alt: string | null }>;
|
||||
@ -151,18 +158,3 @@ export interface CrawledPageResult {
|
||||
crawlDepth: number | null;
|
||||
inSitemap: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Slim per-page summary returned as durable step state from a crawl batch.
|
||||
* Keep this small: full page data lives in D1, not in Workflow step state.
|
||||
*/
|
||||
export interface StepPageSummary {
|
||||
id: string;
|
||||
url: string;
|
||||
statusCode: number;
|
||||
fetchClass: PageFetchClass;
|
||||
redirectUrl: string | null;
|
||||
title: string;
|
||||
/** Normalized same-origin link targets, for frontier expansion. */
|
||||
internalLinks: string[];
|
||||
}
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { AppError } from "@/server/lib/errors";
|
||||
import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
|
||||
import {
|
||||
normalizeAndValidateStartUrl,
|
||||
resolveStartUrlRedirects,
|
||||
} from "@/server/lib/audit/url-policy";
|
||||
|
||||
describe("normalizeAndValidateStartUrl", () => {
|
||||
beforeEach(() => {
|
||||
@ -48,3 +51,86 @@ describe("normalizeAndValidateStartUrl", () => {
|
||||
} satisfies Partial<AppError>);
|
||||
});
|
||||
});
|
||||
|
||||
const dnsOk = () =>
|
||||
new Response(JSON.stringify({ Status: 0, Answer: [] }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/dns-json" },
|
||||
});
|
||||
const redirect = (location: string) =>
|
||||
new Response(null, { status: 301, headers: { location } });
|
||||
|
||||
describe("resolveStartUrlRedirects", () => {
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal("fetch", vi.fn());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
/** Route probe fetches by URL; DoH lookups always resolve clean. */
|
||||
function stubFetch(routes: Record<string, () => Response>) {
|
||||
vi.mocked(fetch).mockImplementation((input) => {
|
||||
const url = String(input instanceof Request ? input.url : input);
|
||||
if (url.includes("dns-query")) return Promise.resolve(dnsOk());
|
||||
const route = routes[url];
|
||||
return route
|
||||
? Promise.resolve(route())
|
||||
: Promise.resolve(new Response(null, { status: 200 }));
|
||||
});
|
||||
}
|
||||
|
||||
it("follows a cross-domain redirect to the real origin", async () => {
|
||||
stubFetch({
|
||||
"https://example.net/": () => redirect("https://example.com/"),
|
||||
});
|
||||
await expect(
|
||||
resolveStartUrlRedirects("https://example.net/"),
|
||||
).resolves.toBe("https://example.com/");
|
||||
});
|
||||
|
||||
it("follows an apex-to-www redirect chain", async () => {
|
||||
stubFetch({
|
||||
"https://example.com/": () => redirect("https://www.example.com/"),
|
||||
});
|
||||
await expect(
|
||||
resolveStartUrlRedirects("https://example.com/"),
|
||||
).resolves.toBe("https://www.example.com/");
|
||||
});
|
||||
|
||||
it("returns the original URL when the site does not redirect", async () => {
|
||||
stubFetch({});
|
||||
await expect(
|
||||
resolveStartUrlRedirects("https://example.com/"),
|
||||
).resolves.toBe("https://example.com/");
|
||||
});
|
||||
|
||||
it("returns the last URL when the probe fails", async () => {
|
||||
vi.mocked(fetch).mockRejectedValue(new Error("network down"));
|
||||
await expect(
|
||||
resolveStartUrlRedirects("https://example.com/"),
|
||||
).resolves.toBe("https://example.com/");
|
||||
});
|
||||
|
||||
it("stops after the hop limit on a redirect loop", async () => {
|
||||
stubFetch({
|
||||
"https://a.example/": () => redirect("https://b.example/"),
|
||||
"https://b.example/": () => redirect("https://a.example/"),
|
||||
});
|
||||
await expect(
|
||||
resolveStartUrlRedirects("https://a.example/"),
|
||||
).resolves.toMatch(/^https:\/\/(a|b)\.example\/$/);
|
||||
});
|
||||
|
||||
it("rejects redirects into blocked targets", async () => {
|
||||
stubFetch({
|
||||
"https://example.com/": () => redirect("http://192.168.0.10/"),
|
||||
});
|
||||
await expect(
|
||||
resolveStartUrlRedirects("https://example.com/"),
|
||||
).rejects.toMatchObject({
|
||||
code: "CRAWL_TARGET_BLOCKED",
|
||||
} satisfies Partial<AppError>);
|
||||
});
|
||||
});
|
||||
|
||||
@ -238,3 +238,48 @@ export async function normalizeAndValidateStartUrl(
|
||||
parsed.hash = "";
|
||||
return parsed.toString();
|
||||
}
|
||||
|
||||
const START_URL_REDIRECT_HOPS = 5;
|
||||
const START_URL_PROBE_TIMEOUT_MS = 10_000;
|
||||
|
||||
/**
|
||||
* Follow redirects on the audit's start URL so the audit anchors to the
|
||||
* site's real origin. Without this, auditing a domain that 301s elsewhere
|
||||
* (…net -> …com, apex -> www) dead-ends after one page: the redirect target
|
||||
* is a different origin, so the same-origin crawl policy can't follow it.
|
||||
*
|
||||
* Every hop re-runs the full start-URL validation (SSRF, blocked hosts), so
|
||||
* a redirect can't smuggle the audit somewhere the user couldn't have
|
||||
* pointed it directly. Probe failures (timeouts, HEAD rejected) fall back
|
||||
* to the last validated URL — the crawl records the real fetch result.
|
||||
*/
|
||||
export async function resolveStartUrlRedirects(
|
||||
startUrl: string,
|
||||
): Promise<string> {
|
||||
let current = startUrl;
|
||||
for (let hop = 0; hop < START_URL_REDIRECT_HOPS; hop++) {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(current, {
|
||||
method: "HEAD",
|
||||
redirect: "manual",
|
||||
headers: { "User-Agent": "OpenSEO-Audit/1.0" },
|
||||
signal: AbortSignal.timeout(START_URL_PROBE_TIMEOUT_MS),
|
||||
});
|
||||
} catch {
|
||||
return current;
|
||||
}
|
||||
if (response.status < 300 || response.status >= 400) return current;
|
||||
const location = response.headers.get("location");
|
||||
if (!location) return current;
|
||||
|
||||
let next: URL;
|
||||
try {
|
||||
next = new URL(location, current);
|
||||
} catch {
|
||||
return current;
|
||||
}
|
||||
current = await normalizeAndValidateStartUrl(next.toString());
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
@ -128,7 +128,7 @@ export const runSiteAuditTool = {
|
||||
});
|
||||
|
||||
return mcpResponse({
|
||||
text: `Audit ${auditId} started for ${args.url}. Poll get_audit_status until it completes, then call get_audit_issues for the prioritized issue report.`,
|
||||
text: `Audit ${auditId} started for ${args.url}. Poll get_audit_status until it finishes, then call get_audit_issues for the prioritized issue report (even a failed audit keeps results for every page it crawled).`,
|
||||
meta: buildProjectMeta(
|
||||
context,
|
||||
args.projectId,
|
||||
@ -177,8 +177,16 @@ export const getAuditStatusTool = {
|
||||
status.lighthouseTotal > 0
|
||||
? `, lighthouse ${status.lighthouseCompleted + status.lighthouseFailed}/${status.lighthouseTotal}`
|
||||
: "";
|
||||
// Failed audits keep partial results — point agents at them instead of
|
||||
// letting a mid-crawl death read as "no data".
|
||||
const nextStep =
|
||||
status.status === "completed"
|
||||
? " Call get_audit_issues for the issue report."
|
||||
: status.status === "failed" && status.pagesCrawled > 0
|
||||
? ` The audit stopped early but kept results for the ${status.pagesCrawled} pages it crawled — call get_audit_issues for the partial issue report.`
|
||||
: "";
|
||||
return mcpResponse({
|
||||
text: `Audit ${status.id} (${status.startUrl}): ${status.status} — phase ${status.currentPhase}, ${status.pagesCrawled}/${status.pagesTotal} pages${lighthouseNote}.${status.status === "completed" ? " Call get_audit_issues for the issue report." : ""}`,
|
||||
text: `Audit ${status.id} (${status.startUrl}): ${status.status} — phase ${status.currentPhase}, ${status.pagesCrawled}/${status.pagesTotal} pages${lighthouseNote}.${nextStep}`,
|
||||
meta: buildProjectMeta(
|
||||
context,
|
||||
args.projectId,
|
||||
|
||||
@ -12,10 +12,12 @@ import {
|
||||
import { withPgClient } from "@/db";
|
||||
import type { BillingCustomerContext } from "@/server/billing/subscription";
|
||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||
import { classifyAuditError } from "@/server/lib/audit/audit-errors";
|
||||
import type { AuditConfig } from "@/server/lib/audit/types";
|
||||
import { captureServerError, captureServerEvent } from "@/server/lib/posthog";
|
||||
import { runAuditPhases } from "@/server/workflows/siteAuditWorkflowPhases";
|
||||
import { pgStep } from "@/server/workflows/pgStep";
|
||||
import { DB_STEP } from "@/server/workflows/auditStepConfigs";
|
||||
|
||||
interface AuditParams {
|
||||
auditId: string;
|
||||
@ -44,7 +46,7 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint<Env, AuditParams> {
|
||||
// Inside a step so the D1 read is retried and replay-cached; a bare
|
||||
// read here would re-execute on every replay and a transient failure
|
||||
// would kill the instance before the catch below exists.
|
||||
await pgStep(step, "validate-context", undefined, async () => {
|
||||
await pgStep(step, "validate-context", DB_STEP, async () => {
|
||||
const audit = await AuditRepository.getAuditForWorkflow(
|
||||
auditId,
|
||||
event.instanceId,
|
||||
@ -86,13 +88,17 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint<Env, AuditParams> {
|
||||
project_id: projectId,
|
||||
});
|
||||
}
|
||||
await pgStep(step, "mark-failed", undefined, async () => {
|
||||
await AuditRepository.failAudit(auditId, event.instanceId);
|
||||
|
||||
const latestAudit = await AuditRepository.getAuditForWorkflow(
|
||||
const errorInfo = classifyAuditError(error);
|
||||
await pgStep(step, "mark-failed", DB_STEP, async () => {
|
||||
// Read the phase before failAudit stamps currentPhase = "failed".
|
||||
const runningAudit = await AuditRepository.getAuditForWorkflow(
|
||||
auditId,
|
||||
event.instanceId,
|
||||
);
|
||||
await AuditRepository.failAudit(auditId, event.instanceId, {
|
||||
...errorInfo,
|
||||
failedPhase: runningAudit?.currentPhase ?? null,
|
||||
});
|
||||
|
||||
await captureServerEvent({
|
||||
distinctId: billingCustomer.userId,
|
||||
@ -101,8 +107,10 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint<Env, AuditParams> {
|
||||
properties: {
|
||||
project_id: projectId,
|
||||
status: "failed",
|
||||
pages_crawled: latestAudit?.pagesCrawled,
|
||||
pages_total: latestAudit?.pagesTotal,
|
||||
error_code: errorInfo.errorCode,
|
||||
failed_phase: runningAudit?.currentPhase,
|
||||
pages_crawled: runningAudit?.pagesCrawled,
|
||||
pages_total: runningAudit?.pagesTotal,
|
||||
run_lighthouse: config.lighthouseStrategy !== "none",
|
||||
},
|
||||
});
|
||||
|
||||
40
src/server/workflows/auditStepConfigs.ts
Normal file
40
src/server/workflows/auditStepConfigs.ts
Normal file
@ -0,0 +1,40 @@
|
||||
import type { WorkflowStepConfig } from "cloudflare:workers";
|
||||
|
||||
/**
|
||||
* Explicit step configs for the site-audit workflow.
|
||||
*
|
||||
* Without these, steps inherit the platform default (10-minute timeout,
|
||||
* multiple retries with backoff) — production audits against slow/hostile
|
||||
* sites burned ~65 minutes replaying a doomed step before erroring. Every
|
||||
* step now has a timeout sized to its real work and a small retry budget.
|
||||
*/
|
||||
|
||||
/** Robots + sitemap walk (bounded at 300 docs / 15s per fetch internally). */
|
||||
export const DISCOVERY_STEP: WorkflowStepConfig = {
|
||||
retries: { limit: 2, delay: "5 seconds", backoff: "exponential" },
|
||||
timeout: "4 minutes",
|
||||
};
|
||||
|
||||
/** One crawl chunk: ~90s soft crawl deadline + persistence headroom. */
|
||||
export const CRAWL_CHUNK_STEP: WorkflowStepConfig = {
|
||||
retries: { limit: 1, delay: "10 seconds", backoff: "constant" },
|
||||
timeout: "5 minutes",
|
||||
};
|
||||
|
||||
/** One Lighthouse batch: 20 DataForSEO calls, each with internal retries. */
|
||||
export const LIGHTHOUSE_BATCH_STEP: WorkflowStepConfig = {
|
||||
retries: { limit: 1, delay: "10 seconds", backoff: "constant" },
|
||||
timeout: "5 minutes",
|
||||
};
|
||||
|
||||
/** Small DB-only steps (validate, select sample, finalize, mark-failed). */
|
||||
export const DB_STEP: WorkflowStepConfig = {
|
||||
retries: { limit: 3, delay: "5 seconds", backoff: "exponential" },
|
||||
timeout: "2 minutes",
|
||||
};
|
||||
|
||||
/** Cross-page checks read every page row of the audit — allow more time. */
|
||||
export const MULTIPAGE_CHECKS_STEP: WorkflowStepConfig = {
|
||||
retries: { limit: 2, delay: "5 seconds", backoff: "exponential" },
|
||||
timeout: "5 minutes",
|
||||
};
|
||||
@ -104,9 +104,8 @@ export async function crawlPage(
|
||||
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
const isHtml = contentType.includes("text/html");
|
||||
// Large pages make Cheerio disproportionately expensive and can exhaust a
|
||||
// crawl step's CPU or isolate memory. The first 2 MiB still contains the
|
||||
// SEO metadata and navigation needed by the audit in normal documents.
|
||||
// Cap what we read: the first 2 MiB still contains the SEO metadata and
|
||||
// navigation needed by the audit in normal documents.
|
||||
const body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";
|
||||
const fetchClass = classifyFetch(
|
||||
statusCode,
|
||||
@ -128,10 +127,10 @@ export async function crawlPage(
|
||||
});
|
||||
}
|
||||
|
||||
// Dynamic import keeps cheerio (page-analyzer's HTML parser) out of the
|
||||
// worker's startup module graph: SiteAuditWorkflow is re-exported from
|
||||
// src/server.ts, so a static import would evaluate cheerio in every
|
||||
// isolate's baseline heap, not just when an audit actually crawls.
|
||||
// Dynamic import keeps the HTML parser out of the worker's startup
|
||||
// module graph: SiteAuditWorkflow is re-exported from src/server.ts, so
|
||||
// a static import would evaluate it in every isolate's baseline heap,
|
||||
// not just when an audit actually crawls.
|
||||
const { analyzeHtml } = await import("@/server/lib/audit/page-analyzer");
|
||||
const analysis = analyzeHtml(body, url, statusCode, responseTimeMs);
|
||||
const robotsDirectives = [analysis.robotsMeta, xRobotsTag]
|
||||
@ -171,6 +170,7 @@ export async function crawlPage(
|
||||
? await sha256Hex(analysis.bodyText)
|
||||
: null,
|
||||
isHtml: true,
|
||||
htmlBytes: body.length,
|
||||
imagesTotal: analysis.images.length,
|
||||
// Only a truly absent alt attribute counts: alt="" is the correct
|
||||
// markup for decorative images.
|
||||
@ -270,6 +270,7 @@ function emptyPageResult(input: {
|
||||
wordCount: 0,
|
||||
contentHash: null,
|
||||
isHtml: false,
|
||||
htmlBytes: 0,
|
||||
imagesTotal: 0,
|
||||
imagesMissingAlt: 0,
|
||||
images: [],
|
||||
|
||||
@ -1,59 +1,77 @@
|
||||
import type { WorkflowStep } from "cloudflare:workers";
|
||||
import type { RobotsResult } from "@/server/lib/audit/discovery";
|
||||
import type { StepPageSummary } from "@/server/lib/audit/types";
|
||||
import { isSameOrigin, normalizeUrl } from "@/server/lib/audit/url-utils";
|
||||
import type { CrawledPageResult } from "@/server/lib/audit/types";
|
||||
import { isSameOrigin } from "@/server/lib/audit/url-utils";
|
||||
import { isCrawlableUrl } from "@/server/lib/audit/url-policy";
|
||||
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
|
||||
import { runPageReporters } from "@/server/lib/audit/issues/page-reporters";
|
||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||
import {
|
||||
getAuditScratchpad,
|
||||
type ClaimedUrl,
|
||||
type FrontierStats,
|
||||
type ScratchpadLinkRow,
|
||||
} from "@/server/features/audit/AuditScratchpad";
|
||||
import { AuditProgressKV } from "@/server/lib/audit/progress-kv";
|
||||
import {
|
||||
adjustCrawlWindow,
|
||||
INITIAL_CRAWL_WINDOW,
|
||||
} from "@/server/lib/audit/crawl-window";
|
||||
import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";
|
||||
import { pgStep } from "@/server/workflows/pgStep";
|
||||
import { CRAWL_CHUNK_STEP } from "@/server/workflows/auditStepConfigs";
|
||||
|
||||
const CRAWL_CONCURRENCY = 25;
|
||||
// Keep durable step state under the ~1MiB limit: full link lists live in D1;
|
||||
// the step return only carries new-to-the-frontier targets, deduped across
|
||||
// the batch and capped.
|
||||
const MAX_FRONTIER_LINKS_PER_BATCH = 2_000;
|
||||
const MAX_SUMMARY_TITLE_CHARS = 300;
|
||||
/**
|
||||
* The crawl runs in chunks: each chunk is one durable step that leases up to
|
||||
* CHUNK_TARGET_PAGES URLs from the scratchpad DO, crawls them with a rolling
|
||||
* concurrency window, and persists results incrementally (full page rows to
|
||||
* the app DB, links/mirror/frontier updates to the DO). Step returns carry
|
||||
* only counters, so step state stays tiny regardless of site size and the
|
||||
* workflow heap stays O(one batch).
|
||||
*/
|
||||
const CHUNK_TARGET_PAGES = 200;
|
||||
/** Stop launching new fetches after this long; leftover leases are released. */
|
||||
const CHUNK_SOFT_DEADLINE_MS = 90_000;
|
||||
/** Crawled pages are persisted in sub-batches of this size. */
|
||||
const PERSIST_BATCH_SIZE = 25;
|
||||
|
||||
/**
|
||||
* Mega-menu/footer-heavy sites can carry 1000+ links per page; cap what we
|
||||
* record so a 10k-page crawl can't produce tens of millions of link rows.
|
||||
*/
|
||||
const MAX_STORED_LINKS_PER_PAGE = 500;
|
||||
/**
|
||||
* Cap newly discovered URLs sent to the scratchpad per persist sub-batch.
|
||||
* Serialized RPC messages are limited to 32 MiB; a crawler-trap page family
|
||||
* (faceted nav, calendars) can emit tens of thousands of unique URLs per
|
||||
* page. Dropped URLs are usually re-discovered from later pages, and a site
|
||||
* generating this many is past maxPages anyway.
|
||||
*/
|
||||
const MAX_DISCOVERED_PER_BATCH = 20_000;
|
||||
const MAX_PROGRESS_TITLE_CHARS = 300;
|
||||
|
||||
function shouldQueueCrawlLink(
|
||||
link: string,
|
||||
origin: string,
|
||||
robots: RobotsResult,
|
||||
visited: Set<string>,
|
||||
queued: Set<string>,
|
||||
): boolean {
|
||||
return (
|
||||
isSameOrigin(link, origin) &&
|
||||
isCrawlableUrl(link) &&
|
||||
robots.isAllowed(link) &&
|
||||
!visited.has(link) &&
|
||||
!queued.has(link)
|
||||
isSameOrigin(link, origin) && isCrawlableUrl(link) && robots.isAllowed(link)
|
||||
);
|
||||
}
|
||||
|
||||
interface QueueEntry {
|
||||
url: string;
|
||||
/** Clicks from the start URL; null when only reachable via sitemap. */
|
||||
depth: number | null;
|
||||
}
|
||||
|
||||
type CrawlPhaseParams = {
|
||||
auditId: string;
|
||||
workflowInstanceId: string;
|
||||
origin: string;
|
||||
startUrl: string;
|
||||
maxPages: number;
|
||||
robots: RobotsResult;
|
||||
sitemapUrls: string[];
|
||||
/** Frontier size after discovery seeding (from the discover-urls step). */
|
||||
seededCount: number;
|
||||
};
|
||||
|
||||
/** What later phases need per page — no link lists (those stay in D1). */
|
||||
export type CrawledPageSummary = Omit<StepPageSummary, "internalLinks">;
|
||||
|
||||
export type CrawlPhaseResult = {
|
||||
pages: CrawledPageSummary[];
|
||||
pagesCrawled: number;
|
||||
/** True when the frontier was exhausted before hitting maxPages. */
|
||||
completed: boolean;
|
||||
};
|
||||
@ -62,282 +80,219 @@ export async function runCrawlPhase(
|
||||
step: WorkflowStep,
|
||||
params: CrawlPhaseParams,
|
||||
): Promise<CrawlPhaseResult> {
|
||||
const {
|
||||
auditId,
|
||||
workflowInstanceId,
|
||||
origin,
|
||||
startUrl,
|
||||
maxPages,
|
||||
robots,
|
||||
sitemapUrls,
|
||||
} = params;
|
||||
const visited = new Set<string>();
|
||||
const queued = new Set<string>();
|
||||
// Link-discovered URLs crawl first (BFS from the start URL); sitemap-only
|
||||
// URLs drain last so link discovery isn't starved of page budget and
|
||||
// orphan detection stays meaningful.
|
||||
const linkQueue: QueueEntry[] = [];
|
||||
const sitemapQueue: QueueEntry[] = [];
|
||||
const sitemapSet = new Set<string>();
|
||||
const summaries: CrawledPageSummary[] = [];
|
||||
let chunkNo = 0;
|
||||
let attemptedTotal = 0;
|
||||
let pending = params.seededCount;
|
||||
let zeroProgressChunks = 0;
|
||||
|
||||
const normalizedStart = normalizeUrl(startUrl) ?? startUrl;
|
||||
if (
|
||||
robots.isAllowed(normalizedStart) &&
|
||||
isSameOrigin(normalizedStart, origin)
|
||||
) {
|
||||
linkQueue.push({ url: normalizedStart, depth: 0 });
|
||||
queued.add(normalizedStart);
|
||||
}
|
||||
|
||||
for (const sitemapUrl of sitemapUrls) {
|
||||
const normalized = normalizeUrl(sitemapUrl);
|
||||
if (!normalized) continue;
|
||||
sitemapSet.add(normalized);
|
||||
if (!shouldQueueCrawlLink(normalized, origin, robots, visited, queued)) {
|
||||
continue;
|
||||
}
|
||||
sitemapQueue.push({ url: normalized, depth: null });
|
||||
queued.add(normalized);
|
||||
}
|
||||
|
||||
let crawlBatchIndex = 0;
|
||||
while (
|
||||
(linkQueue.length > 0 || sitemapQueue.length > 0) &&
|
||||
summaries.length < maxPages
|
||||
) {
|
||||
const batchEntries = selectNextCrawlBatch({
|
||||
linkQueue,
|
||||
sitemapQueue,
|
||||
queued,
|
||||
visited,
|
||||
robots,
|
||||
remaining: maxPages - summaries.length,
|
||||
});
|
||||
if (batchEntries.length === 0) continue;
|
||||
|
||||
crawlBatchIndex += 1;
|
||||
const crawledBatch = await runCrawlBatch(step, {
|
||||
crawlBatchIndex,
|
||||
auditId,
|
||||
batchEntries,
|
||||
sitemapSet,
|
||||
visited,
|
||||
queued,
|
||||
});
|
||||
// Keep only the slim summary in memory: at 10k pages, retaining link
|
||||
// lists for the whole crawl would not fit in the 128MB Worker heap.
|
||||
summaries.push(
|
||||
...crawledBatch.map(({ internalLinks: _links, ...summary }) => summary),
|
||||
);
|
||||
|
||||
enqueueDiscoveredLinks({
|
||||
crawledBatch,
|
||||
batchEntries,
|
||||
linkQueue,
|
||||
queued,
|
||||
visited,
|
||||
origin,
|
||||
robots,
|
||||
});
|
||||
await persistCrawlProgress({
|
||||
while (pending > 0 && attemptedTotal < params.maxPages) {
|
||||
chunkNo += 1;
|
||||
const result = await pgStep(
|
||||
step,
|
||||
crawlBatchIndex,
|
||||
auditId,
|
||||
workflowInstanceId,
|
||||
crawledBatch,
|
||||
pagesCrawled: summaries.length,
|
||||
visitedCount: visited.size,
|
||||
queueLength: linkQueue.length + sitemapQueue.length,
|
||||
maxPages,
|
||||
});
|
||||
`crawl-chunk-${chunkNo}`,
|
||||
CRAWL_CHUNK_STEP,
|
||||
() =>
|
||||
runCrawlChunk({
|
||||
...params,
|
||||
chunkNo,
|
||||
attemptedBefore: attemptedTotal,
|
||||
}),
|
||||
);
|
||||
// Apply the chunk's counters even when it did no new work (a retried
|
||||
// chunk whose earlier attempt persisted everything reports 0 attempts
|
||||
// with up-to-date scratchpad totals) — finalize must not see stale ones.
|
||||
attemptedTotal = result.attempted;
|
||||
pending = result.pending;
|
||||
// One zero-attempt chunk is normal (retry of a completed chunk number);
|
||||
// two in a row means the frontier is unservable — stop with what we
|
||||
// have instead of spinning forever.
|
||||
zeroProgressChunks =
|
||||
result.attemptedInChunk === 0 ? zeroProgressChunks + 1 : 0;
|
||||
if (zeroProgressChunks >= 2) break;
|
||||
}
|
||||
|
||||
return { pagesCrawled: attemptedTotal, completed: pending === 0 };
|
||||
}
|
||||
|
||||
async function runCrawlChunk(
|
||||
input: CrawlPhaseParams & { chunkNo: number; attemptedBefore: number },
|
||||
): Promise<{ attemptedInChunk: number; attempted: number; pending: number }> {
|
||||
const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =
|
||||
input;
|
||||
const scratchpad = getAuditScratchpad(auditId);
|
||||
|
||||
const claimLimit = Math.min(
|
||||
CHUNK_TARGET_PAGES,
|
||||
maxPages - input.attemptedBefore,
|
||||
);
|
||||
const claimed = await scratchpad.claimChunk(chunkNo, claimLimit);
|
||||
if (claimed.length === 0) {
|
||||
const stats = await scratchpad.getStats();
|
||||
return {
|
||||
pages: summaries,
|
||||
completed: linkQueue.length === 0 && sitemapQueue.length === 0,
|
||||
attemptedInChunk: 0,
|
||||
attempted: stats.attempted,
|
||||
pending: stats.pending,
|
||||
};
|
||||
}
|
||||
|
||||
const depthByUrl = new Map(claimed.map((entry) => [entry.url, entry.depth]));
|
||||
const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS;
|
||||
|
||||
let windowSize = INITIAL_CRAWL_WINDOW;
|
||||
let nextIndex = 0;
|
||||
let attemptedInChunk = 0;
|
||||
const inFlight = new Set<Promise<void>>();
|
||||
let batch: CrawledPageResult[] = [];
|
||||
// Persistence runs concurrently with fetching (pipelined) but sequentially
|
||||
// with itself, so DB write pressure stays bounded at one batch at a time.
|
||||
let persistChain: Promise<unknown> = Promise.resolve();
|
||||
|
||||
const flush = () => {
|
||||
if (batch.length === 0) return;
|
||||
const pages = batch;
|
||||
batch = [];
|
||||
windowSize = adjustCrawlWindow(windowSize, pages);
|
||||
persistChain = persistChain.then(() =>
|
||||
persistCrawledPages({
|
||||
auditId,
|
||||
workflowInstanceId,
|
||||
origin,
|
||||
robots,
|
||||
scratchpad,
|
||||
pages,
|
||||
depthByUrl,
|
||||
maxPages,
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const launch = (entry: ClaimedUrl) => {
|
||||
const promise = crawlPage(entry.url, entry.depth, entry.inSitemap)
|
||||
.then((page) => {
|
||||
attemptedInChunk += 1;
|
||||
batch.push(page);
|
||||
if (batch.length >= PERSIST_BATCH_SIZE) flush();
|
||||
})
|
||||
.finally(() => {
|
||||
inFlight.delete(promise);
|
||||
});
|
||||
inFlight.add(promise);
|
||||
};
|
||||
|
||||
while (true) {
|
||||
while (
|
||||
inFlight.size < windowSize &&
|
||||
nextIndex < claimed.length &&
|
||||
Date.now() < deadlineAt
|
||||
) {
|
||||
launch(claimed[nextIndex]);
|
||||
nextIndex += 1;
|
||||
}
|
||||
if (inFlight.size === 0) break;
|
||||
await Promise.race(inFlight);
|
||||
}
|
||||
flush();
|
||||
await persistChain;
|
||||
|
||||
// Leases we never launched (soft deadline) go back to the queue.
|
||||
const unattempted = claimed.slice(nextIndex).map((entry) => entry.url);
|
||||
if (unattempted.length > 0) {
|
||||
await scratchpad.releaseUrls(unattempted);
|
||||
}
|
||||
|
||||
// Progress counters are written per persisted sub-batch (in
|
||||
// persistCrawledPages), so a chunk that dies mid-way underreports by at
|
||||
// most one sub-batch, not a whole chunk.
|
||||
const stats = await scratchpad.getStats();
|
||||
return {
|
||||
attemptedInChunk,
|
||||
attempted: stats.attempted,
|
||||
pending: stats.pending,
|
||||
};
|
||||
}
|
||||
|
||||
function selectNextCrawlBatch(params: {
|
||||
linkQueue: QueueEntry[];
|
||||
sitemapQueue: QueueEntry[];
|
||||
queued: Set<string>;
|
||||
visited: Set<string>;
|
||||
robots: RobotsResult;
|
||||
remaining: number;
|
||||
}) {
|
||||
const { linkQueue, sitemapQueue, queued, visited, robots, remaining } =
|
||||
params;
|
||||
const batchSize = Math.min(CRAWL_CONCURRENCY, remaining);
|
||||
const batchEntries: QueueEntry[] = [];
|
||||
|
||||
while (
|
||||
(linkQueue.length > 0 || sitemapQueue.length > 0) &&
|
||||
batchEntries.length < batchSize
|
||||
) {
|
||||
const entry = (linkQueue.length > 0 ? linkQueue : sitemapQueue).shift()!;
|
||||
queued.delete(entry.url);
|
||||
if (visited.has(entry.url)) continue;
|
||||
if (!robots.isAllowed(entry.url)) continue;
|
||||
visited.add(entry.url);
|
||||
batchEntries.push(entry);
|
||||
}
|
||||
|
||||
return batchEntries;
|
||||
}
|
||||
|
||||
async function runCrawlBatch(
|
||||
step: WorkflowStep,
|
||||
input: {
|
||||
crawlBatchIndex: number;
|
||||
async function persistCrawledPages(input: {
|
||||
auditId: string;
|
||||
batchEntries: QueueEntry[];
|
||||
sitemapSet: Set<string>;
|
||||
visited: Set<string>;
|
||||
queued: Set<string>;
|
||||
},
|
||||
): Promise<StepPageSummary[]> {
|
||||
const {
|
||||
crawlBatchIndex,
|
||||
auditId,
|
||||
batchEntries,
|
||||
sitemapSet,
|
||||
visited,
|
||||
queued,
|
||||
} = input;
|
||||
return pgStep(step, `crawl-batch-${crawlBatchIndex}`, undefined, async () => {
|
||||
const pages = await Promise.all(
|
||||
batchEntries.map((entry) =>
|
||||
crawlPage(entry.url, entry.depth, sitemapSet.has(entry.url)),
|
||||
),
|
||||
);
|
||||
workflowInstanceId: string;
|
||||
origin: string;
|
||||
robots: RobotsResult;
|
||||
scratchpad: ReturnType<typeof getAuditScratchpad>;
|
||||
pages: CrawledPageResult[];
|
||||
depthByUrl: Map<string, number | null>;
|
||||
maxPages: number;
|
||||
}): Promise<FrontierStats> {
|
||||
const { auditId, origin, robots, scratchpad, pages, depthByUrl } = input;
|
||||
|
||||
// Deterministic ids keep the D1 writes idempotent across step retries.
|
||||
// Deterministic ids keep every write idempotent across step retries.
|
||||
for (const page of pages) {
|
||||
page.id = await deterministicAuditRowId(auditId, page.url);
|
||||
}
|
||||
|
||||
const issues = pages.flatMap((page) => runPageReporters(page));
|
||||
await AuditRepository.insertCrawledBatch(auditId, pages, issues);
|
||||
|
||||
// Frontier candidates only: drop targets already visited/queued and
|
||||
// dedupe across the batch, so the step return stays far under the
|
||||
// ~1MiB durable-state limit even on mega-menu sites.
|
||||
const seenTargets = new Set<string>();
|
||||
return pages.map((page) => {
|
||||
const internalLinks: string[] = [];
|
||||
const links: ScratchpadLinkRow[] = [];
|
||||
const discovered = new Map<string, number | null>();
|
||||
for (const page of pages) {
|
||||
const pageDepth = depthByUrl.get(page.url) ?? null;
|
||||
const childDepth = pageDepth === null ? null : pageDepth + 1;
|
||||
|
||||
let storedForPage = 0;
|
||||
for (const link of page.links) {
|
||||
if (!link.isInternal) continue;
|
||||
if (seenTargets.size >= MAX_FRONTIER_LINKS_PER_BATCH) break;
|
||||
if (visited.has(link.targetUrl) || queued.has(link.targetUrl)) continue;
|
||||
if (seenTargets.has(link.targetUrl)) continue;
|
||||
seenTargets.add(link.targetUrl);
|
||||
internalLinks.push(link.targetUrl);
|
||||
if (storedForPage < MAX_STORED_LINKS_PER_PAGE) {
|
||||
storedForPage += 1;
|
||||
links.push({
|
||||
sourcePageId: page.id,
|
||||
sourceUrl: page.url,
|
||||
targetUrl: link.targetUrl,
|
||||
anchor: link.anchor,
|
||||
isNofollow: link.isNofollow,
|
||||
});
|
||||
}
|
||||
return {
|
||||
id: page.id,
|
||||
if (
|
||||
discovered.size < MAX_DISCOVERED_PER_BATCH &&
|
||||
!discovered.has(link.targetUrl) &&
|
||||
shouldQueueCrawlLink(link.targetUrl, origin, robots)
|
||||
) {
|
||||
discovered.set(link.targetUrl, childDepth);
|
||||
}
|
||||
}
|
||||
|
||||
// Redirect targets continue the same navigation path: same depth.
|
||||
if (
|
||||
page.redirectUrl &&
|
||||
!discovered.has(page.redirectUrl) &&
|
||||
shouldQueueCrawlLink(page.redirectUrl, origin, robots)
|
||||
) {
|
||||
discovered.set(page.redirectUrl, pageDepth);
|
||||
}
|
||||
}
|
||||
|
||||
const stats = await scratchpad.recordBatch({
|
||||
crawledUrls: pages.map((page) => page.url),
|
||||
pages: pages.map((page) => ({
|
||||
pageId: page.id,
|
||||
url: page.url,
|
||||
statusCode: page.statusCode,
|
||||
fetchClass: page.fetchClass,
|
||||
redirectUrl: page.redirectUrl,
|
||||
title: page.title.slice(0, MAX_SUMMARY_TITLE_CHARS),
|
||||
internalLinks,
|
||||
};
|
||||
})),
|
||||
links,
|
||||
discovered: Array.from(discovered, ([url, depth]) => ({ url, depth })),
|
||||
});
|
||||
|
||||
await AuditRepository.updateAuditProgress(auditId, input.workflowInstanceId, {
|
||||
pagesCrawled: stats.attempted,
|
||||
pagesTotal: Math.min(stats.seen, input.maxPages),
|
||||
});
|
||||
}
|
||||
|
||||
function enqueueDiscoveredLinks(params: {
|
||||
crawledBatch: StepPageSummary[];
|
||||
batchEntries: QueueEntry[];
|
||||
linkQueue: QueueEntry[];
|
||||
queued: Set<string>;
|
||||
visited: Set<string>;
|
||||
origin: string;
|
||||
robots: RobotsResult;
|
||||
}) {
|
||||
const {
|
||||
crawledBatch,
|
||||
batchEntries,
|
||||
linkQueue,
|
||||
queued,
|
||||
visited,
|
||||
origin,
|
||||
robots,
|
||||
} = params;
|
||||
const depthByUrl = new Map(
|
||||
batchEntries.map((entry) => [entry.url, entry.depth]),
|
||||
);
|
||||
|
||||
for (const pageResult of crawledBatch) {
|
||||
const pageDepth = depthByUrl.get(pageResult.url) ?? null;
|
||||
const childDepth = pageDepth === null ? null : pageDepth + 1;
|
||||
|
||||
for (const link of pageResult.internalLinks) {
|
||||
if (!shouldQueueCrawlLink(link, origin, robots, visited, queued)) {
|
||||
continue;
|
||||
}
|
||||
linkQueue.push({ url: link, depth: childDepth });
|
||||
queued.add(link);
|
||||
}
|
||||
|
||||
// Redirect targets continue the same navigation path: same depth.
|
||||
const redirectTarget = pageResult.redirectUrl;
|
||||
if (
|
||||
redirectTarget &&
|
||||
shouldQueueCrawlLink(redirectTarget, origin, robots, visited, queued)
|
||||
) {
|
||||
linkQueue.push({ url: redirectTarget, depth: pageDepth });
|
||||
queued.add(redirectTarget);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function persistCrawlProgress(params: {
|
||||
step: WorkflowStep;
|
||||
crawlBatchIndex: number;
|
||||
auditId: string;
|
||||
workflowInstanceId: string;
|
||||
crawledBatch: StepPageSummary[];
|
||||
pagesCrawled: number;
|
||||
visitedCount: number;
|
||||
queueLength: number;
|
||||
maxPages: number;
|
||||
}) {
|
||||
const {
|
||||
step,
|
||||
crawlBatchIndex,
|
||||
auditId,
|
||||
workflowInstanceId,
|
||||
crawledBatch,
|
||||
pagesCrawled,
|
||||
visitedCount,
|
||||
queueLength,
|
||||
maxPages,
|
||||
} = params;
|
||||
// KV push + D1 progress in one step — merging them halves the per-batch
|
||||
// step count against the ~1k step budget. The D1 update is idempotent; the
|
||||
// KV push can duplicate entries on a partial retry, which is acceptable for
|
||||
// an ephemeral progress feed (capped list, short TTL).
|
||||
await pgStep(
|
||||
step,
|
||||
`progress-batch-${crawlBatchIndex}`,
|
||||
undefined,
|
||||
async () => {
|
||||
await AuditProgressKV.pushCrawledUrls(
|
||||
auditId,
|
||||
crawledBatch.map((pageResult) => ({
|
||||
url: pageResult.url,
|
||||
statusCode: pageResult.statusCode,
|
||||
title: pageResult.title,
|
||||
pages.map((page) => ({
|
||||
url: page.url,
|
||||
statusCode: page.statusCode,
|
||||
title: page.title.slice(0, MAX_PROGRESS_TITLE_CHARS),
|
||||
crawledAt: Date.now(),
|
||||
})),
|
||||
);
|
||||
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
||||
pagesCrawled,
|
||||
pagesTotal: Math.min(visitedCount + queueLength, maxPages),
|
||||
});
|
||||
},
|
||||
);
|
||||
return stats;
|
||||
}
|
||||
|
||||
@ -5,36 +5,34 @@ import {
|
||||
fetchAndStoreLighthouseResult,
|
||||
selectLighthouseSample,
|
||||
} from "@/server/lib/audit/lighthouse";
|
||||
import { getOrigin } from "@/server/lib/audit/url-utils";
|
||||
import {
|
||||
getOrigin,
|
||||
isSameOrigin,
|
||||
normalizeUrl,
|
||||
} from "@/server/lib/audit/url-utils";
|
||||
import { isCrawlableUrl } from "@/server/lib/audit/url-policy";
|
||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||
import { getAuditScratchpad } from "@/server/features/audit/AuditScratchpad";
|
||||
import { AuditProgressKV } from "@/server/lib/audit/progress-kv";
|
||||
import { runMultipageChecks } from "@/server/lib/audit/issues/multipage";
|
||||
import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters";
|
||||
import type { AuditConfig } from "@/server/lib/audit/types";
|
||||
import { captureServerEvent } from "@/server/lib/posthog";
|
||||
import {
|
||||
runCrawlPhase,
|
||||
type CrawledPageSummary,
|
||||
type CrawlPhaseResult,
|
||||
} from "@/server/workflows/siteAuditWorkflowCrawl";
|
||||
import { pgStep } from "@/server/workflows/pgStep";
|
||||
import {
|
||||
DB_STEP,
|
||||
DISCOVERY_STEP,
|
||||
LIGHTHOUSE_BATCH_STEP,
|
||||
MULTIPAGE_CHECKS_STEP,
|
||||
} from "@/server/workflows/auditStepConfigs";
|
||||
|
||||
const LIGHTHOUSE_URL_BATCH_SIZE = 10;
|
||||
|
||||
// Workflows rejects step outputs over 1MiB; keep the sitemap seed list well
|
||||
// under that. The crawl visits at most maxPages URLs, so extra seeds are moot.
|
||||
const SITEMAP_SEED_BYTE_BUDGET = 768 * 1024;
|
||||
|
||||
function capSitemapSeeds(urls: string[], maxPages: number): string[] {
|
||||
const seeds: string[] = [];
|
||||
let bytes = 0;
|
||||
for (const url of urls) {
|
||||
if (seeds.length >= maxPages) break;
|
||||
bytes += url.length + 3; // JSON quotes + comma
|
||||
if (bytes > SITEMAP_SEED_BYTE_BUDGET) break;
|
||||
seeds.push(url);
|
||||
}
|
||||
return seeds;
|
||||
}
|
||||
/** Frontier seeds per scratchpad RPC call. */
|
||||
const SEED_RPC_BATCH = 2_000;
|
||||
|
||||
type AuditPhasesParams = {
|
||||
auditId: string;
|
||||
@ -60,13 +58,13 @@ export async function runAuditPhases(
|
||||
const origin = getOrigin(startUrl);
|
||||
const maxPages = config.maxPages;
|
||||
|
||||
const discovery = await runDiscoveryPhase(
|
||||
step,
|
||||
const discovery = await runDiscoveryPhase(step, {
|
||||
auditId,
|
||||
workflowInstanceId,
|
||||
origin,
|
||||
startUrl,
|
||||
maxPages,
|
||||
);
|
||||
});
|
||||
// Parsed outside the step from checkpointed text, so replays see the exact
|
||||
// robots rules the original run used (a live re-fetch could differ and
|
||||
// desync the frontier from already-persisted crawl batches).
|
||||
@ -75,10 +73,9 @@ export async function runAuditPhases(
|
||||
auditId,
|
||||
workflowInstanceId,
|
||||
origin,
|
||||
startUrl,
|
||||
maxPages,
|
||||
robots,
|
||||
sitemapUrls: discovery.sitemapUrls,
|
||||
seededCount: discovery.seededCount,
|
||||
});
|
||||
await runLighthousePhase(step, {
|
||||
auditId,
|
||||
@ -87,7 +84,6 @@ export async function runAuditPhases(
|
||||
projectId,
|
||||
startUrl,
|
||||
config,
|
||||
pages: crawl.pages,
|
||||
});
|
||||
await finalizeAudit({
|
||||
step,
|
||||
@ -103,21 +99,61 @@ export async function runAuditPhases(
|
||||
|
||||
async function runDiscoveryPhase(
|
||||
step: WorkflowStep,
|
||||
auditId: string,
|
||||
workflowInstanceId: string,
|
||||
origin: string,
|
||||
maxPages: number,
|
||||
input: {
|
||||
auditId: string;
|
||||
workflowInstanceId: string;
|
||||
origin: string;
|
||||
startUrl: string;
|
||||
maxPages: number;
|
||||
},
|
||||
) {
|
||||
return pgStep(step, "discover-urls", undefined, async () => {
|
||||
const { auditId, workflowInstanceId, origin, startUrl, maxPages } = input;
|
||||
// "-v2": the checkpoint shape changed (seeds now live in the scratchpad DO
|
||||
// instead of the step return). A pre-refactor instance replayed under this
|
||||
// code must re-run discovery — resuming from the old cached {sitemapUrls}
|
||||
// shape would leave the scratchpad empty and finalize a zero-page audit.
|
||||
return pgStep(step, "discover-urls-v2", DISCOVERY_STEP, async () => {
|
||||
const result = await discoverUrls(origin, maxPages);
|
||||
const robots = parseRobotsTxt(origin, result.robotsText);
|
||||
const scratchpad = getAuditScratchpad(auditId);
|
||||
|
||||
// Seeds go straight into the scratchpad frontier — nothing large is
|
||||
// returned as step state (an uncapped seed list used to blow the ~1MiB
|
||||
// step-output limit on big sitemaps).
|
||||
let seededCount = 0;
|
||||
const normalizedStart = normalizeUrl(startUrl) ?? startUrl;
|
||||
if (
|
||||
robots.isAllowed(normalizedStart) &&
|
||||
isSameOrigin(normalizedStart, origin)
|
||||
) {
|
||||
await scratchpad.seedStart(normalizedStart);
|
||||
seededCount += 1;
|
||||
}
|
||||
|
||||
// The start URL is deliberately not excluded here: seedSitemapUrls
|
||||
// upserts, so a start URL that also appears in the sitemap keeps its
|
||||
// link-queue position but gains the in-sitemap flag.
|
||||
const seen = new Set<string>();
|
||||
const seeds: string[] = [];
|
||||
for (const url of result.urls) {
|
||||
const normalized = normalizeUrl(url);
|
||||
if (!normalized || seen.has(normalized)) continue;
|
||||
seen.add(normalized);
|
||||
if (!isSameOrigin(normalized, origin)) continue;
|
||||
if (!isCrawlableUrl(normalized)) continue;
|
||||
if (!robots.isAllowed(normalized)) continue;
|
||||
seeds.push(normalized);
|
||||
}
|
||||
for (let i = 0; i < seeds.length; i += SEED_RPC_BATCH) {
|
||||
await scratchpad.seedSitemapUrls(seeds.slice(i, i + SEED_RPC_BATCH));
|
||||
}
|
||||
seededCount += seeds.filter((seed) => seed !== normalizedStart).length;
|
||||
|
||||
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
||||
pagesTotal: Math.min(result.urls.length + 1, maxPages),
|
||||
pagesTotal: Math.min(seededCount, maxPages),
|
||||
currentPhase: "crawling",
|
||||
});
|
||||
return {
|
||||
sitemapUrls: capSitemapSeeds(result.urls, maxPages),
|
||||
robotsText: result.robotsText,
|
||||
};
|
||||
return { robotsText: result.robotsText, seededCount };
|
||||
});
|
||||
}
|
||||
|
||||
@ -128,7 +164,6 @@ type LighthousePhaseParams = {
|
||||
projectId: string;
|
||||
startUrl: string;
|
||||
config: AuditConfig;
|
||||
pages: CrawledPageSummary[];
|
||||
};
|
||||
|
||||
async function runLighthousePhase(
|
||||
@ -142,7 +177,6 @@ async function runLighthousePhase(
|
||||
projectId,
|
||||
startUrl,
|
||||
config,
|
||||
pages,
|
||||
} = params;
|
||||
if (config.lighthouseStrategy === "none") return;
|
||||
|
||||
@ -150,7 +184,6 @@ async function runLighthousePhase(
|
||||
step,
|
||||
auditId,
|
||||
workflowInstanceId,
|
||||
pages,
|
||||
startUrl,
|
||||
strategy: config.lighthouseStrategy,
|
||||
});
|
||||
@ -165,12 +198,12 @@ async function runLighthousePhase(
|
||||
const priorCompleted = completedChecks;
|
||||
const priorFailed = failedChecks;
|
||||
|
||||
// Fetch, store (R2 + D1) and update progress inside one step. The step
|
||||
// returns only counts; full results live in D1.
|
||||
// Fetch, store (R2 + DB) and update progress inside one step. The step
|
||||
// returns only counts; full results live in the DB.
|
||||
const counts = await pgStep(
|
||||
step,
|
||||
`lighthouse-batch-${lighthouseBatchIndex}`,
|
||||
undefined,
|
||||
LIGHTHOUSE_BATCH_STEP,
|
||||
async () => {
|
||||
const perUrlResults = await Promise.all(
|
||||
batch.map(async ({ url, pageId }) => {
|
||||
@ -217,14 +250,22 @@ async function selectLighthousePages(params: {
|
||||
step: WorkflowStep;
|
||||
auditId: string;
|
||||
workflowInstanceId: string;
|
||||
pages: CrawledPageSummary[];
|
||||
startUrl: string;
|
||||
strategy: AuditConfig["lighthouseStrategy"];
|
||||
}) {
|
||||
const { step, auditId, workflowInstanceId, pages, startUrl, strategy } =
|
||||
params;
|
||||
return pgStep(step, "select-lighthouse-sample", undefined, async () => {
|
||||
const sample = selectLighthouseSample(pages, startUrl, strategy);
|
||||
const { step, auditId, workflowInstanceId, startUrl, strategy } = params;
|
||||
return pgStep(step, "select-lighthouse-sample", DB_STEP, async () => {
|
||||
// Crawled pages come from the DB — the crawl phase no longer holds a
|
||||
// whole-crawl page list in memory.
|
||||
const crawledPages = await AuditRepository.getPagesForAudit(auditId);
|
||||
const sample = selectLighthouseSample(
|
||||
crawledPages.map((page) => ({
|
||||
url: page.url,
|
||||
statusCode: page.statusCode ?? 0,
|
||||
})),
|
||||
startUrl,
|
||||
strategy,
|
||||
);
|
||||
const selectedUrls = new Set(sample);
|
||||
|
||||
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
||||
@ -233,7 +274,7 @@ async function selectLighthousePages(params: {
|
||||
lighthouseCompleted: 0,
|
||||
lighthouseFailed: 0,
|
||||
});
|
||||
return pages.flatMap((page) =>
|
||||
return crawledPages.flatMap((page) =>
|
||||
selectedUrls.has(page.url) ? [{ url: page.url, pageId: page.id }] : [],
|
||||
);
|
||||
});
|
||||
@ -260,37 +301,34 @@ async function finalizeAudit(args: {
|
||||
crawl,
|
||||
} = args;
|
||||
|
||||
await pgStep(step, "multipage-checks", undefined, async () => {
|
||||
await pgStep(step, "multipage-checks", MULTIPAGE_CHECKS_STEP, async () => {
|
||||
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
||||
currentPhase: "finalizing",
|
||||
});
|
||||
|
||||
// Integrity guard: pages are persisted inside crawl-batch steps. If the
|
||||
// crawl claims pages but D1 has none (e.g. an instance started under the
|
||||
// pre-incremental-persistence code was replayed under this code), fail
|
||||
// loudly instead of completing with an empty audit.
|
||||
// Integrity guard: pages are persisted inside crawl-chunk steps. If the
|
||||
// crawl claims pages but the DB has none, fail loudly instead of
|
||||
// completing with an empty audit.
|
||||
if (
|
||||
crawl.pages.length > 0 &&
|
||||
crawl.pagesCrawled > 0 &&
|
||||
!(await AuditRepository.hasPagesForAudit(auditId))
|
||||
) {
|
||||
throw new Error(
|
||||
`Audit ${auditId}: crawl reported ${crawl.pages.length} pages but none were persisted`,
|
||||
`Audit ${auditId}: crawl reported ${crawl.pagesCrawled} pages but none were persisted`,
|
||||
);
|
||||
}
|
||||
|
||||
const issues = await runMultipageChecks({
|
||||
auditId,
|
||||
startUrl,
|
||||
crawlCompleted: crawl.completed,
|
||||
});
|
||||
const issues = await runMultipageChecks({ auditId });
|
||||
issues.push(...(await runScratchpadLinkChecks(auditId, startUrl, crawl)));
|
||||
await AuditRepository.insertIssues(auditId, issues);
|
||||
return { issueCount: issues.length };
|
||||
});
|
||||
|
||||
await pgStep(step, "finalize", undefined, async () => {
|
||||
await pgStep(step, "finalize", DB_STEP, async () => {
|
||||
const blockedPages = await AuditRepository.countBlockedPages(auditId);
|
||||
await AuditRepository.completeAudit(auditId, workflowInstanceId, {
|
||||
pagesCrawled: crawl.pages.length,
|
||||
pagesTotal: crawl.pages.length,
|
||||
pagesCrawled: crawl.pagesCrawled,
|
||||
pagesTotal: crawl.pagesCrawled,
|
||||
});
|
||||
await captureServerEvent({
|
||||
distinctId: billingCustomer.userId,
|
||||
@ -299,15 +337,49 @@ async function finalizeAudit(args: {
|
||||
properties: {
|
||||
project_id: projectId,
|
||||
status: "completed",
|
||||
pages_crawled: crawl.pages.length,
|
||||
pages_total: crawl.pages.length,
|
||||
pages_crawled: crawl.pagesCrawled,
|
||||
pages_total: crawl.pagesCrawled,
|
||||
crawl_completed: crawl.completed,
|
||||
pages_blocked: crawl.pages.filter(
|
||||
(page) => page.fetchClass === "blocked",
|
||||
).length,
|
||||
pages_blocked: blockedPages,
|
||||
run_lighthouse: config.lighthouseStrategy !== "none",
|
||||
},
|
||||
});
|
||||
await AuditProgressKV.clear(auditId);
|
||||
// Crawl scratch state (frontier, links, mirror) is no longer needed.
|
||||
await getAuditScratchpad(auditId).destroy();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The two finalize checks that need link edges run as SQL inside the
|
||||
* audit's scratchpad DO; map their rows onto DetectedIssue.
|
||||
*/
|
||||
async function runScratchpadLinkChecks(
|
||||
auditId: string,
|
||||
startUrl: string,
|
||||
crawl: CrawlPhaseResult,
|
||||
): Promise<DetectedIssue[]> {
|
||||
const scratchpad = getAuditScratchpad(auditId);
|
||||
const { brokenLinks, orphanPages } = await scratchpad.runFinalizeChecks({
|
||||
// Page rows store normalized URLs; normalize the start URL the same way
|
||||
// so the orphan exclusion matches.
|
||||
startUrl: normalizeUrl(startUrl) ?? startUrl,
|
||||
// Orphan detection only makes sense when the crawl wasn't truncated.
|
||||
crawlCompleted: crawl.completed,
|
||||
});
|
||||
|
||||
return [
|
||||
...brokenLinks.map((row) => ({
|
||||
issueType: "broken-internal-link" as const,
|
||||
pageId: row.sourcePageId,
|
||||
pageUrl: row.sourceUrl,
|
||||
dedupeKey: row.targetUrl,
|
||||
details: { targetUrl: row.targetUrl, targetStatus: row.targetStatus },
|
||||
})),
|
||||
...orphanPages.map((row) => ({
|
||||
issueType: "orphan-page" as const,
|
||||
pageId: row.pageId,
|
||||
pageUrl: row.url,
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
@ -33,7 +33,9 @@
|
||||
],
|
||||
// Durable Object backing the onboarding strategy chat (Agents SDK
|
||||
// AIChatAgent). One instance per project; messages persist in the DO's
|
||||
// SQLite. SQLite-backed classes must be declared in `migrations` below.
|
||||
// SQLite. SQLite-backed classes must be declared in `migrations` below for
|
||||
// the wrangler/workerd surfaces (local dev, Docker self-host); Alchemy
|
||||
// deploys derive SQLite backing on their own.
|
||||
"durable_objects": {
|
||||
"bindings": [
|
||||
{
|
||||
@ -46,6 +48,13 @@
|
||||
"name": "SAM_CHAT",
|
||||
"class_name": "SamChatAgent",
|
||||
},
|
||||
// Per-audit crawl scratchpad: frontier, link edges, and a slim page
|
||||
// mirror in the DO's SQLite. Destroyed at finalize; self-cleans via
|
||||
// alarm if the audit dies.
|
||||
{
|
||||
"name": "AUDIT_SCRATCHPAD",
|
||||
"class_name": "AuditScratchpad",
|
||||
},
|
||||
],
|
||||
},
|
||||
"migrations": [
|
||||
@ -57,6 +66,10 @@
|
||||
"tag": "v2",
|
||||
"new_sqlite_classes": ["SamChatAgent"],
|
||||
},
|
||||
{
|
||||
"tag": "v3",
|
||||
"new_sqlite_classes": ["AuditScratchpad"],
|
||||
},
|
||||
],
|
||||
"triggers": {
|
||||
"crons": ["*/15 * * * *"],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user