Site Audit - Improve reliability and performance with Durable Objects
This commit is contained in:
Ben Senescu
GitHub
parent
9d19e43990
commit
1e8a924c4c
@ -20,3 +20,15 @@ data, or sensitive paths.
|
|||||||
## Resolved
|
## Resolved
|
||||||
|
|
||||||
Move fixed entries here, mark them checked, and append the resolving date or commit.
|
Move fixed entries here, mark them checked, and append the resolving date or commit.
|
||||||
|
|
||||||
|
## badseo harness vs `wrangler dev`: sitemap emits badseo.dev locs locally
|
||||||
|
|
||||||
|
`badseo/scripts/run-audit.ts` against a local `wrangler dev --port 8787` fails 4
|
||||||
|
sitemap-dependent checks (orphan page, 500, 403, duplicate-content) with
|
||||||
|
NOT CRAWLED: wrangler dev adopts the `badseo.dev` custom-domain route as the
|
||||||
|
host the worker sees, so `/sitemap.xml` emits `http://badseo.dev/...` locs that
|
||||||
|
the crawler's same-origin filter drops. Run it as
|
||||||
|
`wrangler dev --port 8787 --local-upstream "localhost:8787"` (after
|
||||||
|
`vite build`). Also: `pnpm --filter badseo audit` fails with
|
||||||
|
"Unknown option: 'recursive'" from the repo root — badseo is its own pnpm
|
||||||
|
workspace, not a root workspace member; use `npx tsx badseo/scripts/run-audit.ts`.
|
||||||
|
|||||||
@ -400,8 +400,10 @@ export default Alchemy.Stack(
|
|||||||
// Prod-only: pooled Postgres via the existing Hyperdrive config.
|
// Prod-only: pooled Postgres via the existing Hyperdrive config.
|
||||||
...(prod ? { HYPERDRIVE: makeHyperdrive() } : {}),
|
...(prod ? { HYPERDRIVE: makeHyperdrive() } : {}),
|
||||||
|
|
||||||
// Durable Objects (Agents SDK chat agents). Alchemy backs new DO
|
// Durable Objects (chat agents + the per-audit crawl scratchpad).
|
||||||
// classes with SQLite storage, which both require.
|
// Alchemy backs new DO classes with SQLite storage, which all of
|
||||||
|
// them require; the `migrations` array in wrangler.jsonc only
|
||||||
|
// applies to the wrangler/workerd surfaces (local dev, Docker).
|
||||||
...Object.fromEntries(
|
...Object.fromEntries(
|
||||||
wrangler.durable_objects.bindings.map((binding) => [
|
wrangler.durable_objects.bindings.map((binding) => [
|
||||||
binding.name,
|
binding.name,
|
||||||
|
|||||||
@ -46,7 +46,7 @@ a:focus-visible, button:focus-visible { outline: 3px solid #ff6a57; outline-offs
|
|||||||
.home-copy > p { max-width: 860px; margin: 42px 0 0; font-family: var(--serif); font-size: clamp(28px, 2.2vw, 32px); line-height: 1.08; }
|
.home-copy > p { max-width: 860px; margin: 42px 0 0; font-family: var(--serif); font-size: clamp(28px, 2.2vw, 32px); line-height: 1.08; }
|
||||||
.home-issues { scroll-margin-top: 24px; border-top: 3px solid var(--navy); }
|
.home-issues { scroll-margin-top: 24px; border-top: 3px solid var(--navy); }
|
||||||
.home-issue-group { margin: 22px 0 54px; }
|
.home-issue-group { margin: 22px 0 54px; }
|
||||||
.home-issue-group h3 { margin: 0; padding: 18px 0 14px; border-bottom: 1px solid var(--line); font-family: var(--condensed); font-size: 30px; line-height: 1; }
|
.home-issue-group h2 { margin: 0; padding: 18px 0 14px; border-bottom: 1px solid var(--line); font-family: var(--condensed); font-size: 30px; line-height: 1; }
|
||||||
.home-case-head, .home-case-row { display: grid; grid-template-columns: minmax(220px, .72fr) minmax(320px, 1.28fr); gap: clamp(30px, 6vw, 90px); }
|
.home-case-head, .home-case-row { display: grid; grid-template-columns: minmax(220px, .72fr) minmax(320px, 1.28fr); gap: clamp(30px, 6vw, 90px); }
|
||||||
.home-case-head { padding: 11px 0; border-bottom: 1px solid var(--line-soft); color: var(--muted); font-family: var(--mono); font-size: 8px; }
|
.home-case-head { padding: 11px 0; border-bottom: 1px solid var(--line-soft); color: var(--muted); font-family: var(--mono); font-size: 8px; }
|
||||||
.home-case-row { min-height: 72px; align-items: center; padding: 16px 0; border-bottom: 1px solid var(--line-soft); text-decoration: none; }
|
.home-case-row { min-height: 72px; align-items: center; padding: 16px 0; border-bottom: 1px solid var(--line-soft); text-decoration: none; }
|
||||||
|
|||||||
@ -75,7 +75,12 @@ export interface FileRoutesById {
|
|||||||
export interface FileRouteTypes {
|
export interface FileRouteTypes {
|
||||||
fileRoutesByFullPath: FileRoutesByFullPath
|
fileRoutesByFullPath: FileRoutesByFullPath
|
||||||
fullPaths:
|
fullPaths:
|
||||||
'/' | '/$' | '/catalog' | '/privacy' | '/robots.txt' | '/sitemap.xml'
|
| '/'
|
||||||
|
| '/$'
|
||||||
|
| '/catalog'
|
||||||
|
| '/privacy'
|
||||||
|
| '/robots.txt'
|
||||||
|
| '/sitemap.xml'
|
||||||
fileRoutesByTo: FileRoutesByTo
|
fileRoutesByTo: FileRoutesByTo
|
||||||
to: '/' | '/$' | '/catalog' | '/privacy' | '/robots.txt' | '/sitemap.xml'
|
to: '/' | '/$' | '/catalog' | '/privacy' | '/robots.txt' | '/sitemap.xml'
|
||||||
id:
|
id:
|
||||||
|
|||||||
@ -41,7 +41,7 @@ function HomePage() {
|
|||||||
<div className="home-issues" id="issues">
|
<div className="home-issues" id="issues">
|
||||||
{homepageCategories.map((category) => (
|
{homepageCategories.map((category) => (
|
||||||
<section className="home-issue-group" key={category}>
|
<section className="home-issue-group" key={category}>
|
||||||
<h3>{category}</h3>
|
<h2>{category}</h2>
|
||||||
<div className="home-case-head" aria-hidden="true">
|
<div className="home-case-head" aria-hidden="true">
|
||||||
<span>Issue</span>
|
<span>Issue</span>
|
||||||
<span>What it demonstrates</span>
|
<span>What it demonstrates</span>
|
||||||
|
|||||||
3
drizzle-pg/0014_solid_centennial.sql
Normal file
3
drizzle-pg/0014_solid_centennial.sql
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE "audits" ADD COLUMN "error_code" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "audits" ADD COLUMN "error_detail" text;--> statement-breakpoint
|
||||||
|
ALTER TABLE "audits" ADD COLUMN "failed_phase" text;
|
||||||
3732
drizzle-pg/meta/0014_snapshot.json
Normal file
3732
drizzle-pg/meta/0014_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -99,6 +99,13 @@
|
|||||||
"when": 1784426969390,
|
"when": 1784426969390,
|
||||||
"tag": "0013_sleepy_black_tarantula",
|
"tag": "0013_sleepy_black_tarantula",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 14,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1784588248793,
|
||||||
|
"tag": "0014_solid_centennial",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
3
drizzle/0037_small_caretaker.sql
Normal file
3
drizzle/0037_small_caretaker.sql
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE `audits` ADD `error_code` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `audits` ADD `error_detail` text;--> statement-breakpoint
|
||||||
|
ALTER TABLE `audits` ADD `failed_phase` text;
|
||||||
3401
drizzle/meta/0037_snapshot.json
Normal file
3401
drizzle/meta/0037_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -260,6 +260,13 @@
|
|||||||
"when": 1784426967421,
|
"when": 1784426967421,
|
||||||
"tag": "0036_curvy_silk_fever",
|
"tag": "0036_curvy_silk_fever",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 37,
|
||||||
|
"version": "6",
|
||||||
|
"when": 1784588247803,
|
||||||
|
"tag": "0037_small_caretaker",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@ -91,12 +91,12 @@
|
|||||||
"ai": "^6.0.199",
|
"ai": "^6.0.199",
|
||||||
"autumn-js": "^1.2.33",
|
"autumn-js": "^1.2.33",
|
||||||
"better-auth": "^1.6.22",
|
"better-auth": "^1.6.22",
|
||||||
"cheerio": "^1.2.0",
|
|
||||||
"cloudflare": "^5.2.0",
|
"cloudflare": "^5.2.0",
|
||||||
"daisyui": "^5.5.5",
|
"daisyui": "^5.5.5",
|
||||||
"dataforseo-client": "^2.0.19",
|
"dataforseo-client": "^2.0.19",
|
||||||
"drizzle-orm": "^0.45.2",
|
"drizzle-orm": "^0.45.2",
|
||||||
"fast-xml-parser": "^5.4.1",
|
"fast-xml-parser": "^5.4.1",
|
||||||
|
"htmlparser2": "^10.1.0",
|
||||||
"jose": "^6.0.12",
|
"jose": "^6.0.12",
|
||||||
"lucide-react": "^0.542.0",
|
"lucide-react": "^0.542.0",
|
||||||
"papaparse": "^5.5.3",
|
"papaparse": "^5.5.3",
|
||||||
@ -132,6 +132,7 @@
|
|||||||
"@vitejs/plugin-react": "^4.6.0",
|
"@vitejs/plugin-react": "^4.6.0",
|
||||||
"alchemy": "2.0.0-beta.61",
|
"alchemy": "2.0.0-beta.61",
|
||||||
"chalk": "^5.6.2",
|
"chalk": "^5.6.2",
|
||||||
|
"cheerio": "^1.2.0",
|
||||||
"drizzle-kit": "^0.31.10",
|
"drizzle-kit": "^0.31.10",
|
||||||
"effect": "4.0.0-beta.93",
|
"effect": "4.0.0-beta.93",
|
||||||
"knip": "^5.88.1",
|
"knip": "^5.88.1",
|
||||||
|
|||||||
9
pnpm-lock.yaml
generated
9
pnpm-lock.yaml
generated
@ -75,9 +75,6 @@ importers:
|
|||||||
better-auth:
|
better-auth:
|
||||||
specifier: ^1.6.22
|
specifier: ^1.6.22
|
||||||
version: 1.6.22(@cloudflare/workers-types@4.20260611.1)(@opentelemetry/api@1.9.1)(@tanstack/react-start@1.168.26(esbuild@0.28.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(rolldown@1.0.1)(rollup@4.59.0)(vite@7.3.6(@types/node@22.19.11)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)))(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@4.20260611.1)(@libsql/client@0.15.15)(@opentelemetry/api@1.9.1)(kysely@0.29.2)(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(postgres@3.4.9)(sql.js@1.14.1))(mongodb@7.2.0(@aws-sdk/credential-providers@3.1080.0)(@mongodb-js/zstd@7.0.0))(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.6(@types/debug@4.1.13)(@types/node@22.19.11)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0))
|
version: 1.6.22(@cloudflare/workers-types@4.20260611.1)(@opentelemetry/api@1.9.1)(@tanstack/react-start@1.168.26(esbuild@0.28.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(rolldown@1.0.1)(rollup@4.59.0)(vite@7.3.6(@types/node@22.19.11)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)))(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@4.20260611.1)(@libsql/client@0.15.15)(@opentelemetry/api@1.9.1)(kysely@0.29.2)(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(postgres@3.4.9)(sql.js@1.14.1))(mongodb@7.2.0(@aws-sdk/credential-providers@3.1080.0)(@mongodb-js/zstd@7.0.0))(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.6(@types/debug@4.1.13)(@types/node@22.19.11)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0))
|
||||||
cheerio:
|
|
||||||
specifier: ^1.2.0
|
|
||||||
version: 1.2.0
|
|
||||||
cloudflare:
|
cloudflare:
|
||||||
specifier: ^5.2.0
|
specifier: ^5.2.0
|
||||||
version: 5.2.0
|
version: 5.2.0
|
||||||
@ -93,6 +90,9 @@ importers:
|
|||||||
fast-xml-parser:
|
fast-xml-parser:
|
||||||
specifier: ^5.4.1
|
specifier: ^5.4.1
|
||||||
version: 5.8.0
|
version: 5.8.0
|
||||||
|
htmlparser2:
|
||||||
|
specifier: ^10.1.0
|
||||||
|
version: 10.1.0
|
||||||
jose:
|
jose:
|
||||||
specifier: ^6.0.12
|
specifier: ^6.0.12
|
||||||
version: 6.1.3
|
version: 6.1.3
|
||||||
@ -193,6 +193,9 @@ importers:
|
|||||||
chalk:
|
chalk:
|
||||||
specifier: ^5.6.2
|
specifier: ^5.6.2
|
||||||
version: 5.6.2
|
version: 5.6.2
|
||||||
|
cheerio:
|
||||||
|
specifier: ^1.2.0
|
||||||
|
version: 1.2.0
|
||||||
drizzle-kit:
|
drizzle-kit:
|
||||||
specifier: ^0.31.10
|
specifier: ^0.31.10
|
||||||
version: 0.31.10
|
version: 0.31.10
|
||||||
|
|||||||
125
specs/0009-site-audit-crawl-architecture.md
Normal file
125
specs/0009-site-audit-crawl-architecture.md
Normal file
@ -0,0 +1,125 @@
|
|||||||
|
# Site audit crawl architecture
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
The site audit runs as a Cloudflare Workflow. The original design kept all
|
||||||
|
crawl state inside the workflow run: the URL frontier and page summaries lived
|
||||||
|
in workflow memory and step returns, pages were parsed with a full DOM parser
|
||||||
|
(cheerio), and link edges were bulk-inserted into the app database during the
|
||||||
|
crawl. On large or slow sites this collided with platform limits:
|
||||||
|
|
||||||
|
- **Isolate memory.** DOM parsing expands a page several times over, multiplied
|
||||||
|
by concurrent parses, plus a whole-crawl summaries array that was rebuilt on
|
||||||
|
every workflow replay.
|
||||||
|
- **Step-output cap (~1 MiB).** Sitemap seed lists flowed through step returns
|
||||||
|
and could exceed it outright.
|
||||||
|
- **Step timeout × retry.** The default 10-minute step timeout with default
|
||||||
|
retries meant a deterministically failing step burned about an hour before
|
||||||
|
the audit died.
|
||||||
|
- **Batch head-of-line blocking.** Fixed-size crawl batches waited on their
|
||||||
|
slowest fetch; one tarpit URL stalled a whole batch.
|
||||||
|
- **Invisible failure.** Workflow-level kills (OOM, CPU) skip the error
|
||||||
|
handler, so audit rows could stay `running` forever with no recorded reason,
|
||||||
|
and the UI refused to show the pages that had already been crawled and
|
||||||
|
persisted.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Split the audit into a control plane and a data plane.
|
||||||
|
|
||||||
|
### Orchestration stays in Workflows; crawl state moves to a Durable Object
|
||||||
|
|
||||||
|
`SiteAuditWorkflow` remains the orchestrator (phase ordering, durable steps,
|
||||||
|
retries). All transient crawl state lives in `AuditScratchpad`, a SQLite-backed
|
||||||
|
Durable Object, one instance per audit (`idFromName(auditId)`):
|
||||||
|
|
||||||
|
- **frontier** — URL queue and dedup set (URL primary key), with
|
||||||
|
`pending / leased / crawled` states keyed by chunk number;
|
||||||
|
- **links** — internal link edges, primary-keyed for idempotent re-insert;
|
||||||
|
- **page_mirror** — the few columns the finalize link checks need.
|
||||||
|
|
||||||
|
The app database (D1/Postgres) keeps only what the product reads: `audits`
|
||||||
|
(plus failure columns), `audit_pages`, `audit_issues`,
|
||||||
|
`audit_lighthouse_results`. Link edges are never persisted to the app DB; the
|
||||||
|
cross-page link checks (broken internal links, orphan pages) run as SQL inside
|
||||||
|
the DO at finalize.
|
||||||
|
|
||||||
|
Rejected alternatives: Cloudflare Queues (no dedup, no counters, no completion
|
||||||
|
signal — a frontier is mostly those things), a Postgres frontier table (chatty
|
||||||
|
hot-path writes, dual-dialect surface), R2 spill files (not queryable).
|
||||||
|
|
||||||
|
### Chunked rolling crawl
|
||||||
|
|
||||||
|
The crawl phase is a loop of `crawl-chunk-N` steps. Each chunk leases up to
|
||||||
|
~200 URLs from the DO and crawls them with a rolling concurrency window: the
|
||||||
|
moment a fetch settles, the next URL launches. The window self-adjusts between
|
||||||
|
5 and 40 based on recent fetch health (errors, blocked fetches, slow responses,
|
||||||
|
oversized bodies shrink it; clean fast batches grow it). A soft deadline ends
|
||||||
|
the chunk early and releases unfetched leases. Persistence is pipelined with
|
||||||
|
fetching but serialized with itself: page rows and per-page issues go to the
|
||||||
|
app DB, link edges and frontier updates go to the DO, and progress counters
|
||||||
|
update per sub-batch. Step returns carry only counters, so no step output
|
||||||
|
scales with site size.
|
||||||
|
|
||||||
|
Everything is idempotent under step retries: chunk-keyed leases (a retried
|
||||||
|
step re-receives exactly the URLs its failed attempt held, and refuses a fresh
|
||||||
|
claim for an already-crawled chunk number), deterministic page-row ids, and
|
||||||
|
insert-or-ignore/replace writes on stable keys.
|
||||||
|
|
||||||
|
### Streaming HTML parsing
|
||||||
|
|
||||||
|
Pages are parsed with htmlparser2's streaming tokenizer — no DOM is built, so
|
||||||
|
per-page memory is constant. cheerio (which uses the same tokenizer
|
||||||
|
internally) remains only as a test reference: the parser test suite asserts
|
||||||
|
extraction parity against it, and the badseo fixture harness asserts identical
|
||||||
|
issue output end to end.
|
||||||
|
|
||||||
|
### Failure handling and graceful degradation
|
||||||
|
|
||||||
|
- `audits` carries `error_code` (a closed vocabulary mapped from real platform
|
||||||
|
error strings — `step_timeout`, `oom`, `cpu_limit`, `db_error`,
|
||||||
|
`step_output_too_large`, `workflow_internal`, `instance_lost`, `unknown`),
|
||||||
|
`error_detail`, and `failed_phase`.
|
||||||
|
- A watchdog on the existing `*/15` cron reconciles audit rows stuck in
|
||||||
|
`running` against the Workflows API; the audit status read path does the
|
||||||
|
same lazily. Only confirmed instance-not-found errors (after a grace period)
|
||||||
|
count as a lost instance, so transient API failures never fail a live audit.
|
||||||
|
- A failed audit shows everything crawled before the failure ("stopped early
|
||||||
|
after N pages") instead of hiding results.
|
||||||
|
- The start URL follows redirects (with per-hop SSRF revalidation) before the
|
||||||
|
crawl anchors its origin, so apex→www and cross-TLD redirects don't dead-end
|
||||||
|
the crawl after one page.
|
||||||
|
|
||||||
|
### DO lifecycle and platform-limit guards
|
||||||
|
|
||||||
|
- `destroy()` (delete alarm, then all storage) runs on audit success and on
|
||||||
|
audit deletion. Every DO construction schedules a 7-day self-cleanup alarm,
|
||||||
|
so any instantiation — including a write racing in after destroy, or a
|
||||||
|
workflow that dies before seeding — is eventually wiped. Failed audits keep
|
||||||
|
their scratchpad for those 7 days as the resume/debug artifact.
|
||||||
|
- Guards where the platform has hard edges: link writes stop at a storage
|
||||||
|
budget (below the free-plan per-object SQLite cap; orphan detection is
|
||||||
|
skipped when the link graph was truncated), discovered-URL batches are
|
||||||
|
capped below the serialized-RPC limit, and sitemap documents are read up to
|
||||||
|
a byte cap and skipped whole beyond it.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Workflow heap and step state are O(one chunk) regardless of site size; the
|
||||||
|
OOM/step-output failure classes are gone by construction.
|
||||||
|
- Hostile or slow sites degrade to a slower window and finish (or fail with a
|
||||||
|
classified reason and visible partials) instead of burning an hour of
|
||||||
|
retries.
|
||||||
|
- Every failure is aggregable by `error_code` in plain SQL, and zombie
|
||||||
|
`running` rows self-heal.
|
||||||
|
- Crawl-state code is provider-independent (DO SQLite exists in workerd), so
|
||||||
|
self-host does not add a dialect surface for the frontier or links.
|
||||||
|
- The scratchpad is opaque from outside the DO; the 7-day retention of failed
|
||||||
|
audits' state is the debugging window.
|
||||||
|
- Resume-from-frontier ("retry finishes the last N pages instead of
|
||||||
|
recrawling") is enabled by the retained frontier but intentionally not
|
||||||
|
built yet.
|
||||||
@ -1,6 +1,6 @@
|
|||||||
import { AlertCircle, CheckCircle, Loader2 } from "lucide-react";
|
import { AlertCircle, CheckCircle, Loader2 } from "lucide-react";
|
||||||
|
|
||||||
export const SUPPORT_URL = "https://everyapp.dev/support";
|
export const SUPPORT_EMAIL = "ben@openseo.so";
|
||||||
|
|
||||||
export function extractPathname(url: string): string {
|
export function extractPathname(url: string): string {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@ -37,6 +37,13 @@ export const audits = sqliteTable(
|
|||||||
lighthouseCompleted: integer("lighthouse_completed").notNull().default(0),
|
lighthouseCompleted: integer("lighthouse_completed").notNull().default(0),
|
||||||
lighthouseFailed: integer("lighthouse_failed").notNull().default(0),
|
lighthouseFailed: integer("lighthouse_failed").notNull().default(0),
|
||||||
currentPhase: text("current_phase").default("discovery"),
|
currentPhase: text("current_phase").default("discovery"),
|
||||||
|
// Failure diagnostics; null unless status = "failed". errorCode is a
|
||||||
|
// closed vocabulary (see classifyAuditError) so failures are aggregable;
|
||||||
|
// errorDetail is the raw message, truncated. failedPhase records which
|
||||||
|
// currentPhase the audit was in when it died.
|
||||||
|
errorCode: text("error_code"),
|
||||||
|
errorDetail: text("error_detail"),
|
||||||
|
failedPhase: text("failed_phase"),
|
||||||
startedAt: text("started_at")
|
startedAt: text("started_at")
|
||||||
.notNull()
|
.notNull()
|
||||||
.default(sql`(current_timestamp)`),
|
.default(sql`(current_timestamp)`),
|
||||||
|
|||||||
@ -44,6 +44,13 @@ export const audits = pgTable(
|
|||||||
lighthouseCompleted: integer("lighthouse_completed").notNull().default(0),
|
lighthouseCompleted: integer("lighthouse_completed").notNull().default(0),
|
||||||
lighthouseFailed: integer("lighthouse_failed").notNull().default(0),
|
lighthouseFailed: integer("lighthouse_failed").notNull().default(0),
|
||||||
currentPhase: text("current_phase").default("discovery"),
|
currentPhase: text("current_phase").default("discovery"),
|
||||||
|
// Failure diagnostics; null unless status = "failed". errorCode is a
|
||||||
|
// closed vocabulary (see classifyAuditError) so failures are aggregable;
|
||||||
|
// errorDetail is the raw message, truncated. failedPhase records which
|
||||||
|
// currentPhase the audit was in when it died.
|
||||||
|
errorCode: text("error_code"),
|
||||||
|
errorDetail: text("error_detail"),
|
||||||
|
failedPhase: text("failed_phase"),
|
||||||
startedAt: timestampColumn("started_at").notNull().default(isoNow),
|
startedAt: timestampColumn("started_at").notNull().default(isoNow),
|
||||||
completedAt: timestampColumn("completed_at"),
|
completedAt: timestampColumn("completed_at"),
|
||||||
},
|
},
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import { getTableConfig as getSqliteTableConfig } from "drizzle-orm/sqlite-core"
|
|||||||
import { getTableConfig as getPgTableConfig } from "drizzle-orm/pg-core";
|
import { getTableConfig as getPgTableConfig } from "drizzle-orm/pg-core";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import * as sqliteApp from "./app.schema";
|
import * as sqliteApp from "./app.schema";
|
||||||
|
import * as sqliteAudit from "./audit.schema";
|
||||||
import * as sqliteSam from "./sam.schema";
|
import * as sqliteSam from "./sam.schema";
|
||||||
import * as sqliteAuth from "./better-auth-schema";
|
import * as sqliteAuth from "./better-auth-schema";
|
||||||
import * as sqliteBilling from "./billing.schema";
|
import * as sqliteBilling from "./billing.schema";
|
||||||
@ -12,6 +13,7 @@ import * as sqliteGsc from "./gsc.schema";
|
|||||||
import * as sqliteReddit from "./reddit-attribution.schema";
|
import * as sqliteReddit from "./reddit-attribution.schema";
|
||||||
import * as sqliteTelemetry from "./telemetry.schema";
|
import * as sqliteTelemetry from "./telemetry.schema";
|
||||||
import * as pgApp from "./pg/app.schema";
|
import * as pgApp from "./pg/app.schema";
|
||||||
|
import * as pgAudit from "./pg/audit.schema";
|
||||||
import * as pgSam from "./pg/sam.schema";
|
import * as pgSam from "./pg/sam.schema";
|
||||||
import * as pgAuth from "./pg/better-auth-schema";
|
import * as pgAuth from "./pg/better-auth-schema";
|
||||||
import * as pgBilling from "./pg/billing.schema";
|
import * as pgBilling from "./pg/billing.schema";
|
||||||
@ -135,6 +137,7 @@ function foreignKeys(table: Table, dialect: Dialect): string[] {
|
|||||||
|
|
||||||
const sqliteAppTables = tablesFrom(
|
const sqliteAppTables = tablesFrom(
|
||||||
sqliteApp,
|
sqliteApp,
|
||||||
|
sqliteAudit,
|
||||||
sqliteSam,
|
sqliteSam,
|
||||||
sqliteBilling,
|
sqliteBilling,
|
||||||
sqliteGsc,
|
sqliteGsc,
|
||||||
@ -143,6 +146,7 @@ const sqliteAppTables = tablesFrom(
|
|||||||
);
|
);
|
||||||
const pgAppTables = tablesFrom(
|
const pgAppTables = tablesFrom(
|
||||||
pgApp,
|
pgApp,
|
||||||
|
pgAudit,
|
||||||
pgSam,
|
pgSam,
|
||||||
pgBilling,
|
pgBilling,
|
||||||
pgGsc,
|
pgGsc,
|
||||||
|
|||||||
4
src/env.d.ts
vendored
4
src/env.d.ts
vendored
@ -12,6 +12,10 @@ declare namespace Cloudflare {
|
|||||||
// Durable Object backing the SAM in-app agent (see wrangler.jsonc).
|
// Durable Object backing the SAM in-app agent (see wrangler.jsonc).
|
||||||
SAM_CHAT: DurableObjectNamespace;
|
SAM_CHAT: DurableObjectNamespace;
|
||||||
|
|
||||||
|
// Durable Object holding per-audit crawl scratch state (frontier, link
|
||||||
|
// edges, page mirror). Untyped here; getAuditScratchpad narrows the stub.
|
||||||
|
AUDIT_SCRATCHPAD: DurableObjectNamespace;
|
||||||
|
|
||||||
AUTH_MODE?: "cloudflare_access" | "local_noauth" | "hosted";
|
AUTH_MODE?: "cloudflare_access" | "local_noauth" | "hosted";
|
||||||
BYPASS_EMAIL_VERIFICATION?: string;
|
BYPASS_EMAIL_VERIFICATION?: string;
|
||||||
TEAM_DOMAIN?: string;
|
TEAM_DOMAIN?: string;
|
||||||
|
|||||||
@ -16,7 +16,7 @@ import {
|
|||||||
formatStartedAt,
|
formatStartedAt,
|
||||||
HttpStatusBadge,
|
HttpStatusBadge,
|
||||||
StatusBadge,
|
StatusBadge,
|
||||||
SUPPORT_URL,
|
SUPPORT_EMAIL,
|
||||||
} from "@/client/features/audit/shared";
|
} from "@/client/features/audit/shared";
|
||||||
|
|
||||||
export const Route = createFileRoute<"/_project/p/$projectId/audit/">(
|
export const Route = createFileRoute<"/_project/p/$projectId/audit/">(
|
||||||
@ -87,10 +87,13 @@ function AuditDetail({
|
|||||||
const isFailed = statusQuery.data?.status === "failed";
|
const isFailed = statusQuery.data?.status === "failed";
|
||||||
const isRunning = statusQuery.data?.status === "running";
|
const isRunning = statusQuery.data?.status === "running";
|
||||||
|
|
||||||
|
// Failed audits keep whatever pages were crawled before the failure
|
||||||
|
// (persistence is per-batch), so fetch results for them too and show the
|
||||||
|
// partial crawl instead of a dead end.
|
||||||
const resultsQuery = useQuery({
|
const resultsQuery = useQuery({
|
||||||
queryKey: ["audit-results", projectId, auditId],
|
queryKey: ["audit-results", projectId, auditId],
|
||||||
queryFn: () => getAuditResults({ data: { projectId, auditId } }),
|
queryFn: () => getAuditResults({ data: { projectId, auditId } }),
|
||||||
enabled: isComplete,
|
enabled: isComplete || isFailed,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (statusQuery.isLoading) {
|
if (statusQuery.isLoading) {
|
||||||
@ -118,8 +121,15 @@ function AuditDetail({
|
|||||||
}
|
}
|
||||||
|
|
||||||
const status = statusQuery.data;
|
const status = statusQuery.data;
|
||||||
|
const partialPageCount = isFailed
|
||||||
|
? (resultsQuery.data?.pages.length ?? 0)
|
||||||
|
: 0;
|
||||||
|
const failedWithResults = isFailed && partialPageCount > 0;
|
||||||
|
// Wait for the results fetch before choosing between the "partial results"
|
||||||
|
// banner and the zero-page support CTA, so the CTA doesn't flash first.
|
||||||
const showSupportCta =
|
const showSupportCta =
|
||||||
isFailed || (isComplete && status && status.pagesCrawled <= 1);
|
(isFailed && resultsQuery.isSuccess && !failedWithResults) ||
|
||||||
|
(isComplete && status && status.pagesCrawled <= 1);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="px-4 py-4 md:px-6 md:py-6 pb-24 md:pb-8 overflow-auto">
|
<div className="px-4 py-4 md:px-6 md:py-6 pb-24 md:pb-8 overflow-auto">
|
||||||
@ -161,15 +171,12 @@ function AuditDetail({
|
|||||||
Site audit couldn't fully crawl this website.
|
Site audit couldn't fully crawl this website.
|
||||||
</p>
|
</p>
|
||||||
<p>
|
<p>
|
||||||
This is often caused by anti-bot or firewall settings. Reach out
|
This is often caused by anti-bot or firewall settings. Email{" "}
|
||||||
at{" "}
|
|
||||||
<a
|
<a
|
||||||
className="link link-primary"
|
className="link link-primary"
|
||||||
href={SUPPORT_URL}
|
href={`mailto:${SUPPORT_EMAIL}`}
|
||||||
target="_blank"
|
|
||||||
rel="noreferrer"
|
|
||||||
>
|
>
|
||||||
everyapp.dev/support
|
{SUPPORT_EMAIL}
|
||||||
</a>{" "}
|
</a>{" "}
|
||||||
and we'll help configure auditing for your site.
|
and we'll help configure auditing for your site.
|
||||||
</p>
|
</p>
|
||||||
@ -177,7 +184,30 @@ function AuditDetail({
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{isComplete && resultsQuery.data && (
|
{failedWithResults && (
|
||||||
|
<div className="alert alert-warning">
|
||||||
|
<AlertCircle className="size-5" />
|
||||||
|
<div className="space-y-1">
|
||||||
|
<p className="font-medium">
|
||||||
|
This audit stopped early after {partialPageCount} page
|
||||||
|
{partialPageCount === 1 ? "" : "s"}.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
The results below cover everything crawled before it stopped.
|
||||||
|
Run a new audit to try again, or email{" "}
|
||||||
|
<a
|
||||||
|
className="link link-primary"
|
||||||
|
href={`mailto:${SUPPORT_EMAIL}`}
|
||||||
|
>
|
||||||
|
{SUPPORT_EMAIL}
|
||||||
|
</a>{" "}
|
||||||
|
if this keeps happening.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{(isComplete || failedWithResults) && resultsQuery.data && (
|
||||||
<ResultsView
|
<ResultsView
|
||||||
projectId={projectId}
|
projectId={projectId}
|
||||||
data={resultsQuery.data}
|
data={resultsQuery.data}
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import { resolveUserContextFromHeaders } from "@/middleware/ensure-user/resolve"
|
|||||||
import { ProjectRepository } from "@/server/features/projects/repositories/ProjectRepository";
|
import { ProjectRepository } from "@/server/features/projects/repositories/ProjectRepository";
|
||||||
import { SamSessionRepository } from "@/server/features/sam/SamSessionRepository";
|
import { SamSessionRepository } from "@/server/features/sam/SamSessionRepository";
|
||||||
import { runScheduledRankChecks } from "@/server/features/rank-tracking/services/scheduledRankChecks";
|
import { runScheduledRankChecks } from "@/server/features/rank-tracking/services/scheduledRankChecks";
|
||||||
|
import { reconcileStaleAudits } from "@/server/features/audit/services/auditReconciler";
|
||||||
import { getOrCreateOrganizationCustomer } from "@/server/billing/subscription";
|
import { getOrCreateOrganizationCustomer } from "@/server/billing/subscription";
|
||||||
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
|
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
|
||||||
import { getAuthMode, isHostedAuthMode } from "@/lib/auth-mode";
|
import { getAuthMode, isHostedAuthMode } from "@/lib/auth-mode";
|
||||||
@ -177,6 +178,8 @@ export { RankCheckWorkflow } from "./server/workflows/RankCheckWorkflow";
|
|||||||
export { OnboardingChatAgent } from "./server/features/onboarding/OnboardingChatAgent";
|
export { OnboardingChatAgent } from "./server/features/onboarding/OnboardingChatAgent";
|
||||||
// Durable Object class for the SAM in-app agent (Agents SDK).
|
// Durable Object class for the SAM in-app agent (Agents SDK).
|
||||||
export { SamChatAgent } from "./server/features/sam/SamChatAgent";
|
export { SamChatAgent } from "./server/features/sam/SamChatAgent";
|
||||||
|
// Durable Object class for the per-audit crawl scratchpad.
|
||||||
|
export { AuditScratchpad } from "./server/features/audit/AuditScratchpad";
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
fetch,
|
fetch,
|
||||||
@ -187,5 +190,8 @@ export default {
|
|||||||
) {
|
) {
|
||||||
// Scope a per-request Postgres client for the cron run (no-op in D1 mode).
|
// Scope a per-request Postgres client for the cron run (no-op in D1 mode).
|
||||||
await withPgClient(() => runScheduledRankChecks(env));
|
await withPgClient(() => runScheduledRankChecks(env));
|
||||||
|
// Watchdog: reconcile audits stuck in "running" whose workflow died
|
||||||
|
// without reaching mark-failed (OOM/CPU kills, expired instances).
|
||||||
|
await withPgClient(() => reconcileStaleAudits());
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|||||||
372
src/server/features/audit/AuditScratchpad.ts
Normal file
372
src/server/features/audit/AuditScratchpad.ts
Normal file
@ -0,0 +1,372 @@
|
|||||||
|
/**
|
||||||
|
* Per-audit crawl scratchpad — a SQLite-backed Durable Object.
|
||||||
|
*
|
||||||
|
* Holds all transient crawl state for one audit (id = auditId): the frontier
|
||||||
|
* (URL queue + seen-set), the internal-link edges, and a slim mirror of page
|
||||||
|
* rows. This keeps chatty crawl-loop writes off Postgres, removes the ~1MiB
|
||||||
|
* Workflow step-output limits (nothing large flows through step returns
|
||||||
|
* anymore), and gives the finalize link checks a local SQL database.
|
||||||
|
*
|
||||||
|
* Lifecycle: seeded by the discover-urls step, read/written by every crawl
|
||||||
|
* chunk, queried once at finalize, then destroyed on success. A self-cleanup
|
||||||
|
* alarm set on every construction guarantees any instantiation — including a
|
||||||
|
* write racing in after destroy() — is wiped after 7 days (failed audits'
|
||||||
|
* state doubles as the resume/debug artifact until then).
|
||||||
|
*
|
||||||
|
* All methods are synchronous inside (SQLite in DOs is sync), so each RPC is
|
||||||
|
* effectively atomic. Writes are idempotent: the workflow retries steps, so
|
||||||
|
* every insert is OR IGNORE / OR REPLACE on a stable key.
|
||||||
|
*/
|
||||||
|
import { DurableObject, env } from "cloudflare:workers";
|
||||||
|
|
||||||
|
export interface ClaimedUrl {
|
||||||
|
url: string;
|
||||||
|
/** Clicks from the start URL; null when only reachable via sitemap. */
|
||||||
|
depth: number | null;
|
||||||
|
inSitemap: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FrontierStats {
|
||||||
|
/** Pages attempted (crawled or errored) so far. */
|
||||||
|
attempted: number;
|
||||||
|
/** URLs still waiting to be crawled. */
|
||||||
|
pending: number;
|
||||||
|
/** Every URL ever enqueued (attempted + pending + leased). */
|
||||||
|
seen: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ScratchpadPageRow {
|
||||||
|
pageId: string;
|
||||||
|
url: string;
|
||||||
|
statusCode: number;
|
||||||
|
fetchClass: string;
|
||||||
|
redirectUrl: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ScratchpadLinkRow {
|
||||||
|
sourcePageId: string;
|
||||||
|
sourceUrl: string;
|
||||||
|
targetUrl: string;
|
||||||
|
anchor: string | null;
|
||||||
|
isNofollow: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface RecordBatchInput {
|
||||||
|
/** URLs whose crawl attempt finished (successfully or not). */
|
||||||
|
crawledUrls: string[];
|
||||||
|
pages: ScratchpadPageRow[];
|
||||||
|
links: ScratchpadLinkRow[];
|
||||||
|
/** Newly discovered same-origin URLs to enqueue (already policy-filtered). */
|
||||||
|
discovered: Array<{ url: string; depth: number | null }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BrokenLinkRow {
|
||||||
|
sourcePageId: string;
|
||||||
|
sourceUrl: string;
|
||||||
|
targetUrl: string;
|
||||||
|
targetStatus: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface OrphanPageRow {
|
||||||
|
pageId: string;
|
||||||
|
url: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const BROKEN_LINK_ISSUE_CAP = 2_000;
|
||||||
|
const CLEANUP_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
|
||||||
|
/**
|
||||||
|
* Stop storing link edges once the database reaches this size. Link rows are
|
||||||
|
* the only unbounded-per-page data; a pathological link-dense site could
|
||||||
|
* otherwise hit the platform's per-object SQLite cap (1 GB on the free plan
|
||||||
|
* self-hosters may run on) and fail the crawl with SQLITE_FULL. Past the
|
||||||
|
* budget the crawl continues — the audit just loses link-graph issues.
|
||||||
|
*/
|
||||||
|
const LINK_STORAGE_BUDGET_BYTES = 500 * 1024 * 1024;
|
||||||
|
|
||||||
|
export class AuditScratchpad extends DurableObject {
|
||||||
|
constructor(ctx: DurableObjectState, workerEnv: Env) {
|
||||||
|
super(ctx, workerEnv);
|
||||||
|
this.ctx.storage.sql.exec(`
|
||||||
|
CREATE TABLE IF NOT EXISTS frontier (
|
||||||
|
url TEXT PRIMARY KEY,
|
||||||
|
depth INTEGER,
|
||||||
|
source TEXT NOT NULL,
|
||||||
|
in_sitemap INTEGER NOT NULL DEFAULT 0,
|
||||||
|
state TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
chunk_no INTEGER
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS frontier_claim_idx ON frontier (state, source);
|
||||||
|
CREATE TABLE IF NOT EXISTS links (
|
||||||
|
source_page_id TEXT NOT NULL,
|
||||||
|
source_url TEXT NOT NULL,
|
||||||
|
target_url TEXT NOT NULL,
|
||||||
|
anchor TEXT,
|
||||||
|
is_nofollow INTEGER NOT NULL DEFAULT 0,
|
||||||
|
PRIMARY KEY (source_page_id, target_url)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS links_target_idx ON links (target_url);
|
||||||
|
CREATE TABLE IF NOT EXISTS page_mirror (
|
||||||
|
page_id TEXT PRIMARY KEY,
|
||||||
|
url TEXT NOT NULL UNIQUE,
|
||||||
|
status_code INTEGER,
|
||||||
|
fetch_class TEXT NOT NULL,
|
||||||
|
redirect_url TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS page_mirror_redirect_idx ON page_mirror (redirect_url);
|
||||||
|
`);
|
||||||
|
// Guarantee the cleanup alarm on EVERY instantiation, not just at seed:
|
||||||
|
// any RPC (even one racing in right after destroy()) re-creates the
|
||||||
|
// tables above, and without an alarm that storage would leak forever.
|
||||||
|
// blockConcurrencyWhile gates RPC delivery on its own; the returned
|
||||||
|
// promise doesn't need observing (constructors can't await).
|
||||||
|
void this.ctx.blockConcurrencyWhile(() => this.ensureCleanupAlarm());
|
||||||
|
}
|
||||||
|
|
||||||
|
async seedStart(url: string): Promise<void> {
|
||||||
|
this.ctx.storage.sql.exec(
|
||||||
|
`INSERT OR IGNORE INTO frontier (url, depth, source, in_sitemap) VALUES (?, 0, 'link', 0)`,
|
||||||
|
url,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async seedSitemapUrls(urls: string[]): Promise<void> {
|
||||||
|
for (const url of urls) {
|
||||||
|
// Upsert so a URL that already exists (e.g. the start URL) still gets
|
||||||
|
// its in-sitemap flag; queue position and depth stay as first seen.
|
||||||
|
this.ctx.storage.sql.exec(
|
||||||
|
`INSERT INTO frontier (url, depth, source, in_sitemap) VALUES (?, NULL, 'sitemap', 1)
|
||||||
|
ON CONFLICT(url) DO UPDATE SET in_sitemap = 1`,
|
||||||
|
url,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lease the next batch of pending URLs for one crawl chunk. Idempotent per
|
||||||
|
* chunkNo: a retried step gets back exactly the URLs its failed attempt
|
||||||
|
* had leased (link-discovered URLs drain before sitemap-only ones, FIFO
|
||||||
|
* within each class — same ordering as the old in-memory queues).
|
||||||
|
*/
|
||||||
|
async claimChunk(chunkNo: number, limit: number): Promise<ClaimedUrl[]> {
|
||||||
|
const existing = this.selectClaimed(
|
||||||
|
`SELECT url, depth, in_sitemap FROM frontier WHERE state = 'leased' AND chunk_no = ?`,
|
||||||
|
chunkNo,
|
||||||
|
);
|
||||||
|
if (existing.length > 0) return existing;
|
||||||
|
// A retried step whose earlier attempt already crawled this chunk's
|
||||||
|
// leases must not claim a fresh set under the same chunk number — that
|
||||||
|
// would duplicate work and overshoot the page budget.
|
||||||
|
const done = this.ctx.storage.sql
|
||||||
|
.exec<{
|
||||||
|
n: number;
|
||||||
|
}>(
|
||||||
|
`SELECT COUNT(*) AS n FROM frontier WHERE chunk_no = ? AND state = 'crawled'`,
|
||||||
|
chunkNo,
|
||||||
|
)
|
||||||
|
.one();
|
||||||
|
if (done.n > 0) return [];
|
||||||
|
if (limit <= 0) return [];
|
||||||
|
|
||||||
|
const fresh = this.selectClaimed(
|
||||||
|
`SELECT url, depth, in_sitemap FROM frontier WHERE state = 'pending'
|
||||||
|
ORDER BY CASE source WHEN 'link' THEN 0 ELSE 1 END, rowid LIMIT ?`,
|
||||||
|
limit,
|
||||||
|
);
|
||||||
|
for (const row of fresh) {
|
||||||
|
this.ctx.storage.sql.exec(
|
||||||
|
`UPDATE frontier SET state = 'leased', chunk_no = ? WHERE url = ?`,
|
||||||
|
chunkNo,
|
||||||
|
row.url,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return fresh;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Persist one crawled sub-batch: completions, mirror rows, links, frontier. */
|
||||||
|
async recordBatch(input: RecordBatchInput): Promise<FrontierStats> {
|
||||||
|
for (const url of input.crawledUrls) {
|
||||||
|
this.ctx.storage.sql.exec(
|
||||||
|
`UPDATE frontier SET state = 'crawled' WHERE url = ?`,
|
||||||
|
url,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const page of input.pages) {
|
||||||
|
this.ctx.storage.sql.exec(
|
||||||
|
`INSERT OR REPLACE INTO page_mirror (page_id, url, status_code, fetch_class, redirect_url)
|
||||||
|
VALUES (?, ?, ?, ?, ?)`,
|
||||||
|
page.pageId,
|
||||||
|
page.url,
|
||||||
|
page.statusCode,
|
||||||
|
page.fetchClass,
|
||||||
|
page.redirectUrl,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Database size only ever grows (deletes happen solely via destroy), so
|
||||||
|
// once the budget trips it stays tripped — runFinalizeChecks uses the
|
||||||
|
// same comparison to know the link graph is incomplete.
|
||||||
|
if (this.ctx.storage.sql.databaseSize < LINK_STORAGE_BUDGET_BYTES) {
|
||||||
|
for (const link of input.links) {
|
||||||
|
this.ctx.storage.sql.exec(
|
||||||
|
`INSERT OR IGNORE INTO links (source_page_id, source_url, target_url, anchor, is_nofollow)
|
||||||
|
VALUES (?, ?, ?, ?, ?)`,
|
||||||
|
link.sourcePageId,
|
||||||
|
link.sourceUrl,
|
||||||
|
link.targetUrl,
|
||||||
|
link.anchor,
|
||||||
|
link.isNofollow ? 1 : 0,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const found of input.discovered) {
|
||||||
|
// OR IGNORE: already-seen URLs (crawled, leased, or pending) keep
|
||||||
|
// their existing row — this is the global dedup.
|
||||||
|
this.ctx.storage.sql.exec(
|
||||||
|
`INSERT OR IGNORE INTO frontier (url, depth, source, in_sitemap) VALUES (?, ?, 'link', 0)`,
|
||||||
|
found.url,
|
||||||
|
found.depth,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return this.stats();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return unattempted leases to the queue (chunk soft-deadline hit). */
|
||||||
|
async releaseUrls(urls: string[]): Promise<void> {
|
||||||
|
for (const url of urls) {
|
||||||
|
this.ctx.storage.sql.exec(
|
||||||
|
`UPDATE frontier SET state = 'pending', chunk_no = NULL WHERE url = ? AND state = 'leased'`,
|
||||||
|
url,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getStats(): Promise<FrontierStats> {
|
||||||
|
return this.stats();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The two finalize checks that need link edges, as local SQL. Mirrors the
|
||||||
|
* former Postgres implementations in multipage.ts exactly.
|
||||||
|
*/
|
||||||
|
async runFinalizeChecks(input: {
|
||||||
|
startUrl: string;
|
||||||
|
crawlCompleted: boolean;
|
||||||
|
}): Promise<{ brokenLinks: BrokenLinkRow[]; orphanPages: OrphanPageRow[] }> {
|
||||||
|
// Only flag targets we actually crawled and saw fail — never inferred
|
||||||
|
// from absence. Blocked targets (WAF challenges) are excluded: a 403
|
||||||
|
// from bot protection is not evidence of a broken link.
|
||||||
|
const brokenLinks = this.ctx.storage.sql
|
||||||
|
.exec<{
|
||||||
|
source_page_id: string;
|
||||||
|
source_url: string;
|
||||||
|
target_url: string;
|
||||||
|
target_status: number;
|
||||||
|
}>(
|
||||||
|
`SELECT l.source_page_id, l.source_url, l.target_url, m.status_code AS target_status
|
||||||
|
FROM links l JOIN page_mirror m ON m.url = l.target_url
|
||||||
|
WHERE m.status_code >= 400 AND m.fetch_class = 'ok'
|
||||||
|
ORDER BY l.source_page_id, l.target_url
|
||||||
|
LIMIT ?`,
|
||||||
|
BROKEN_LINK_ISSUE_CAP,
|
||||||
|
)
|
||||||
|
.toArray()
|
||||||
|
.map((row) => ({
|
||||||
|
sourcePageId: row.source_page_id,
|
||||||
|
sourceUrl: row.source_url,
|
||||||
|
targetUrl: row.target_url,
|
||||||
|
targetStatus: row.target_status,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// A live 2xx page is an orphan when no OTHER crawled page links to it
|
||||||
|
// and nothing redirects to it. Only meaningful on a completed crawl with
|
||||||
|
// a complete link graph — if the storage budget truncated link writes,
|
||||||
|
// "no inbound edge" is missing data, not evidence of orphanhood.
|
||||||
|
const linkGraphComplete =
|
||||||
|
this.ctx.storage.sql.databaseSize < LINK_STORAGE_BUDGET_BYTES;
|
||||||
|
const orphanPages =
|
||||||
|
input.crawlCompleted && linkGraphComplete
|
||||||
|
? this.ctx.storage.sql
|
||||||
|
.exec<{ page_id: string; url: string }>(
|
||||||
|
`SELECT m.page_id, m.url FROM page_mirror m
|
||||||
|
WHERE m.url != ?
|
||||||
|
AND m.fetch_class = 'ok'
|
||||||
|
AND m.status_code >= 200 AND m.status_code < 300
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM links l
|
||||||
|
WHERE l.target_url = m.url AND l.source_page_id != m.page_id
|
||||||
|
)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM page_mirror r WHERE r.redirect_url = m.url
|
||||||
|
)`,
|
||||||
|
input.startUrl,
|
||||||
|
)
|
||||||
|
.toArray()
|
||||||
|
.map((row) => ({ pageId: row.page_id, url: row.url }))
|
||||||
|
: [];
|
||||||
|
|
||||||
|
return { brokenLinks, orphanPages };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Wipe all state (success path, or explicit audit deletion). */
|
||||||
|
async destroy(): Promise<void> {
|
||||||
|
await this.ctx.storage.deleteAlarm();
|
||||||
|
await this.ctx.storage.deleteAll();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Self-cleanup for audits whose workflow died without reaching finalize.
|
||||||
|
* Full destroy(), not just deleteAll(): under our compatibility date,
|
||||||
|
* deleteAll() does not clear alarm state, and a leftover alarm would keep
|
||||||
|
* the object (and its billing) alive.
|
||||||
|
*/
|
||||||
|
async alarm(): Promise<void> {
|
||||||
|
await this.destroy();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async ensureCleanupAlarm(): Promise<void> {
|
||||||
|
const existing = await this.ctx.storage.getAlarm();
|
||||||
|
if (existing === null) {
|
||||||
|
await this.ctx.storage.setAlarm(Date.now() + CLEANUP_AFTER_MS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private selectClaimed(query: string, param: number): ClaimedUrl[] {
|
||||||
|
return this.ctx.storage.sql
|
||||||
|
.exec<{ url: string; depth: number | null; in_sitemap: number }>(
|
||||||
|
query,
|
||||||
|
param,
|
||||||
|
)
|
||||||
|
.toArray()
|
||||||
|
.map((row) => ({
|
||||||
|
url: row.url,
|
||||||
|
depth: row.depth,
|
||||||
|
inSitemap: row.in_sitemap === 1,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private stats(): FrontierStats {
|
||||||
|
const row = this.ctx.storage.sql
|
||||||
|
.exec<{ attempted: number; pending: number; seen: number }>(
|
||||||
|
`SELECT
|
||||||
|
COUNT(*) FILTER (WHERE state = 'crawled') AS attempted,
|
||||||
|
COUNT(*) FILTER (WHERE state = 'pending') AS pending,
|
||||||
|
COUNT(*) AS seen
|
||||||
|
FROM frontier`,
|
||||||
|
)
|
||||||
|
.one();
|
||||||
|
return {
|
||||||
|
attempted: row.attempted,
|
||||||
|
pending: row.pending,
|
||||||
|
seen: row.seen,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stub for the audit's scratchpad DO (one instance per audit id). */
|
||||||
|
export function getAuditScratchpad(auditId: string) {
|
||||||
|
// env.d.ts declares the binding untyped (ambient contexts can't import the
|
||||||
|
// class); narrow here so callers get typed RPC methods.
|
||||||
|
const namespace =
|
||||||
|
// oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the binding is declared as this class in wrangler.jsonc
|
||||||
|
env.AUDIT_SCRATCHPAD as unknown as DurableObjectNamespace<AuditScratchpad>;
|
||||||
|
return namespace.get(namespace.idFromName(auditId));
|
||||||
|
}
|
||||||
@ -1,18 +1,17 @@
|
|||||||
/**
|
/**
|
||||||
* Data access layer for site audit tables.
|
* Data access layer for site audit tables.
|
||||||
* Provider-aware (D1 or Postgres) via the `@/db` handle. Covers audits,
|
* Provider-aware (D1 or Postgres) via the `@/db` handle. Covers audits,
|
||||||
* audit_pages, audit_links, audit_issues, and stored Lighthouse results.
|
* audit_pages, audit_issues, and stored Lighthouse results. Link edges live
|
||||||
|
* in the per-audit scratchpad Durable Object, not here.
|
||||||
*/
|
*/
|
||||||
import { and, desc, eq } from "drizzle-orm";
|
import { and, count, desc, eq } from "drizzle-orm";
|
||||||
import { db } from "@/db";
|
import { db } from "@/db";
|
||||||
import {
|
import {
|
||||||
audits,
|
audits,
|
||||||
auditIssues,
|
auditIssues,
|
||||||
auditLighthouseResults,
|
auditLighthouseResults,
|
||||||
auditLinks,
|
|
||||||
auditPages,
|
auditPages,
|
||||||
} from "@/db/schema";
|
} from "@/db/schema";
|
||||||
import { getDatabaseProvider } from "@/db/provider";
|
|
||||||
import { executeInBatches } from "@/db/runBatch";
|
import { executeInBatches } from "@/db/runBatch";
|
||||||
import { AUDIT_ISSUE_TYPES } from "@/shared/audit-issues";
|
import { AUDIT_ISSUE_TYPES } from "@/shared/audit-issues";
|
||||||
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
|
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
|
||||||
@ -23,14 +22,6 @@ import type {
|
|||||||
LighthouseResult,
|
LighthouseResult,
|
||||||
} from "@/server/lib/audit/types";
|
} from "@/server/lib/audit/types";
|
||||||
|
|
||||||
// Only internal links are stored: both consumers (broken-internal-link and
|
|
||||||
// orphan checks) filter on isInternal, and per-page external counts already
|
|
||||||
// live on audit_pages. External rows come back when P1 adds external-link
|
|
||||||
// checks. Mega-menu/footer-heavy sites can carry 1000+ links per page; cap
|
|
||||||
// what we store so a 10k-page crawl can't write tens of millions of link rows.
|
|
||||||
const MAX_STORED_LINKS_PER_PAGE = 500;
|
|
||||||
const POSTGRES_LINK_INSERT_SIZE = 500;
|
|
||||||
|
|
||||||
async function createAudit(data: {
|
async function createAudit(data: {
|
||||||
id: string;
|
id: string;
|
||||||
projectId: string;
|
projectId: string;
|
||||||
@ -102,7 +93,15 @@ async function completeAudit(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function failAudit(auditId: string, workflowInstanceId: string) {
|
async function failAudit(
|
||||||
|
auditId: string,
|
||||||
|
workflowInstanceId: string,
|
||||||
|
errorInfo?: {
|
||||||
|
errorCode: string;
|
||||||
|
errorDetail: string;
|
||||||
|
failedPhase: string | null;
|
||||||
|
},
|
||||||
|
) {
|
||||||
// Only a running audit can transition to failed: the getStatus reconciler
|
// Only a running audit can transition to failed: the getStatus reconciler
|
||||||
// races the workflow's own finalize, and without this guard it could flip
|
// races the workflow's own finalize, and without this guard it could flip
|
||||||
// a just-completed audit to failed.
|
// a just-completed audit to failed.
|
||||||
@ -112,6 +111,11 @@ async function failAudit(auditId: string, workflowInstanceId: string) {
|
|||||||
status: "failed",
|
status: "failed",
|
||||||
completedAt: new Date().toISOString(),
|
completedAt: new Date().toISOString(),
|
||||||
currentPhase: "failed",
|
currentPhase: "failed",
|
||||||
|
...(errorInfo && {
|
||||||
|
errorCode: errorInfo.errorCode,
|
||||||
|
errorDetail: errorInfo.errorDetail,
|
||||||
|
failedPhase: errorInfo.failedPhase,
|
||||||
|
}),
|
||||||
})
|
})
|
||||||
.where(
|
.where(
|
||||||
and(
|
and(
|
||||||
@ -135,15 +139,15 @@ async function getAuditForWorkflow(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Persist one crawl batch (pages + link edges + per-page issues).
|
* Persist one crawled sub-batch (pages + per-page issues). Called inside the
|
||||||
* Called inside the crawl-batch Workflow step so results land in D1
|
* crawl-chunk Workflow step so results land in the app DB incrementally
|
||||||
* incrementally instead of accumulating in memory until finalize.
|
* instead of accumulating in memory until finalize. Link edges go to the
|
||||||
|
* audit's scratchpad DO, not here.
|
||||||
*
|
*
|
||||||
* Idempotent on step retry: callers assign deterministic page ids
|
* Idempotent on step retry: callers assign deterministic page ids
|
||||||
* (deterministicAuditRowId) and link/issue ids are derived from stable
|
* (deterministicAuditRowId) and issue ids are derived from stable content.
|
||||||
* content. Page rows upsert (a retried fetch may legitimately differ — last
|
* Page rows upsert (a retried fetch may legitimately differ — last attempt
|
||||||
* attempt wins, matching what the step returns); links and issues are
|
* wins); issues are insert-or-ignore.
|
||||||
* insert-or-ignore.
|
|
||||||
*/
|
*/
|
||||||
async function insertCrawledBatch(
|
async function insertCrawledBatch(
|
||||||
auditId: string,
|
auditId: string,
|
||||||
@ -192,38 +196,6 @@ async function insertCrawledBatch(
|
|||||||
.onConflictDoUpdate({ target: auditPages.id, set: dataColumns });
|
.onConflictDoUpdate({ target: auditPages.id, set: dataColumns });
|
||||||
});
|
});
|
||||||
|
|
||||||
const linkRows = await Promise.all(
|
|
||||||
pages.flatMap((page) =>
|
|
||||||
page.links
|
|
||||||
.filter((link) => link.isInternal)
|
|
||||||
.slice(0, MAX_STORED_LINKS_PER_PAGE)
|
|
||||||
.map(async (link) => ({
|
|
||||||
id: await deterministicAuditRowId(auditId, page.url, link.targetUrl),
|
|
||||||
auditId,
|
|
||||||
sourcePageId: page.id,
|
|
||||||
sourceUrl: page.url,
|
|
||||||
targetUrl: link.targetUrl,
|
|
||||||
anchor: link.anchor,
|
|
||||||
isInternal: link.isInternal,
|
|
||||||
isNofollow: link.isNofollow,
|
|
||||||
})),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
if (getDatabaseProvider() === "postgres") {
|
|
||||||
// A Postgres transaction executes runBatch statements sequentially. Bulk
|
|
||||||
// values avoid thousands of Hyperdrive round trips on link-heavy pages.
|
|
||||||
for (let i = 0; i < linkRows.length; i += POSTGRES_LINK_INSERT_SIZE) {
|
|
||||||
await db
|
|
||||||
.insert(auditLinks)
|
|
||||||
.values(linkRows.slice(i, i + POSTGRES_LINK_INSERT_SIZE))
|
|
||||||
.onConflictDoNothing();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
await executeInBatches(linkRows, (tx, row) =>
|
|
||||||
tx.insert(auditLinks).values(row).onConflictDoNothing(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
await insertIssues(auditId, issues);
|
await insertIssues(auditId, issues);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -336,6 +308,19 @@ async function getPagesForAudit(auditId: string) {
|
|||||||
.where(eq(auditPages.auditId, auditId));
|
.where(eq(auditPages.auditId, auditId));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function countBlockedPages(auditId: string): Promise<number> {
|
||||||
|
const rows = await db
|
||||||
|
.select({ blocked: count() })
|
||||||
|
.from(auditPages)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(auditPages.auditId, auditId),
|
||||||
|
eq(auditPages.fetchClass, "blocked"),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return rows[0]?.blocked ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
async function hasPagesForAudit(auditId: string): Promise<boolean> {
|
async function hasPagesForAudit(auditId: string): Promise<boolean> {
|
||||||
const rows = await db
|
const rows = await db
|
||||||
.select({ id: auditPages.id })
|
.select({ id: auditPages.id })
|
||||||
@ -449,6 +434,7 @@ export const AuditRepository = {
|
|||||||
getLatestAuditForProject,
|
getLatestAuditForProject,
|
||||||
getIssuesForAudit,
|
getIssuesForAudit,
|
||||||
getPagesForAudit,
|
getPagesForAudit,
|
||||||
|
countBlockedPages,
|
||||||
hasPagesForAudit,
|
hasPagesForAudit,
|
||||||
getAuditsByProject,
|
getAuditsByProject,
|
||||||
getAuditUsageForUser,
|
getAuditUsageForUser,
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import {
|
|||||||
type BillingCustomerContext,
|
type BillingCustomerContext,
|
||||||
} from "@/server/billing/subscription";
|
} from "@/server/billing/subscription";
|
||||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||||
|
import { getAuditScratchpad } from "@/server/features/audit/AuditScratchpad";
|
||||||
import {
|
import {
|
||||||
AUDIT_LIMITS,
|
AUDIT_LIMITS,
|
||||||
clampAuditMaxPages,
|
clampAuditMaxPages,
|
||||||
@ -18,7 +19,11 @@ import {
|
|||||||
type AuditConfig,
|
type AuditConfig,
|
||||||
type LighthouseStrategy,
|
type LighthouseStrategy,
|
||||||
} from "@/server/lib/audit/types";
|
} from "@/server/lib/audit/types";
|
||||||
import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
|
import {
|
||||||
|
normalizeAndValidateStartUrl,
|
||||||
|
resolveStartUrlRedirects,
|
||||||
|
} from "@/server/lib/audit/url-policy";
|
||||||
|
import { reconcileRunningAudit } from "@/server/features/audit/services/auditReconciler";
|
||||||
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
|
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
|
||||||
|
|
||||||
// Plan-tier limits are the abuse bound in hosted mode: free accounts get one
|
// Plan-tier limits are the abuse bound in hosted mode: free accounts get one
|
||||||
@ -61,7 +66,12 @@ async function startAudit(input: {
|
|||||||
|
|
||||||
const auditId = crypto.randomUUID();
|
const auditId = crypto.randomUUID();
|
||||||
const config: AuditConfig = { maxPages, lighthouseStrategy };
|
const config: AuditConfig = { maxPages, lighthouseStrategy };
|
||||||
const startUrl = await normalizeAndValidateStartUrl(input.startUrl);
|
// Anchor the audit to the site's real origin: a start domain that 301s
|
||||||
|
// elsewhere (…net -> …com, apex -> www) would otherwise dead-end after
|
||||||
|
// one page at the same-origin crawl boundary.
|
||||||
|
const startUrl = await resolveStartUrlRedirects(
|
||||||
|
await normalizeAndValidateStartUrl(input.startUrl),
|
||||||
|
);
|
||||||
|
|
||||||
await AuditRepository.createAudit({
|
await AuditRepository.createAudit({
|
||||||
id: auditId,
|
id: auditId,
|
||||||
@ -125,22 +135,13 @@ async function getStatus(auditId: string, projectId: string) {
|
|||||||
throw new AppError("NOT_FOUND", "Audit not found in this project.");
|
throw new AppError("NOT_FOUND", "Audit not found in this project.");
|
||||||
|
|
||||||
// Self-heal audits whose workflow died without reaching the mark-failed
|
// Self-heal audits whose workflow died without reaching the mark-failed
|
||||||
// step (instance terminated, mark-failed itself failed, deploys, ...).
|
// step (instance terminated/errored, instance expired from retention, ...).
|
||||||
// Without this they stay "running" forever and hold capacity.
|
// Without this they stay "running" forever and hold capacity.
|
||||||
if (audit.status === "running" && audit.workflowInstanceId) {
|
if (audit.status === "running") {
|
||||||
try {
|
const reconciled = await reconcileRunningAudit(audit);
|
||||||
const instance = await env.SITE_AUDIT_WORKFLOW.get(
|
if (reconciled) {
|
||||||
audit.workflowInstanceId,
|
|
||||||
);
|
|
||||||
const { status } = await instance.status();
|
|
||||||
if (status === "errored" || status === "terminated") {
|
|
||||||
await AuditRepository.failAudit(audit.id, audit.workflowInstanceId);
|
|
||||||
audit =
|
audit =
|
||||||
(await AuditRepository.getAuditForProject(auditId, projectId)) ??
|
(await AuditRepository.getAuditForProject(auditId, projectId)) ?? audit;
|
||||||
audit;
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Instance not found or status unavailable — leave the audit as-is.
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -154,6 +155,7 @@ async function getStatus(auditId: string, projectId: string) {
|
|||||||
lighthouseCompleted: audit.lighthouseCompleted,
|
lighthouseCompleted: audit.lighthouseCompleted,
|
||||||
lighthouseFailed: audit.lighthouseFailed,
|
lighthouseFailed: audit.lighthouseFailed,
|
||||||
currentPhase: audit.currentPhase,
|
currentPhase: audit.currentPhase,
|
||||||
|
errorCode: audit.errorCode,
|
||||||
startedAt: audit.startedAt,
|
startedAt: audit.startedAt,
|
||||||
completedAt: audit.completedAt,
|
completedAt: audit.completedAt,
|
||||||
};
|
};
|
||||||
@ -257,6 +259,13 @@ async function remove(auditId: string, projectId: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
await AuditRepository.deleteAuditForProject(auditId, projectId);
|
await AuditRepository.deleteAuditForProject(auditId, projectId);
|
||||||
|
// Best-effort: drop the crawl scratchpad DO with the audit. A missed
|
||||||
|
// destroy self-cleans via the DO's 7-day alarm.
|
||||||
|
try {
|
||||||
|
await getAuditScratchpad(auditId).destroy();
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(`Failed to destroy audit scratchpad ${auditId}:`, error);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const AuditService = {
|
export const AuditService = {
|
||||||
|
|||||||
176
src/server/features/audit/services/auditReconciler.ts
Normal file
176
src/server/features/audit/services/auditReconciler.ts
Normal file
@ -0,0 +1,176 @@
|
|||||||
|
/**
|
||||||
|
* Reconciles audits stuck in "running" with the actual state of their
|
||||||
|
* Cloudflare Workflow instance.
|
||||||
|
*
|
||||||
|
* A workflow killed by the platform (OOM, CPU limit, deploy reset) never
|
||||||
|
* reaches its own mark-failed step, and an instance can expire from Workflows
|
||||||
|
* retention entirely. Two callers close that gap:
|
||||||
|
* - AuditService.getStatus (lazy: whenever the UI polls a running audit)
|
||||||
|
* - the scheduled watchdog (reconcileStaleAudits, every cron tick) — so
|
||||||
|
* zombie rows die even if nobody ever reopens the page, and the instance
|
||||||
|
* error is copied into the row before Workflows retention deletes it.
|
||||||
|
*/
|
||||||
|
import { env } from "cloudflare:workers";
|
||||||
|
import { and, eq, lt } from "drizzle-orm";
|
||||||
|
import { db } from "@/db";
|
||||||
|
import { audits, projects } from "@/db/schema";
|
||||||
|
import { getDatabaseProvider } from "@/db/provider";
|
||||||
|
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||||
|
import {
|
||||||
|
classifyAuditError,
|
||||||
|
type AuditErrorInfo,
|
||||||
|
} from "@/server/lib/audit/audit-errors";
|
||||||
|
import { captureServerEvent } from "@/server/lib/posthog";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Don't declare an instance "lost" until the audit is comfortably older than
|
||||||
|
* any legitimate create/start delay: startAudit inserts the row before
|
||||||
|
* creating the workflow, so a brand-new audit can briefly have no instance.
|
||||||
|
*/
|
||||||
|
const INSTANCE_LOST_GRACE_MS = 10 * 60 * 1000;
|
||||||
|
|
||||||
|
/** Audits still "running" after this long get reconciled by the watchdog. */
|
||||||
|
const STALE_RUNNING_AFTER_MS = 15 * 60 * 1000;
|
||||||
|
|
||||||
|
const WATCHDOG_BATCH_LIMIT = 100;
|
||||||
|
|
||||||
|
type RunningAudit = {
|
||||||
|
id: string;
|
||||||
|
workflowInstanceId: string | null;
|
||||||
|
startedAt: string;
|
||||||
|
currentPhase: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If the audit's workflow instance is dead (errored/terminated) or gone,
|
||||||
|
* flip the row to failed with a classified error. Returns the error info
|
||||||
|
* when the row was flipped, null when the audit is genuinely still running.
|
||||||
|
*/
|
||||||
|
export async function reconcileRunningAudit(
|
||||||
|
audit: RunningAudit,
|
||||||
|
): Promise<AuditErrorInfo | null> {
|
||||||
|
if (!audit.workflowInstanceId) return null;
|
||||||
|
|
||||||
|
let errorInfo: AuditErrorInfo | null = null;
|
||||||
|
try {
|
||||||
|
const instance = await env.SITE_AUDIT_WORKFLOW.get(
|
||||||
|
audit.workflowInstanceId,
|
||||||
|
);
|
||||||
|
const status = await instance.status();
|
||||||
|
if (status.status === "errored" || status.status === "terminated") {
|
||||||
|
errorInfo = status.error
|
||||||
|
? classifyAuditError(
|
||||||
|
typeof status.error === "string"
|
||||||
|
? status.error
|
||||||
|
: (status.error.message ?? JSON.stringify(status.error)),
|
||||||
|
)
|
||||||
|
: {
|
||||||
|
errorCode: "unknown",
|
||||||
|
errorDetail: `Workflow instance ${status.status}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// Instance not found (never created, or expired from Workflows
|
||||||
|
// retention). Past the grace window that means the audit can never
|
||||||
|
// finish — without this branch such rows stay "running" forever.
|
||||||
|
// Transient status/control-plane errors must NOT fail a live audit,
|
||||||
|
// so only a confirmed not-found counts.
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
if (!/not[ _]?found/i.test(message)) return null;
|
||||||
|
if (!isOlderThan(audit.startedAt, INSTANCE_LOST_GRACE_MS)) return null;
|
||||||
|
errorInfo = {
|
||||||
|
errorCode: "instance_lost",
|
||||||
|
errorDetail: "Workflow instance not found",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (!errorInfo) return null;
|
||||||
|
|
||||||
|
await AuditRepository.failAudit(audit.id, audit.workflowInstanceId, {
|
||||||
|
...errorInfo,
|
||||||
|
failedPhase: audit.currentPhase,
|
||||||
|
});
|
||||||
|
return errorInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Cron watchdog: sweep stale running audits and reconcile each. */
|
||||||
|
export async function reconcileStaleAudits() {
|
||||||
|
const cutoff = new Date(Date.now() - STALE_RUNNING_AFTER_MS);
|
||||||
|
const stale = await getStaleRunningAudits(cutoff, WATCHDOG_BATCH_LIMIT);
|
||||||
|
|
||||||
|
for (const audit of stale) {
|
||||||
|
try {
|
||||||
|
const errorInfo = await reconcileRunningAudit(audit);
|
||||||
|
if (!errorInfo) continue;
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`Audit watchdog: marked ${audit.id} failed (${errorInfo.errorCode}, phase=${audit.currentPhase})`,
|
||||||
|
);
|
||||||
|
const project = await db.query.projects.findFirst({
|
||||||
|
where: eq(projects.id, audit.projectId),
|
||||||
|
});
|
||||||
|
if (!project) continue;
|
||||||
|
await captureServerEvent({
|
||||||
|
distinctId: audit.startedByUserId,
|
||||||
|
event: "site_audit:complete",
|
||||||
|
organizationId: project.organizationId,
|
||||||
|
properties: {
|
||||||
|
project_id: audit.projectId,
|
||||||
|
status: "failed",
|
||||||
|
reconciled_by: "watchdog",
|
||||||
|
error_code: errorInfo.errorCode,
|
||||||
|
pages_crawled: audit.pagesCrawled,
|
||||||
|
pages_total: audit.pagesTotal,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
// One unreconcilable audit must not stop the sweep.
|
||||||
|
console.error(`Audit watchdog: failed to reconcile ${audit.id}:`, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Running audits started before the cutoff — watchdog candidates. Timestamps
|
||||||
|
* are stored as text, so this compares lexicographically: PG rows use ISO
|
||||||
|
* with a "T", D1-default rows use "YYYY-MM-DD HH:MM:SS" (space < "T", so a
|
||||||
|
* same-day ISO cutoff would misorder against D1 rows without the reformat).
|
||||||
|
*/
|
||||||
|
async function getStaleRunningAudits(cutoff: Date, limit: number) {
|
||||||
|
const iso = cutoff.toISOString();
|
||||||
|
const startedBefore =
|
||||||
|
getDatabaseProvider() === "postgres"
|
||||||
|
? iso
|
||||||
|
: iso.replace("T", " ").slice(0, 19);
|
||||||
|
// Oldest first: genuinely dead audits age past the cutoff and stay there,
|
||||||
|
// while long-but-live crawls are the newest of the stale set — without a
|
||||||
|
// deterministic order they could occupy the whole batch every sweep and
|
||||||
|
// starve real zombies.
|
||||||
|
return db.query.audits.findMany({
|
||||||
|
where: and(
|
||||||
|
eq(audits.status, "running"),
|
||||||
|
lt(audits.startedAt, startedBefore),
|
||||||
|
),
|
||||||
|
orderBy: audits.startedAt,
|
||||||
|
columns: {
|
||||||
|
id: true,
|
||||||
|
workflowInstanceId: true,
|
||||||
|
projectId: true,
|
||||||
|
startedByUserId: true,
|
||||||
|
startedAt: true,
|
||||||
|
currentPhase: true,
|
||||||
|
pagesCrawled: true,
|
||||||
|
pagesTotal: true,
|
||||||
|
},
|
||||||
|
limit,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function isOlderThan(startedAt: string, ageMs: number): boolean {
|
||||||
|
// D1-default timestamps ("YYYY-MM-DD HH:MM:SS") lack the T/Z; normalize so
|
||||||
|
// Date.parse reads them as UTC, matching the PG ISO format.
|
||||||
|
const parsed = Date.parse(
|
||||||
|
startedAt.includes("T") ? startedAt : `${startedAt.replace(" ", "T")}Z`,
|
||||||
|
);
|
||||||
|
if (Number.isNaN(parsed)) return true;
|
||||||
|
return parsed < Date.now() - ageMs;
|
||||||
|
}
|
||||||
79
src/server/lib/audit/audit-errors.ts
Normal file
79
src/server/lib/audit/audit-errors.ts
Normal file
@ -0,0 +1,79 @@
|
|||||||
|
/**
|
||||||
|
* Classification of audit failures into a closed error-code vocabulary.
|
||||||
|
*
|
||||||
|
* Codes are written to audits.error_code by the workflow's mark-failed step
|
||||||
|
* and by the stale-audit reconciler (which reads the message off the dead
|
||||||
|
* Workflows instance). Keeping the vocabulary closed makes failures
|
||||||
|
* aggregable in SQL and lets the UI map codes to friendly copy without
|
||||||
|
* leaking raw infrastructure errors to users.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const AUDIT_ERROR_CODES = [
|
||||||
|
// A workflow step exceeded its timeout (typically a site that hangs
|
||||||
|
// connections or a stalled DB write).
|
||||||
|
"step_timeout",
|
||||||
|
// The Worker isolate was killed for exceeding its 128MB memory limit.
|
||||||
|
"oom",
|
||||||
|
// The Worker isolate was killed for exceeding its CPU time limit.
|
||||||
|
"cpu_limit",
|
||||||
|
// A database query failed permanently.
|
||||||
|
"db_error",
|
||||||
|
// A workflow step tried to persist more than the ~1MiB durable-state cap.
|
||||||
|
"step_output_too_large",
|
||||||
|
// Cloudflare Workflows internal error (not caused by our code).
|
||||||
|
"workflow_internal",
|
||||||
|
// The workflow instance no longer exists (expired from retention or never
|
||||||
|
// created) while the audit row still said "running".
|
||||||
|
"instance_lost",
|
||||||
|
// Anything we could not classify.
|
||||||
|
"unknown",
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
export type AuditErrorCode = (typeof AUDIT_ERROR_CODES)[number];
|
||||||
|
|
||||||
|
export interface AuditErrorInfo {
|
||||||
|
errorCode: AuditErrorCode;
|
||||||
|
errorDetail: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ERROR_DETAIL_MAX_CHARS = 500;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify an error (thrown in the workflow, or read back from a dead
|
||||||
|
* Workflows instance) into an AuditErrorCode. Matches the exact failure
|
||||||
|
* messages observed in production instances.
|
||||||
|
*/
|
||||||
|
export function classifyAuditError(error: unknown): AuditErrorInfo {
|
||||||
|
const message =
|
||||||
|
error instanceof Error
|
||||||
|
? error.message
|
||||||
|
: typeof error === "string"
|
||||||
|
? error
|
||||||
|
: String(error);
|
||||||
|
|
||||||
|
return {
|
||||||
|
errorCode: classifyMessage(message),
|
||||||
|
errorDetail: message.slice(0, ERROR_DETAIL_MAX_CHARS),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function classifyMessage(message: string): AuditErrorCode {
|
||||||
|
if (message.includes("exceeded memory limit")) return "oom";
|
||||||
|
if (message.includes("exceeded CPU time limit")) return "cpu_limit";
|
||||||
|
if (
|
||||||
|
message.includes("WorkflowTimeoutError") ||
|
||||||
|
message.includes("Execution timed out")
|
||||||
|
) {
|
||||||
|
return "step_timeout";
|
||||||
|
}
|
||||||
|
if (message.includes("output is too large")) return "step_output_too_large";
|
||||||
|
if (message.includes("WorkflowInternalError")) return "workflow_internal";
|
||||||
|
if (
|
||||||
|
message.startsWith("Failed query:") ||
|
||||||
|
message.includes("D1_ERROR") ||
|
||||||
|
message.includes("Postgres database accessed outside a request scope")
|
||||||
|
) {
|
||||||
|
return "db_error";
|
||||||
|
}
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
100
src/server/lib/audit/crawl-window.test.ts
Normal file
100
src/server/lib/audit/crawl-window.test.ts
Normal file
@ -0,0 +1,100 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { adjustCrawlWindow } from "@/server/lib/audit/crawl-window";
|
||||||
|
import type {
|
||||||
|
CrawledPageResult,
|
||||||
|
PageFetchClass,
|
||||||
|
} from "@/server/lib/audit/types";
|
||||||
|
|
||||||
|
function page(
|
||||||
|
fetchClass: PageFetchClass,
|
||||||
|
responseTimeMs: number,
|
||||||
|
htmlBytes = 10_000,
|
||||||
|
): CrawledPageResult {
|
||||||
|
return {
|
||||||
|
id: "",
|
||||||
|
url: "https://example.com/",
|
||||||
|
statusCode: fetchClass === "ok" ? 200 : 0,
|
||||||
|
fetchClass,
|
||||||
|
redirectUrl: null,
|
||||||
|
title: "",
|
||||||
|
metaDescription: "",
|
||||||
|
canonicalUrl: null,
|
||||||
|
robotsMeta: null,
|
||||||
|
xRobotsTag: null,
|
||||||
|
headerCanonicalUrl: null,
|
||||||
|
ogTitle: null,
|
||||||
|
ogDescription: null,
|
||||||
|
ogImage: null,
|
||||||
|
h1Count: 0,
|
||||||
|
h2Count: 0,
|
||||||
|
h3Count: 0,
|
||||||
|
h4Count: 0,
|
||||||
|
h5Count: 0,
|
||||||
|
h6Count: 0,
|
||||||
|
headingOrder: [],
|
||||||
|
wordCount: 0,
|
||||||
|
contentHash: null,
|
||||||
|
isHtml: true,
|
||||||
|
htmlBytes,
|
||||||
|
imagesTotal: 0,
|
||||||
|
imagesMissingAlt: 0,
|
||||||
|
images: [],
|
||||||
|
links: [],
|
||||||
|
hasStructuredData: false,
|
||||||
|
hreflangTags: [],
|
||||||
|
isIndexable: true,
|
||||||
|
responseTimeMs,
|
||||||
|
crawlDepth: 0,
|
||||||
|
inSitemap: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("adjustCrawlWindow", () => {
|
||||||
|
it("keeps the window on an empty batch", () => {
|
||||||
|
expect(adjustCrawlWindow(25, [])).toBe(25);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("halves the window when a third of the batch is troubled", () => {
|
||||||
|
const recent = [
|
||||||
|
...Array.from({ length: 9 }, () => page("error", 15_000)),
|
||||||
|
...Array.from({ length: 16 }, () => page("ok", 500)),
|
||||||
|
];
|
||||||
|
expect(adjustCrawlWindow(25, recent)).toBe(12);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats blocked fetches as trouble", () => {
|
||||||
|
const recent = Array.from({ length: 10 }, () => page("blocked", 300));
|
||||||
|
expect(adjustCrawlWindow(20, recent)).toBe(10);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never shrinks below the minimum", () => {
|
||||||
|
const recent = Array.from({ length: 10 }, () => page("error", 15_000));
|
||||||
|
expect(adjustCrawlWindow(6, recent)).toBe(5);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("grows on a clean, fast batch up to the cap", () => {
|
||||||
|
const recent = Array.from({ length: 25 }, () => page("ok", 400));
|
||||||
|
expect(adjustCrawlWindow(25, recent)).toBe(30);
|
||||||
|
expect(adjustCrawlWindow(40, recent)).toBe(40);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats heavy pages as trouble even when they respond fast", () => {
|
||||||
|
const recent = Array.from({ length: 25 }, () =>
|
||||||
|
page("ok", 300, 2 * 1024 * 1024),
|
||||||
|
);
|
||||||
|
expect(adjustCrawlWindow(25, recent)).toBe(12);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not grow when some pages are heavy", () => {
|
||||||
|
const recent = [
|
||||||
|
...Array.from({ length: 4 }, () => page("ok", 300, 1024 * 1024)),
|
||||||
|
...Array.from({ length: 21 }, () => page("ok", 300)),
|
||||||
|
];
|
||||||
|
expect(adjustCrawlWindow(25, recent)).toBe(25);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("holds steady on a clean but slow batch", () => {
|
||||||
|
const recent = Array.from({ length: 25 }, () => page("ok", 5_000));
|
||||||
|
expect(adjustCrawlWindow(25, recent)).toBe(25);
|
||||||
|
});
|
||||||
|
});
|
||||||
50
src/server/lib/audit/crawl-window.ts
Normal file
50
src/server/lib/audit/crawl-window.ts
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
import type { CrawledPageResult } from "@/server/lib/audit/types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rolling fetch-concurrency window for the crawl. Unlike fixed batches, a
|
||||||
|
* slow page only occupies one slot instead of stalling a whole batch. The
|
||||||
|
* window adapts to the site: it shrinks when fetches error/block/crawl
|
||||||
|
* slowly (politeness toward struggling or defensive sites) and grows when
|
||||||
|
* the site answers fast.
|
||||||
|
*/
|
||||||
|
export const INITIAL_CRAWL_WINDOW = 25;
|
||||||
|
const MIN_WINDOW = 5;
|
||||||
|
const MAX_WINDOW = 40;
|
||||||
|
const SLOW_RESPONSE_MS = 10_000;
|
||||||
|
const FAST_RESPONSE_MS = 1_500;
|
||||||
|
/**
|
||||||
|
* Pages at/above this HTML size count as trouble: each in-flight page
|
||||||
|
* buffers its body, so a wide window on a heavy-page site is memory
|
||||||
|
* pressure the response time can't see (it's measured at headers).
|
||||||
|
*/
|
||||||
|
const HEAVY_PAGE_BYTES = 1024 * 1024;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adapt the window to the last persisted sub-batch. Shrinks on trouble
|
||||||
|
* (errors, blocks, very slow responses, heavy bodies), grows only on a
|
||||||
|
* clean and mostly-fast batch.
|
||||||
|
*/
|
||||||
|
export function adjustCrawlWindow(
|
||||||
|
windowSize: number,
|
||||||
|
recent: CrawledPageResult[],
|
||||||
|
): number {
|
||||||
|
if (recent.length === 0) return windowSize;
|
||||||
|
const troubled = recent.filter(
|
||||||
|
(page) =>
|
||||||
|
page.fetchClass !== "ok" ||
|
||||||
|
(page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS ||
|
||||||
|
page.htmlBytes >= HEAVY_PAGE_BYTES,
|
||||||
|
).length;
|
||||||
|
if (troubled * 3 >= recent.length) {
|
||||||
|
return Math.max(MIN_WINDOW, Math.floor(windowSize / 2));
|
||||||
|
}
|
||||||
|
const fast = recent.filter(
|
||||||
|
(page) =>
|
||||||
|
page.fetchClass === "ok" &&
|
||||||
|
(page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS,
|
||||||
|
).length;
|
||||||
|
if (troubled === 0 && fast * 2 >= recent.length) {
|
||||||
|
return Math.min(MAX_WINDOW, windowSize + 5);
|
||||||
|
}
|
||||||
|
return windowSize;
|
||||||
|
}
|
||||||
@ -6,10 +6,21 @@ import { XMLParser } from "fast-xml-parser";
|
|||||||
import { isSameOrigin, normalizeUrl } from "./url-utils";
|
import { isSameOrigin, normalizeUrl } from "./url-utils";
|
||||||
|
|
||||||
const SITEMAP_FETCH_TIMEOUT_MS = 15_000;
|
const SITEMAP_FETCH_TIMEOUT_MS = 15_000;
|
||||||
|
// robots.txt is checkpointed as durable Workflow step state (~1MiB cap, shared
|
||||||
|
// with the rest of the step's return). RFC 9309 requires parsers to handle at
|
||||||
|
// least 500 KiB and permits ignoring anything beyond it — Google does exactly
|
||||||
|
// that — so this cap matches standard crawler behavior while keeping a
|
||||||
|
// misbehaving server (e.g. HTML at /robots.txt) from blowing the step limit.
|
||||||
|
const MAX_ROBOTS_TXT_BYTES = 500 * 1024;
|
||||||
const MAX_SITEMAP_DEPTH = 3;
|
const MAX_SITEMAP_DEPTH = 3;
|
||||||
const MAX_SITEMAP_DOCS = 300;
|
const MAX_SITEMAP_DOCS = 300;
|
||||||
const SITEMAP_CONCURRENCY = 5;
|
const SITEMAP_CONCURRENCY = 5;
|
||||||
const SITEMAP_RETRIES = 1;
|
const SITEMAP_RETRIES = 1;
|
||||||
|
// Sitemap shards can legally reach 50 MB and SITEMAP_CONCURRENCY of them are
|
||||||
|
// read at once, so unbounded reads can exhaust Worker memory. Oversized
|
||||||
|
// shards are skipped whole — truncated XML would not parse anyway, and real
|
||||||
|
// generators shard far below this.
|
||||||
|
const MAX_SITEMAP_BYTES = 10 * 1024 * 1024;
|
||||||
|
|
||||||
const xmlParser = new XMLParser({
|
const xmlParser = new XMLParser({
|
||||||
ignoreAttributes: false,
|
ignoreAttributes: false,
|
||||||
@ -34,7 +45,7 @@ async function fetchRobotsTxtText(origin: string): Promise<string | null> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) return null;
|
if (!response.ok) return null;
|
||||||
return await response.text();
|
return (await response.text()).slice(0, MAX_ROBOTS_TXT_BYTES);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("Failed to fetch robots.txt:", error);
|
console.warn("Failed to fetch robots.txt:", error);
|
||||||
return null;
|
return null;
|
||||||
@ -119,6 +130,34 @@ function isTimeoutError(error: unknown): boolean {
|
|||||||
return "name" in error && error.name === "TimeoutError";
|
return "name" in error && error.name === "TimeoutError";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Read a response body up to maxBytes; null when the body exceeds it. */
|
||||||
|
async function readBodyCapped(
|
||||||
|
response: Response,
|
||||||
|
maxBytes: number,
|
||||||
|
): Promise<string | null> {
|
||||||
|
if (!response.body) return "";
|
||||||
|
const reader = response.body.getReader();
|
||||||
|
const chunks: Uint8Array[] = [];
|
||||||
|
let total = 0;
|
||||||
|
for (;;) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
total += value.byteLength;
|
||||||
|
if (total > maxBytes) {
|
||||||
|
await reader.cancel();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
chunks.push(value);
|
||||||
|
}
|
||||||
|
const joined = new Uint8Array(total);
|
||||||
|
let offset = 0;
|
||||||
|
for (const chunk of chunks) {
|
||||||
|
joined.set(chunk, offset);
|
||||||
|
offset += chunk.byteLength;
|
||||||
|
}
|
||||||
|
return new TextDecoder().decode(joined);
|
||||||
|
}
|
||||||
|
|
||||||
async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{
|
async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{
|
||||||
nestedSitemaps: string[];
|
nestedSitemaps: string[];
|
||||||
pageUrls: string[];
|
pageUrls: string[];
|
||||||
@ -147,8 +186,11 @@ async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{
|
|||||||
return { nestedSitemaps: [], pageUrls: [], timedOut: false };
|
return { nestedSitemaps: [], pageUrls: [], timedOut: false };
|
||||||
}
|
}
|
||||||
|
|
||||||
const body = await response.text();
|
const body = await readBodyCapped(response, MAX_SITEMAP_BYTES);
|
||||||
if (!isProbablySitemapXml(response.headers.get("content-type"), body)) {
|
if (
|
||||||
|
body === null ||
|
||||||
|
!isProbablySitemapXml(response.headers.get("content-type"), body)
|
||||||
|
) {
|
||||||
return { nestedSitemaps: [], pageUrls: [], timedOut: false };
|
return { nestedSitemaps: [], pageUrls: [], timedOut: false };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,15 +1,15 @@
|
|||||||
/**
|
/**
|
||||||
* Cross-page (multipage) issue checks.
|
* Cross-page (multipage) issue checks over the app DB's page rows:
|
||||||
|
* duplicates and redirect chains/loops. Pure set-queries over crawl data —
|
||||||
|
* no fetching, no DOM.
|
||||||
*
|
*
|
||||||
* These run once after the crawl, against the rows the crawl wrote to D1:
|
* The two link-edge checks (broken-internal-link, orphan-page) live in the
|
||||||
* duplicates, broken internal links, redirect chains/loops, orphan pages.
|
* audit's scratchpad Durable Object (AuditScratchpad.runFinalizeChecks),
|
||||||
* Pure set-queries over crawl data — no fetching, no DOM.
|
* next to the link edges themselves — link rows never touch the app DB.
|
||||||
*/
|
*/
|
||||||
import { and, eq, gte, lt, ne, notExists, sql } from "drizzle-orm";
|
import { eq } from "drizzle-orm";
|
||||||
import { alias } from "drizzle-orm/sqlite-core";
|
|
||||||
import { db } from "@/db";
|
import { db } from "@/db";
|
||||||
import { auditLinks, auditPages } from "@/db/schema";
|
import { auditPages } from "@/db/schema";
|
||||||
import { normalizeUrl } from "@/server/lib/audit/url-utils";
|
|
||||||
import {
|
import {
|
||||||
findDuplicates,
|
findDuplicates,
|
||||||
findRedirectChainsAndLoops,
|
findRedirectChainsAndLoops,
|
||||||
@ -17,13 +17,8 @@ import {
|
|||||||
} from "@/server/lib/audit/issues/multipage-checks";
|
} from "@/server/lib/audit/issues/multipage-checks";
|
||||||
import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters";
|
import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters";
|
||||||
|
|
||||||
const BROKEN_LINK_ISSUE_CAP = 2_000;
|
|
||||||
|
|
||||||
export async function runMultipageChecks(input: {
|
export async function runMultipageChecks(input: {
|
||||||
auditId: string;
|
auditId: string;
|
||||||
startUrl: string;
|
|
||||||
/** Orphan detection only makes sense when the crawl wasn't truncated. */
|
|
||||||
crawlCompleted: boolean;
|
|
||||||
}): Promise<DetectedIssue[]> {
|
}): Promise<DetectedIssue[]> {
|
||||||
const pages: SlimPage[] = await db
|
const pages: SlimPage[] = await db
|
||||||
.select({
|
.select({
|
||||||
@ -43,112 +38,5 @@ export async function runMultipageChecks(input: {
|
|||||||
.from(auditPages)
|
.from(auditPages)
|
||||||
.where(eq(auditPages.auditId, input.auditId));
|
.where(eq(auditPages.auditId, input.auditId));
|
||||||
|
|
||||||
const issues: DetectedIssue[] = [
|
return [...findDuplicates(pages), ...findRedirectChainsAndLoops(pages)];
|
||||||
...findDuplicates(pages),
|
|
||||||
...findRedirectChainsAndLoops(pages),
|
|
||||||
...(await findBrokenInternalLinks(input.auditId)),
|
|
||||||
];
|
|
||||||
|
|
||||||
if (input.crawlCompleted) {
|
|
||||||
// Page rows store normalized URLs; normalize the start URL the same way
|
|
||||||
// so the orphan exclusion matches.
|
|
||||||
const normalizedStart = normalizeUrl(input.startUrl) ?? input.startUrl;
|
|
||||||
issues.push(...(await findOrphanPages(input.auditId, normalizedStart)));
|
|
||||||
}
|
|
||||||
|
|
||||||
return issues;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function findBrokenInternalLinks(
|
|
||||||
auditId: string,
|
|
||||||
): Promise<DetectedIssue[]> {
|
|
||||||
// Only flag targets we actually crawled and saw fail — never inferred from
|
|
||||||
// absence. Blocked targets (WAF challenges) are excluded: a 403 from bot
|
|
||||||
// protection is not evidence of a broken link.
|
|
||||||
const rows = await db
|
|
||||||
.select({
|
|
||||||
sourcePageId: auditLinks.sourcePageId,
|
|
||||||
sourceUrl: auditLinks.sourceUrl,
|
|
||||||
targetUrl: auditLinks.targetUrl,
|
|
||||||
targetStatus: auditPages.statusCode,
|
|
||||||
})
|
|
||||||
.from(auditLinks)
|
|
||||||
.innerJoin(
|
|
||||||
auditPages,
|
|
||||||
and(
|
|
||||||
eq(auditPages.auditId, auditLinks.auditId),
|
|
||||||
eq(auditPages.url, auditLinks.targetUrl),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.where(
|
|
||||||
and(
|
|
||||||
eq(auditLinks.auditId, auditId),
|
|
||||||
eq(auditLinks.isInternal, true),
|
|
||||||
gte(auditPages.statusCode, 400),
|
|
||||||
eq(auditPages.fetchClass, "ok"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.limit(BROKEN_LINK_ISSUE_CAP);
|
|
||||||
|
|
||||||
return rows.map((row) => ({
|
|
||||||
issueType: "broken-internal-link" as const,
|
|
||||||
pageId: row.sourcePageId,
|
|
||||||
pageUrl: row.sourceUrl,
|
|
||||||
dedupeKey: row.targetUrl,
|
|
||||||
details: { targetUrl: row.targetUrl, targetStatus: row.targetStatus },
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
async function findOrphanPages(
|
|
||||||
auditId: string,
|
|
||||||
startUrl: string,
|
|
||||||
): Promise<DetectedIssue[]> {
|
|
||||||
// A live 2xx page is an orphan when no OTHER crawled page links to it and
|
|
||||||
// nothing redirects to it. Only meaningful on a completed crawl: on a
|
|
||||||
// truncated one, "no observed inlinks" is true of nearly everything.
|
|
||||||
// Error/redirect rows aren't orphans — they already get their own issues.
|
|
||||||
const inlink = db
|
|
||||||
.select({ one: sql`1` })
|
|
||||||
.from(auditLinks)
|
|
||||||
.where(
|
|
||||||
and(
|
|
||||||
eq(auditLinks.auditId, auditId),
|
|
||||||
eq(auditLinks.isInternal, true),
|
|
||||||
eq(auditLinks.targetUrl, auditPages.url),
|
|
||||||
// Self-links (breadcrumbs, anchors) don't make a page reachable.
|
|
||||||
ne(auditLinks.sourcePageId, auditPages.id),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
const redirectSourcePages = alias(auditPages, "redirect_sources");
|
|
||||||
const redirectSources = db
|
|
||||||
.select({ one: sql`1` })
|
|
||||||
.from(redirectSourcePages)
|
|
||||||
.where(
|
|
||||||
and(
|
|
||||||
eq(redirectSourcePages.auditId, auditId),
|
|
||||||
eq(redirectSourcePages.redirectUrl, auditPages.url),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
const rows = await db
|
|
||||||
.select({ id: auditPages.id, url: auditPages.url })
|
|
||||||
.from(auditPages)
|
|
||||||
.where(
|
|
||||||
and(
|
|
||||||
eq(auditPages.auditId, auditId),
|
|
||||||
ne(auditPages.url, startUrl),
|
|
||||||
eq(auditPages.fetchClass, "ok"),
|
|
||||||
gte(auditPages.statusCode, 200),
|
|
||||||
lt(auditPages.statusCode, 300),
|
|
||||||
notExists(inlink),
|
|
||||||
notExists(redirectSources),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
return rows.map((row) => ({
|
|
||||||
issueType: "orphan-page" as const,
|
|
||||||
pageId: row.id,
|
|
||||||
pageUrl: row.url,
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -41,6 +41,7 @@ function makePage(overrides: Partial<CrawledPageResult>): CrawledPageResult {
|
|||||||
wordCount: 500,
|
wordCount: 500,
|
||||||
contentHash: "abc123",
|
contentHash: "abc123",
|
||||||
isHtml: true,
|
isHtml: true,
|
||||||
|
htmlBytes: 10_000,
|
||||||
imagesTotal: 0,
|
imagesTotal: 0,
|
||||||
imagesMissingAlt: 0,
|
imagesMissingAlt: 0,
|
||||||
images: [],
|
images: [],
|
||||||
|
|||||||
@ -2,7 +2,7 @@
|
|||||||
* Per-page issue reporters.
|
* Per-page issue reporters.
|
||||||
*
|
*
|
||||||
* Each reporter is a pure function over a single crawled page record —
|
* Each reporter is a pure function over a single crawled page record —
|
||||||
* DOM-free by design (cheerio runs once in crawlPage), so the engine works
|
* DOM-free by design (HTML parsing runs once in crawlPage), so the engine works
|
||||||
* over any crawl source that can produce a CrawledPageResult.
|
* over any crawl source that can produce a CrawledPageResult.
|
||||||
*
|
*
|
||||||
* Cross-page checks (duplicates, broken links, orphans, redirect chains)
|
* Cross-page checks (duplicates, broken links, orphans, redirect chains)
|
||||||
|
|||||||
208
src/server/lib/audit/page-analyzer.test.ts
Normal file
208
src/server/lib/audit/page-analyzer.test.ts
Normal file
@ -0,0 +1,208 @@
|
|||||||
|
/**
|
||||||
|
* Parity tests for the streaming (htmlparser2) page analyzer against a
|
||||||
|
* cheerio/DOM reference implementation — the exact logic the analyzer
|
||||||
|
* replaced. Cheerio stays as a devDependency for this test only.
|
||||||
|
*/
|
||||||
|
import * as cheerio from "cheerio";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { analyzeHtml } from "@/server/lib/audit/page-analyzer";
|
||||||
|
import { normalizeUrl, isSameOrigin } from "@/server/lib/audit/url-utils";
|
||||||
|
import type { PageAnalysis, PageLink } from "@/server/lib/audit/types";
|
||||||
|
|
||||||
|
/** The previous cheerio implementation, verbatim (minus passthrough fields). */
|
||||||
|
function analyzeHtmlWithCheerio(html: string, pageUrl: string): PageAnalysis {
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
|
||||||
|
const title = $("title").first().text().trim();
|
||||||
|
const metaDescription =
|
||||||
|
$('meta[name="description"]').first().attr("content")?.trim() ?? "";
|
||||||
|
const canonical = $('link[rel="canonical"]').first().attr("href") ?? null;
|
||||||
|
const robotsMeta = $('meta[name="robots"]').first().attr("content") ?? null;
|
||||||
|
const ogTitle =
|
||||||
|
$('meta[property="og:title"]').first().attr("content") ?? null;
|
||||||
|
const ogDescription =
|
||||||
|
$('meta[property="og:description"]').first().attr("content") ?? null;
|
||||||
|
const ogImage =
|
||||||
|
$('meta[property="og:image"]').first().attr("content") ?? null;
|
||||||
|
|
||||||
|
const h1s: string[] = [];
|
||||||
|
$("h1").each((_, el) => {
|
||||||
|
h1s.push($(el).text().trim());
|
||||||
|
});
|
||||||
|
|
||||||
|
const headingOrder: number[] = [];
|
||||||
|
$("h1, h2, h3, h4, h5, h6").each((_, el) => {
|
||||||
|
const tag =
|
||||||
|
"tagName" in el && typeof el.tagName === "string"
|
||||||
|
? el.tagName.toLowerCase()
|
||||||
|
: null;
|
||||||
|
if (tag) {
|
||||||
|
const level = parseInt(tag.charAt(1), 10);
|
||||||
|
if (!isNaN(level)) headingOrder.push(level);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const bodyClone = $("body").clone();
|
||||||
|
bodyClone.find("script, style, noscript, svg").remove();
|
||||||
|
const bodyText = bodyClone.text().replace(/\s+/g, " ").trim();
|
||||||
|
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
||||||
|
|
||||||
|
const images: Array<{ src: string | null; alt: string | null }> = [];
|
||||||
|
$("img").each((_, el) => {
|
||||||
|
images.push({
|
||||||
|
src: $(el).attr("src") ?? null,
|
||||||
|
alt: $(el).attr("alt") ?? null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const linksByTarget = new Map<string, PageLink>();
|
||||||
|
$("a[href]").each((_, el) => {
|
||||||
|
const href = $(el).attr("href");
|
||||||
|
if (!href) return;
|
||||||
|
if (/^(javascript:|mailto:|tel:|#)/.test(href)) return;
|
||||||
|
const resolved = normalizeUrl(href, pageUrl);
|
||||||
|
if (!resolved) return;
|
||||||
|
if (linksByTarget.has(resolved)) return;
|
||||||
|
const anchor = $(el).text().replace(/\s+/g, " ").trim().slice(0, 200);
|
||||||
|
const rel = $(el).attr("rel")?.toLowerCase() ?? "";
|
||||||
|
linksByTarget.set(resolved, {
|
||||||
|
targetUrl: resolved,
|
||||||
|
anchor: anchor || null,
|
||||||
|
isInternal: isSameOrigin(resolved, pageUrl),
|
||||||
|
isNofollow: rel.split(/\s+/).includes("nofollow"),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
let hasStructuredData = false;
|
||||||
|
$('script[type="application/ld+json"]').each(() => {
|
||||||
|
hasStructuredData = true;
|
||||||
|
});
|
||||||
|
|
||||||
|
const hreflangTags: string[] = [];
|
||||||
|
$('link[rel="alternate"][hreflang]').each((_, el) => {
|
||||||
|
const hreflang = $(el).attr("hreflang");
|
||||||
|
if (hreflang) hreflangTags.push(hreflang);
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: pageUrl,
|
||||||
|
statusCode: 200,
|
||||||
|
redirectUrl: null,
|
||||||
|
responseTimeMs: 0,
|
||||||
|
title,
|
||||||
|
metaDescription,
|
||||||
|
canonical,
|
||||||
|
robotsMeta,
|
||||||
|
ogTitle,
|
||||||
|
ogDescription,
|
||||||
|
ogImage,
|
||||||
|
h1s,
|
||||||
|
headingOrder,
|
||||||
|
wordCount,
|
||||||
|
bodyText,
|
||||||
|
images,
|
||||||
|
links: Array.from(linksByTarget.values()),
|
||||||
|
hasStructuredData,
|
||||||
|
hreflangTags,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const PAGE_URL = "https://example.com/blog/post";
|
||||||
|
|
||||||
|
function expectParity(html: string) {
|
||||||
|
const streamed = analyzeHtml(html, PAGE_URL, 200, 0);
|
||||||
|
const reference = analyzeHtmlWithCheerio(html, PAGE_URL);
|
||||||
|
expect(streamed).toEqual(reference);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("analyzeHtml parity with the DOM reference", () => {
|
||||||
|
it("matches on a full, well-formed document", () => {
|
||||||
|
expectParity(`<!DOCTYPE html>
|
||||||
|
<html><head>
|
||||||
|
<title> The Title </title>
|
||||||
|
<meta name="description" content=" A description. ">
|
||||||
|
<meta name="robots" content="index, follow">
|
||||||
|
<meta property="og:title" content="OG Title">
|
||||||
|
<meta property="og:description" content="OG Desc">
|
||||||
|
<meta property="og:image" content="/og.png">
|
||||||
|
<link rel="canonical" href="https://example.com/blog/post">
|
||||||
|
<link rel="alternate" hreflang="en" href="/en">
|
||||||
|
<link rel="alternate" hreflang="de" href="/de">
|
||||||
|
<script type="application/ld+json">{"@type":"Article"}</script>
|
||||||
|
</head><body>
|
||||||
|
<h1>Main <em>Heading</em></h1>
|
||||||
|
<h2>Sub</h2><h3>Deep</h3><h2>Sub 2</h2>
|
||||||
|
<p>Some visible body text with <strong>bold words</strong> here.</p>
|
||||||
|
<script>ignored();</script>
|
||||||
|
<style>.x{}</style>
|
||||||
|
<svg><title>icon</title><text>svg text</text></svg>
|
||||||
|
<noscript><img src="/ns.png"><p>noscript text</p></noscript>
|
||||||
|
<img src="/a.png" alt="A picture">
|
||||||
|
<img src="/b.png" alt="">
|
||||||
|
<img src="/c.png">
|
||||||
|
<img alt="no src">
|
||||||
|
<a href="/relative">Relative <span>link</span></a>
|
||||||
|
<a href="https://example.com/relative">Duplicate target</a>
|
||||||
|
<a href="https://other.example/x" rel="NoFollow sponsored">External</a>
|
||||||
|
<a href="mailto:x@example.com">Mail</a>
|
||||||
|
<a href="#frag">Fragment</a>
|
||||||
|
<a href="javascript:void(0)">JS</a>
|
||||||
|
<a href="/empty-anchor"><img src="/img-link.png" alt=""></a>
|
||||||
|
</body></html>`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches on documents with no head, body, or title", () => {
|
||||||
|
expectParity(
|
||||||
|
`<h1>Bare fragment</h1><p>Just text and a <a href="/x">link</a>.</p>`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches on an empty document", () => {
|
||||||
|
expectParity("");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches on a document with only a head", () => {
|
||||||
|
expectParity(
|
||||||
|
`<html><head><title>Head only</title><meta name="description" content="d"></head></html>`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches with duplicate metas and titles (first wins)", () => {
|
||||||
|
expectParity(`<html><head>
|
||||||
|
<title>First</title><title>Second</title>
|
||||||
|
<meta name="description" content="first desc">
|
||||||
|
<meta name="description" content="second desc">
|
||||||
|
<link rel="canonical" href="/first"><link rel="canonical" href="/second">
|
||||||
|
</head><body><p>text</p></body></html>`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches on unclosed and misnested tags", () => {
|
||||||
|
expectParity(`<html><body>
|
||||||
|
<h1>Unclosed heading
|
||||||
|
<p>Paragraph <b>bold <i>both</b> italic?</i>
|
||||||
|
<a href="/one">first <a href="/two">second</a>
|
||||||
|
<div>trailing text`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches on entity-heavy content", () => {
|
||||||
|
expectParity(`<html><head><title>A & B <C></title></head>
|
||||||
|
<body><h1>Café "menu"</h1>
|
||||||
|
<p>1 < 2 && 3 > 2</p>
|
||||||
|
<a href="/x?a=1&b=2">Query & anchor</a></body></html>`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches heading order across nesting", () => {
|
||||||
|
expectParity(`<body><h3>three</h3><div><h1>one</h1><section><h2>two</h2>
|
||||||
|
<h6>six</h6></section></div><h4>four</h4></body>`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches word counts with whitespace-heavy markup", () => {
|
||||||
|
expectParity(`<body>
|
||||||
|
<p>
|
||||||
|
one
|
||||||
|
two three
|
||||||
|
</p>
|
||||||
|
<ul><li>four</li><li>five</li></ul>
|
||||||
|
</body>`);
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -1,14 +1,38 @@
|
|||||||
/**
|
/**
|
||||||
* HTML page analyzer using cheerio.
|
* HTML page analyzer using htmlparser2's streaming tokenizer.
|
||||||
*
|
*
|
||||||
* Extracts SEO-relevant data from a page's HTML:
|
* Extracts SEO-relevant data from a page's HTML: title, meta description,
|
||||||
* title, meta description, headings, images, links, canonical, OG tags,
|
* headings, images, links, canonical, OG tags, structured data, robots meta,
|
||||||
* structured data, robots meta, word count, hreflang.
|
* word count, hreflang.
|
||||||
|
*
|
||||||
|
* Deliberately NOT a DOM parser: the previous cheerio implementation built a
|
||||||
|
* full DOM (~5-10x the HTML's size) per page, and with 25 concurrent parses
|
||||||
|
* on a 128MB isolate that was the audit engine's dominant OOM cause. The
|
||||||
|
* tokenizer keeps only the accumulated text and extracted fields in memory.
|
||||||
*/
|
*/
|
||||||
import * as cheerio from "cheerio";
|
import { Parser } from "htmlparser2";
|
||||||
import { normalizeUrl, isSameOrigin } from "./url-utils";
|
import { normalizeUrl, isSameOrigin } from "./url-utils";
|
||||||
import type { PageAnalysis, PageLink } from "./types";
|
import type { PageAnalysis, PageLink } from "./types";
|
||||||
|
|
||||||
|
const SKIPPED_LINK_PROTOCOLS = /^(javascript:|mailto:|tel:|#)/;
|
||||||
|
/** Subtrees whose text is not visible content. */
|
||||||
|
const NON_CONTENT_TAGS = new Set(["script", "style", "noscript", "svg"]);
|
||||||
|
const HEADING_LEVELS: Record<string, number> = {
|
||||||
|
h1: 1,
|
||||||
|
h2: 2,
|
||||||
|
h3: 3,
|
||||||
|
h4: 4,
|
||||||
|
h5: 5,
|
||||||
|
h6: 6,
|
||||||
|
};
|
||||||
|
const MAX_ANCHOR_CHARS = 200;
|
||||||
|
|
||||||
|
interface OpenAnchor {
|
||||||
|
href: string;
|
||||||
|
rel: string;
|
||||||
|
text: string[];
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Analyze an HTML string and extract all SEO-relevant data.
|
* Analyze an HTML string and extract all SEO-relevant data.
|
||||||
*/
|
*/
|
||||||
@ -19,102 +43,195 @@ export function analyzeHtml(
|
|||||||
responseTimeMs: number,
|
responseTimeMs: number,
|
||||||
redirectUrl: string | null = null,
|
redirectUrl: string | null = null,
|
||||||
): PageAnalysis {
|
): PageAnalysis {
|
||||||
const $ = cheerio.load(html);
|
let title: string | null = null;
|
||||||
|
let titleDepth = 0;
|
||||||
|
let titleDone = false;
|
||||||
|
// parse5 (the old DOM path) treats <noscript> content as raw text when
|
||||||
|
// scripting is enabled; skip element extraction inside it to match.
|
||||||
|
let noscriptDepth = 0;
|
||||||
|
let metaDescription: string | null = null;
|
||||||
|
let canonical: string | null = null;
|
||||||
|
let robotsMeta: string | null = null;
|
||||||
|
let ogTitle: string | null = null;
|
||||||
|
let ogDescription: string | null = null;
|
||||||
|
let ogImage: string | null = null;
|
||||||
|
let hasStructuredData = false;
|
||||||
|
const hreflangTags: string[] = [];
|
||||||
|
|
||||||
const title = $("title").first().text().trim();
|
|
||||||
|
|
||||||
const metaDescription =
|
|
||||||
$('meta[name="description"]').first().attr("content")?.trim() ?? "";
|
|
||||||
|
|
||||||
const canonical = $('link[rel="canonical"]').first().attr("href") ?? null;
|
|
||||||
|
|
||||||
const robotsMeta = $('meta[name="robots"]').first().attr("content") ?? null;
|
|
||||||
|
|
||||||
// --- Open Graph ---
|
|
||||||
const ogTitle =
|
|
||||||
$('meta[property="og:title"]').first().attr("content") ?? null;
|
|
||||||
const ogDescription =
|
|
||||||
$('meta[property="og:description"]').first().attr("content") ?? null;
|
|
||||||
const ogImage =
|
|
||||||
$('meta[property="og:image"]').first().attr("content") ?? null;
|
|
||||||
|
|
||||||
// --- Headings ---
|
|
||||||
const h1s: string[] = [];
|
const h1s: string[] = [];
|
||||||
$("h1").each((_, el) => {
|
|
||||||
h1s.push($(el).text().trim());
|
|
||||||
});
|
|
||||||
|
|
||||||
const headingOrder: number[] = [];
|
const headingOrder: number[] = [];
|
||||||
$("h1, h2, h3, h4, h5, h6").each((_, el) => {
|
let openH1: string[] | null = null;
|
||||||
const tag =
|
|
||||||
"tagName" in el && typeof el.tagName === "string"
|
|
||||||
? el.tagName.toLowerCase()
|
|
||||||
: null;
|
|
||||||
if (tag) {
|
|
||||||
const level = parseInt(tag.charAt(1), 10);
|
|
||||||
if (!isNaN(level)) headingOrder.push(level);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// --- Word count (visible text in body) ---
|
|
||||||
// Remove script/style/noscript tags, then count words in remaining text
|
|
||||||
const bodyClone = $("body").clone();
|
|
||||||
bodyClone.find("script, style, noscript, svg").remove();
|
|
||||||
const bodyText = bodyClone.text().replace(/\s+/g, " ").trim();
|
|
||||||
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
|
||||||
|
|
||||||
const images: Array<{ src: string | null; alt: string | null }> = [];
|
const images: Array<{ src: string | null; alt: string | null }> = [];
|
||||||
$("img").each((_, el) => {
|
|
||||||
images.push({
|
|
||||||
src: $(el).attr("src") ?? null,
|
|
||||||
alt: $(el).attr("alt") ?? null,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// --- Links (deduped by target URL; first anchor wins) ---
|
|
||||||
const linksByTarget = new Map<string, PageLink>();
|
const linksByTarget = new Map<string, PageLink>();
|
||||||
|
let openAnchor: OpenAnchor | null = null;
|
||||||
|
|
||||||
$("a[href]").each((_, el) => {
|
// Visible text: prefer text inside an explicit <body>; when the document
|
||||||
const href = $(el).attr("href");
|
// never opens one (fragments), fall back to all non-head text. Both
|
||||||
if (!href) return;
|
// exclude NON_CONTENT_TAGS subtrees.
|
||||||
|
let suppressDepth = 0;
|
||||||
|
let bodyDepth = 0;
|
||||||
|
let headDepth = 0;
|
||||||
|
let sawBody = false;
|
||||||
|
const bodyParts: string[] = [];
|
||||||
|
const fallbackParts: string[] = [];
|
||||||
|
|
||||||
// Skip javascript:, mailto:, tel:, #anchors
|
const handleMetaTag = (attribs: Record<string, string>) => {
|
||||||
if (/^(javascript:|mailto:|tel:|#)/.test(href)) return;
|
const content = attribs["content"];
|
||||||
|
if (attribs["name"] === "description") {
|
||||||
|
metaDescription ??= content?.trim() ?? "";
|
||||||
|
} else if (attribs["name"] === "robots") {
|
||||||
|
robotsMeta ??= content ?? null;
|
||||||
|
} else if (attribs["property"] === "og:title") {
|
||||||
|
ogTitle ??= content ?? null;
|
||||||
|
} else if (attribs["property"] === "og:description") {
|
||||||
|
ogDescription ??= content ?? null;
|
||||||
|
} else if (attribs["property"] === "og:image") {
|
||||||
|
ogImage ??= content ?? null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleLinkTag = (attribs: Record<string, string>) => {
|
||||||
|
if (attribs["rel"] === "canonical") {
|
||||||
|
canonical ??= attribs["href"] ?? null;
|
||||||
|
} else if (attribs["rel"] === "alternate" && attribs["hreflang"]) {
|
||||||
|
hreflangTags.push(attribs["hreflang"]);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const closeAnchor = () => {
|
||||||
|
if (!openAnchor) return;
|
||||||
|
const { href, rel, text } = openAnchor;
|
||||||
|
openAnchor = null;
|
||||||
const resolved = normalizeUrl(href, pageUrl);
|
const resolved = normalizeUrl(href, pageUrl);
|
||||||
if (!resolved) return;
|
if (!resolved || linksByTarget.has(resolved)) return;
|
||||||
if (linksByTarget.has(resolved)) return;
|
const anchor = text
|
||||||
|
.join("")
|
||||||
const anchor = $(el).text().replace(/\s+/g, " ").trim().slice(0, 200);
|
.replace(/\s+/g, " ")
|
||||||
const rel = $(el).attr("rel")?.toLowerCase() ?? "";
|
.trim()
|
||||||
|
.slice(0, MAX_ANCHOR_CHARS);
|
||||||
linksByTarget.set(resolved, {
|
linksByTarget.set(resolved, {
|
||||||
targetUrl: resolved,
|
targetUrl: resolved,
|
||||||
anchor: anchor || null,
|
anchor: anchor || null,
|
||||||
isInternal: isSameOrigin(resolved, pageUrl),
|
isInternal: isSameOrigin(resolved, pageUrl),
|
||||||
isNofollow: rel.split(/\s+/).includes("nofollow"),
|
isNofollow: rel.split(/\s+/).includes("nofollow"),
|
||||||
});
|
});
|
||||||
});
|
};
|
||||||
const links = Array.from(linksByTarget.values());
|
|
||||||
|
|
||||||
// --- Structured data (JSON-LD) ---
|
const parser = new Parser(
|
||||||
let hasStructuredData = false;
|
{
|
||||||
$('script[type="application/ld+json"]').each(() => {
|
onopentag(name, attribs) {
|
||||||
|
if (NON_CONTENT_TAGS.has(name)) {
|
||||||
|
suppressDepth += 1;
|
||||||
|
}
|
||||||
|
if (name === "noscript") noscriptDepth += 1;
|
||||||
|
if (noscriptDepth > 0) return;
|
||||||
|
switch (name) {
|
||||||
|
case "title":
|
||||||
|
// Ignore <title> inside <svg> — only the document title counts.
|
||||||
|
if (!titleDone && suppressDepth === 0) {
|
||||||
|
titleDepth += 1;
|
||||||
|
if (title === null) title = "";
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case "head":
|
||||||
|
headDepth += 1;
|
||||||
|
break;
|
||||||
|
case "body":
|
||||||
|
bodyDepth += 1;
|
||||||
|
sawBody = true;
|
||||||
|
break;
|
||||||
|
case "meta":
|
||||||
|
handleMetaTag(attribs);
|
||||||
|
break;
|
||||||
|
case "link":
|
||||||
|
handleLinkTag(attribs);
|
||||||
|
break;
|
||||||
|
case "img":
|
||||||
|
images.push({
|
||||||
|
src: attribs["src"] ?? null,
|
||||||
|
alt: "alt" in attribs ? attribs["alt"] : null,
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
case "script":
|
||||||
|
if (attribs["type"] === "application/ld+json") {
|
||||||
hasStructuredData = true;
|
hasStructuredData = true;
|
||||||
});
|
}
|
||||||
|
break;
|
||||||
|
case "a": {
|
||||||
|
// HTML forbids nested <a>; browsers implicitly close the open
|
||||||
|
// one, and the tokenizer has no tree correction, so mirror that.
|
||||||
|
closeAnchor();
|
||||||
|
const href = attribs["href"];
|
||||||
|
if (href && !SKIPPED_LINK_PROTOCOLS.test(href)) {
|
||||||
|
openAnchor = {
|
||||||
|
href,
|
||||||
|
rel: attribs["rel"]?.toLowerCase() ?? "",
|
||||||
|
text: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const headingLevel = HEADING_LEVELS[name];
|
||||||
|
if (headingLevel !== undefined) {
|
||||||
|
headingOrder.push(headingLevel);
|
||||||
|
if (headingLevel === 1 && openH1 === null) openH1 = [];
|
||||||
|
}
|
||||||
|
},
|
||||||
|
ontext(text) {
|
||||||
|
if (suppressDepth > 0) return;
|
||||||
|
if (titleDepth > 0) {
|
||||||
|
if (title !== null) title += text;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (openH1) openH1.push(text);
|
||||||
|
if (openAnchor) openAnchor.text.push(text);
|
||||||
|
if (bodyDepth > 0) {
|
||||||
|
bodyParts.push(text);
|
||||||
|
} else if (headDepth === 0) {
|
||||||
|
fallbackParts.push(text);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onclosetag(name) {
|
||||||
|
if (NON_CONTENT_TAGS.has(name) && suppressDepth > 0) {
|
||||||
|
suppressDepth -= 1;
|
||||||
|
}
|
||||||
|
if (name === "noscript" && noscriptDepth > 0) {
|
||||||
|
noscriptDepth -= 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (noscriptDepth > 0) return;
|
||||||
|
if (name === "title" && titleDepth > 0) {
|
||||||
|
titleDepth -= 1;
|
||||||
|
if (titleDepth === 0) titleDone = true;
|
||||||
|
}
|
||||||
|
if (name === "head" && headDepth > 0) headDepth -= 1;
|
||||||
|
if (name === "body" && bodyDepth > 0) bodyDepth -= 1;
|
||||||
|
if (name === "a") closeAnchor();
|
||||||
|
if (name === "h1" && openH1) {
|
||||||
|
h1s.push(openH1.join("").trim());
|
||||||
|
openH1 = null;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
// Defaults (non-XML mode): lowercased tag/attribute names, decoded
|
||||||
|
// entities — matching what the DOM-based implementation saw.
|
||||||
|
);
|
||||||
|
parser.write(html);
|
||||||
|
parser.end();
|
||||||
|
|
||||||
const hreflangTags: string[] = [];
|
const rawText = (sawBody ? bodyParts : fallbackParts).join("");
|
||||||
$('link[rel="alternate"][hreflang]').each((_, el) => {
|
const bodyText = rawText.replace(/\s+/g, " ").trim();
|
||||||
const hreflang = $(el).attr("hreflang");
|
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
||||||
if (hreflang) hreflangTags.push(hreflang);
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
url: pageUrl,
|
url: pageUrl,
|
||||||
statusCode,
|
statusCode,
|
||||||
redirectUrl,
|
redirectUrl,
|
||||||
responseTimeMs,
|
responseTimeMs,
|
||||||
title,
|
title: (title ?? "").trim(),
|
||||||
metaDescription,
|
metaDescription: metaDescription ?? "",
|
||||||
canonical,
|
canonical,
|
||||||
robotsMeta,
|
robotsMeta,
|
||||||
ogTitle,
|
ogTitle,
|
||||||
@ -125,7 +242,7 @@ export function analyzeHtml(
|
|||||||
wordCount,
|
wordCount,
|
||||||
bodyText,
|
bodyText,
|
||||||
images,
|
images,
|
||||||
links,
|
links: Array.from(linksByTarget.values()),
|
||||||
hasStructuredData,
|
hasStructuredData,
|
||||||
hreflangTags,
|
hreflangTags,
|
||||||
};
|
};
|
||||||
|
|||||||
@ -50,7 +50,7 @@ export interface PageLink {
|
|||||||
isNofollow: boolean;
|
isNofollow: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Data extracted from a single page via cheerio. */
|
/** Data extracted from a single page's HTML. */
|
||||||
export interface PageAnalysis {
|
export interface PageAnalysis {
|
||||||
url: string;
|
url: string;
|
||||||
statusCode: number;
|
statusCode: number;
|
||||||
@ -106,8 +106,9 @@ export interface LighthouseResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Full result of crawling one page. Persisted to D1 inside the crawl-batch
|
* Full result of crawling one page. Persisted to the app DB inside the
|
||||||
* step; never accumulated in memory or returned as durable step state.
|
* crawl-chunk step; never accumulated in memory or returned as durable
|
||||||
|
* step state.
|
||||||
*/
|
*/
|
||||||
export interface CrawledPageResult {
|
export interface CrawledPageResult {
|
||||||
id: string;
|
id: string;
|
||||||
@ -139,6 +140,12 @@ export interface CrawledPageResult {
|
|||||||
* checked; a PDF must not). Transient — not persisted.
|
* checked; a PDF must not). Transient — not persisted.
|
||||||
*/
|
*/
|
||||||
isHtml: boolean;
|
isHtml: boolean;
|
||||||
|
/**
|
||||||
|
* HTML size read for this page (approximate; capped at MAX_HTML_BYTES).
|
||||||
|
* Transient — feeds the crawl window's memory-pressure signal, since
|
||||||
|
* response time is measured at headers and says nothing about body size.
|
||||||
|
*/
|
||||||
|
htmlBytes: number;
|
||||||
imagesTotal: number;
|
imagesTotal: number;
|
||||||
imagesMissingAlt: number;
|
imagesMissingAlt: number;
|
||||||
images: Array<{ src: string | null; alt: string | null }>;
|
images: Array<{ src: string | null; alt: string | null }>;
|
||||||
@ -151,18 +158,3 @@ export interface CrawledPageResult {
|
|||||||
crawlDepth: number | null;
|
crawlDepth: number | null;
|
||||||
inSitemap: boolean;
|
inSitemap: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Slim per-page summary returned as durable step state from a crawl batch.
|
|
||||||
* Keep this small: full page data lives in D1, not in Workflow step state.
|
|
||||||
*/
|
|
||||||
export interface StepPageSummary {
|
|
||||||
id: string;
|
|
||||||
url: string;
|
|
||||||
statusCode: number;
|
|
||||||
fetchClass: PageFetchClass;
|
|
||||||
redirectUrl: string | null;
|
|
||||||
title: string;
|
|
||||||
/** Normalized same-origin link targets, for frontier expansion. */
|
|
||||||
internalLinks: string[];
|
|
||||||
}
|
|
||||||
|
|||||||
@ -1,6 +1,9 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import type { AppError } from "@/server/lib/errors";
|
import type { AppError } from "@/server/lib/errors";
|
||||||
import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
|
import {
|
||||||
|
normalizeAndValidateStartUrl,
|
||||||
|
resolveStartUrlRedirects,
|
||||||
|
} from "@/server/lib/audit/url-policy";
|
||||||
|
|
||||||
describe("normalizeAndValidateStartUrl", () => {
|
describe("normalizeAndValidateStartUrl", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
@ -48,3 +51,86 @@ describe("normalizeAndValidateStartUrl", () => {
|
|||||||
} satisfies Partial<AppError>);
|
} satisfies Partial<AppError>);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const dnsOk = () =>
|
||||||
|
new Response(JSON.stringify({ Status: 0, Answer: [] }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { "content-type": "application/dns-json" },
|
||||||
|
});
|
||||||
|
const redirect = (location: string) =>
|
||||||
|
new Response(null, { status: 301, headers: { location } });
|
||||||
|
|
||||||
|
describe("resolveStartUrlRedirects", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.stubGlobal("fetch", vi.fn());
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
});
|
||||||
|
|
||||||
|
/** Route probe fetches by URL; DoH lookups always resolve clean. */
|
||||||
|
function stubFetch(routes: Record<string, () => Response>) {
|
||||||
|
vi.mocked(fetch).mockImplementation((input) => {
|
||||||
|
const url = String(input instanceof Request ? input.url : input);
|
||||||
|
if (url.includes("dns-query")) return Promise.resolve(dnsOk());
|
||||||
|
const route = routes[url];
|
||||||
|
return route
|
||||||
|
? Promise.resolve(route())
|
||||||
|
: Promise.resolve(new Response(null, { status: 200 }));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
it("follows a cross-domain redirect to the real origin", async () => {
|
||||||
|
stubFetch({
|
||||||
|
"https://example.net/": () => redirect("https://example.com/"),
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
resolveStartUrlRedirects("https://example.net/"),
|
||||||
|
).resolves.toBe("https://example.com/");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("follows an apex-to-www redirect chain", async () => {
|
||||||
|
stubFetch({
|
||||||
|
"https://example.com/": () => redirect("https://www.example.com/"),
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
resolveStartUrlRedirects("https://example.com/"),
|
||||||
|
).resolves.toBe("https://www.example.com/");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns the original URL when the site does not redirect", async () => {
|
||||||
|
stubFetch({});
|
||||||
|
await expect(
|
||||||
|
resolveStartUrlRedirects("https://example.com/"),
|
||||||
|
).resolves.toBe("https://example.com/");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns the last URL when the probe fails", async () => {
|
||||||
|
vi.mocked(fetch).mockRejectedValue(new Error("network down"));
|
||||||
|
await expect(
|
||||||
|
resolveStartUrlRedirects("https://example.com/"),
|
||||||
|
).resolves.toBe("https://example.com/");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stops after the hop limit on a redirect loop", async () => {
|
||||||
|
stubFetch({
|
||||||
|
"https://a.example/": () => redirect("https://b.example/"),
|
||||||
|
"https://b.example/": () => redirect("https://a.example/"),
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
resolveStartUrlRedirects("https://a.example/"),
|
||||||
|
).resolves.toMatch(/^https:\/\/(a|b)\.example\/$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects redirects into blocked targets", async () => {
|
||||||
|
stubFetch({
|
||||||
|
"https://example.com/": () => redirect("http://192.168.0.10/"),
|
||||||
|
});
|
||||||
|
await expect(
|
||||||
|
resolveStartUrlRedirects("https://example.com/"),
|
||||||
|
).rejects.toMatchObject({
|
||||||
|
code: "CRAWL_TARGET_BLOCKED",
|
||||||
|
} satisfies Partial<AppError>);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@ -238,3 +238,48 @@ export async function normalizeAndValidateStartUrl(
|
|||||||
parsed.hash = "";
|
parsed.hash = "";
|
||||||
return parsed.toString();
|
return parsed.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const START_URL_REDIRECT_HOPS = 5;
|
||||||
|
const START_URL_PROBE_TIMEOUT_MS = 10_000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Follow redirects on the audit's start URL so the audit anchors to the
|
||||||
|
* site's real origin. Without this, auditing a domain that 301s elsewhere
|
||||||
|
* (…net -> …com, apex -> www) dead-ends after one page: the redirect target
|
||||||
|
* is a different origin, so the same-origin crawl policy can't follow it.
|
||||||
|
*
|
||||||
|
* Every hop re-runs the full start-URL validation (SSRF, blocked hosts), so
|
||||||
|
* a redirect can't smuggle the audit somewhere the user couldn't have
|
||||||
|
* pointed it directly. Probe failures (timeouts, HEAD rejected) fall back
|
||||||
|
* to the last validated URL — the crawl records the real fetch result.
|
||||||
|
*/
|
||||||
|
export async function resolveStartUrlRedirects(
|
||||||
|
startUrl: string,
|
||||||
|
): Promise<string> {
|
||||||
|
let current = startUrl;
|
||||||
|
for (let hop = 0; hop < START_URL_REDIRECT_HOPS; hop++) {
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetch(current, {
|
||||||
|
method: "HEAD",
|
||||||
|
redirect: "manual",
|
||||||
|
headers: { "User-Agent": "OpenSEO-Audit/1.0" },
|
||||||
|
signal: AbortSignal.timeout(START_URL_PROBE_TIMEOUT_MS),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
if (response.status < 300 || response.status >= 400) return current;
|
||||||
|
const location = response.headers.get("location");
|
||||||
|
if (!location) return current;
|
||||||
|
|
||||||
|
let next: URL;
|
||||||
|
try {
|
||||||
|
next = new URL(location, current);
|
||||||
|
} catch {
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
current = await normalizeAndValidateStartUrl(next.toString());
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
|||||||
@ -128,7 +128,7 @@ export const runSiteAuditTool = {
|
|||||||
});
|
});
|
||||||
|
|
||||||
return mcpResponse({
|
return mcpResponse({
|
||||||
text: `Audit ${auditId} started for ${args.url}. Poll get_audit_status until it completes, then call get_audit_issues for the prioritized issue report.`,
|
text: `Audit ${auditId} started for ${args.url}. Poll get_audit_status until it finishes, then call get_audit_issues for the prioritized issue report (even a failed audit keeps results for every page it crawled).`,
|
||||||
meta: buildProjectMeta(
|
meta: buildProjectMeta(
|
||||||
context,
|
context,
|
||||||
args.projectId,
|
args.projectId,
|
||||||
@ -177,8 +177,16 @@ export const getAuditStatusTool = {
|
|||||||
status.lighthouseTotal > 0
|
status.lighthouseTotal > 0
|
||||||
? `, lighthouse ${status.lighthouseCompleted + status.lighthouseFailed}/${status.lighthouseTotal}`
|
? `, lighthouse ${status.lighthouseCompleted + status.lighthouseFailed}/${status.lighthouseTotal}`
|
||||||
: "";
|
: "";
|
||||||
|
// Failed audits keep partial results — point agents at them instead of
|
||||||
|
// letting a mid-crawl death read as "no data".
|
||||||
|
const nextStep =
|
||||||
|
status.status === "completed"
|
||||||
|
? " Call get_audit_issues for the issue report."
|
||||||
|
: status.status === "failed" && status.pagesCrawled > 0
|
||||||
|
? ` The audit stopped early but kept results for the ${status.pagesCrawled} pages it crawled — call get_audit_issues for the partial issue report.`
|
||||||
|
: "";
|
||||||
return mcpResponse({
|
return mcpResponse({
|
||||||
text: `Audit ${status.id} (${status.startUrl}): ${status.status} — phase ${status.currentPhase}, ${status.pagesCrawled}/${status.pagesTotal} pages${lighthouseNote}.${status.status === "completed" ? " Call get_audit_issues for the issue report." : ""}`,
|
text: `Audit ${status.id} (${status.startUrl}): ${status.status} — phase ${status.currentPhase}, ${status.pagesCrawled}/${status.pagesTotal} pages${lighthouseNote}.${nextStep}`,
|
||||||
meta: buildProjectMeta(
|
meta: buildProjectMeta(
|
||||||
context,
|
context,
|
||||||
args.projectId,
|
args.projectId,
|
||||||
|
|||||||
@ -12,10 +12,12 @@ import {
|
|||||||
import { withPgClient } from "@/db";
|
import { withPgClient } from "@/db";
|
||||||
import type { BillingCustomerContext } from "@/server/billing/subscription";
|
import type { BillingCustomerContext } from "@/server/billing/subscription";
|
||||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||||
|
import { classifyAuditError } from "@/server/lib/audit/audit-errors";
|
||||||
import type { AuditConfig } from "@/server/lib/audit/types";
|
import type { AuditConfig } from "@/server/lib/audit/types";
|
||||||
import { captureServerError, captureServerEvent } from "@/server/lib/posthog";
|
import { captureServerError, captureServerEvent } from "@/server/lib/posthog";
|
||||||
import { runAuditPhases } from "@/server/workflows/siteAuditWorkflowPhases";
|
import { runAuditPhases } from "@/server/workflows/siteAuditWorkflowPhases";
|
||||||
import { pgStep } from "@/server/workflows/pgStep";
|
import { pgStep } from "@/server/workflows/pgStep";
|
||||||
|
import { DB_STEP } from "@/server/workflows/auditStepConfigs";
|
||||||
|
|
||||||
interface AuditParams {
|
interface AuditParams {
|
||||||
auditId: string;
|
auditId: string;
|
||||||
@ -44,7 +46,7 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint<Env, AuditParams> {
|
|||||||
// Inside a step so the D1 read is retried and replay-cached; a bare
|
// Inside a step so the D1 read is retried and replay-cached; a bare
|
||||||
// read here would re-execute on every replay and a transient failure
|
// read here would re-execute on every replay and a transient failure
|
||||||
// would kill the instance before the catch below exists.
|
// would kill the instance before the catch below exists.
|
||||||
await pgStep(step, "validate-context", undefined, async () => {
|
await pgStep(step, "validate-context", DB_STEP, async () => {
|
||||||
const audit = await AuditRepository.getAuditForWorkflow(
|
const audit = await AuditRepository.getAuditForWorkflow(
|
||||||
auditId,
|
auditId,
|
||||||
event.instanceId,
|
event.instanceId,
|
||||||
@ -86,13 +88,17 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint<Env, AuditParams> {
|
|||||||
project_id: projectId,
|
project_id: projectId,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
await pgStep(step, "mark-failed", undefined, async () => {
|
const errorInfo = classifyAuditError(error);
|
||||||
await AuditRepository.failAudit(auditId, event.instanceId);
|
await pgStep(step, "mark-failed", DB_STEP, async () => {
|
||||||
|
// Read the phase before failAudit stamps currentPhase = "failed".
|
||||||
const latestAudit = await AuditRepository.getAuditForWorkflow(
|
const runningAudit = await AuditRepository.getAuditForWorkflow(
|
||||||
auditId,
|
auditId,
|
||||||
event.instanceId,
|
event.instanceId,
|
||||||
);
|
);
|
||||||
|
await AuditRepository.failAudit(auditId, event.instanceId, {
|
||||||
|
...errorInfo,
|
||||||
|
failedPhase: runningAudit?.currentPhase ?? null,
|
||||||
|
});
|
||||||
|
|
||||||
await captureServerEvent({
|
await captureServerEvent({
|
||||||
distinctId: billingCustomer.userId,
|
distinctId: billingCustomer.userId,
|
||||||
@ -101,8 +107,10 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint<Env, AuditParams> {
|
|||||||
properties: {
|
properties: {
|
||||||
project_id: projectId,
|
project_id: projectId,
|
||||||
status: "failed",
|
status: "failed",
|
||||||
pages_crawled: latestAudit?.pagesCrawled,
|
error_code: errorInfo.errorCode,
|
||||||
pages_total: latestAudit?.pagesTotal,
|
failed_phase: runningAudit?.currentPhase,
|
||||||
|
pages_crawled: runningAudit?.pagesCrawled,
|
||||||
|
pages_total: runningAudit?.pagesTotal,
|
||||||
run_lighthouse: config.lighthouseStrategy !== "none",
|
run_lighthouse: config.lighthouseStrategy !== "none",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
40
src/server/workflows/auditStepConfigs.ts
Normal file
40
src/server/workflows/auditStepConfigs.ts
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
import type { WorkflowStepConfig } from "cloudflare:workers";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Explicit step configs for the site-audit workflow.
|
||||||
|
*
|
||||||
|
* Without these, steps inherit the platform default (10-minute timeout,
|
||||||
|
* multiple retries with backoff) — production audits against slow/hostile
|
||||||
|
* sites burned ~65 minutes replaying a doomed step before erroring. Every
|
||||||
|
* step now has a timeout sized to its real work and a small retry budget.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Robots + sitemap walk (bounded at 300 docs / 15s per fetch internally). */
|
||||||
|
export const DISCOVERY_STEP: WorkflowStepConfig = {
|
||||||
|
retries: { limit: 2, delay: "5 seconds", backoff: "exponential" },
|
||||||
|
timeout: "4 minutes",
|
||||||
|
};
|
||||||
|
|
||||||
|
/** One crawl chunk: ~90s soft crawl deadline + persistence headroom. */
|
||||||
|
export const CRAWL_CHUNK_STEP: WorkflowStepConfig = {
|
||||||
|
retries: { limit: 1, delay: "10 seconds", backoff: "constant" },
|
||||||
|
timeout: "5 minutes",
|
||||||
|
};
|
||||||
|
|
||||||
|
/** One Lighthouse batch: 20 DataForSEO calls, each with internal retries. */
|
||||||
|
export const LIGHTHOUSE_BATCH_STEP: WorkflowStepConfig = {
|
||||||
|
retries: { limit: 1, delay: "10 seconds", backoff: "constant" },
|
||||||
|
timeout: "5 minutes",
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Small DB-only steps (validate, select sample, finalize, mark-failed). */
|
||||||
|
export const DB_STEP: WorkflowStepConfig = {
|
||||||
|
retries: { limit: 3, delay: "5 seconds", backoff: "exponential" },
|
||||||
|
timeout: "2 minutes",
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Cross-page checks read every page row of the audit — allow more time. */
|
||||||
|
export const MULTIPAGE_CHECKS_STEP: WorkflowStepConfig = {
|
||||||
|
retries: { limit: 2, delay: "5 seconds", backoff: "exponential" },
|
||||||
|
timeout: "5 minutes",
|
||||||
|
};
|
||||||
@ -104,9 +104,8 @@ export async function crawlPage(
|
|||||||
|
|
||||||
const contentType = response.headers.get("content-type") ?? "";
|
const contentType = response.headers.get("content-type") ?? "";
|
||||||
const isHtml = contentType.includes("text/html");
|
const isHtml = contentType.includes("text/html");
|
||||||
// Large pages make Cheerio disproportionately expensive and can exhaust a
|
// Cap what we read: the first 2 MiB still contains the SEO metadata and
|
||||||
// crawl step's CPU or isolate memory. The first 2 MiB still contains the
|
// navigation needed by the audit in normal documents.
|
||||||
// SEO metadata and navigation needed by the audit in normal documents.
|
|
||||||
const body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";
|
const body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";
|
||||||
const fetchClass = classifyFetch(
|
const fetchClass = classifyFetch(
|
||||||
statusCode,
|
statusCode,
|
||||||
@ -128,10 +127,10 @@ export async function crawlPage(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dynamic import keeps cheerio (page-analyzer's HTML parser) out of the
|
// Dynamic import keeps the HTML parser out of the worker's startup
|
||||||
// worker's startup module graph: SiteAuditWorkflow is re-exported from
|
// module graph: SiteAuditWorkflow is re-exported from src/server.ts, so
|
||||||
// src/server.ts, so a static import would evaluate cheerio in every
|
// a static import would evaluate it in every isolate's baseline heap,
|
||||||
// isolate's baseline heap, not just when an audit actually crawls.
|
// not just when an audit actually crawls.
|
||||||
const { analyzeHtml } = await import("@/server/lib/audit/page-analyzer");
|
const { analyzeHtml } = await import("@/server/lib/audit/page-analyzer");
|
||||||
const analysis = analyzeHtml(body, url, statusCode, responseTimeMs);
|
const analysis = analyzeHtml(body, url, statusCode, responseTimeMs);
|
||||||
const robotsDirectives = [analysis.robotsMeta, xRobotsTag]
|
const robotsDirectives = [analysis.robotsMeta, xRobotsTag]
|
||||||
@ -171,6 +170,7 @@ export async function crawlPage(
|
|||||||
? await sha256Hex(analysis.bodyText)
|
? await sha256Hex(analysis.bodyText)
|
||||||
: null,
|
: null,
|
||||||
isHtml: true,
|
isHtml: true,
|
||||||
|
htmlBytes: body.length,
|
||||||
imagesTotal: analysis.images.length,
|
imagesTotal: analysis.images.length,
|
||||||
// Only a truly absent alt attribute counts: alt="" is the correct
|
// Only a truly absent alt attribute counts: alt="" is the correct
|
||||||
// markup for decorative images.
|
// markup for decorative images.
|
||||||
@ -270,6 +270,7 @@ function emptyPageResult(input: {
|
|||||||
wordCount: 0,
|
wordCount: 0,
|
||||||
contentHash: null,
|
contentHash: null,
|
||||||
isHtml: false,
|
isHtml: false,
|
||||||
|
htmlBytes: 0,
|
||||||
imagesTotal: 0,
|
imagesTotal: 0,
|
||||||
imagesMissingAlt: 0,
|
imagesMissingAlt: 0,
|
||||||
images: [],
|
images: [],
|
||||||
|
|||||||
@ -1,59 +1,77 @@
|
|||||||
import type { WorkflowStep } from "cloudflare:workers";
|
import type { WorkflowStep } from "cloudflare:workers";
|
||||||
import type { RobotsResult } from "@/server/lib/audit/discovery";
|
import type { RobotsResult } from "@/server/lib/audit/discovery";
|
||||||
import type { StepPageSummary } from "@/server/lib/audit/types";
|
import type { CrawledPageResult } from "@/server/lib/audit/types";
|
||||||
import { isSameOrigin, normalizeUrl } from "@/server/lib/audit/url-utils";
|
import { isSameOrigin } from "@/server/lib/audit/url-utils";
|
||||||
import { isCrawlableUrl } from "@/server/lib/audit/url-policy";
|
import { isCrawlableUrl } from "@/server/lib/audit/url-policy";
|
||||||
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
|
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
|
||||||
import { runPageReporters } from "@/server/lib/audit/issues/page-reporters";
|
import { runPageReporters } from "@/server/lib/audit/issues/page-reporters";
|
||||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||||
|
import {
|
||||||
|
getAuditScratchpad,
|
||||||
|
type ClaimedUrl,
|
||||||
|
type FrontierStats,
|
||||||
|
type ScratchpadLinkRow,
|
||||||
|
} from "@/server/features/audit/AuditScratchpad";
|
||||||
import { AuditProgressKV } from "@/server/lib/audit/progress-kv";
|
import { AuditProgressKV } from "@/server/lib/audit/progress-kv";
|
||||||
|
import {
|
||||||
|
adjustCrawlWindow,
|
||||||
|
INITIAL_CRAWL_WINDOW,
|
||||||
|
} from "@/server/lib/audit/crawl-window";
|
||||||
import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";
|
import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";
|
||||||
import { pgStep } from "@/server/workflows/pgStep";
|
import { pgStep } from "@/server/workflows/pgStep";
|
||||||
|
import { CRAWL_CHUNK_STEP } from "@/server/workflows/auditStepConfigs";
|
||||||
|
|
||||||
const CRAWL_CONCURRENCY = 25;
|
/**
|
||||||
// Keep durable step state under the ~1MiB limit: full link lists live in D1;
|
* The crawl runs in chunks: each chunk is one durable step that leases up to
|
||||||
// the step return only carries new-to-the-frontier targets, deduped across
|
* CHUNK_TARGET_PAGES URLs from the scratchpad DO, crawls them with a rolling
|
||||||
// the batch and capped.
|
* concurrency window, and persists results incrementally (full page rows to
|
||||||
const MAX_FRONTIER_LINKS_PER_BATCH = 2_000;
|
* the app DB, links/mirror/frontier updates to the DO). Step returns carry
|
||||||
const MAX_SUMMARY_TITLE_CHARS = 300;
|
* only counters, so step state stays tiny regardless of site size and the
|
||||||
|
* workflow heap stays O(one batch).
|
||||||
|
*/
|
||||||
|
const CHUNK_TARGET_PAGES = 200;
|
||||||
|
/** Stop launching new fetches after this long; leftover leases are released. */
|
||||||
|
const CHUNK_SOFT_DEADLINE_MS = 90_000;
|
||||||
|
/** Crawled pages are persisted in sub-batches of this size. */
|
||||||
|
const PERSIST_BATCH_SIZE = 25;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mega-menu/footer-heavy sites can carry 1000+ links per page; cap what we
|
||||||
|
* record so a 10k-page crawl can't produce tens of millions of link rows.
|
||||||
|
*/
|
||||||
|
const MAX_STORED_LINKS_PER_PAGE = 500;
|
||||||
|
/**
|
||||||
|
* Cap newly discovered URLs sent to the scratchpad per persist sub-batch.
|
||||||
|
* Serialized RPC messages are limited to 32 MiB; a crawler-trap page family
|
||||||
|
* (faceted nav, calendars) can emit tens of thousands of unique URLs per
|
||||||
|
* page. Dropped URLs are usually re-discovered from later pages, and a site
|
||||||
|
* generating this many is past maxPages anyway.
|
||||||
|
*/
|
||||||
|
const MAX_DISCOVERED_PER_BATCH = 20_000;
|
||||||
|
const MAX_PROGRESS_TITLE_CHARS = 300;
|
||||||
|
|
||||||
function shouldQueueCrawlLink(
|
function shouldQueueCrawlLink(
|
||||||
link: string,
|
link: string,
|
||||||
origin: string,
|
origin: string,
|
||||||
robots: RobotsResult,
|
robots: RobotsResult,
|
||||||
visited: Set<string>,
|
|
||||||
queued: Set<string>,
|
|
||||||
): boolean {
|
): boolean {
|
||||||
return (
|
return (
|
||||||
isSameOrigin(link, origin) &&
|
isSameOrigin(link, origin) && isCrawlableUrl(link) && robots.isAllowed(link)
|
||||||
isCrawlableUrl(link) &&
|
|
||||||
robots.isAllowed(link) &&
|
|
||||||
!visited.has(link) &&
|
|
||||||
!queued.has(link)
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
interface QueueEntry {
|
|
||||||
url: string;
|
|
||||||
/** Clicks from the start URL; null when only reachable via sitemap. */
|
|
||||||
depth: number | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
type CrawlPhaseParams = {
|
type CrawlPhaseParams = {
|
||||||
auditId: string;
|
auditId: string;
|
||||||
workflowInstanceId: string;
|
workflowInstanceId: string;
|
||||||
origin: string;
|
origin: string;
|
||||||
startUrl: string;
|
|
||||||
maxPages: number;
|
maxPages: number;
|
||||||
robots: RobotsResult;
|
robots: RobotsResult;
|
||||||
sitemapUrls: string[];
|
/** Frontier size after discovery seeding (from the discover-urls step). */
|
||||||
|
seededCount: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
/** What later phases need per page — no link lists (those stay in D1). */
|
|
||||||
export type CrawledPageSummary = Omit<StepPageSummary, "internalLinks">;
|
|
||||||
|
|
||||||
export type CrawlPhaseResult = {
|
export type CrawlPhaseResult = {
|
||||||
pages: CrawledPageSummary[];
|
pagesCrawled: number;
|
||||||
/** True when the frontier was exhausted before hitting maxPages. */
|
/** True when the frontier was exhausted before hitting maxPages. */
|
||||||
completed: boolean;
|
completed: boolean;
|
||||||
};
|
};
|
||||||
@ -62,282 +80,219 @@ export async function runCrawlPhase(
|
|||||||
step: WorkflowStep,
|
step: WorkflowStep,
|
||||||
params: CrawlPhaseParams,
|
params: CrawlPhaseParams,
|
||||||
): Promise<CrawlPhaseResult> {
|
): Promise<CrawlPhaseResult> {
|
||||||
const {
|
let chunkNo = 0;
|
||||||
auditId,
|
let attemptedTotal = 0;
|
||||||
workflowInstanceId,
|
let pending = params.seededCount;
|
||||||
origin,
|
let zeroProgressChunks = 0;
|
||||||
startUrl,
|
|
||||||
maxPages,
|
|
||||||
robots,
|
|
||||||
sitemapUrls,
|
|
||||||
} = params;
|
|
||||||
const visited = new Set<string>();
|
|
||||||
const queued = new Set<string>();
|
|
||||||
// Link-discovered URLs crawl first (BFS from the start URL); sitemap-only
|
|
||||||
// URLs drain last so link discovery isn't starved of page budget and
|
|
||||||
// orphan detection stays meaningful.
|
|
||||||
const linkQueue: QueueEntry[] = [];
|
|
||||||
const sitemapQueue: QueueEntry[] = [];
|
|
||||||
const sitemapSet = new Set<string>();
|
|
||||||
const summaries: CrawledPageSummary[] = [];
|
|
||||||
|
|
||||||
const normalizedStart = normalizeUrl(startUrl) ?? startUrl;
|
while (pending > 0 && attemptedTotal < params.maxPages) {
|
||||||
if (
|
chunkNo += 1;
|
||||||
robots.isAllowed(normalizedStart) &&
|
const result = await pgStep(
|
||||||
isSameOrigin(normalizedStart, origin)
|
|
||||||
) {
|
|
||||||
linkQueue.push({ url: normalizedStart, depth: 0 });
|
|
||||||
queued.add(normalizedStart);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const sitemapUrl of sitemapUrls) {
|
|
||||||
const normalized = normalizeUrl(sitemapUrl);
|
|
||||||
if (!normalized) continue;
|
|
||||||
sitemapSet.add(normalized);
|
|
||||||
if (!shouldQueueCrawlLink(normalized, origin, robots, visited, queued)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
sitemapQueue.push({ url: normalized, depth: null });
|
|
||||||
queued.add(normalized);
|
|
||||||
}
|
|
||||||
|
|
||||||
let crawlBatchIndex = 0;
|
|
||||||
while (
|
|
||||||
(linkQueue.length > 0 || sitemapQueue.length > 0) &&
|
|
||||||
summaries.length < maxPages
|
|
||||||
) {
|
|
||||||
const batchEntries = selectNextCrawlBatch({
|
|
||||||
linkQueue,
|
|
||||||
sitemapQueue,
|
|
||||||
queued,
|
|
||||||
visited,
|
|
||||||
robots,
|
|
||||||
remaining: maxPages - summaries.length,
|
|
||||||
});
|
|
||||||
if (batchEntries.length === 0) continue;
|
|
||||||
|
|
||||||
crawlBatchIndex += 1;
|
|
||||||
const crawledBatch = await runCrawlBatch(step, {
|
|
||||||
crawlBatchIndex,
|
|
||||||
auditId,
|
|
||||||
batchEntries,
|
|
||||||
sitemapSet,
|
|
||||||
visited,
|
|
||||||
queued,
|
|
||||||
});
|
|
||||||
// Keep only the slim summary in memory: at 10k pages, retaining link
|
|
||||||
// lists for the whole crawl would not fit in the 128MB Worker heap.
|
|
||||||
summaries.push(
|
|
||||||
...crawledBatch.map(({ internalLinks: _links, ...summary }) => summary),
|
|
||||||
);
|
|
||||||
|
|
||||||
enqueueDiscoveredLinks({
|
|
||||||
crawledBatch,
|
|
||||||
batchEntries,
|
|
||||||
linkQueue,
|
|
||||||
queued,
|
|
||||||
visited,
|
|
||||||
origin,
|
|
||||||
robots,
|
|
||||||
});
|
|
||||||
await persistCrawlProgress({
|
|
||||||
step,
|
step,
|
||||||
crawlBatchIndex,
|
`crawl-chunk-${chunkNo}`,
|
||||||
auditId,
|
CRAWL_CHUNK_STEP,
|
||||||
workflowInstanceId,
|
() =>
|
||||||
crawledBatch,
|
runCrawlChunk({
|
||||||
pagesCrawled: summaries.length,
|
...params,
|
||||||
visitedCount: visited.size,
|
chunkNo,
|
||||||
queueLength: linkQueue.length + sitemapQueue.length,
|
attemptedBefore: attemptedTotal,
|
||||||
maxPages,
|
}),
|
||||||
});
|
);
|
||||||
|
// Apply the chunk's counters even when it did no new work (a retried
|
||||||
|
// chunk whose earlier attempt persisted everything reports 0 attempts
|
||||||
|
// with up-to-date scratchpad totals) — finalize must not see stale ones.
|
||||||
|
attemptedTotal = result.attempted;
|
||||||
|
pending = result.pending;
|
||||||
|
// One zero-attempt chunk is normal (retry of a completed chunk number);
|
||||||
|
// two in a row means the frontier is unservable — stop with what we
|
||||||
|
// have instead of spinning forever.
|
||||||
|
zeroProgressChunks =
|
||||||
|
result.attemptedInChunk === 0 ? zeroProgressChunks + 1 : 0;
|
||||||
|
if (zeroProgressChunks >= 2) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return { pagesCrawled: attemptedTotal, completed: pending === 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runCrawlChunk(
|
||||||
|
input: CrawlPhaseParams & { chunkNo: number; attemptedBefore: number },
|
||||||
|
): Promise<{ attemptedInChunk: number; attempted: number; pending: number }> {
|
||||||
|
const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =
|
||||||
|
input;
|
||||||
|
const scratchpad = getAuditScratchpad(auditId);
|
||||||
|
|
||||||
|
const claimLimit = Math.min(
|
||||||
|
CHUNK_TARGET_PAGES,
|
||||||
|
maxPages - input.attemptedBefore,
|
||||||
|
);
|
||||||
|
const claimed = await scratchpad.claimChunk(chunkNo, claimLimit);
|
||||||
|
if (claimed.length === 0) {
|
||||||
|
const stats = await scratchpad.getStats();
|
||||||
return {
|
return {
|
||||||
pages: summaries,
|
attemptedInChunk: 0,
|
||||||
completed: linkQueue.length === 0 && sitemapQueue.length === 0,
|
attempted: stats.attempted,
|
||||||
|
pending: stats.pending,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const depthByUrl = new Map(claimed.map((entry) => [entry.url, entry.depth]));
|
||||||
|
const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS;
|
||||||
|
|
||||||
|
let windowSize = INITIAL_CRAWL_WINDOW;
|
||||||
|
let nextIndex = 0;
|
||||||
|
let attemptedInChunk = 0;
|
||||||
|
const inFlight = new Set<Promise<void>>();
|
||||||
|
let batch: CrawledPageResult[] = [];
|
||||||
|
// Persistence runs concurrently with fetching (pipelined) but sequentially
|
||||||
|
// with itself, so DB write pressure stays bounded at one batch at a time.
|
||||||
|
let persistChain: Promise<unknown> = Promise.resolve();
|
||||||
|
|
||||||
|
const flush = () => {
|
||||||
|
if (batch.length === 0) return;
|
||||||
|
const pages = batch;
|
||||||
|
batch = [];
|
||||||
|
windowSize = adjustCrawlWindow(windowSize, pages);
|
||||||
|
persistChain = persistChain.then(() =>
|
||||||
|
persistCrawledPages({
|
||||||
|
auditId,
|
||||||
|
workflowInstanceId,
|
||||||
|
origin,
|
||||||
|
robots,
|
||||||
|
scratchpad,
|
||||||
|
pages,
|
||||||
|
depthByUrl,
|
||||||
|
maxPages,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const launch = (entry: ClaimedUrl) => {
|
||||||
|
const promise = crawlPage(entry.url, entry.depth, entry.inSitemap)
|
||||||
|
.then((page) => {
|
||||||
|
attemptedInChunk += 1;
|
||||||
|
batch.push(page);
|
||||||
|
if (batch.length >= PERSIST_BATCH_SIZE) flush();
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
inFlight.delete(promise);
|
||||||
|
});
|
||||||
|
inFlight.add(promise);
|
||||||
|
};
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
while (
|
||||||
|
inFlight.size < windowSize &&
|
||||||
|
nextIndex < claimed.length &&
|
||||||
|
Date.now() < deadlineAt
|
||||||
|
) {
|
||||||
|
launch(claimed[nextIndex]);
|
||||||
|
nextIndex += 1;
|
||||||
|
}
|
||||||
|
if (inFlight.size === 0) break;
|
||||||
|
await Promise.race(inFlight);
|
||||||
|
}
|
||||||
|
flush();
|
||||||
|
await persistChain;
|
||||||
|
|
||||||
|
// Leases we never launched (soft deadline) go back to the queue.
|
||||||
|
const unattempted = claimed.slice(nextIndex).map((entry) => entry.url);
|
||||||
|
if (unattempted.length > 0) {
|
||||||
|
await scratchpad.releaseUrls(unattempted);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress counters are written per persisted sub-batch (in
|
||||||
|
// persistCrawledPages), so a chunk that dies mid-way underreports by at
|
||||||
|
// most one sub-batch, not a whole chunk.
|
||||||
|
const stats = await scratchpad.getStats();
|
||||||
|
return {
|
||||||
|
attemptedInChunk,
|
||||||
|
attempted: stats.attempted,
|
||||||
|
pending: stats.pending,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function selectNextCrawlBatch(params: {
|
async function persistCrawledPages(input: {
|
||||||
linkQueue: QueueEntry[];
|
|
||||||
sitemapQueue: QueueEntry[];
|
|
||||||
queued: Set<string>;
|
|
||||||
visited: Set<string>;
|
|
||||||
robots: RobotsResult;
|
|
||||||
remaining: number;
|
|
||||||
}) {
|
|
||||||
const { linkQueue, sitemapQueue, queued, visited, robots, remaining } =
|
|
||||||
params;
|
|
||||||
const batchSize = Math.min(CRAWL_CONCURRENCY, remaining);
|
|
||||||
const batchEntries: QueueEntry[] = [];
|
|
||||||
|
|
||||||
while (
|
|
||||||
(linkQueue.length > 0 || sitemapQueue.length > 0) &&
|
|
||||||
batchEntries.length < batchSize
|
|
||||||
) {
|
|
||||||
const entry = (linkQueue.length > 0 ? linkQueue : sitemapQueue).shift()!;
|
|
||||||
queued.delete(entry.url);
|
|
||||||
if (visited.has(entry.url)) continue;
|
|
||||||
if (!robots.isAllowed(entry.url)) continue;
|
|
||||||
visited.add(entry.url);
|
|
||||||
batchEntries.push(entry);
|
|
||||||
}
|
|
||||||
|
|
||||||
return batchEntries;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function runCrawlBatch(
|
|
||||||
step: WorkflowStep,
|
|
||||||
input: {
|
|
||||||
crawlBatchIndex: number;
|
|
||||||
auditId: string;
|
auditId: string;
|
||||||
batchEntries: QueueEntry[];
|
workflowInstanceId: string;
|
||||||
sitemapSet: Set<string>;
|
origin: string;
|
||||||
visited: Set<string>;
|
robots: RobotsResult;
|
||||||
queued: Set<string>;
|
scratchpad: ReturnType<typeof getAuditScratchpad>;
|
||||||
},
|
pages: CrawledPageResult[];
|
||||||
): Promise<StepPageSummary[]> {
|
depthByUrl: Map<string, number | null>;
|
||||||
const {
|
maxPages: number;
|
||||||
crawlBatchIndex,
|
}): Promise<FrontierStats> {
|
||||||
auditId,
|
const { auditId, origin, robots, scratchpad, pages, depthByUrl } = input;
|
||||||
batchEntries,
|
|
||||||
sitemapSet,
|
|
||||||
visited,
|
|
||||||
queued,
|
|
||||||
} = input;
|
|
||||||
return pgStep(step, `crawl-batch-${crawlBatchIndex}`, undefined, async () => {
|
|
||||||
const pages = await Promise.all(
|
|
||||||
batchEntries.map((entry) =>
|
|
||||||
crawlPage(entry.url, entry.depth, sitemapSet.has(entry.url)),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
// Deterministic ids keep the D1 writes idempotent across step retries.
|
// Deterministic ids keep every write idempotent across step retries.
|
||||||
for (const page of pages) {
|
for (const page of pages) {
|
||||||
page.id = await deterministicAuditRowId(auditId, page.url);
|
page.id = await deterministicAuditRowId(auditId, page.url);
|
||||||
}
|
}
|
||||||
|
|
||||||
const issues = pages.flatMap((page) => runPageReporters(page));
|
const issues = pages.flatMap((page) => runPageReporters(page));
|
||||||
await AuditRepository.insertCrawledBatch(auditId, pages, issues);
|
await AuditRepository.insertCrawledBatch(auditId, pages, issues);
|
||||||
|
|
||||||
// Frontier candidates only: drop targets already visited/queued and
|
const links: ScratchpadLinkRow[] = [];
|
||||||
// dedupe across the batch, so the step return stays far under the
|
const discovered = new Map<string, number | null>();
|
||||||
// ~1MiB durable-state limit even on mega-menu sites.
|
for (const page of pages) {
|
||||||
const seenTargets = new Set<string>();
|
const pageDepth = depthByUrl.get(page.url) ?? null;
|
||||||
return pages.map((page) => {
|
const childDepth = pageDepth === null ? null : pageDepth + 1;
|
||||||
const internalLinks: string[] = [];
|
|
||||||
|
let storedForPage = 0;
|
||||||
for (const link of page.links) {
|
for (const link of page.links) {
|
||||||
if (!link.isInternal) continue;
|
if (!link.isInternal) continue;
|
||||||
if (seenTargets.size >= MAX_FRONTIER_LINKS_PER_BATCH) break;
|
if (storedForPage < MAX_STORED_LINKS_PER_PAGE) {
|
||||||
if (visited.has(link.targetUrl) || queued.has(link.targetUrl)) continue;
|
storedForPage += 1;
|
||||||
if (seenTargets.has(link.targetUrl)) continue;
|
links.push({
|
||||||
seenTargets.add(link.targetUrl);
|
sourcePageId: page.id,
|
||||||
internalLinks.push(link.targetUrl);
|
sourceUrl: page.url,
|
||||||
|
targetUrl: link.targetUrl,
|
||||||
|
anchor: link.anchor,
|
||||||
|
isNofollow: link.isNofollow,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return {
|
if (
|
||||||
id: page.id,
|
discovered.size < MAX_DISCOVERED_PER_BATCH &&
|
||||||
|
!discovered.has(link.targetUrl) &&
|
||||||
|
shouldQueueCrawlLink(link.targetUrl, origin, robots)
|
||||||
|
) {
|
||||||
|
discovered.set(link.targetUrl, childDepth);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Redirect targets continue the same navigation path: same depth.
|
||||||
|
if (
|
||||||
|
page.redirectUrl &&
|
||||||
|
!discovered.has(page.redirectUrl) &&
|
||||||
|
shouldQueueCrawlLink(page.redirectUrl, origin, robots)
|
||||||
|
) {
|
||||||
|
discovered.set(page.redirectUrl, pageDepth);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const stats = await scratchpad.recordBatch({
|
||||||
|
crawledUrls: pages.map((page) => page.url),
|
||||||
|
pages: pages.map((page) => ({
|
||||||
|
pageId: page.id,
|
||||||
url: page.url,
|
url: page.url,
|
||||||
statusCode: page.statusCode,
|
statusCode: page.statusCode,
|
||||||
fetchClass: page.fetchClass,
|
fetchClass: page.fetchClass,
|
||||||
redirectUrl: page.redirectUrl,
|
redirectUrl: page.redirectUrl,
|
||||||
title: page.title.slice(0, MAX_SUMMARY_TITLE_CHARS),
|
})),
|
||||||
internalLinks,
|
links,
|
||||||
};
|
discovered: Array.from(discovered, ([url, depth]) => ({ url, depth })),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await AuditRepository.updateAuditProgress(auditId, input.workflowInstanceId, {
|
||||||
|
pagesCrawled: stats.attempted,
|
||||||
|
pagesTotal: Math.min(stats.seen, input.maxPages),
|
||||||
});
|
});
|
||||||
}
|
|
||||||
|
|
||||||
function enqueueDiscoveredLinks(params: {
|
|
||||||
crawledBatch: StepPageSummary[];
|
|
||||||
batchEntries: QueueEntry[];
|
|
||||||
linkQueue: QueueEntry[];
|
|
||||||
queued: Set<string>;
|
|
||||||
visited: Set<string>;
|
|
||||||
origin: string;
|
|
||||||
robots: RobotsResult;
|
|
||||||
}) {
|
|
||||||
const {
|
|
||||||
crawledBatch,
|
|
||||||
batchEntries,
|
|
||||||
linkQueue,
|
|
||||||
queued,
|
|
||||||
visited,
|
|
||||||
origin,
|
|
||||||
robots,
|
|
||||||
} = params;
|
|
||||||
const depthByUrl = new Map(
|
|
||||||
batchEntries.map((entry) => [entry.url, entry.depth]),
|
|
||||||
);
|
|
||||||
|
|
||||||
for (const pageResult of crawledBatch) {
|
|
||||||
const pageDepth = depthByUrl.get(pageResult.url) ?? null;
|
|
||||||
const childDepth = pageDepth === null ? null : pageDepth + 1;
|
|
||||||
|
|
||||||
for (const link of pageResult.internalLinks) {
|
|
||||||
if (!shouldQueueCrawlLink(link, origin, robots, visited, queued)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
linkQueue.push({ url: link, depth: childDepth });
|
|
||||||
queued.add(link);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Redirect targets continue the same navigation path: same depth.
|
|
||||||
const redirectTarget = pageResult.redirectUrl;
|
|
||||||
if (
|
|
||||||
redirectTarget &&
|
|
||||||
shouldQueueCrawlLink(redirectTarget, origin, robots, visited, queued)
|
|
||||||
) {
|
|
||||||
linkQueue.push({ url: redirectTarget, depth: pageDepth });
|
|
||||||
queued.add(redirectTarget);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function persistCrawlProgress(params: {
|
|
||||||
step: WorkflowStep;
|
|
||||||
crawlBatchIndex: number;
|
|
||||||
auditId: string;
|
|
||||||
workflowInstanceId: string;
|
|
||||||
crawledBatch: StepPageSummary[];
|
|
||||||
pagesCrawled: number;
|
|
||||||
visitedCount: number;
|
|
||||||
queueLength: number;
|
|
||||||
maxPages: number;
|
|
||||||
}) {
|
|
||||||
const {
|
|
||||||
step,
|
|
||||||
crawlBatchIndex,
|
|
||||||
auditId,
|
|
||||||
workflowInstanceId,
|
|
||||||
crawledBatch,
|
|
||||||
pagesCrawled,
|
|
||||||
visitedCount,
|
|
||||||
queueLength,
|
|
||||||
maxPages,
|
|
||||||
} = params;
|
|
||||||
// KV push + D1 progress in one step — merging them halves the per-batch
|
|
||||||
// step count against the ~1k step budget. The D1 update is idempotent; the
|
|
||||||
// KV push can duplicate entries on a partial retry, which is acceptable for
|
|
||||||
// an ephemeral progress feed (capped list, short TTL).
|
|
||||||
await pgStep(
|
|
||||||
step,
|
|
||||||
`progress-batch-${crawlBatchIndex}`,
|
|
||||||
undefined,
|
|
||||||
async () => {
|
|
||||||
await AuditProgressKV.pushCrawledUrls(
|
await AuditProgressKV.pushCrawledUrls(
|
||||||
auditId,
|
auditId,
|
||||||
crawledBatch.map((pageResult) => ({
|
pages.map((page) => ({
|
||||||
url: pageResult.url,
|
url: page.url,
|
||||||
statusCode: pageResult.statusCode,
|
statusCode: page.statusCode,
|
||||||
title: pageResult.title,
|
title: page.title.slice(0, MAX_PROGRESS_TITLE_CHARS),
|
||||||
crawledAt: Date.now(),
|
crawledAt: Date.now(),
|
||||||
})),
|
})),
|
||||||
);
|
);
|
||||||
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
return stats;
|
||||||
pagesCrawled,
|
|
||||||
pagesTotal: Math.min(visitedCount + queueLength, maxPages),
|
|
||||||
});
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,36 +5,34 @@ import {
|
|||||||
fetchAndStoreLighthouseResult,
|
fetchAndStoreLighthouseResult,
|
||||||
selectLighthouseSample,
|
selectLighthouseSample,
|
||||||
} from "@/server/lib/audit/lighthouse";
|
} from "@/server/lib/audit/lighthouse";
|
||||||
import { getOrigin } from "@/server/lib/audit/url-utils";
|
import {
|
||||||
|
getOrigin,
|
||||||
|
isSameOrigin,
|
||||||
|
normalizeUrl,
|
||||||
|
} from "@/server/lib/audit/url-utils";
|
||||||
|
import { isCrawlableUrl } from "@/server/lib/audit/url-policy";
|
||||||
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
|
||||||
|
import { getAuditScratchpad } from "@/server/features/audit/AuditScratchpad";
|
||||||
import { AuditProgressKV } from "@/server/lib/audit/progress-kv";
|
import { AuditProgressKV } from "@/server/lib/audit/progress-kv";
|
||||||
import { runMultipageChecks } from "@/server/lib/audit/issues/multipage";
|
import { runMultipageChecks } from "@/server/lib/audit/issues/multipage";
|
||||||
|
import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters";
|
||||||
import type { AuditConfig } from "@/server/lib/audit/types";
|
import type { AuditConfig } from "@/server/lib/audit/types";
|
||||||
import { captureServerEvent } from "@/server/lib/posthog";
|
import { captureServerEvent } from "@/server/lib/posthog";
|
||||||
import {
|
import {
|
||||||
runCrawlPhase,
|
runCrawlPhase,
|
||||||
type CrawledPageSummary,
|
|
||||||
type CrawlPhaseResult,
|
type CrawlPhaseResult,
|
||||||
} from "@/server/workflows/siteAuditWorkflowCrawl";
|
} from "@/server/workflows/siteAuditWorkflowCrawl";
|
||||||
import { pgStep } from "@/server/workflows/pgStep";
|
import { pgStep } from "@/server/workflows/pgStep";
|
||||||
|
import {
|
||||||
|
DB_STEP,
|
||||||
|
DISCOVERY_STEP,
|
||||||
|
LIGHTHOUSE_BATCH_STEP,
|
||||||
|
MULTIPAGE_CHECKS_STEP,
|
||||||
|
} from "@/server/workflows/auditStepConfigs";
|
||||||
|
|
||||||
const LIGHTHOUSE_URL_BATCH_SIZE = 10;
|
const LIGHTHOUSE_URL_BATCH_SIZE = 10;
|
||||||
|
/** Frontier seeds per scratchpad RPC call. */
|
||||||
// Workflows rejects step outputs over 1MiB; keep the sitemap seed list well
|
const SEED_RPC_BATCH = 2_000;
|
||||||
// under that. The crawl visits at most maxPages URLs, so extra seeds are moot.
|
|
||||||
const SITEMAP_SEED_BYTE_BUDGET = 768 * 1024;
|
|
||||||
|
|
||||||
function capSitemapSeeds(urls: string[], maxPages: number): string[] {
|
|
||||||
const seeds: string[] = [];
|
|
||||||
let bytes = 0;
|
|
||||||
for (const url of urls) {
|
|
||||||
if (seeds.length >= maxPages) break;
|
|
||||||
bytes += url.length + 3; // JSON quotes + comma
|
|
||||||
if (bytes > SITEMAP_SEED_BYTE_BUDGET) break;
|
|
||||||
seeds.push(url);
|
|
||||||
}
|
|
||||||
return seeds;
|
|
||||||
}
|
|
||||||
|
|
||||||
type AuditPhasesParams = {
|
type AuditPhasesParams = {
|
||||||
auditId: string;
|
auditId: string;
|
||||||
@ -60,13 +58,13 @@ export async function runAuditPhases(
|
|||||||
const origin = getOrigin(startUrl);
|
const origin = getOrigin(startUrl);
|
||||||
const maxPages = config.maxPages;
|
const maxPages = config.maxPages;
|
||||||
|
|
||||||
const discovery = await runDiscoveryPhase(
|
const discovery = await runDiscoveryPhase(step, {
|
||||||
step,
|
|
||||||
auditId,
|
auditId,
|
||||||
workflowInstanceId,
|
workflowInstanceId,
|
||||||
origin,
|
origin,
|
||||||
|
startUrl,
|
||||||
maxPages,
|
maxPages,
|
||||||
);
|
});
|
||||||
// Parsed outside the step from checkpointed text, so replays see the exact
|
// Parsed outside the step from checkpointed text, so replays see the exact
|
||||||
// robots rules the original run used (a live re-fetch could differ and
|
// robots rules the original run used (a live re-fetch could differ and
|
||||||
// desync the frontier from already-persisted crawl batches).
|
// desync the frontier from already-persisted crawl batches).
|
||||||
@ -75,10 +73,9 @@ export async function runAuditPhases(
|
|||||||
auditId,
|
auditId,
|
||||||
workflowInstanceId,
|
workflowInstanceId,
|
||||||
origin,
|
origin,
|
||||||
startUrl,
|
|
||||||
maxPages,
|
maxPages,
|
||||||
robots,
|
robots,
|
||||||
sitemapUrls: discovery.sitemapUrls,
|
seededCount: discovery.seededCount,
|
||||||
});
|
});
|
||||||
await runLighthousePhase(step, {
|
await runLighthousePhase(step, {
|
||||||
auditId,
|
auditId,
|
||||||
@ -87,7 +84,6 @@ export async function runAuditPhases(
|
|||||||
projectId,
|
projectId,
|
||||||
startUrl,
|
startUrl,
|
||||||
config,
|
config,
|
||||||
pages: crawl.pages,
|
|
||||||
});
|
});
|
||||||
await finalizeAudit({
|
await finalizeAudit({
|
||||||
step,
|
step,
|
||||||
@ -103,21 +99,61 @@ export async function runAuditPhases(
|
|||||||
|
|
||||||
async function runDiscoveryPhase(
|
async function runDiscoveryPhase(
|
||||||
step: WorkflowStep,
|
step: WorkflowStep,
|
||||||
auditId: string,
|
input: {
|
||||||
workflowInstanceId: string,
|
auditId: string;
|
||||||
origin: string,
|
workflowInstanceId: string;
|
||||||
maxPages: number,
|
origin: string;
|
||||||
|
startUrl: string;
|
||||||
|
maxPages: number;
|
||||||
|
},
|
||||||
) {
|
) {
|
||||||
return pgStep(step, "discover-urls", undefined, async () => {
|
const { auditId, workflowInstanceId, origin, startUrl, maxPages } = input;
|
||||||
|
// "-v2": the checkpoint shape changed (seeds now live in the scratchpad DO
|
||||||
|
// instead of the step return). A pre-refactor instance replayed under this
|
||||||
|
// code must re-run discovery — resuming from the old cached {sitemapUrls}
|
||||||
|
// shape would leave the scratchpad empty and finalize a zero-page audit.
|
||||||
|
return pgStep(step, "discover-urls-v2", DISCOVERY_STEP, async () => {
|
||||||
const result = await discoverUrls(origin, maxPages);
|
const result = await discoverUrls(origin, maxPages);
|
||||||
|
const robots = parseRobotsTxt(origin, result.robotsText);
|
||||||
|
const scratchpad = getAuditScratchpad(auditId);
|
||||||
|
|
||||||
|
// Seeds go straight into the scratchpad frontier — nothing large is
|
||||||
|
// returned as step state (an uncapped seed list used to blow the ~1MiB
|
||||||
|
// step-output limit on big sitemaps).
|
||||||
|
let seededCount = 0;
|
||||||
|
const normalizedStart = normalizeUrl(startUrl) ?? startUrl;
|
||||||
|
if (
|
||||||
|
robots.isAllowed(normalizedStart) &&
|
||||||
|
isSameOrigin(normalizedStart, origin)
|
||||||
|
) {
|
||||||
|
await scratchpad.seedStart(normalizedStart);
|
||||||
|
seededCount += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The start URL is deliberately not excluded here: seedSitemapUrls
|
||||||
|
// upserts, so a start URL that also appears in the sitemap keeps its
|
||||||
|
// link-queue position but gains the in-sitemap flag.
|
||||||
|
const seen = new Set<string>();
|
||||||
|
const seeds: string[] = [];
|
||||||
|
for (const url of result.urls) {
|
||||||
|
const normalized = normalizeUrl(url);
|
||||||
|
if (!normalized || seen.has(normalized)) continue;
|
||||||
|
seen.add(normalized);
|
||||||
|
if (!isSameOrigin(normalized, origin)) continue;
|
||||||
|
if (!isCrawlableUrl(normalized)) continue;
|
||||||
|
if (!robots.isAllowed(normalized)) continue;
|
||||||
|
seeds.push(normalized);
|
||||||
|
}
|
||||||
|
for (let i = 0; i < seeds.length; i += SEED_RPC_BATCH) {
|
||||||
|
await scratchpad.seedSitemapUrls(seeds.slice(i, i + SEED_RPC_BATCH));
|
||||||
|
}
|
||||||
|
seededCount += seeds.filter((seed) => seed !== normalizedStart).length;
|
||||||
|
|
||||||
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
||||||
pagesTotal: Math.min(result.urls.length + 1, maxPages),
|
pagesTotal: Math.min(seededCount, maxPages),
|
||||||
currentPhase: "crawling",
|
currentPhase: "crawling",
|
||||||
});
|
});
|
||||||
return {
|
return { robotsText: result.robotsText, seededCount };
|
||||||
sitemapUrls: capSitemapSeeds(result.urls, maxPages),
|
|
||||||
robotsText: result.robotsText,
|
|
||||||
};
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -128,7 +164,6 @@ type LighthousePhaseParams = {
|
|||||||
projectId: string;
|
projectId: string;
|
||||||
startUrl: string;
|
startUrl: string;
|
||||||
config: AuditConfig;
|
config: AuditConfig;
|
||||||
pages: CrawledPageSummary[];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
async function runLighthousePhase(
|
async function runLighthousePhase(
|
||||||
@ -142,7 +177,6 @@ async function runLighthousePhase(
|
|||||||
projectId,
|
projectId,
|
||||||
startUrl,
|
startUrl,
|
||||||
config,
|
config,
|
||||||
pages,
|
|
||||||
} = params;
|
} = params;
|
||||||
if (config.lighthouseStrategy === "none") return;
|
if (config.lighthouseStrategy === "none") return;
|
||||||
|
|
||||||
@ -150,7 +184,6 @@ async function runLighthousePhase(
|
|||||||
step,
|
step,
|
||||||
auditId,
|
auditId,
|
||||||
workflowInstanceId,
|
workflowInstanceId,
|
||||||
pages,
|
|
||||||
startUrl,
|
startUrl,
|
||||||
strategy: config.lighthouseStrategy,
|
strategy: config.lighthouseStrategy,
|
||||||
});
|
});
|
||||||
@ -165,12 +198,12 @@ async function runLighthousePhase(
|
|||||||
const priorCompleted = completedChecks;
|
const priorCompleted = completedChecks;
|
||||||
const priorFailed = failedChecks;
|
const priorFailed = failedChecks;
|
||||||
|
|
||||||
// Fetch, store (R2 + D1) and update progress inside one step. The step
|
// Fetch, store (R2 + DB) and update progress inside one step. The step
|
||||||
// returns only counts; full results live in D1.
|
// returns only counts; full results live in the DB.
|
||||||
const counts = await pgStep(
|
const counts = await pgStep(
|
||||||
step,
|
step,
|
||||||
`lighthouse-batch-${lighthouseBatchIndex}`,
|
`lighthouse-batch-${lighthouseBatchIndex}`,
|
||||||
undefined,
|
LIGHTHOUSE_BATCH_STEP,
|
||||||
async () => {
|
async () => {
|
||||||
const perUrlResults = await Promise.all(
|
const perUrlResults = await Promise.all(
|
||||||
batch.map(async ({ url, pageId }) => {
|
batch.map(async ({ url, pageId }) => {
|
||||||
@ -217,14 +250,22 @@ async function selectLighthousePages(params: {
|
|||||||
step: WorkflowStep;
|
step: WorkflowStep;
|
||||||
auditId: string;
|
auditId: string;
|
||||||
workflowInstanceId: string;
|
workflowInstanceId: string;
|
||||||
pages: CrawledPageSummary[];
|
|
||||||
startUrl: string;
|
startUrl: string;
|
||||||
strategy: AuditConfig["lighthouseStrategy"];
|
strategy: AuditConfig["lighthouseStrategy"];
|
||||||
}) {
|
}) {
|
||||||
const { step, auditId, workflowInstanceId, pages, startUrl, strategy } =
|
const { step, auditId, workflowInstanceId, startUrl, strategy } = params;
|
||||||
params;
|
return pgStep(step, "select-lighthouse-sample", DB_STEP, async () => {
|
||||||
return pgStep(step, "select-lighthouse-sample", undefined, async () => {
|
// Crawled pages come from the DB — the crawl phase no longer holds a
|
||||||
const sample = selectLighthouseSample(pages, startUrl, strategy);
|
// whole-crawl page list in memory.
|
||||||
|
const crawledPages = await AuditRepository.getPagesForAudit(auditId);
|
||||||
|
const sample = selectLighthouseSample(
|
||||||
|
crawledPages.map((page) => ({
|
||||||
|
url: page.url,
|
||||||
|
statusCode: page.statusCode ?? 0,
|
||||||
|
})),
|
||||||
|
startUrl,
|
||||||
|
strategy,
|
||||||
|
);
|
||||||
const selectedUrls = new Set(sample);
|
const selectedUrls = new Set(sample);
|
||||||
|
|
||||||
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
||||||
@ -233,7 +274,7 @@ async function selectLighthousePages(params: {
|
|||||||
lighthouseCompleted: 0,
|
lighthouseCompleted: 0,
|
||||||
lighthouseFailed: 0,
|
lighthouseFailed: 0,
|
||||||
});
|
});
|
||||||
return pages.flatMap((page) =>
|
return crawledPages.flatMap((page) =>
|
||||||
selectedUrls.has(page.url) ? [{ url: page.url, pageId: page.id }] : [],
|
selectedUrls.has(page.url) ? [{ url: page.url, pageId: page.id }] : [],
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@ -260,37 +301,34 @@ async function finalizeAudit(args: {
|
|||||||
crawl,
|
crawl,
|
||||||
} = args;
|
} = args;
|
||||||
|
|
||||||
await pgStep(step, "multipage-checks", undefined, async () => {
|
await pgStep(step, "multipage-checks", MULTIPAGE_CHECKS_STEP, async () => {
|
||||||
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
|
||||||
currentPhase: "finalizing",
|
currentPhase: "finalizing",
|
||||||
});
|
});
|
||||||
|
|
||||||
// Integrity guard: pages are persisted inside crawl-batch steps. If the
|
// Integrity guard: pages are persisted inside crawl-chunk steps. If the
|
||||||
// crawl claims pages but D1 has none (e.g. an instance started under the
|
// crawl claims pages but the DB has none, fail loudly instead of
|
||||||
// pre-incremental-persistence code was replayed under this code), fail
|
// completing with an empty audit.
|
||||||
// loudly instead of completing with an empty audit.
|
|
||||||
if (
|
if (
|
||||||
crawl.pages.length > 0 &&
|
crawl.pagesCrawled > 0 &&
|
||||||
!(await AuditRepository.hasPagesForAudit(auditId))
|
!(await AuditRepository.hasPagesForAudit(auditId))
|
||||||
) {
|
) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Audit ${auditId}: crawl reported ${crawl.pages.length} pages but none were persisted`,
|
`Audit ${auditId}: crawl reported ${crawl.pagesCrawled} pages but none were persisted`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const issues = await runMultipageChecks({
|
const issues = await runMultipageChecks({ auditId });
|
||||||
auditId,
|
issues.push(...(await runScratchpadLinkChecks(auditId, startUrl, crawl)));
|
||||||
startUrl,
|
|
||||||
crawlCompleted: crawl.completed,
|
|
||||||
});
|
|
||||||
await AuditRepository.insertIssues(auditId, issues);
|
await AuditRepository.insertIssues(auditId, issues);
|
||||||
return { issueCount: issues.length };
|
return { issueCount: issues.length };
|
||||||
});
|
});
|
||||||
|
|
||||||
await pgStep(step, "finalize", undefined, async () => {
|
await pgStep(step, "finalize", DB_STEP, async () => {
|
||||||
|
const blockedPages = await AuditRepository.countBlockedPages(auditId);
|
||||||
await AuditRepository.completeAudit(auditId, workflowInstanceId, {
|
await AuditRepository.completeAudit(auditId, workflowInstanceId, {
|
||||||
pagesCrawled: crawl.pages.length,
|
pagesCrawled: crawl.pagesCrawled,
|
||||||
pagesTotal: crawl.pages.length,
|
pagesTotal: crawl.pagesCrawled,
|
||||||
});
|
});
|
||||||
await captureServerEvent({
|
await captureServerEvent({
|
||||||
distinctId: billingCustomer.userId,
|
distinctId: billingCustomer.userId,
|
||||||
@ -299,15 +337,49 @@ async function finalizeAudit(args: {
|
|||||||
properties: {
|
properties: {
|
||||||
project_id: projectId,
|
project_id: projectId,
|
||||||
status: "completed",
|
status: "completed",
|
||||||
pages_crawled: crawl.pages.length,
|
pages_crawled: crawl.pagesCrawled,
|
||||||
pages_total: crawl.pages.length,
|
pages_total: crawl.pagesCrawled,
|
||||||
crawl_completed: crawl.completed,
|
crawl_completed: crawl.completed,
|
||||||
pages_blocked: crawl.pages.filter(
|
pages_blocked: blockedPages,
|
||||||
(page) => page.fetchClass === "blocked",
|
|
||||||
).length,
|
|
||||||
run_lighthouse: config.lighthouseStrategy !== "none",
|
run_lighthouse: config.lighthouseStrategy !== "none",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
await AuditProgressKV.clear(auditId);
|
await AuditProgressKV.clear(auditId);
|
||||||
|
// Crawl scratch state (frontier, links, mirror) is no longer needed.
|
||||||
|
await getAuditScratchpad(auditId).destroy();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The two finalize checks that need link edges run as SQL inside the
|
||||||
|
* audit's scratchpad DO; map their rows onto DetectedIssue.
|
||||||
|
*/
|
||||||
|
async function runScratchpadLinkChecks(
|
||||||
|
auditId: string,
|
||||||
|
startUrl: string,
|
||||||
|
crawl: CrawlPhaseResult,
|
||||||
|
): Promise<DetectedIssue[]> {
|
||||||
|
const scratchpad = getAuditScratchpad(auditId);
|
||||||
|
const { brokenLinks, orphanPages } = await scratchpad.runFinalizeChecks({
|
||||||
|
// Page rows store normalized URLs; normalize the start URL the same way
|
||||||
|
// so the orphan exclusion matches.
|
||||||
|
startUrl: normalizeUrl(startUrl) ?? startUrl,
|
||||||
|
// Orphan detection only makes sense when the crawl wasn't truncated.
|
||||||
|
crawlCompleted: crawl.completed,
|
||||||
|
});
|
||||||
|
|
||||||
|
return [
|
||||||
|
...brokenLinks.map((row) => ({
|
||||||
|
issueType: "broken-internal-link" as const,
|
||||||
|
pageId: row.sourcePageId,
|
||||||
|
pageUrl: row.sourceUrl,
|
||||||
|
dedupeKey: row.targetUrl,
|
||||||
|
details: { targetUrl: row.targetUrl, targetStatus: row.targetStatus },
|
||||||
|
})),
|
||||||
|
...orphanPages.map((row) => ({
|
||||||
|
issueType: "orphan-page" as const,
|
||||||
|
pageId: row.pageId,
|
||||||
|
pageUrl: row.url,
|
||||||
|
})),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|||||||
@ -33,7 +33,9 @@
|
|||||||
],
|
],
|
||||||
// Durable Object backing the onboarding strategy chat (Agents SDK
|
// Durable Object backing the onboarding strategy chat (Agents SDK
|
||||||
// AIChatAgent). One instance per project; messages persist in the DO's
|
// AIChatAgent). One instance per project; messages persist in the DO's
|
||||||
// SQLite. SQLite-backed classes must be declared in `migrations` below.
|
// SQLite. SQLite-backed classes must be declared in `migrations` below for
|
||||||
|
// the wrangler/workerd surfaces (local dev, Docker self-host); Alchemy
|
||||||
|
// deploys derive SQLite backing on their own.
|
||||||
"durable_objects": {
|
"durable_objects": {
|
||||||
"bindings": [
|
"bindings": [
|
||||||
{
|
{
|
||||||
@ -46,6 +48,13 @@
|
|||||||
"name": "SAM_CHAT",
|
"name": "SAM_CHAT",
|
||||||
"class_name": "SamChatAgent",
|
"class_name": "SamChatAgent",
|
||||||
},
|
},
|
||||||
|
// Per-audit crawl scratchpad: frontier, link edges, and a slim page
|
||||||
|
// mirror in the DO's SQLite. Destroyed at finalize; self-cleans via
|
||||||
|
// alarm if the audit dies.
|
||||||
|
{
|
||||||
|
"name": "AUDIT_SCRATCHPAD",
|
||||||
|
"class_name": "AuditScratchpad",
|
||||||
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
"migrations": [
|
"migrations": [
|
||||||
@ -57,6 +66,10 @@
|
|||||||
"tag": "v2",
|
"tag": "v2",
|
||||||
"new_sqlite_classes": ["SamChatAgent"],
|
"new_sqlite_classes": ["SamChatAgent"],
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"tag": "v3",
|
||||||
|
"new_sqlite_classes": ["AuditScratchpad"],
|
||||||
|
},
|
||||||
],
|
],
|
||||||
"triggers": {
|
"triggers": {
|
||||||
"crons": ["*/15 * * * *"],
|
"crons": ["*/15 * * * *"],
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user