Issue
What it demonstrates
diff --git a/drizzle-pg/0014_solid_centennial.sql b/drizzle-pg/0014_solid_centennial.sql
new file mode 100644
index 0000000..c018e71
--- /dev/null
+++ b/drizzle-pg/0014_solid_centennial.sql
@@ -0,0 +1,3 @@
+ALTER TABLE "audits" ADD COLUMN "error_code" text;--> statement-breakpoint
+ALTER TABLE "audits" ADD COLUMN "error_detail" text;--> statement-breakpoint
+ALTER TABLE "audits" ADD COLUMN "failed_phase" text;
\ No newline at end of file
diff --git a/drizzle-pg/meta/0014_snapshot.json b/drizzle-pg/meta/0014_snapshot.json
new file mode 100644
index 0000000..73fdbbf
--- /dev/null
+++ b/drizzle-pg/meta/0014_snapshot.json
@@ -0,0 +1,3732 @@
+{
+ "id": "c425b63a-cdeb-42d2-8ee2-b8805fa85a42",
+ "prevId": "d61539b1-c4dd-4438-8a8b-7840b9122a72",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.backlink_snapshots": {
+ "name": "backlink_snapshots",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "domain": {
+ "name": "domain",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "rank": {
+ "name": "rank",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "backlinks": {
+ "name": "backlinks",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "referring_domains": {
+ "name": "referring_domains",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "broken_backlinks": {
+ "name": "broken_backlinks",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "new_backlinks": {
+ "name": "new_backlinks",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "lost_backlinks": {
+ "name": "lost_backlinks",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "new_referring_domains": {
+ "name": "new_referring_domains",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "lost_referring_domains": {
+ "name": "lost_referring_domains",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "captured_at": {
+ "name": "captured_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "backlink_snapshots_project_captured_idx": {
+ "name": "backlink_snapshots_project_captured_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "captured_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "backlink_snapshots_project_id_projects_id_fk": {
+ "name": "backlink_snapshots_project_id_projects_id_fk",
+ "tableFrom": "backlink_snapshots",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.keyword_metrics": {
+ "name": "keyword_metrics",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "keyword": {
+ "name": "keyword",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "location_code": {
+ "name": "location_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "language_code": {
+ "name": "language_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'en'"
+ },
+ "search_volume": {
+ "name": "search_volume",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "cpc": {
+ "name": "cpc",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "competition": {
+ "name": "competition",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "keyword_difficulty": {
+ "name": "keyword_difficulty",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "intent": {
+ "name": "intent",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "monthly_searches": {
+ "name": "monthly_searches",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "fetched_at": {
+ "name": "fetched_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "keyword_metrics_unique_project_keyword_location_language": {
+ "name": "keyword_metrics_unique_project_keyword_location_language",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "keyword",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "location_code",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "language_code",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "keyword_metrics_lookup_idx": {
+ "name": "keyword_metrics_lookup_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "keyword",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "location_code",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "language_code",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "fetched_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "keyword_metrics_project_id_projects_id_fk": {
+ "name": "keyword_metrics_project_id_projects_id_fk",
+ "tableFrom": "keyword_metrics",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.organization_activation_state": {
+ "name": "organization_activation_state",
+ "schema": "",
+ "columns": {
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "first_mcp_authorized_at": {
+ "name": "first_mcp_authorized_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "first_mcp_tool_call_at": {
+ "name": "first_mcp_tool_call_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "organization_activation_state_organization_id_organization_id_fk": {
+ "name": "organization_activation_state_organization_id_organization_id_fk",
+ "tableFrom": "organization_activation_state",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.project_activation_state": {
+ "name": "project_activation_state",
+ "schema": "",
+ "columns": {
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "competitor_step_clicked_at": {
+ "name": "competitor_step_clicked_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "mcp_card_dismissed_at": {
+ "name": "mcp_card_dismissed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "project_activation_state_project_id_projects_id_fk": {
+ "name": "project_activation_state_project_id_projects_id_fk",
+ "tableFrom": "project_activation_state",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.projects": {
+ "name": "projects",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "domain": {
+ "name": "domain",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "location_code": {
+ "name": "location_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 2840
+ },
+ "language_code": {
+ "name": "language_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'en'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "projects_one_default_per_organization_idx": {
+ "name": "projects_one_default_per_organization_idx",
+ "columns": [
+ {
+ "expression": "organization_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"projects\".\"name\" = 'Default' AND \"projects\".\"domain\" IS NULL AND \"projects\".\"archived_at\" IS NULL",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "projects_organization_id_idx": {
+ "name": "projects_organization_id_idx",
+ "columns": [
+ {
+ "expression": "organization_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "projects_organization_id_organization_id_fk": {
+ "name": "projects_organization_id_organization_id_fk",
+ "tableFrom": "projects",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.rank_check_runs": {
+ "name": "rank_check_runs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "config_id": {
+ "name": "config_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending'"
+ },
+ "keywords_total": {
+ "name": "keywords_total",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "keywords_checked": {
+ "name": "keywords_checked",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "is_subset_run": {
+ "name": "is_subset_run",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "error_message": {
+ "name": "error_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "started_at": {
+ "name": "started_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "rank_check_runs_config_idx": {
+ "name": "rank_check_runs_config_idx",
+ "columns": [
+ {
+ "expression": "config_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "started_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "rank_check_runs_project_idx": {
+ "name": "rank_check_runs_project_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "started_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "rank_check_runs_one_active_per_config_idx": {
+ "name": "rank_check_runs_one_active_per_config_idx",
+ "columns": [
+ {
+ "expression": "config_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"rank_check_runs\".\"status\" IN ('pending', 'running')",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "rank_check_runs_config_id_rank_tracking_configs_id_fk": {
+ "name": "rank_check_runs_config_id_rank_tracking_configs_id_fk",
+ "tableFrom": "rank_check_runs",
+ "tableTo": "rank_tracking_configs",
+ "columnsFrom": [
+ "config_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "rank_check_runs_project_id_projects_id_fk": {
+ "name": "rank_check_runs_project_id_projects_id_fk",
+ "tableFrom": "rank_check_runs",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.rank_snapshots": {
+ "name": "rank_snapshots",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "run_id": {
+ "name": "run_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "tracking_keyword_id": {
+ "name": "tracking_keyword_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "keyword": {
+ "name": "keyword",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "device": {
+ "name": "device",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "position": {
+ "name": "position",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "serp_features": {
+ "name": "serp_features",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "checked_at": {
+ "name": "checked_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "rank_snapshots_keyword_device_idx": {
+ "name": "rank_snapshots_keyword_device_idx",
+ "columns": [
+ {
+ "expression": "tracking_keyword_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "device",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "checked_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "rank_snapshots_run_keyword_device_idx": {
+ "name": "rank_snapshots_run_keyword_device_idx",
+ "columns": [
+ {
+ "expression": "run_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "tracking_keyword_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "device",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "rank_snapshots_run_id_rank_check_runs_id_fk": {
+ "name": "rank_snapshots_run_id_rank_check_runs_id_fk",
+ "tableFrom": "rank_snapshots",
+ "tableTo": "rank_check_runs",
+ "columnsFrom": [
+ "run_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.rank_tracking_configs": {
+ "name": "rank_tracking_configs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "domain": {
+ "name": "domain",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "location_code": {
+ "name": "location_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 2840
+ },
+ "language_code": {
+ "name": "language_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'en'"
+ },
+ "devices": {
+ "name": "devices",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'both'"
+ },
+ "serp_depth": {
+ "name": "serp_depth",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "schedule_interval": {
+ "name": "schedule_interval",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'weekly'"
+ },
+ "location_name": {
+ "name": "location_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "is_active": {
+ "name": "is_active",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": true
+ },
+ "last_checked_at": {
+ "name": "last_checked_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "next_check_at": {
+ "name": "next_check_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_skip_reason": {
+ "name": "last_skip_reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "rank_tracking_configs_project_active_created_idx": {
+ "name": "rank_tracking_configs_project_active_created_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "is_active",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "created_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "rank_tracking_configs_national_idx": {
+ "name": "rank_tracking_configs_national_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "domain",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "location_code",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"rank_tracking_configs\".\"location_name\" IS NULL",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "rank_tracking_configs_local_idx": {
+ "name": "rank_tracking_configs_local_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "domain",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "location_code",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "location_name",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "where": "\"rank_tracking_configs\".\"location_name\" IS NOT NULL",
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "rank_tracking_configs_project_id_projects_id_fk": {
+ "name": "rank_tracking_configs_project_id_projects_id_fk",
+ "tableFrom": "rank_tracking_configs",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.rank_tracking_keywords": {
+ "name": "rank_tracking_keywords",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "config_id": {
+ "name": "config_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "keyword": {
+ "name": "keyword",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "search_volume": {
+ "name": "search_volume",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "keyword_difficulty": {
+ "name": "keyword_difficulty",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "cpc": {
+ "name": "cpc",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "metrics_fetched_at": {
+ "name": "metrics_fetched_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "rank_tracking_keywords_config_keyword_idx": {
+ "name": "rank_tracking_keywords_config_keyword_idx",
+ "columns": [
+ {
+ "expression": "config_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "keyword",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "rank_tracking_keywords_config_id_rank_tracking_configs_id_fk": {
+ "name": "rank_tracking_keywords_config_id_rank_tracking_configs_id_fk",
+ "tableFrom": "rank_tracking_keywords",
+ "tableTo": "rank_tracking_configs",
+ "columnsFrom": [
+ "config_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.saved_keyword_tag_assignments": {
+ "name": "saved_keyword_tag_assignments",
+ "schema": "",
+ "columns": {
+ "saved_keyword_id": {
+ "name": "saved_keyword_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "tag_id": {
+ "name": "tag_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "saved_keyword_tag_assignments_unique_idx": {
+ "name": "saved_keyword_tag_assignments_unique_idx",
+ "columns": [
+ {
+ "expression": "saved_keyword_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "tag_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "saved_keyword_tag_assignments_tag_idx": {
+ "name": "saved_keyword_tag_assignments_tag_idx",
+ "columns": [
+ {
+ "expression": "tag_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "saved_keyword_tag_assignments_saved_keyword_id_saved_keywords_id_fk": {
+ "name": "saved_keyword_tag_assignments_saved_keyword_id_saved_keywords_id_fk",
+ "tableFrom": "saved_keyword_tag_assignments",
+ "tableTo": "saved_keywords",
+ "columnsFrom": [
+ "saved_keyword_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "saved_keyword_tag_assignments_tag_id_saved_keyword_tags_id_fk": {
+ "name": "saved_keyword_tag_assignments_tag_id_saved_keyword_tags_id_fk",
+ "tableFrom": "saved_keyword_tag_assignments",
+ "tableTo": "saved_keyword_tags",
+ "columnsFrom": [
+ "tag_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.saved_keyword_tags": {
+ "name": "saved_keyword_tags",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "normalized_name": {
+ "name": "normalized_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "color": {
+ "name": "color",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "saved_keyword_tags_project_normalized_name_idx": {
+ "name": "saved_keyword_tags_project_normalized_name_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "normalized_name",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "saved_keyword_tags_project_name_idx": {
+ "name": "saved_keyword_tags_project_name_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "name",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "saved_keyword_tags_project_id_projects_id_fk": {
+ "name": "saved_keyword_tags_project_id_projects_id_fk",
+ "tableFrom": "saved_keyword_tags",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.saved_keywords": {
+ "name": "saved_keywords",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "keyword": {
+ "name": "keyword",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "location_code": {
+ "name": "location_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 2840
+ },
+ "language_code": {
+ "name": "language_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'en'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "saved_keywords_unique_project_keyword_location_language": {
+ "name": "saved_keywords_unique_project_keyword_location_language",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "keyword",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "location_code",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "language_code",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "saved_keywords_project_created_idx": {
+ "name": "saved_keywords_project_created_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "created_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "saved_keywords_project_id_projects_id_fk": {
+ "name": "saved_keywords_project_id_projects_id_fk",
+ "tableFrom": "saved_keywords",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.user_onboarding_answers": {
+ "name": "user_onboarding_answers",
+ "schema": "",
+ "columns": {
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "interested_features": {
+ "name": "interested_features",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'[]'"
+ },
+ "work_for": {
+ "name": "work_for",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "client_website_count": {
+ "name": "client_website_count",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "found_via": {
+ "name": "found_via",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "mcp_setup_intent": {
+ "name": "mcp_setup_intent",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "gsc_nudge_dismissed_at": {
+ "name": "gsc_nudge_dismissed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "user_onboarding_answers_organization_idx": {
+ "name": "user_onboarding_answers_organization_idx",
+ "columns": [
+ {
+ "expression": "organization_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "user_onboarding_answers_user_id_user_id_fk": {
+ "name": "user_onboarding_answers_user_id_user_id_fk",
+ "tableFrom": "user_onboarding_answers",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "user_onboarding_answers_organization_id_organization_id_fk": {
+ "name": "user_onboarding_answers_organization_id_organization_id_fk",
+ "tableFrom": "user_onboarding_answers",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_issues": {
+ "name": "audit_issues",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "audit_id": {
+ "name": "audit_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "page_id": {
+ "name": "page_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "page_url": {
+ "name": "page_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "issue_type": {
+ "name": "issue_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "severity": {
+ "name": "severity",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'info'"
+ },
+ "details_json": {
+ "name": "details_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "audit_issues_audit_type_idx": {
+ "name": "audit_issues_audit_type_idx",
+ "columns": [
+ {
+ "expression": "audit_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "issue_type",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "audit_issues_page_id_idx": {
+ "name": "audit_issues_page_id_idx",
+ "columns": [
+ {
+ "expression": "page_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "audit_issues_audit_id_audits_id_fk": {
+ "name": "audit_issues_audit_id_audits_id_fk",
+ "tableFrom": "audit_issues",
+ "tableTo": "audits",
+ "columnsFrom": [
+ "audit_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "audit_issues_page_id_audit_pages_id_fk": {
+ "name": "audit_issues_page_id_audit_pages_id_fk",
+ "tableFrom": "audit_issues",
+ "tableTo": "audit_pages",
+ "columnsFrom": [
+ "page_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_lighthouse_results": {
+ "name": "audit_lighthouse_results",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "audit_id": {
+ "name": "audit_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "page_id": {
+ "name": "page_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "strategy": {
+ "name": "strategy",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "performance_score": {
+ "name": "performance_score",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "accessibility_score": {
+ "name": "accessibility_score",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "best_practices_score": {
+ "name": "best_practices_score",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "seo_score": {
+ "name": "seo_score",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "lcp_ms": {
+ "name": "lcp_ms",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "cls": {
+ "name": "cls",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "inp_ms": {
+ "name": "inp_ms",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "ttfb_ms": {
+ "name": "ttfb_ms",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "error_message": {
+ "name": "error_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "r2_key": {
+ "name": "r2_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "payload_size_bytes": {
+ "name": "payload_size_bytes",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "audit_lighthouse_results_audit_id_idx": {
+ "name": "audit_lighthouse_results_audit_id_idx",
+ "columns": [
+ {
+ "expression": "audit_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "audit_lighthouse_results_page_id_idx": {
+ "name": "audit_lighthouse_results_page_id_idx",
+ "columns": [
+ {
+ "expression": "page_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "audit_lighthouse_results_audit_id_audits_id_fk": {
+ "name": "audit_lighthouse_results_audit_id_audits_id_fk",
+ "tableFrom": "audit_lighthouse_results",
+ "tableTo": "audits",
+ "columnsFrom": [
+ "audit_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "audit_lighthouse_results_page_id_audit_pages_id_fk": {
+ "name": "audit_lighthouse_results_page_id_audit_pages_id_fk",
+ "tableFrom": "audit_lighthouse_results",
+ "tableTo": "audit_pages",
+ "columnsFrom": [
+ "page_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_links": {
+ "name": "audit_links",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "audit_id": {
+ "name": "audit_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_page_id": {
+ "name": "source_page_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "source_url": {
+ "name": "source_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "target_url": {
+ "name": "target_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "anchor": {
+ "name": "anchor",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "is_internal": {
+ "name": "is_internal",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": true
+ },
+ "is_nofollow": {
+ "name": "is_nofollow",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ }
+ },
+ "indexes": {
+ "audit_links_audit_target_idx": {
+ "name": "audit_links_audit_target_idx",
+ "columns": [
+ {
+ "expression": "audit_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "target_url",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "audit_links_source_page_id_idx": {
+ "name": "audit_links_source_page_id_idx",
+ "columns": [
+ {
+ "expression": "source_page_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "audit_links_audit_id_audits_id_fk": {
+ "name": "audit_links_audit_id_audits_id_fk",
+ "tableFrom": "audit_links",
+ "tableTo": "audits",
+ "columnsFrom": [
+ "audit_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "audit_links_source_page_id_audit_pages_id_fk": {
+ "name": "audit_links_source_page_id_audit_pages_id_fk",
+ "tableFrom": "audit_links",
+ "tableTo": "audit_pages",
+ "columnsFrom": [
+ "source_page_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_pages": {
+ "name": "audit_pages",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "audit_id": {
+ "name": "audit_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status_code": {
+ "name": "status_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "redirect_url": {
+ "name": "redirect_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "meta_description": {
+ "name": "meta_description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "canonical_url": {
+ "name": "canonical_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "robots_meta": {
+ "name": "robots_meta",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "og_title": {
+ "name": "og_title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "og_description": {
+ "name": "og_description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "og_image": {
+ "name": "og_image",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "h1_count": {
+ "name": "h1_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "h2_count": {
+ "name": "h2_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "h3_count": {
+ "name": "h3_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "h4_count": {
+ "name": "h4_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "h5_count": {
+ "name": "h5_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "h6_count": {
+ "name": "h6_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "heading_order_json": {
+ "name": "heading_order_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "word_count": {
+ "name": "word_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "images_total": {
+ "name": "images_total",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "images_missing_alt": {
+ "name": "images_missing_alt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "images_json": {
+ "name": "images_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "internal_link_count": {
+ "name": "internal_link_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "external_link_count": {
+ "name": "external_link_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "has_structured_data": {
+ "name": "has_structured_data",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "hreflang_tags_json": {
+ "name": "hreflang_tags_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "is_indexable": {
+ "name": "is_indexable",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": true
+ },
+ "x_robots_tag": {
+ "name": "x_robots_tag",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "header_canonical_url": {
+ "name": "header_canonical_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "crawl_depth": {
+ "name": "crawl_depth",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "in_sitemap": {
+ "name": "in_sitemap",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "content_hash": {
+ "name": "content_hash",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "fetch_class": {
+ "name": "fetch_class",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'ok'"
+ },
+ "response_time_ms": {
+ "name": "response_time_ms",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "audit_pages_audit_url_idx": {
+ "name": "audit_pages_audit_url_idx",
+ "columns": [
+ {
+ "expression": "audit_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "url",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "audit_pages_audit_id_audits_id_fk": {
+ "name": "audit_pages_audit_id_audits_id_fk",
+ "tableFrom": "audit_pages",
+ "tableTo": "audits",
+ "columnsFrom": [
+ "audit_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audits": {
+ "name": "audits",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "started_by_user_id": {
+ "name": "started_by_user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "start_url": {
+ "name": "start_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'running'"
+ },
+ "workflow_instance_id": {
+ "name": "workflow_instance_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "config": {
+ "name": "config",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'{}'"
+ },
+ "pages_crawled": {
+ "name": "pages_crawled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "pages_total": {
+ "name": "pages_total",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "lighthouse_total": {
+ "name": "lighthouse_total",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "lighthouse_completed": {
+ "name": "lighthouse_completed",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "lighthouse_failed": {
+ "name": "lighthouse_failed",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ },
+ "current_phase": {
+ "name": "current_phase",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'discovery'"
+ },
+ "error_code": {
+ "name": "error_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "error_detail": {
+ "name": "error_detail",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "failed_phase": {
+ "name": "failed_phase",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "started_at": {
+ "name": "started_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "audits_project_id_idx": {
+ "name": "audits_project_id_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "audits_started_by_user_id_idx": {
+ "name": "audits_started_by_user_id_idx",
+ "columns": [
+ {
+ "expression": "started_by_user_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "audits_project_id_projects_id_fk": {
+ "name": "audits_project_id_projects_id_fk",
+ "tableFrom": "audits",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.sam_project_memory": {
+ "name": "sam_project_memory",
+ "schema": "",
+ "columns": {
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "label": {
+ "name": "label",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "content": {
+ "name": "content",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "sam_project_memory_project_id_projects_id_fk": {
+ "name": "sam_project_memory_project_id_projects_id_fk",
+ "tableFrom": "sam_project_memory",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "sam_project_memory_project_id_label_pk": {
+ "name": "sam_project_memory_project_id_label_pk",
+ "columns": [
+ "project_id",
+ "label"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.sam_sessions": {
+ "name": "sam_sessions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'New chat'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "sam_sessions_project_updated_idx": {
+ "name": "sam_sessions_project_updated_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "updated_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "sam_sessions_project_id_projects_id_fk": {
+ "name": "sam_sessions_project_id_projects_id_fk",
+ "tableFrom": "sam_sessions",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "sam_sessions_user_id_user_id_fk": {
+ "name": "sam_sessions_user_id_user_id_fk",
+ "tableFrom": "sam_sessions",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.account": {
+ "name": "account",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "account_id": {
+ "name": "account_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "provider_id": {
+ "name": "provider_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "access_token": {
+ "name": "access_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "refresh_token": {
+ "name": "refresh_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "id_token": {
+ "name": "id_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "access_token_expires_at": {
+ "name": "access_token_expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "refresh_token_expires_at": {
+ "name": "refresh_token_expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "scope": {
+ "name": "scope",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {
+ "account_userId_idx": {
+ "name": "account_userId_idx",
+ "columns": [
+ {
+ "expression": "user_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "account_accountId_providerId_idx": {
+ "name": "account_accountId_providerId_idx",
+ "columns": [
+ {
+ "expression": "account_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ },
+ {
+ "expression": "provider_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "account_user_id_user_id_fk": {
+ "name": "account_user_id_user_id_fk",
+ "tableFrom": "account",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.invitation": {
+ "name": "invitation",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "role": {
+ "name": "role",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending'"
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "inviter_id": {
+ "name": "inviter_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {
+ "invitation_organizationId_idx": {
+ "name": "invitation_organizationId_idx",
+ "columns": [
+ {
+ "expression": "organization_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "invitation_email_idx": {
+ "name": "invitation_email_idx",
+ "columns": [
+ {
+ "expression": "email",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "invitation_organization_id_organization_id_fk": {
+ "name": "invitation_organization_id_organization_id_fk",
+ "tableFrom": "invitation",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "invitation_inviter_id_user_id_fk": {
+ "name": "invitation_inviter_id_user_id_fk",
+ "tableFrom": "invitation",
+ "tableTo": "user",
+ "columnsFrom": [
+ "inviter_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.member": {
+ "name": "member",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "role": {
+ "name": "role",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'member'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {
+ "member_organizationId_idx": {
+ "name": "member_organizationId_idx",
+ "columns": [
+ {
+ "expression": "organization_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "member_userId_idx": {
+ "name": "member_userId_idx",
+ "columns": [
+ {
+ "expression": "user_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "member_organization_id_organization_id_fk": {
+ "name": "member_organization_id_organization_id_fk",
+ "tableFrom": "member",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "member_user_id_user_id_fk": {
+ "name": "member_user_id_user_id_fk",
+ "tableFrom": "member",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.organization": {
+ "name": "organization",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "slug": {
+ "name": "slug",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "logo": {
+ "name": "logo",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "metadata": {
+ "name": "metadata",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "organization_slug_uidx": {
+ "name": "organization_slug_uidx",
+ "columns": [
+ {
+ "expression": "slug",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "organization_slug_unique": {
+ "name": "organization_slug_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "slug"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.session": {
+ "name": "session",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "token": {
+ "name": "token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "ip_address": {
+ "name": "ip_address",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "user_agent": {
+ "name": "user_agent",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "active_organization_id": {
+ "name": "active_organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "session_userId_idx": {
+ "name": "session_userId_idx",
+ "columns": [
+ {
+ "expression": "user_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "session_user_id_user_id_fk": {
+ "name": "session_user_id_user_id_fk",
+ "tableFrom": "session",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "session_token_unique": {
+ "name": "session_token_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "token"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.user": {
+ "name": "user",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "email_verified": {
+ "name": "email_verified",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "image": {
+ "name": "image",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "analytics_opted_out": {
+ "name": "analytics_opted_out",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "user_email_unique": {
+ "name": "user_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.verification": {
+ "name": "verification",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "identifier": {
+ "name": "identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "value": {
+ "name": "value",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "verification_identifier_idx": {
+ "name": "verification_identifier_idx",
+ "columns": [
+ {
+ "expression": "identifier",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "verification_expiresAt_idx": {
+ "name": "verification_expiresAt_idx",
+ "columns": [
+ {
+ "expression": "expires_at",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.billing_customer_status": {
+ "name": "billing_customer_status",
+ "schema": "",
+ "columns": {
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "is_paying": {
+ "name": "is_paying",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "paid_plan_id": {
+ "name": "paid_plan_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "paid_plan_status": {
+ "name": "paid_plan_status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "customer_json": {
+ "name": "customer_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "synced_at": {
+ "name": "synced_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "billing_customer_status_organization_id_organization_id_fk": {
+ "name": "billing_customer_status_organization_id_organization_id_fk",
+ "tableFrom": "billing_customer_status",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.gsc_connections": {
+ "name": "gsc_connections",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "site_url": {
+ "name": "site_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "connected_by_user_id": {
+ "name": "connected_by_user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "gsc_account_id": {
+ "name": "gsc_account_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "connected_account_email": {
+ "name": "connected_account_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "gsc_connections_project_idx": {
+ "name": "gsc_connections_project_idx",
+ "columns": [
+ {
+ "expression": "project_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "gsc_connections_organization_idx": {
+ "name": "gsc_connections_organization_idx",
+ "columns": [
+ {
+ "expression": "organization_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "gsc_connections_project_id_projects_id_fk": {
+ "name": "gsc_connections_project_id_projects_id_fk",
+ "tableFrom": "gsc_connections",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "gsc_connections_organization_id_organization_id_fk": {
+ "name": "gsc_connections_organization_id_organization_id_fk",
+ "tableFrom": "gsc_connections",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.reddit_attributions": {
+ "name": "reddit_attributions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "click_id": {
+ "name": "click_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "uuid": {
+ "name": "uuid",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "landing_page": {
+ "name": "landing_page",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "referrer": {
+ "name": "referrer",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "utm_source": {
+ "name": "utm_source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "utm_medium": {
+ "name": "utm_medium",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "utm_campaign": {
+ "name": "utm_campaign",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "utm_term": {
+ "name": "utm_term",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "utm_content": {
+ "name": "utm_content",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "signup_sent_at": {
+ "name": "signup_sent_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "purchase_sent_at": {
+ "name": "purchase_sent_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "to_char(now() AT TIME ZONE 'utc', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"')"
+ }
+ },
+ "indexes": {
+ "reddit_attributions_user_idx": {
+ "name": "reddit_attributions_user_idx",
+ "columns": [
+ {
+ "expression": "user_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": true,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ },
+ "reddit_attributions_organization_idx": {
+ "name": "reddit_attributions_organization_idx",
+ "columns": [
+ {
+ "expression": "organization_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "reddit_attributions_user_id_user_id_fk": {
+ "name": "reddit_attributions_user_id_user_id_fk",
+ "tableFrom": "reddit_attributions",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "reddit_attributions_organization_id_organization_id_fk": {
+ "name": "reddit_attributions_organization_id_organization_id_fk",
+ "tableFrom": "reddit_attributions",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.telemetry_state": {
+ "name": "telemetry_state",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "default": 1
+ },
+ "install_id": {
+ "name": "install_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "installed_at": {
+ "name": "installed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_heartbeat_at": {
+ "name": "last_heartbeat_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_version": {
+ "name": "last_version",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "mcp_tool_call_count": {
+ "name": "mcp_tool_call_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "default": 0
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {},
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/drizzle-pg/meta/_journal.json b/drizzle-pg/meta/_journal.json
index d9b0014..bb52c7b 100644
--- a/drizzle-pg/meta/_journal.json
+++ b/drizzle-pg/meta/_journal.json
@@ -99,6 +99,13 @@
"when": 1784426969390,
"tag": "0013_sleepy_black_tarantula",
"breakpoints": true
+ },
+ {
+ "idx": 14,
+ "version": "7",
+ "when": 1784588248793,
+ "tag": "0014_solid_centennial",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/drizzle/0037_small_caretaker.sql b/drizzle/0037_small_caretaker.sql
new file mode 100644
index 0000000..4048072
--- /dev/null
+++ b/drizzle/0037_small_caretaker.sql
@@ -0,0 +1,3 @@
+ALTER TABLE `audits` ADD `error_code` text;--> statement-breakpoint
+ALTER TABLE `audits` ADD `error_detail` text;--> statement-breakpoint
+ALTER TABLE `audits` ADD `failed_phase` text;
\ No newline at end of file
diff --git a/drizzle/meta/0037_snapshot.json b/drizzle/meta/0037_snapshot.json
new file mode 100644
index 0000000..c0199fd
--- /dev/null
+++ b/drizzle/meta/0037_snapshot.json
@@ -0,0 +1,3401 @@
+{
+ "version": "6",
+ "dialect": "sqlite",
+ "id": "e193cf24-9947-4bfb-a04f-8a98b247fa2f",
+ "prevId": "3eed5245-3458-4f6b-b6ce-b970ca141bd8",
+ "tables": {
+ "backlink_snapshots": {
+ "name": "backlink_snapshots",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "domain": {
+ "name": "domain",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "rank": {
+ "name": "rank",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "backlinks": {
+ "name": "backlinks",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "referring_domains": {
+ "name": "referring_domains",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "broken_backlinks": {
+ "name": "broken_backlinks",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "new_backlinks": {
+ "name": "new_backlinks",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "lost_backlinks": {
+ "name": "lost_backlinks",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "new_referring_domains": {
+ "name": "new_referring_domains",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "lost_referring_domains": {
+ "name": "lost_referring_domains",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "captured_at": {
+ "name": "captured_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "backlink_snapshots_project_captured_idx": {
+ "name": "backlink_snapshots_project_captured_idx",
+ "columns": [
+ "project_id",
+ "captured_at"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "backlink_snapshots_project_id_projects_id_fk": {
+ "name": "backlink_snapshots_project_id_projects_id_fk",
+ "tableFrom": "backlink_snapshots",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "keyword_metrics": {
+ "name": "keyword_metrics",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "keyword": {
+ "name": "keyword",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "location_code": {
+ "name": "location_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "language_code": {
+ "name": "language_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'en'"
+ },
+ "search_volume": {
+ "name": "search_volume",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "cpc": {
+ "name": "cpc",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "competition": {
+ "name": "competition",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "keyword_difficulty": {
+ "name": "keyword_difficulty",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "intent": {
+ "name": "intent",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "monthly_searches": {
+ "name": "monthly_searches",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "fetched_at": {
+ "name": "fetched_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "keyword_metrics_unique_project_keyword_location_language": {
+ "name": "keyword_metrics_unique_project_keyword_location_language",
+ "columns": [
+ "project_id",
+ "keyword",
+ "location_code",
+ "language_code"
+ ],
+ "isUnique": true
+ },
+ "keyword_metrics_lookup_idx": {
+ "name": "keyword_metrics_lookup_idx",
+ "columns": [
+ "project_id",
+ "keyword",
+ "location_code",
+ "language_code",
+ "fetched_at"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "keyword_metrics_project_id_projects_id_fk": {
+ "name": "keyword_metrics_project_id_projects_id_fk",
+ "tableFrom": "keyword_metrics",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "organization_activation_state": {
+ "name": "organization_activation_state",
+ "columns": {
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "first_mcp_authorized_at": {
+ "name": "first_mcp_authorized_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "first_mcp_tool_call_at": {
+ "name": "first_mcp_tool_call_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "organization_activation_state_organization_id_organization_id_fk": {
+ "name": "organization_activation_state_organization_id_organization_id_fk",
+ "tableFrom": "organization_activation_state",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "project_activation_state": {
+ "name": "project_activation_state",
+ "columns": {
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "competitor_step_clicked_at": {
+ "name": "competitor_step_clicked_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "mcp_card_dismissed_at": {
+ "name": "mcp_card_dismissed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "project_activation_state_project_id_projects_id_fk": {
+ "name": "project_activation_state_project_id_projects_id_fk",
+ "tableFrom": "project_activation_state",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "projects": {
+ "name": "projects",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "domain": {
+ "name": "domain",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "location_code": {
+ "name": "location_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 2840
+ },
+ "language_code": {
+ "name": "language_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'en'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "projects_one_default_per_organization_idx": {
+ "name": "projects_one_default_per_organization_idx",
+ "columns": [
+ "organization_id"
+ ],
+ "isUnique": true,
+ "where": "\"projects\".\"name\" = 'Default' AND \"projects\".\"domain\" IS NULL AND \"projects\".\"archived_at\" IS NULL"
+ },
+ "projects_organization_id_idx": {
+ "name": "projects_organization_id_idx",
+ "columns": [
+ "organization_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "projects_organization_id_organization_id_fk": {
+ "name": "projects_organization_id_organization_id_fk",
+ "tableFrom": "projects",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "rank_check_runs": {
+ "name": "rank_check_runs",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "config_id": {
+ "name": "config_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "keywords_total": {
+ "name": "keywords_total",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "keywords_checked": {
+ "name": "keywords_checked",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "is_subset_run": {
+ "name": "is_subset_run",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "error_message": {
+ "name": "error_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "started_at": {
+ "name": "started_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "rank_check_runs_config_idx": {
+ "name": "rank_check_runs_config_idx",
+ "columns": [
+ "config_id",
+ "started_at"
+ ],
+ "isUnique": false
+ },
+ "rank_check_runs_project_idx": {
+ "name": "rank_check_runs_project_idx",
+ "columns": [
+ "project_id",
+ "started_at"
+ ],
+ "isUnique": false
+ },
+ "rank_check_runs_one_active_per_config_idx": {
+ "name": "rank_check_runs_one_active_per_config_idx",
+ "columns": [
+ "config_id"
+ ],
+ "isUnique": true,
+ "where": "\"rank_check_runs\".\"status\" IN ('pending', 'running')"
+ }
+ },
+ "foreignKeys": {
+ "rank_check_runs_config_id_rank_tracking_configs_id_fk": {
+ "name": "rank_check_runs_config_id_rank_tracking_configs_id_fk",
+ "tableFrom": "rank_check_runs",
+ "tableTo": "rank_tracking_configs",
+ "columnsFrom": [
+ "config_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "rank_check_runs_project_id_projects_id_fk": {
+ "name": "rank_check_runs_project_id_projects_id_fk",
+ "tableFrom": "rank_check_runs",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "rank_snapshots": {
+ "name": "rank_snapshots",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": true
+ },
+ "run_id": {
+ "name": "run_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "tracking_keyword_id": {
+ "name": "tracking_keyword_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "keyword": {
+ "name": "keyword",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "device": {
+ "name": "device",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "position": {
+ "name": "position",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "serp_features": {
+ "name": "serp_features",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "checked_at": {
+ "name": "checked_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "rank_snapshots_keyword_device_idx": {
+ "name": "rank_snapshots_keyword_device_idx",
+ "columns": [
+ "tracking_keyword_id",
+ "device",
+ "checked_at"
+ ],
+ "isUnique": false
+ },
+ "rank_snapshots_run_keyword_device_idx": {
+ "name": "rank_snapshots_run_keyword_device_idx",
+ "columns": [
+ "run_id",
+ "tracking_keyword_id",
+ "device"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "rank_snapshots_run_id_rank_check_runs_id_fk": {
+ "name": "rank_snapshots_run_id_rank_check_runs_id_fk",
+ "tableFrom": "rank_snapshots",
+ "tableTo": "rank_check_runs",
+ "columnsFrom": [
+ "run_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "rank_tracking_configs": {
+ "name": "rank_tracking_configs",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "domain": {
+ "name": "domain",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "location_code": {
+ "name": "location_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 2840
+ },
+ "language_code": {
+ "name": "language_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'en'"
+ },
+ "devices": {
+ "name": "devices",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'both'"
+ },
+ "serp_depth": {
+ "name": "serp_depth",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "schedule_interval": {
+ "name": "schedule_interval",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'weekly'"
+ },
+ "location_name": {
+ "name": "location_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "is_active": {
+ "name": "is_active",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": true
+ },
+ "last_checked_at": {
+ "name": "last_checked_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "next_check_at": {
+ "name": "next_check_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_skip_reason": {
+ "name": "last_skip_reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "rank_tracking_configs_project_active_created_idx": {
+ "name": "rank_tracking_configs_project_active_created_idx",
+ "columns": [
+ "project_id",
+ "is_active",
+ "created_at"
+ ],
+ "isUnique": false
+ },
+ "rank_tracking_configs_national_idx": {
+ "name": "rank_tracking_configs_national_idx",
+ "columns": [
+ "project_id",
+ "domain",
+ "location_code"
+ ],
+ "isUnique": true,
+ "where": "\"rank_tracking_configs\".\"location_name\" IS NULL"
+ },
+ "rank_tracking_configs_local_idx": {
+ "name": "rank_tracking_configs_local_idx",
+ "columns": [
+ "project_id",
+ "domain",
+ "location_code",
+ "location_name"
+ ],
+ "isUnique": true,
+ "where": "\"rank_tracking_configs\".\"location_name\" IS NOT NULL"
+ }
+ },
+ "foreignKeys": {
+ "rank_tracking_configs_project_id_projects_id_fk": {
+ "name": "rank_tracking_configs_project_id_projects_id_fk",
+ "tableFrom": "rank_tracking_configs",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "rank_tracking_keywords": {
+ "name": "rank_tracking_keywords",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "config_id": {
+ "name": "config_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "keyword": {
+ "name": "keyword",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "search_volume": {
+ "name": "search_volume",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "keyword_difficulty": {
+ "name": "keyword_difficulty",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "cpc": {
+ "name": "cpc",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "metrics_fetched_at": {
+ "name": "metrics_fetched_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "rank_tracking_keywords_config_keyword_idx": {
+ "name": "rank_tracking_keywords_config_keyword_idx",
+ "columns": [
+ "config_id",
+ "keyword"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {
+ "rank_tracking_keywords_config_id_rank_tracking_configs_id_fk": {
+ "name": "rank_tracking_keywords_config_id_rank_tracking_configs_id_fk",
+ "tableFrom": "rank_tracking_keywords",
+ "tableTo": "rank_tracking_configs",
+ "columnsFrom": [
+ "config_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "saved_keyword_tag_assignments": {
+ "name": "saved_keyword_tag_assignments",
+ "columns": {
+ "saved_keyword_id": {
+ "name": "saved_keyword_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "tag_id": {
+ "name": "tag_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "saved_keyword_tag_assignments_unique_idx": {
+ "name": "saved_keyword_tag_assignments_unique_idx",
+ "columns": [
+ "saved_keyword_id",
+ "tag_id"
+ ],
+ "isUnique": true
+ },
+ "saved_keyword_tag_assignments_tag_idx": {
+ "name": "saved_keyword_tag_assignments_tag_idx",
+ "columns": [
+ "tag_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "saved_keyword_tag_assignments_saved_keyword_id_saved_keywords_id_fk": {
+ "name": "saved_keyword_tag_assignments_saved_keyword_id_saved_keywords_id_fk",
+ "tableFrom": "saved_keyword_tag_assignments",
+ "tableTo": "saved_keywords",
+ "columnsFrom": [
+ "saved_keyword_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "saved_keyword_tag_assignments_tag_id_saved_keyword_tags_id_fk": {
+ "name": "saved_keyword_tag_assignments_tag_id_saved_keyword_tags_id_fk",
+ "tableFrom": "saved_keyword_tag_assignments",
+ "tableTo": "saved_keyword_tags",
+ "columnsFrom": [
+ "tag_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "saved_keyword_tags": {
+ "name": "saved_keyword_tags",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "normalized_name": {
+ "name": "normalized_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "color": {
+ "name": "color",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "saved_keyword_tags_project_normalized_name_idx": {
+ "name": "saved_keyword_tags_project_normalized_name_idx",
+ "columns": [
+ "project_id",
+ "normalized_name"
+ ],
+ "isUnique": true
+ },
+ "saved_keyword_tags_project_name_idx": {
+ "name": "saved_keyword_tags_project_name_idx",
+ "columns": [
+ "project_id",
+ "name"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "saved_keyword_tags_project_id_projects_id_fk": {
+ "name": "saved_keyword_tags_project_id_projects_id_fk",
+ "tableFrom": "saved_keyword_tags",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "saved_keywords": {
+ "name": "saved_keywords",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "keyword": {
+ "name": "keyword",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "location_code": {
+ "name": "location_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 2840
+ },
+ "language_code": {
+ "name": "language_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'en'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "saved_keywords_unique_project_keyword_location_language": {
+ "name": "saved_keywords_unique_project_keyword_location_language",
+ "columns": [
+ "project_id",
+ "keyword",
+ "location_code",
+ "language_code"
+ ],
+ "isUnique": true
+ },
+ "saved_keywords_project_created_idx": {
+ "name": "saved_keywords_project_created_idx",
+ "columns": [
+ "project_id",
+ "created_at"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "saved_keywords_project_id_projects_id_fk": {
+ "name": "saved_keywords_project_id_projects_id_fk",
+ "tableFrom": "saved_keywords",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "user_onboarding_answers": {
+ "name": "user_onboarding_answers",
+ "columns": {
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "interested_features": {
+ "name": "interested_features",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'[]'"
+ },
+ "work_for": {
+ "name": "work_for",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "client_website_count": {
+ "name": "client_website_count",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "found_via": {
+ "name": "found_via",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "mcp_setup_intent": {
+ "name": "mcp_setup_intent",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "gsc_nudge_dismissed_at": {
+ "name": "gsc_nudge_dismissed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "user_onboarding_answers_organization_idx": {
+ "name": "user_onboarding_answers_organization_idx",
+ "columns": [
+ "organization_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "user_onboarding_answers_user_id_user_id_fk": {
+ "name": "user_onboarding_answers_user_id_user_id_fk",
+ "tableFrom": "user_onboarding_answers",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "user_onboarding_answers_organization_id_organization_id_fk": {
+ "name": "user_onboarding_answers_organization_id_organization_id_fk",
+ "tableFrom": "user_onboarding_answers",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "audit_issues": {
+ "name": "audit_issues",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "audit_id": {
+ "name": "audit_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "page_id": {
+ "name": "page_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "page_url": {
+ "name": "page_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "issue_type": {
+ "name": "issue_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "severity": {
+ "name": "severity",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'info'"
+ },
+ "details_json": {
+ "name": "details_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "audit_issues_audit_type_idx": {
+ "name": "audit_issues_audit_type_idx",
+ "columns": [
+ "audit_id",
+ "issue_type"
+ ],
+ "isUnique": false
+ },
+ "audit_issues_page_id_idx": {
+ "name": "audit_issues_page_id_idx",
+ "columns": [
+ "page_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "audit_issues_audit_id_audits_id_fk": {
+ "name": "audit_issues_audit_id_audits_id_fk",
+ "tableFrom": "audit_issues",
+ "tableTo": "audits",
+ "columnsFrom": [
+ "audit_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "audit_issues_page_id_audit_pages_id_fk": {
+ "name": "audit_issues_page_id_audit_pages_id_fk",
+ "tableFrom": "audit_issues",
+ "tableTo": "audit_pages",
+ "columnsFrom": [
+ "page_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "audit_lighthouse_results": {
+ "name": "audit_lighthouse_results",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "audit_id": {
+ "name": "audit_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "page_id": {
+ "name": "page_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "strategy": {
+ "name": "strategy",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "performance_score": {
+ "name": "performance_score",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "accessibility_score": {
+ "name": "accessibility_score",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "best_practices_score": {
+ "name": "best_practices_score",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "seo_score": {
+ "name": "seo_score",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "lcp_ms": {
+ "name": "lcp_ms",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "cls": {
+ "name": "cls",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "inp_ms": {
+ "name": "inp_ms",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "ttfb_ms": {
+ "name": "ttfb_ms",
+ "type": "real",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "error_message": {
+ "name": "error_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "r2_key": {
+ "name": "r2_key",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "payload_size_bytes": {
+ "name": "payload_size_bytes",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "audit_lighthouse_results_audit_id_idx": {
+ "name": "audit_lighthouse_results_audit_id_idx",
+ "columns": [
+ "audit_id"
+ ],
+ "isUnique": false
+ },
+ "audit_lighthouse_results_page_id_idx": {
+ "name": "audit_lighthouse_results_page_id_idx",
+ "columns": [
+ "page_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "audit_lighthouse_results_audit_id_audits_id_fk": {
+ "name": "audit_lighthouse_results_audit_id_audits_id_fk",
+ "tableFrom": "audit_lighthouse_results",
+ "tableTo": "audits",
+ "columnsFrom": [
+ "audit_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "audit_lighthouse_results_page_id_audit_pages_id_fk": {
+ "name": "audit_lighthouse_results_page_id_audit_pages_id_fk",
+ "tableFrom": "audit_lighthouse_results",
+ "tableTo": "audit_pages",
+ "columnsFrom": [
+ "page_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "audit_links": {
+ "name": "audit_links",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "audit_id": {
+ "name": "audit_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "source_page_id": {
+ "name": "source_page_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "source_url": {
+ "name": "source_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "target_url": {
+ "name": "target_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "anchor": {
+ "name": "anchor",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "is_internal": {
+ "name": "is_internal",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": true
+ },
+ "is_nofollow": {
+ "name": "is_nofollow",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ }
+ },
+ "indexes": {
+ "audit_links_audit_target_idx": {
+ "name": "audit_links_audit_target_idx",
+ "columns": [
+ "audit_id",
+ "target_url"
+ ],
+ "isUnique": false
+ },
+ "audit_links_source_page_id_idx": {
+ "name": "audit_links_source_page_id_idx",
+ "columns": [
+ "source_page_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "audit_links_audit_id_audits_id_fk": {
+ "name": "audit_links_audit_id_audits_id_fk",
+ "tableFrom": "audit_links",
+ "tableTo": "audits",
+ "columnsFrom": [
+ "audit_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "audit_links_source_page_id_audit_pages_id_fk": {
+ "name": "audit_links_source_page_id_audit_pages_id_fk",
+ "tableFrom": "audit_links",
+ "tableTo": "audit_pages",
+ "columnsFrom": [
+ "source_page_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "audit_pages": {
+ "name": "audit_pages",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "audit_id": {
+ "name": "audit_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "url": {
+ "name": "url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "status_code": {
+ "name": "status_code",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "redirect_url": {
+ "name": "redirect_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "meta_description": {
+ "name": "meta_description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "canonical_url": {
+ "name": "canonical_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "robots_meta": {
+ "name": "robots_meta",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "og_title": {
+ "name": "og_title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "og_description": {
+ "name": "og_description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "og_image": {
+ "name": "og_image",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "h1_count": {
+ "name": "h1_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "h2_count": {
+ "name": "h2_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "h3_count": {
+ "name": "h3_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "h4_count": {
+ "name": "h4_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "h5_count": {
+ "name": "h5_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "h6_count": {
+ "name": "h6_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "heading_order_json": {
+ "name": "heading_order_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "word_count": {
+ "name": "word_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "images_total": {
+ "name": "images_total",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "images_missing_alt": {
+ "name": "images_missing_alt",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "images_json": {
+ "name": "images_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "internal_link_count": {
+ "name": "internal_link_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "external_link_count": {
+ "name": "external_link_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "has_structured_data": {
+ "name": "has_structured_data",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "hreflang_tags_json": {
+ "name": "hreflang_tags_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "is_indexable": {
+ "name": "is_indexable",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": true
+ },
+ "x_robots_tag": {
+ "name": "x_robots_tag",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "header_canonical_url": {
+ "name": "header_canonical_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "crawl_depth": {
+ "name": "crawl_depth",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "in_sitemap": {
+ "name": "in_sitemap",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "content_hash": {
+ "name": "content_hash",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "fetch_class": {
+ "name": "fetch_class",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'ok'"
+ },
+ "response_time_ms": {
+ "name": "response_time_ms",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "audit_pages_audit_url_idx": {
+ "name": "audit_pages_audit_url_idx",
+ "columns": [
+ "audit_id",
+ "url"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "audit_pages_audit_id_audits_id_fk": {
+ "name": "audit_pages_audit_id_audits_id_fk",
+ "tableFrom": "audit_pages",
+ "tableTo": "audits",
+ "columnsFrom": [
+ "audit_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "audits": {
+ "name": "audits",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "started_by_user_id": {
+ "name": "started_by_user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "start_url": {
+ "name": "start_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'running'"
+ },
+ "workflow_instance_id": {
+ "name": "workflow_instance_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "config": {
+ "name": "config",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'{}'"
+ },
+ "pages_crawled": {
+ "name": "pages_crawled",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "pages_total": {
+ "name": "pages_total",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "lighthouse_total": {
+ "name": "lighthouse_total",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "lighthouse_completed": {
+ "name": "lighthouse_completed",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "lighthouse_failed": {
+ "name": "lighthouse_failed",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ },
+ "current_phase": {
+ "name": "current_phase",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false,
+ "default": "'discovery'"
+ },
+ "error_code": {
+ "name": "error_code",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "error_detail": {
+ "name": "error_detail",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "failed_phase": {
+ "name": "failed_phase",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "started_at": {
+ "name": "started_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "audits_project_id_idx": {
+ "name": "audits_project_id_idx",
+ "columns": [
+ "project_id"
+ ],
+ "isUnique": false
+ },
+ "audits_started_by_user_id_idx": {
+ "name": "audits_started_by_user_id_idx",
+ "columns": [
+ "started_by_user_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "audits_project_id_projects_id_fk": {
+ "name": "audits_project_id_projects_id_fk",
+ "tableFrom": "audits",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "sam_project_memory": {
+ "name": "sam_project_memory",
+ "columns": {
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "label": {
+ "name": "label",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "content": {
+ "name": "content",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "sam_project_memory_project_id_projects_id_fk": {
+ "name": "sam_project_memory_project_id_projects_id_fk",
+ "tableFrom": "sam_project_memory",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "sam_project_memory_project_id_label_pk": {
+ "columns": [
+ "project_id",
+ "label"
+ ],
+ "name": "sam_project_memory_project_id_label_pk"
+ }
+ },
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "sam_sessions": {
+ "name": "sam_sessions",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "title": {
+ "name": "title",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'New chat'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "sam_sessions_project_updated_idx": {
+ "name": "sam_sessions_project_updated_idx",
+ "columns": [
+ "project_id",
+ "updated_at"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "sam_sessions_project_id_projects_id_fk": {
+ "name": "sam_sessions_project_id_projects_id_fk",
+ "tableFrom": "sam_sessions",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "sam_sessions_user_id_user_id_fk": {
+ "name": "sam_sessions_user_id_user_id_fk",
+ "tableFrom": "sam_sessions",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "account": {
+ "name": "account",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "account_id": {
+ "name": "account_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "provider_id": {
+ "name": "provider_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "access_token": {
+ "name": "access_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "refresh_token": {
+ "name": "refresh_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "id_token": {
+ "name": "id_token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "access_token_expires_at": {
+ "name": "access_token_expires_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "refresh_token_expires_at": {
+ "name": "refresh_token_expires_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "scope": {
+ "name": "scope",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(cast(unixepoch('subsecond') * 1000 as integer))"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "account_userId_idx": {
+ "name": "account_userId_idx",
+ "columns": [
+ "user_id"
+ ],
+ "isUnique": false
+ },
+ "account_accountId_providerId_idx": {
+ "name": "account_accountId_providerId_idx",
+ "columns": [
+ "account_id",
+ "provider_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "account_user_id_user_id_fk": {
+ "name": "account_user_id_user_id_fk",
+ "tableFrom": "account",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "invitation": {
+ "name": "invitation",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "role": {
+ "name": "role",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'pending'"
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(cast(unixepoch('subsecond') * 1000 as integer))"
+ },
+ "inviter_id": {
+ "name": "inviter_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "invitation_organizationId_idx": {
+ "name": "invitation_organizationId_idx",
+ "columns": [
+ "organization_id"
+ ],
+ "isUnique": false
+ },
+ "invitation_email_idx": {
+ "name": "invitation_email_idx",
+ "columns": [
+ "email"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "invitation_organization_id_organization_id_fk": {
+ "name": "invitation_organization_id_organization_id_fk",
+ "tableFrom": "invitation",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "invitation_inviter_id_user_id_fk": {
+ "name": "invitation_inviter_id_user_id_fk",
+ "tableFrom": "invitation",
+ "tableTo": "user",
+ "columnsFrom": [
+ "inviter_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "member": {
+ "name": "member",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "role": {
+ "name": "role",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "'member'"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "member_organizationId_idx": {
+ "name": "member_organizationId_idx",
+ "columns": [
+ "organization_id"
+ ],
+ "isUnique": false
+ },
+ "member_userId_idx": {
+ "name": "member_userId_idx",
+ "columns": [
+ "user_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "member_organization_id_organization_id_fk": {
+ "name": "member_organization_id_organization_id_fk",
+ "tableFrom": "member",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "member_user_id_user_id_fk": {
+ "name": "member_user_id_user_id_fk",
+ "tableFrom": "member",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "organization": {
+ "name": "organization",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "slug": {
+ "name": "slug",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "logo": {
+ "name": "logo",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "metadata": {
+ "name": "metadata",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "organization_slug_unique": {
+ "name": "organization_slug_unique",
+ "columns": [
+ "slug"
+ ],
+ "isUnique": true
+ },
+ "organization_slug_uidx": {
+ "name": "organization_slug_uidx",
+ "columns": [
+ "slug"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "session": {
+ "name": "session",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "token": {
+ "name": "token",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(cast(unixepoch('subsecond') * 1000 as integer))"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "ip_address": {
+ "name": "ip_address",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "user_agent": {
+ "name": "user_agent",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "active_organization_id": {
+ "name": "active_organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "session_token_unique": {
+ "name": "session_token_unique",
+ "columns": [
+ "token"
+ ],
+ "isUnique": true
+ },
+ "session_userId_idx": {
+ "name": "session_userId_idx",
+ "columns": [
+ "user_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "session_user_id_user_id_fk": {
+ "name": "session_user_id_user_id_fk",
+ "tableFrom": "session",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "user": {
+ "name": "user",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "email_verified": {
+ "name": "email_verified",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "image": {
+ "name": "image",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(cast(unixepoch('subsecond') * 1000 as integer))"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(cast(unixepoch('subsecond') * 1000 as integer))"
+ },
+ "analytics_opted_out": {
+ "name": "analytics_opted_out",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ }
+ },
+ "indexes": {
+ "user_email_unique": {
+ "name": "user_email_unique",
+ "columns": [
+ "email"
+ ],
+ "isUnique": true
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "verification": {
+ "name": "verification",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "identifier": {
+ "name": "identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "value": {
+ "name": "value",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(cast(unixepoch('subsecond') * 1000 as integer))"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(cast(unixepoch('subsecond') * 1000 as integer))"
+ }
+ },
+ "indexes": {
+ "verification_identifier_idx": {
+ "name": "verification_identifier_idx",
+ "columns": [
+ "identifier"
+ ],
+ "isUnique": false
+ },
+ "verification_expiresAt_idx": {
+ "name": "verification_expiresAt_idx",
+ "columns": [
+ "expires_at"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "billing_customer_status": {
+ "name": "billing_customer_status",
+ "columns": {
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "is_paying": {
+ "name": "is_paying",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": false
+ },
+ "paid_plan_id": {
+ "name": "paid_plan_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "paid_plan_status": {
+ "name": "paid_plan_status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "customer_json": {
+ "name": "customer_json",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "synced_at": {
+ "name": "synced_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "billing_customer_status_organization_id_organization_id_fk": {
+ "name": "billing_customer_status_organization_id_organization_id_fk",
+ "tableFrom": "billing_customer_status",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "gsc_connections": {
+ "name": "gsc_connections",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "project_id": {
+ "name": "project_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "site_url": {
+ "name": "site_url",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "connected_by_user_id": {
+ "name": "connected_by_user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "gsc_account_id": {
+ "name": "gsc_account_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "connected_account_email": {
+ "name": "connected_account_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "gsc_connections_project_idx": {
+ "name": "gsc_connections_project_idx",
+ "columns": [
+ "project_id"
+ ],
+ "isUnique": true
+ },
+ "gsc_connections_organization_idx": {
+ "name": "gsc_connections_organization_idx",
+ "columns": [
+ "organization_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "gsc_connections_project_id_projects_id_fk": {
+ "name": "gsc_connections_project_id_projects_id_fk",
+ "tableFrom": "gsc_connections",
+ "tableTo": "projects",
+ "columnsFrom": [
+ "project_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "gsc_connections_organization_id_organization_id_fk": {
+ "name": "gsc_connections_organization_id_organization_id_fk",
+ "tableFrom": "gsc_connections",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "reddit_attributions": {
+ "name": "reddit_attributions",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "organization_id": {
+ "name": "organization_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "click_id": {
+ "name": "click_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "uuid": {
+ "name": "uuid",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "landing_page": {
+ "name": "landing_page",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "referrer": {
+ "name": "referrer",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "utm_source": {
+ "name": "utm_source",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "utm_medium": {
+ "name": "utm_medium",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "utm_campaign": {
+ "name": "utm_campaign",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "utm_term": {
+ "name": "utm_term",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "utm_content": {
+ "name": "utm_content",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "signup_sent_at": {
+ "name": "signup_sent_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "purchase_sent_at": {
+ "name": "purchase_sent_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": "(current_timestamp)"
+ }
+ },
+ "indexes": {
+ "reddit_attributions_user_idx": {
+ "name": "reddit_attributions_user_idx",
+ "columns": [
+ "user_id"
+ ],
+ "isUnique": true
+ },
+ "reddit_attributions_organization_idx": {
+ "name": "reddit_attributions_organization_idx",
+ "columns": [
+ "organization_id"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "reddit_attributions_user_id_user_id_fk": {
+ "name": "reddit_attributions_user_id_user_id_fk",
+ "tableFrom": "reddit_attributions",
+ "tableTo": "user",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "reddit_attributions_organization_id_organization_id_fk": {
+ "name": "reddit_attributions_organization_id_organization_id_fk",
+ "tableFrom": "reddit_attributions",
+ "tableTo": "organization",
+ "columnsFrom": [
+ "organization_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ },
+ "telemetry_state": {
+ "name": "telemetry_state",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "integer",
+ "primaryKey": true,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 1
+ },
+ "install_id": {
+ "name": "install_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false
+ },
+ "installed_at": {
+ "name": "installed_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_heartbeat_at": {
+ "name": "last_heartbeat_at",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "last_version": {
+ "name": "last_version",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "autoincrement": false
+ },
+ "mcp_tool_call_count": {
+ "name": "mcp_tool_call_count",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true,
+ "autoincrement": false,
+ "default": 0
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "checkConstraints": {}
+ }
+ },
+ "views": {},
+ "enums": {},
+ "_meta": {
+ "schemas": {},
+ "tables": {},
+ "columns": {}
+ },
+ "internal": {
+ "indexes": {}
+ }
+}
\ No newline at end of file
diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json
index 8711f67..e0c742a 100644
--- a/drizzle/meta/_journal.json
+++ b/drizzle/meta/_journal.json
@@ -260,6 +260,13 @@
"when": 1784426967421,
"tag": "0036_curvy_silk_fever",
"breakpoints": true
+ },
+ {
+ "idx": 37,
+ "version": "6",
+ "when": 1784588247803,
+ "tag": "0037_small_caretaker",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/package.json b/package.json
index 511816f..c0db0c7 100644
--- a/package.json
+++ b/package.json
@@ -91,12 +91,12 @@
"ai": "^6.0.199",
"autumn-js": "^1.2.33",
"better-auth": "^1.6.22",
- "cheerio": "^1.2.0",
"cloudflare": "^5.2.0",
"daisyui": "^5.5.5",
"dataforseo-client": "^2.0.19",
"drizzle-orm": "^0.45.2",
"fast-xml-parser": "^5.4.1",
+ "htmlparser2": "^10.1.0",
"jose": "^6.0.12",
"lucide-react": "^0.542.0",
"papaparse": "^5.5.3",
@@ -132,6 +132,7 @@
"@vitejs/plugin-react": "^4.6.0",
"alchemy": "2.0.0-beta.61",
"chalk": "^5.6.2",
+ "cheerio": "^1.2.0",
"drizzle-kit": "^0.31.10",
"effect": "4.0.0-beta.93",
"knip": "^5.88.1",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 94d6070..4ae6b3e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -75,9 +75,6 @@ importers:
better-auth:
specifier: ^1.6.22
version: 1.6.22(@cloudflare/workers-types@4.20260611.1)(@opentelemetry/api@1.9.1)(@tanstack/react-start@1.168.26(esbuild@0.28.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(rolldown@1.0.1)(rollup@4.59.0)(vite@7.3.6(@types/node@22.19.11)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0)))(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@cloudflare/workers-types@4.20260611.1)(@libsql/client@0.15.15)(@opentelemetry/api@1.9.1)(kysely@0.29.2)(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(postgres@3.4.9)(sql.js@1.14.1))(mongodb@7.2.0(@aws-sdk/credential-providers@3.1080.0)(@mongodb-js/zstd@7.0.0))(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.6(@types/debug@4.1.13)(@types/node@22.19.11)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)(yaml@2.9.0))
- cheerio:
- specifier: ^1.2.0
- version: 1.2.0
cloudflare:
specifier: ^5.2.0
version: 5.2.0
@@ -93,6 +90,9 @@ importers:
fast-xml-parser:
specifier: ^5.4.1
version: 5.8.0
+ htmlparser2:
+ specifier: ^10.1.0
+ version: 10.1.0
jose:
specifier: ^6.0.12
version: 6.1.3
@@ -193,6 +193,9 @@ importers:
chalk:
specifier: ^5.6.2
version: 5.6.2
+ cheerio:
+ specifier: ^1.2.0
+ version: 1.2.0
drizzle-kit:
specifier: ^0.31.10
version: 0.31.10
diff --git a/specs/0009-site-audit-crawl-architecture.md b/specs/0009-site-audit-crawl-architecture.md
new file mode 100644
index 0000000..dd85d73
--- /dev/null
+++ b/specs/0009-site-audit-crawl-architecture.md
@@ -0,0 +1,125 @@
+# Site audit crawl architecture
+
+## Status
+
+Accepted
+
+## Context
+
+The site audit runs as a Cloudflare Workflow. The original design kept all
+crawl state inside the workflow run: the URL frontier and page summaries lived
+in workflow memory and step returns, pages were parsed with a full DOM parser
+(cheerio), and link edges were bulk-inserted into the app database during the
+crawl. On large or slow sites this collided with platform limits:
+
+- **Isolate memory.** DOM parsing expands a page several times over, multiplied
+ by concurrent parses, plus a whole-crawl summaries array that was rebuilt on
+ every workflow replay.
+- **Step-output cap (~1 MiB).** Sitemap seed lists flowed through step returns
+ and could exceed it outright.
+- **Step timeout × retry.** The default 10-minute step timeout with default
+ retries meant a deterministically failing step burned about an hour before
+ the audit died.
+- **Batch head-of-line blocking.** Fixed-size crawl batches waited on their
+ slowest fetch; one tarpit URL stalled a whole batch.
+- **Invisible failure.** Workflow-level kills (OOM, CPU) skip the error
+ handler, so audit rows could stay `running` forever with no recorded reason,
+ and the UI refused to show the pages that had already been crawled and
+ persisted.
+
+## Decision
+
+Split the audit into a control plane and a data plane.
+
+### Orchestration stays in Workflows; crawl state moves to a Durable Object
+
+`SiteAuditWorkflow` remains the orchestrator (phase ordering, durable steps,
+retries). All transient crawl state lives in `AuditScratchpad`, a SQLite-backed
+Durable Object, one instance per audit (`idFromName(auditId)`):
+
+- **frontier** — URL queue and dedup set (URL primary key), with
+ `pending / leased / crawled` states keyed by chunk number;
+- **links** — internal link edges, primary-keyed for idempotent re-insert;
+- **page_mirror** — the few columns the finalize link checks need.
+
+The app database (D1/Postgres) keeps only what the product reads: `audits`
+(plus failure columns), `audit_pages`, `audit_issues`,
+`audit_lighthouse_results`. Link edges are never persisted to the app DB; the
+cross-page link checks (broken internal links, orphan pages) run as SQL inside
+the DO at finalize.
+
+Rejected alternatives: Cloudflare Queues (no dedup, no counters, no completion
+signal — a frontier is mostly those things), a Postgres frontier table (chatty
+hot-path writes, dual-dialect surface), R2 spill files (not queryable).
+
+### Chunked rolling crawl
+
+The crawl phase is a loop of `crawl-chunk-N` steps. Each chunk leases up to
+~200 URLs from the DO and crawls them with a rolling concurrency window: the
+moment a fetch settles, the next URL launches. The window self-adjusts between
+5 and 40 based on recent fetch health (errors, blocked fetches, slow responses,
+oversized bodies shrink it; clean fast batches grow it). A soft deadline ends
+the chunk early and releases unfetched leases. Persistence is pipelined with
+fetching but serialized with itself: page rows and per-page issues go to the
+app DB, link edges and frontier updates go to the DO, and progress counters
+update per sub-batch. Step returns carry only counters, so no step output
+scales with site size.
+
+Everything is idempotent under step retries: chunk-keyed leases (a retried
+step re-receives exactly the URLs its failed attempt held, and refuses a fresh
+claim for an already-crawled chunk number), deterministic page-row ids, and
+insert-or-ignore/replace writes on stable keys.
+
+### Streaming HTML parsing
+
+Pages are parsed with htmlparser2's streaming tokenizer — no DOM is built, so
+per-page memory is constant. cheerio (which uses the same tokenizer
+internally) remains only as a test reference: the parser test suite asserts
+extraction parity against it, and the badseo fixture harness asserts identical
+issue output end to end.
+
+### Failure handling and graceful degradation
+
+- `audits` carries `error_code` (a closed vocabulary mapped from real platform
+ error strings — `step_timeout`, `oom`, `cpu_limit`, `db_error`,
+ `step_output_too_large`, `workflow_internal`, `instance_lost`, `unknown`),
+ `error_detail`, and `failed_phase`.
+- A watchdog on the existing `*/15` cron reconciles audit rows stuck in
+ `running` against the Workflows API; the audit status read path does the
+ same lazily. Only confirmed instance-not-found errors (after a grace period)
+ count as a lost instance, so transient API failures never fail a live audit.
+- A failed audit shows everything crawled before the failure ("stopped early
+ after N pages") instead of hiding results.
+- The start URL follows redirects (with per-hop SSRF revalidation) before the
+ crawl anchors its origin, so apex→www and cross-TLD redirects don't dead-end
+ the crawl after one page.
+
+### DO lifecycle and platform-limit guards
+
+- `destroy()` (delete alarm, then all storage) runs on audit success and on
+ audit deletion. Every DO construction schedules a 7-day self-cleanup alarm,
+ so any instantiation — including a write racing in after destroy, or a
+ workflow that dies before seeding — is eventually wiped. Failed audits keep
+ their scratchpad for those 7 days as the resume/debug artifact.
+- Guards where the platform has hard edges: link writes stop at a storage
+ budget (below the free-plan per-object SQLite cap; orphan detection is
+ skipped when the link graph was truncated), discovered-URL batches are
+ capped below the serialized-RPC limit, and sitemap documents are read up to
+ a byte cap and skipped whole beyond it.
+
+## Consequences
+
+- Workflow heap and step state are O(one chunk) regardless of site size; the
+ OOM/step-output failure classes are gone by construction.
+- Hostile or slow sites degrade to a slower window and finish (or fail with a
+ classified reason and visible partials) instead of burning an hour of
+ retries.
+- Every failure is aggregable by `error_code` in plain SQL, and zombie
+ `running` rows self-heal.
+- Crawl-state code is provider-independent (DO SQLite exists in workerd), so
+ self-host does not add a dialect surface for the frontier or links.
+- The scratchpad is opaque from outside the DO; the 7-day retention of failed
+ audits' state is the debugging window.
+- Resume-from-frontier ("retry finishes the last N pages instead of
+ recrawling") is enabled by the retained frontier but intentionally not
+ built yet.
diff --git a/src/client/features/audit/shared.tsx b/src/client/features/audit/shared.tsx
index b999e6e..0ecf39d 100644
--- a/src/client/features/audit/shared.tsx
+++ b/src/client/features/audit/shared.tsx
@@ -1,6 +1,6 @@
import { AlertCircle, CheckCircle, Loader2 } from "lucide-react";
-export const SUPPORT_URL = "https://everyapp.dev/support";
+export const SUPPORT_EMAIL = "ben@openseo.so";
export function extractPathname(url: string): string {
try {
diff --git a/src/db/audit.schema.ts b/src/db/audit.schema.ts
index 7ba9598..6aad650 100644
--- a/src/db/audit.schema.ts
+++ b/src/db/audit.schema.ts
@@ -37,6 +37,13 @@ export const audits = sqliteTable(
lighthouseCompleted: integer("lighthouse_completed").notNull().default(0),
lighthouseFailed: integer("lighthouse_failed").notNull().default(0),
currentPhase: text("current_phase").default("discovery"),
+ // Failure diagnostics; null unless status = "failed". errorCode is a
+ // closed vocabulary (see classifyAuditError) so failures are aggregable;
+ // errorDetail is the raw message, truncated. failedPhase records which
+ // currentPhase the audit was in when it died.
+ errorCode: text("error_code"),
+ errorDetail: text("error_detail"),
+ failedPhase: text("failed_phase"),
startedAt: text("started_at")
.notNull()
.default(sql`(current_timestamp)`),
diff --git a/src/db/pg/audit.schema.ts b/src/db/pg/audit.schema.ts
index 4e6d4c8..5666573 100644
--- a/src/db/pg/audit.schema.ts
+++ b/src/db/pg/audit.schema.ts
@@ -44,6 +44,13 @@ export const audits = pgTable(
lighthouseCompleted: integer("lighthouse_completed").notNull().default(0),
lighthouseFailed: integer("lighthouse_failed").notNull().default(0),
currentPhase: text("current_phase").default("discovery"),
+ // Failure diagnostics; null unless status = "failed". errorCode is a
+ // closed vocabulary (see classifyAuditError) so failures are aggregable;
+ // errorDetail is the raw message, truncated. failedPhase records which
+ // currentPhase the audit was in when it died.
+ errorCode: text("error_code"),
+ errorDetail: text("error_detail"),
+ failedPhase: text("failed_phase"),
startedAt: timestampColumn("started_at").notNull().default(isoNow),
completedAt: timestampColumn("completed_at"),
},
diff --git a/src/db/schema-parity.test.ts b/src/db/schema-parity.test.ts
index c62c0b6..206a009 100644
--- a/src/db/schema-parity.test.ts
+++ b/src/db/schema-parity.test.ts
@@ -5,6 +5,7 @@ import { getTableConfig as getSqliteTableConfig } from "drizzle-orm/sqlite-core"
import { getTableConfig as getPgTableConfig } from "drizzle-orm/pg-core";
import { describe, expect, it } from "vitest";
import * as sqliteApp from "./app.schema";
+import * as sqliteAudit from "./audit.schema";
import * as sqliteSam from "./sam.schema";
import * as sqliteAuth from "./better-auth-schema";
import * as sqliteBilling from "./billing.schema";
@@ -12,6 +13,7 @@ import * as sqliteGsc from "./gsc.schema";
import * as sqliteReddit from "./reddit-attribution.schema";
import * as sqliteTelemetry from "./telemetry.schema";
import * as pgApp from "./pg/app.schema";
+import * as pgAudit from "./pg/audit.schema";
import * as pgSam from "./pg/sam.schema";
import * as pgAuth from "./pg/better-auth-schema";
import * as pgBilling from "./pg/billing.schema";
@@ -135,6 +137,7 @@ function foreignKeys(table: Table, dialect: Dialect): string[] {
const sqliteAppTables = tablesFrom(
sqliteApp,
+ sqliteAudit,
sqliteSam,
sqliteBilling,
sqliteGsc,
@@ -143,6 +146,7 @@ const sqliteAppTables = tablesFrom(
);
const pgAppTables = tablesFrom(
pgApp,
+ pgAudit,
pgSam,
pgBilling,
pgGsc,
diff --git a/src/env.d.ts b/src/env.d.ts
index 6f420dd..3402041 100644
--- a/src/env.d.ts
+++ b/src/env.d.ts
@@ -12,6 +12,10 @@ declare namespace Cloudflare {
// Durable Object backing the SAM in-app agent (see wrangler.jsonc).
SAM_CHAT: DurableObjectNamespace;
+ // Durable Object holding per-audit crawl scratch state (frontier, link
+ // edges, page mirror). Untyped here; getAuditScratchpad narrows the stub.
+ AUDIT_SCRATCHPAD: DurableObjectNamespace;
+
AUTH_MODE?: "cloudflare_access" | "local_noauth" | "hosted";
BYPASS_EMAIL_VERIFICATION?: string;
TEAM_DOMAIN?: string;
diff --git a/src/routes/_project/p/$projectId/audit/index.tsx b/src/routes/_project/p/$projectId/audit/index.tsx
index e0dce99..11e4d28 100644
--- a/src/routes/_project/p/$projectId/audit/index.tsx
+++ b/src/routes/_project/p/$projectId/audit/index.tsx
@@ -16,7 +16,7 @@ import {
formatStartedAt,
HttpStatusBadge,
StatusBadge,
- SUPPORT_URL,
+ SUPPORT_EMAIL,
} from "@/client/features/audit/shared";
export const Route = createFileRoute<"/_project/p/$projectId/audit/">(
@@ -87,10 +87,13 @@ function AuditDetail({
const isFailed = statusQuery.data?.status === "failed";
const isRunning = statusQuery.data?.status === "running";
+ // Failed audits keep whatever pages were crawled before the failure
+ // (persistence is per-batch), so fetch results for them too and show the
+ // partial crawl instead of a dead end.
const resultsQuery = useQuery({
queryKey: ["audit-results", projectId, auditId],
queryFn: () => getAuditResults({ data: { projectId, auditId } }),
- enabled: isComplete,
+ enabled: isComplete || isFailed,
});
if (statusQuery.isLoading) {
@@ -118,8 +121,15 @@ function AuditDetail({
}
const status = statusQuery.data;
+ const partialPageCount = isFailed
+ ? (resultsQuery.data?.pages.length ?? 0)
+ : 0;
+ const failedWithResults = isFailed && partialPageCount > 0;
+ // Wait for the results fetch before choosing between the "partial results"
+ // banner and the zero-page support CTA, so the CTA doesn't flash first.
const showSupportCta =
- isFailed || (isComplete && status && status.pagesCrawled <= 1);
+ (isFailed && resultsQuery.isSuccess && !failedWithResults) ||
+ (isComplete && status && status.pagesCrawled <= 1);
return (
@@ -161,15 +171,12 @@ function AuditDetail({
Site audit couldn't fully crawl this website.
- This is often caused by anti-bot or firewall settings. Reach out
- at{" "}
+ This is often caused by anti-bot or firewall settings. Email{" "}
- everyapp.dev/support
+ {SUPPORT_EMAIL}
{" "}
and we'll help configure auditing for your site.
@@ -177,7 +184,30 @@ function AuditDetail({
)}
- {isComplete && resultsQuery.data && (
+ {failedWithResults && (
+
+
+
+
+ This audit stopped early after {partialPageCount} page
+ {partialPageCount === 1 ? "" : "s"}.
+
+
+ The results below cover everything crawled before it stopped.
+ Run a new audit to try again, or email{" "}
+
+ {SUPPORT_EMAIL}
+ {" "}
+ if this keeps happening.
+
+
+
+ )}
+
+ {(isComplete || failedWithResults) && resultsQuery.data && (
runScheduledRankChecks(env));
+ // Watchdog: reconcile audits stuck in "running" whose workflow died
+ // without reaching mark-failed (OOM/CPU kills, expired instances).
+ await withPgClient(() => reconcileStaleAudits());
},
};
diff --git a/src/server/features/audit/AuditScratchpad.ts b/src/server/features/audit/AuditScratchpad.ts
new file mode 100644
index 0000000..1bf3319
--- /dev/null
+++ b/src/server/features/audit/AuditScratchpad.ts
@@ -0,0 +1,372 @@
+/**
+ * Per-audit crawl scratchpad — a SQLite-backed Durable Object.
+ *
+ * Holds all transient crawl state for one audit (id = auditId): the frontier
+ * (URL queue + seen-set), the internal-link edges, and a slim mirror of page
+ * rows. This keeps chatty crawl-loop writes off Postgres, removes the ~1MiB
+ * Workflow step-output limits (nothing large flows through step returns
+ * anymore), and gives the finalize link checks a local SQL database.
+ *
+ * Lifecycle: seeded by the discover-urls step, read/written by every crawl
+ * chunk, queried once at finalize, then destroyed on success. A self-cleanup
+ * alarm set on every construction guarantees any instantiation — including a
+ * write racing in after destroy() — is wiped after 7 days (failed audits'
+ * state doubles as the resume/debug artifact until then).
+ *
+ * All methods are synchronous inside (SQLite in DOs is sync), so each RPC is
+ * effectively atomic. Writes are idempotent: the workflow retries steps, so
+ * every insert is OR IGNORE / OR REPLACE on a stable key.
+ */
+import { DurableObject, env } from "cloudflare:workers";
+
+export interface ClaimedUrl {
+ url: string;
+ /** Clicks from the start URL; null when only reachable via sitemap. */
+ depth: number | null;
+ inSitemap: boolean;
+}
+
+export interface FrontierStats {
+ /** Pages attempted (crawled or errored) so far. */
+ attempted: number;
+ /** URLs still waiting to be crawled. */
+ pending: number;
+ /** Every URL ever enqueued (attempted + pending + leased). */
+ seen: number;
+}
+
+interface ScratchpadPageRow {
+ pageId: string;
+ url: string;
+ statusCode: number;
+ fetchClass: string;
+ redirectUrl: string | null;
+}
+
+export interface ScratchpadLinkRow {
+ sourcePageId: string;
+ sourceUrl: string;
+ targetUrl: string;
+ anchor: string | null;
+ isNofollow: boolean;
+}
+
+interface RecordBatchInput {
+ /** URLs whose crawl attempt finished (successfully or not). */
+ crawledUrls: string[];
+ pages: ScratchpadPageRow[];
+ links: ScratchpadLinkRow[];
+ /** Newly discovered same-origin URLs to enqueue (already policy-filtered). */
+ discovered: Array<{ url: string; depth: number | null }>;
+}
+
+interface BrokenLinkRow {
+ sourcePageId: string;
+ sourceUrl: string;
+ targetUrl: string;
+ targetStatus: number;
+}
+
+interface OrphanPageRow {
+ pageId: string;
+ url: string;
+}
+
+const BROKEN_LINK_ISSUE_CAP = 2_000;
+const CLEANUP_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
+/**
+ * Stop storing link edges once the database reaches this size. Link rows are
+ * the only unbounded-per-page data; a pathological link-dense site could
+ * otherwise hit the platform's per-object SQLite cap (1 GB on the free plan
+ * self-hosters may run on) and fail the crawl with SQLITE_FULL. Past the
+ * budget the crawl continues — the audit just loses link-graph issues.
+ */
+const LINK_STORAGE_BUDGET_BYTES = 500 * 1024 * 1024;
+
+export class AuditScratchpad extends DurableObject {
+ constructor(ctx: DurableObjectState, workerEnv: Env) {
+ super(ctx, workerEnv);
+ this.ctx.storage.sql.exec(`
+ CREATE TABLE IF NOT EXISTS frontier (
+ url TEXT PRIMARY KEY,
+ depth INTEGER,
+ source TEXT NOT NULL,
+ in_sitemap INTEGER NOT NULL DEFAULT 0,
+ state TEXT NOT NULL DEFAULT 'pending',
+ chunk_no INTEGER
+ );
+ CREATE INDEX IF NOT EXISTS frontier_claim_idx ON frontier (state, source);
+ CREATE TABLE IF NOT EXISTS links (
+ source_page_id TEXT NOT NULL,
+ source_url TEXT NOT NULL,
+ target_url TEXT NOT NULL,
+ anchor TEXT,
+ is_nofollow INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (source_page_id, target_url)
+ );
+ CREATE INDEX IF NOT EXISTS links_target_idx ON links (target_url);
+ CREATE TABLE IF NOT EXISTS page_mirror (
+ page_id TEXT PRIMARY KEY,
+ url TEXT NOT NULL UNIQUE,
+ status_code INTEGER,
+ fetch_class TEXT NOT NULL,
+ redirect_url TEXT
+ );
+ CREATE INDEX IF NOT EXISTS page_mirror_redirect_idx ON page_mirror (redirect_url);
+ `);
+ // Guarantee the cleanup alarm on EVERY instantiation, not just at seed:
+ // any RPC (even one racing in right after destroy()) re-creates the
+ // tables above, and without an alarm that storage would leak forever.
+ // blockConcurrencyWhile gates RPC delivery on its own; the returned
+ // promise doesn't need observing (constructors can't await).
+ void this.ctx.blockConcurrencyWhile(() => this.ensureCleanupAlarm());
+ }
+
+ async seedStart(url: string): Promise {
+ this.ctx.storage.sql.exec(
+ `INSERT OR IGNORE INTO frontier (url, depth, source, in_sitemap) VALUES (?, 0, 'link', 0)`,
+ url,
+ );
+ }
+
+ async seedSitemapUrls(urls: string[]): Promise {
+ for (const url of urls) {
+ // Upsert so a URL that already exists (e.g. the start URL) still gets
+ // its in-sitemap flag; queue position and depth stay as first seen.
+ this.ctx.storage.sql.exec(
+ `INSERT INTO frontier (url, depth, source, in_sitemap) VALUES (?, NULL, 'sitemap', 1)
+ ON CONFLICT(url) DO UPDATE SET in_sitemap = 1`,
+ url,
+ );
+ }
+ }
+
+ /**
+ * Lease the next batch of pending URLs for one crawl chunk. Idempotent per
+ * chunkNo: a retried step gets back exactly the URLs its failed attempt
+ * had leased (link-discovered URLs drain before sitemap-only ones, FIFO
+ * within each class — same ordering as the old in-memory queues).
+ */
+ async claimChunk(chunkNo: number, limit: number): Promise {
+ const existing = this.selectClaimed(
+ `SELECT url, depth, in_sitemap FROM frontier WHERE state = 'leased' AND chunk_no = ?`,
+ chunkNo,
+ );
+ if (existing.length > 0) return existing;
+ // A retried step whose earlier attempt already crawled this chunk's
+ // leases must not claim a fresh set under the same chunk number — that
+ // would duplicate work and overshoot the page budget.
+ const done = this.ctx.storage.sql
+ .exec<{
+ n: number;
+ }>(
+ `SELECT COUNT(*) AS n FROM frontier WHERE chunk_no = ? AND state = 'crawled'`,
+ chunkNo,
+ )
+ .one();
+ if (done.n > 0) return [];
+ if (limit <= 0) return [];
+
+ const fresh = this.selectClaimed(
+ `SELECT url, depth, in_sitemap FROM frontier WHERE state = 'pending'
+ ORDER BY CASE source WHEN 'link' THEN 0 ELSE 1 END, rowid LIMIT ?`,
+ limit,
+ );
+ for (const row of fresh) {
+ this.ctx.storage.sql.exec(
+ `UPDATE frontier SET state = 'leased', chunk_no = ? WHERE url = ?`,
+ chunkNo,
+ row.url,
+ );
+ }
+ return fresh;
+ }
+
+ /** Persist one crawled sub-batch: completions, mirror rows, links, frontier. */
+ async recordBatch(input: RecordBatchInput): Promise {
+ for (const url of input.crawledUrls) {
+ this.ctx.storage.sql.exec(
+ `UPDATE frontier SET state = 'crawled' WHERE url = ?`,
+ url,
+ );
+ }
+ for (const page of input.pages) {
+ this.ctx.storage.sql.exec(
+ `INSERT OR REPLACE INTO page_mirror (page_id, url, status_code, fetch_class, redirect_url)
+ VALUES (?, ?, ?, ?, ?)`,
+ page.pageId,
+ page.url,
+ page.statusCode,
+ page.fetchClass,
+ page.redirectUrl,
+ );
+ }
+ // Database size only ever grows (deletes happen solely via destroy), so
+ // once the budget trips it stays tripped — runFinalizeChecks uses the
+ // same comparison to know the link graph is incomplete.
+ if (this.ctx.storage.sql.databaseSize < LINK_STORAGE_BUDGET_BYTES) {
+ for (const link of input.links) {
+ this.ctx.storage.sql.exec(
+ `INSERT OR IGNORE INTO links (source_page_id, source_url, target_url, anchor, is_nofollow)
+ VALUES (?, ?, ?, ?, ?)`,
+ link.sourcePageId,
+ link.sourceUrl,
+ link.targetUrl,
+ link.anchor,
+ link.isNofollow ? 1 : 0,
+ );
+ }
+ }
+ for (const found of input.discovered) {
+ // OR IGNORE: already-seen URLs (crawled, leased, or pending) keep
+ // their existing row — this is the global dedup.
+ this.ctx.storage.sql.exec(
+ `INSERT OR IGNORE INTO frontier (url, depth, source, in_sitemap) VALUES (?, ?, 'link', 0)`,
+ found.url,
+ found.depth,
+ );
+ }
+ return this.stats();
+ }
+
+ /** Return unattempted leases to the queue (chunk soft-deadline hit). */
+ async releaseUrls(urls: string[]): Promise {
+ for (const url of urls) {
+ this.ctx.storage.sql.exec(
+ `UPDATE frontier SET state = 'pending', chunk_no = NULL WHERE url = ? AND state = 'leased'`,
+ url,
+ );
+ }
+ }
+
+ async getStats(): Promise {
+ return this.stats();
+ }
+
+ /**
+ * The two finalize checks that need link edges, as local SQL. Mirrors the
+ * former Postgres implementations in multipage.ts exactly.
+ */
+ async runFinalizeChecks(input: {
+ startUrl: string;
+ crawlCompleted: boolean;
+ }): Promise<{ brokenLinks: BrokenLinkRow[]; orphanPages: OrphanPageRow[] }> {
+ // Only flag targets we actually crawled and saw fail — never inferred
+ // from absence. Blocked targets (WAF challenges) are excluded: a 403
+ // from bot protection is not evidence of a broken link.
+ const brokenLinks = this.ctx.storage.sql
+ .exec<{
+ source_page_id: string;
+ source_url: string;
+ target_url: string;
+ target_status: number;
+ }>(
+ `SELECT l.source_page_id, l.source_url, l.target_url, m.status_code AS target_status
+ FROM links l JOIN page_mirror m ON m.url = l.target_url
+ WHERE m.status_code >= 400 AND m.fetch_class = 'ok'
+ ORDER BY l.source_page_id, l.target_url
+ LIMIT ?`,
+ BROKEN_LINK_ISSUE_CAP,
+ )
+ .toArray()
+ .map((row) => ({
+ sourcePageId: row.source_page_id,
+ sourceUrl: row.source_url,
+ targetUrl: row.target_url,
+ targetStatus: row.target_status,
+ }));
+
+ // A live 2xx page is an orphan when no OTHER crawled page links to it
+ // and nothing redirects to it. Only meaningful on a completed crawl with
+ // a complete link graph — if the storage budget truncated link writes,
+ // "no inbound edge" is missing data, not evidence of orphanhood.
+ const linkGraphComplete =
+ this.ctx.storage.sql.databaseSize < LINK_STORAGE_BUDGET_BYTES;
+ const orphanPages =
+ input.crawlCompleted && linkGraphComplete
+ ? this.ctx.storage.sql
+ .exec<{ page_id: string; url: string }>(
+ `SELECT m.page_id, m.url FROM page_mirror m
+ WHERE m.url != ?
+ AND m.fetch_class = 'ok'
+ AND m.status_code >= 200 AND m.status_code < 300
+ AND NOT EXISTS (
+ SELECT 1 FROM links l
+ WHERE l.target_url = m.url AND l.source_page_id != m.page_id
+ )
+ AND NOT EXISTS (
+ SELECT 1 FROM page_mirror r WHERE r.redirect_url = m.url
+ )`,
+ input.startUrl,
+ )
+ .toArray()
+ .map((row) => ({ pageId: row.page_id, url: row.url }))
+ : [];
+
+ return { brokenLinks, orphanPages };
+ }
+
+ /** Wipe all state (success path, or explicit audit deletion). */
+ async destroy(): Promise {
+ await this.ctx.storage.deleteAlarm();
+ await this.ctx.storage.deleteAll();
+ }
+
+ /**
+ * Self-cleanup for audits whose workflow died without reaching finalize.
+ * Full destroy(), not just deleteAll(): under our compatibility date,
+ * deleteAll() does not clear alarm state, and a leftover alarm would keep
+ * the object (and its billing) alive.
+ */
+ async alarm(): Promise {
+ await this.destroy();
+ }
+
+ private async ensureCleanupAlarm(): Promise {
+ const existing = await this.ctx.storage.getAlarm();
+ if (existing === null) {
+ await this.ctx.storage.setAlarm(Date.now() + CLEANUP_AFTER_MS);
+ }
+ }
+
+ private selectClaimed(query: string, param: number): ClaimedUrl[] {
+ return this.ctx.storage.sql
+ .exec<{ url: string; depth: number | null; in_sitemap: number }>(
+ query,
+ param,
+ )
+ .toArray()
+ .map((row) => ({
+ url: row.url,
+ depth: row.depth,
+ inSitemap: row.in_sitemap === 1,
+ }));
+ }
+
+ private stats(): FrontierStats {
+ const row = this.ctx.storage.sql
+ .exec<{ attempted: number; pending: number; seen: number }>(
+ `SELECT
+ COUNT(*) FILTER (WHERE state = 'crawled') AS attempted,
+ COUNT(*) FILTER (WHERE state = 'pending') AS pending,
+ COUNT(*) AS seen
+ FROM frontier`,
+ )
+ .one();
+ return {
+ attempted: row.attempted,
+ pending: row.pending,
+ seen: row.seen,
+ };
+ }
+}
+
+/** Stub for the audit's scratchpad DO (one instance per audit id). */
+export function getAuditScratchpad(auditId: string) {
+ // env.d.ts declares the binding untyped (ambient contexts can't import the
+ // class); narrow here so callers get typed RPC methods.
+ const namespace =
+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the binding is declared as this class in wrangler.jsonc
+ env.AUDIT_SCRATCHPAD as unknown as DurableObjectNamespace;
+ return namespace.get(namespace.idFromName(auditId));
+}
diff --git a/src/server/features/audit/repositories/AuditRepository.ts b/src/server/features/audit/repositories/AuditRepository.ts
index 28c2767..221cd80 100644
--- a/src/server/features/audit/repositories/AuditRepository.ts
+++ b/src/server/features/audit/repositories/AuditRepository.ts
@@ -1,18 +1,17 @@
/**
* Data access layer for site audit tables.
* Provider-aware (D1 or Postgres) via the `@/db` handle. Covers audits,
- * audit_pages, audit_links, audit_issues, and stored Lighthouse results.
+ * audit_pages, audit_issues, and stored Lighthouse results. Link edges live
+ * in the per-audit scratchpad Durable Object, not here.
*/
-import { and, desc, eq } from "drizzle-orm";
+import { and, count, desc, eq } from "drizzle-orm";
import { db } from "@/db";
import {
audits,
auditIssues,
auditLighthouseResults,
- auditLinks,
auditPages,
} from "@/db/schema";
-import { getDatabaseProvider } from "@/db/provider";
import { executeInBatches } from "@/db/runBatch";
import { AUDIT_ISSUE_TYPES } from "@/shared/audit-issues";
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
@@ -23,14 +22,6 @@ import type {
LighthouseResult,
} from "@/server/lib/audit/types";
-// Only internal links are stored: both consumers (broken-internal-link and
-// orphan checks) filter on isInternal, and per-page external counts already
-// live on audit_pages. External rows come back when P1 adds external-link
-// checks. Mega-menu/footer-heavy sites can carry 1000+ links per page; cap
-// what we store so a 10k-page crawl can't write tens of millions of link rows.
-const MAX_STORED_LINKS_PER_PAGE = 500;
-const POSTGRES_LINK_INSERT_SIZE = 500;
-
async function createAudit(data: {
id: string;
projectId: string;
@@ -102,7 +93,15 @@ async function completeAudit(
);
}
-async function failAudit(auditId: string, workflowInstanceId: string) {
+async function failAudit(
+ auditId: string,
+ workflowInstanceId: string,
+ errorInfo?: {
+ errorCode: string;
+ errorDetail: string;
+ failedPhase: string | null;
+ },
+) {
// Only a running audit can transition to failed: the getStatus reconciler
// races the workflow's own finalize, and without this guard it could flip
// a just-completed audit to failed.
@@ -112,6 +111,11 @@ async function failAudit(auditId: string, workflowInstanceId: string) {
status: "failed",
completedAt: new Date().toISOString(),
currentPhase: "failed",
+ ...(errorInfo && {
+ errorCode: errorInfo.errorCode,
+ errorDetail: errorInfo.errorDetail,
+ failedPhase: errorInfo.failedPhase,
+ }),
})
.where(
and(
@@ -135,15 +139,15 @@ async function getAuditForWorkflow(
}
/**
- * Persist one crawl batch (pages + link edges + per-page issues).
- * Called inside the crawl-batch Workflow step so results land in D1
- * incrementally instead of accumulating in memory until finalize.
+ * Persist one crawled sub-batch (pages + per-page issues). Called inside the
+ * crawl-chunk Workflow step so results land in the app DB incrementally
+ * instead of accumulating in memory until finalize. Link edges go to the
+ * audit's scratchpad DO, not here.
*
* Idempotent on step retry: callers assign deterministic page ids
- * (deterministicAuditRowId) and link/issue ids are derived from stable
- * content. Page rows upsert (a retried fetch may legitimately differ — last
- * attempt wins, matching what the step returns); links and issues are
- * insert-or-ignore.
+ * (deterministicAuditRowId) and issue ids are derived from stable content.
+ * Page rows upsert (a retried fetch may legitimately differ — last attempt
+ * wins); issues are insert-or-ignore.
*/
async function insertCrawledBatch(
auditId: string,
@@ -192,38 +196,6 @@ async function insertCrawledBatch(
.onConflictDoUpdate({ target: auditPages.id, set: dataColumns });
});
- const linkRows = await Promise.all(
- pages.flatMap((page) =>
- page.links
- .filter((link) => link.isInternal)
- .slice(0, MAX_STORED_LINKS_PER_PAGE)
- .map(async (link) => ({
- id: await deterministicAuditRowId(auditId, page.url, link.targetUrl),
- auditId,
- sourcePageId: page.id,
- sourceUrl: page.url,
- targetUrl: link.targetUrl,
- anchor: link.anchor,
- isInternal: link.isInternal,
- isNofollow: link.isNofollow,
- })),
- ),
- );
- if (getDatabaseProvider() === "postgres") {
- // A Postgres transaction executes runBatch statements sequentially. Bulk
- // values avoid thousands of Hyperdrive round trips on link-heavy pages.
- for (let i = 0; i < linkRows.length; i += POSTGRES_LINK_INSERT_SIZE) {
- await db
- .insert(auditLinks)
- .values(linkRows.slice(i, i + POSTGRES_LINK_INSERT_SIZE))
- .onConflictDoNothing();
- }
- } else {
- await executeInBatches(linkRows, (tx, row) =>
- tx.insert(auditLinks).values(row).onConflictDoNothing(),
- );
- }
-
await insertIssues(auditId, issues);
}
@@ -336,6 +308,19 @@ async function getPagesForAudit(auditId: string) {
.where(eq(auditPages.auditId, auditId));
}
+async function countBlockedPages(auditId: string): Promise {
+ const rows = await db
+ .select({ blocked: count() })
+ .from(auditPages)
+ .where(
+ and(
+ eq(auditPages.auditId, auditId),
+ eq(auditPages.fetchClass, "blocked"),
+ ),
+ );
+ return rows[0]?.blocked ?? 0;
+}
+
async function hasPagesForAudit(auditId: string): Promise {
const rows = await db
.select({ id: auditPages.id })
@@ -449,6 +434,7 @@ export const AuditRepository = {
getLatestAuditForProject,
getIssuesForAudit,
getPagesForAudit,
+ countBlockedPages,
hasPagesForAudit,
getAuditsByProject,
getAuditUsageForUser,
diff --git a/src/server/features/audit/services/AuditService.ts b/src/server/features/audit/services/AuditService.ts
index f36146c..459ec4c 100644
--- a/src/server/features/audit/services/AuditService.ts
+++ b/src/server/features/audit/services/AuditService.ts
@@ -5,6 +5,7 @@ import {
type BillingCustomerContext,
} from "@/server/billing/subscription";
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
+import { getAuditScratchpad } from "@/server/features/audit/AuditScratchpad";
import {
AUDIT_LIMITS,
clampAuditMaxPages,
@@ -18,7 +19,11 @@ import {
type AuditConfig,
type LighthouseStrategy,
} from "@/server/lib/audit/types";
-import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
+import {
+ normalizeAndValidateStartUrl,
+ resolveStartUrlRedirects,
+} from "@/server/lib/audit/url-policy";
+import { reconcileRunningAudit } from "@/server/features/audit/services/auditReconciler";
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
// Plan-tier limits are the abuse bound in hosted mode: free accounts get one
@@ -61,7 +66,12 @@ async function startAudit(input: {
const auditId = crypto.randomUUID();
const config: AuditConfig = { maxPages, lighthouseStrategy };
- const startUrl = await normalizeAndValidateStartUrl(input.startUrl);
+ // Anchor the audit to the site's real origin: a start domain that 301s
+ // elsewhere (…net -> …com, apex -> www) would otherwise dead-end after
+ // one page at the same-origin crawl boundary.
+ const startUrl = await resolveStartUrlRedirects(
+ await normalizeAndValidateStartUrl(input.startUrl),
+ );
await AuditRepository.createAudit({
id: auditId,
@@ -125,22 +135,13 @@ async function getStatus(auditId: string, projectId: string) {
throw new AppError("NOT_FOUND", "Audit not found in this project.");
// Self-heal audits whose workflow died without reaching the mark-failed
- // step (instance terminated, mark-failed itself failed, deploys, ...).
+ // step (instance terminated/errored, instance expired from retention, ...).
// Without this they stay "running" forever and hold capacity.
- if (audit.status === "running" && audit.workflowInstanceId) {
- try {
- const instance = await env.SITE_AUDIT_WORKFLOW.get(
- audit.workflowInstanceId,
- );
- const { status } = await instance.status();
- if (status === "errored" || status === "terminated") {
- await AuditRepository.failAudit(audit.id, audit.workflowInstanceId);
- audit =
- (await AuditRepository.getAuditForProject(auditId, projectId)) ??
- audit;
- }
- } catch {
- // Instance not found or status unavailable — leave the audit as-is.
+ if (audit.status === "running") {
+ const reconciled = await reconcileRunningAudit(audit);
+ if (reconciled) {
+ audit =
+ (await AuditRepository.getAuditForProject(auditId, projectId)) ?? audit;
}
}
@@ -154,6 +155,7 @@ async function getStatus(auditId: string, projectId: string) {
lighthouseCompleted: audit.lighthouseCompleted,
lighthouseFailed: audit.lighthouseFailed,
currentPhase: audit.currentPhase,
+ errorCode: audit.errorCode,
startedAt: audit.startedAt,
completedAt: audit.completedAt,
};
@@ -257,6 +259,13 @@ async function remove(auditId: string, projectId: string) {
}
await AuditRepository.deleteAuditForProject(auditId, projectId);
+ // Best-effort: drop the crawl scratchpad DO with the audit. A missed
+ // destroy self-cleans via the DO's 7-day alarm.
+ try {
+ await getAuditScratchpad(auditId).destroy();
+ } catch (error) {
+ console.warn(`Failed to destroy audit scratchpad ${auditId}:`, error);
+ }
}
export const AuditService = {
diff --git a/src/server/features/audit/services/auditReconciler.ts b/src/server/features/audit/services/auditReconciler.ts
new file mode 100644
index 0000000..c13f86a
--- /dev/null
+++ b/src/server/features/audit/services/auditReconciler.ts
@@ -0,0 +1,176 @@
+/**
+ * Reconciles audits stuck in "running" with the actual state of their
+ * Cloudflare Workflow instance.
+ *
+ * A workflow killed by the platform (OOM, CPU limit, deploy reset) never
+ * reaches its own mark-failed step, and an instance can expire from Workflows
+ * retention entirely. Two callers close that gap:
+ * - AuditService.getStatus (lazy: whenever the UI polls a running audit)
+ * - the scheduled watchdog (reconcileStaleAudits, every cron tick) — so
+ * zombie rows die even if nobody ever reopens the page, and the instance
+ * error is copied into the row before Workflows retention deletes it.
+ */
+import { env } from "cloudflare:workers";
+import { and, eq, lt } from "drizzle-orm";
+import { db } from "@/db";
+import { audits, projects } from "@/db/schema";
+import { getDatabaseProvider } from "@/db/provider";
+import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
+import {
+ classifyAuditError,
+ type AuditErrorInfo,
+} from "@/server/lib/audit/audit-errors";
+import { captureServerEvent } from "@/server/lib/posthog";
+
+/**
+ * Don't declare an instance "lost" until the audit is comfortably older than
+ * any legitimate create/start delay: startAudit inserts the row before
+ * creating the workflow, so a brand-new audit can briefly have no instance.
+ */
+const INSTANCE_LOST_GRACE_MS = 10 * 60 * 1000;
+
+/** Audits still "running" after this long get reconciled by the watchdog. */
+const STALE_RUNNING_AFTER_MS = 15 * 60 * 1000;
+
+const WATCHDOG_BATCH_LIMIT = 100;
+
+type RunningAudit = {
+ id: string;
+ workflowInstanceId: string | null;
+ startedAt: string;
+ currentPhase: string | null;
+};
+
+/**
+ * If the audit's workflow instance is dead (errored/terminated) or gone,
+ * flip the row to failed with a classified error. Returns the error info
+ * when the row was flipped, null when the audit is genuinely still running.
+ */
+export async function reconcileRunningAudit(
+ audit: RunningAudit,
+): Promise {
+ if (!audit.workflowInstanceId) return null;
+
+ let errorInfo: AuditErrorInfo | null = null;
+ try {
+ const instance = await env.SITE_AUDIT_WORKFLOW.get(
+ audit.workflowInstanceId,
+ );
+ const status = await instance.status();
+ if (status.status === "errored" || status.status === "terminated") {
+ errorInfo = status.error
+ ? classifyAuditError(
+ typeof status.error === "string"
+ ? status.error
+ : (status.error.message ?? JSON.stringify(status.error)),
+ )
+ : {
+ errorCode: "unknown",
+ errorDetail: `Workflow instance ${status.status}`,
+ };
+ }
+ } catch (error) {
+ // Instance not found (never created, or expired from Workflows
+ // retention). Past the grace window that means the audit can never
+ // finish — without this branch such rows stay "running" forever.
+ // Transient status/control-plane errors must NOT fail a live audit,
+ // so only a confirmed not-found counts.
+ const message = error instanceof Error ? error.message : String(error);
+ if (!/not[ _]?found/i.test(message)) return null;
+ if (!isOlderThan(audit.startedAt, INSTANCE_LOST_GRACE_MS)) return null;
+ errorInfo = {
+ errorCode: "instance_lost",
+ errorDetail: "Workflow instance not found",
+ };
+ }
+ if (!errorInfo) return null;
+
+ await AuditRepository.failAudit(audit.id, audit.workflowInstanceId, {
+ ...errorInfo,
+ failedPhase: audit.currentPhase,
+ });
+ return errorInfo;
+}
+
+/** Cron watchdog: sweep stale running audits and reconcile each. */
+export async function reconcileStaleAudits() {
+ const cutoff = new Date(Date.now() - STALE_RUNNING_AFTER_MS);
+ const stale = await getStaleRunningAudits(cutoff, WATCHDOG_BATCH_LIMIT);
+
+ for (const audit of stale) {
+ try {
+ const errorInfo = await reconcileRunningAudit(audit);
+ if (!errorInfo) continue;
+
+ console.log(
+ `Audit watchdog: marked ${audit.id} failed (${errorInfo.errorCode}, phase=${audit.currentPhase})`,
+ );
+ const project = await db.query.projects.findFirst({
+ where: eq(projects.id, audit.projectId),
+ });
+ if (!project) continue;
+ await captureServerEvent({
+ distinctId: audit.startedByUserId,
+ event: "site_audit:complete",
+ organizationId: project.organizationId,
+ properties: {
+ project_id: audit.projectId,
+ status: "failed",
+ reconciled_by: "watchdog",
+ error_code: errorInfo.errorCode,
+ pages_crawled: audit.pagesCrawled,
+ pages_total: audit.pagesTotal,
+ },
+ });
+ } catch (error) {
+ // One unreconcilable audit must not stop the sweep.
+ console.error(`Audit watchdog: failed to reconcile ${audit.id}:`, error);
+ }
+ }
+}
+
+/**
+ * Running audits started before the cutoff — watchdog candidates. Timestamps
+ * are stored as text, so this compares lexicographically: PG rows use ISO
+ * with a "T", D1-default rows use "YYYY-MM-DD HH:MM:SS" (space < "T", so a
+ * same-day ISO cutoff would misorder against D1 rows without the reformat).
+ */
+async function getStaleRunningAudits(cutoff: Date, limit: number) {
+ const iso = cutoff.toISOString();
+ const startedBefore =
+ getDatabaseProvider() === "postgres"
+ ? iso
+ : iso.replace("T", " ").slice(0, 19);
+ // Oldest first: genuinely dead audits age past the cutoff and stay there,
+ // while long-but-live crawls are the newest of the stale set — without a
+ // deterministic order they could occupy the whole batch every sweep and
+ // starve real zombies.
+ return db.query.audits.findMany({
+ where: and(
+ eq(audits.status, "running"),
+ lt(audits.startedAt, startedBefore),
+ ),
+ orderBy: audits.startedAt,
+ columns: {
+ id: true,
+ workflowInstanceId: true,
+ projectId: true,
+ startedByUserId: true,
+ startedAt: true,
+ currentPhase: true,
+ pagesCrawled: true,
+ pagesTotal: true,
+ },
+ limit,
+ });
+}
+
+function isOlderThan(startedAt: string, ageMs: number): boolean {
+ // D1-default timestamps ("YYYY-MM-DD HH:MM:SS") lack the T/Z; normalize so
+ // Date.parse reads them as UTC, matching the PG ISO format.
+ const parsed = Date.parse(
+ startedAt.includes("T") ? startedAt : `${startedAt.replace(" ", "T")}Z`,
+ );
+ if (Number.isNaN(parsed)) return true;
+ return parsed < Date.now() - ageMs;
+}
diff --git a/src/server/lib/audit/audit-errors.ts b/src/server/lib/audit/audit-errors.ts
new file mode 100644
index 0000000..7b8d074
--- /dev/null
+++ b/src/server/lib/audit/audit-errors.ts
@@ -0,0 +1,79 @@
+/**
+ * Classification of audit failures into a closed error-code vocabulary.
+ *
+ * Codes are written to audits.error_code by the workflow's mark-failed step
+ * and by the stale-audit reconciler (which reads the message off the dead
+ * Workflows instance). Keeping the vocabulary closed makes failures
+ * aggregable in SQL and lets the UI map codes to friendly copy without
+ * leaking raw infrastructure errors to users.
+ */
+
+const AUDIT_ERROR_CODES = [
+ // A workflow step exceeded its timeout (typically a site that hangs
+ // connections or a stalled DB write).
+ "step_timeout",
+ // The Worker isolate was killed for exceeding its 128MB memory limit.
+ "oom",
+ // The Worker isolate was killed for exceeding its CPU time limit.
+ "cpu_limit",
+ // A database query failed permanently.
+ "db_error",
+ // A workflow step tried to persist more than the ~1MiB durable-state cap.
+ "step_output_too_large",
+ // Cloudflare Workflows internal error (not caused by our code).
+ "workflow_internal",
+ // The workflow instance no longer exists (expired from retention or never
+ // created) while the audit row still said "running".
+ "instance_lost",
+ // Anything we could not classify.
+ "unknown",
+] as const;
+
+export type AuditErrorCode = (typeof AUDIT_ERROR_CODES)[number];
+
+export interface AuditErrorInfo {
+ errorCode: AuditErrorCode;
+ errorDetail: string;
+}
+
+const ERROR_DETAIL_MAX_CHARS = 500;
+
+/**
+ * Classify an error (thrown in the workflow, or read back from a dead
+ * Workflows instance) into an AuditErrorCode. Matches the exact failure
+ * messages observed in production instances.
+ */
+export function classifyAuditError(error: unknown): AuditErrorInfo {
+ const message =
+ error instanceof Error
+ ? error.message
+ : typeof error === "string"
+ ? error
+ : String(error);
+
+ return {
+ errorCode: classifyMessage(message),
+ errorDetail: message.slice(0, ERROR_DETAIL_MAX_CHARS),
+ };
+}
+
+function classifyMessage(message: string): AuditErrorCode {
+ if (message.includes("exceeded memory limit")) return "oom";
+ if (message.includes("exceeded CPU time limit")) return "cpu_limit";
+ if (
+ message.includes("WorkflowTimeoutError") ||
+ message.includes("Execution timed out")
+ ) {
+ return "step_timeout";
+ }
+ if (message.includes("output is too large")) return "step_output_too_large";
+ if (message.includes("WorkflowInternalError")) return "workflow_internal";
+ if (
+ message.startsWith("Failed query:") ||
+ message.includes("D1_ERROR") ||
+ message.includes("Postgres database accessed outside a request scope")
+ ) {
+ return "db_error";
+ }
+ return "unknown";
+}
diff --git a/src/server/lib/audit/crawl-window.test.ts b/src/server/lib/audit/crawl-window.test.ts
new file mode 100644
index 0000000..c5c5a39
--- /dev/null
+++ b/src/server/lib/audit/crawl-window.test.ts
@@ -0,0 +1,100 @@
+import { describe, expect, it } from "vitest";
+import { adjustCrawlWindow } from "@/server/lib/audit/crawl-window";
+import type {
+ CrawledPageResult,
+ PageFetchClass,
+} from "@/server/lib/audit/types";
+
+function page(
+ fetchClass: PageFetchClass,
+ responseTimeMs: number,
+ htmlBytes = 10_000,
+): CrawledPageResult {
+ return {
+ id: "",
+ url: "https://example.com/",
+ statusCode: fetchClass === "ok" ? 200 : 0,
+ fetchClass,
+ redirectUrl: null,
+ title: "",
+ metaDescription: "",
+ canonicalUrl: null,
+ robotsMeta: null,
+ xRobotsTag: null,
+ headerCanonicalUrl: null,
+ ogTitle: null,
+ ogDescription: null,
+ ogImage: null,
+ h1Count: 0,
+ h2Count: 0,
+ h3Count: 0,
+ h4Count: 0,
+ h5Count: 0,
+ h6Count: 0,
+ headingOrder: [],
+ wordCount: 0,
+ contentHash: null,
+ isHtml: true,
+ htmlBytes,
+ imagesTotal: 0,
+ imagesMissingAlt: 0,
+ images: [],
+ links: [],
+ hasStructuredData: false,
+ hreflangTags: [],
+ isIndexable: true,
+ responseTimeMs,
+ crawlDepth: 0,
+ inSitemap: false,
+ };
+}
+
+describe("adjustCrawlWindow", () => {
+ it("keeps the window on an empty batch", () => {
+ expect(adjustCrawlWindow(25, [])).toBe(25);
+ });
+
+ it("halves the window when a third of the batch is troubled", () => {
+ const recent = [
+ ...Array.from({ length: 9 }, () => page("error", 15_000)),
+ ...Array.from({ length: 16 }, () => page("ok", 500)),
+ ];
+ expect(adjustCrawlWindow(25, recent)).toBe(12);
+ });
+
+ it("treats blocked fetches as trouble", () => {
+ const recent = Array.from({ length: 10 }, () => page("blocked", 300));
+ expect(adjustCrawlWindow(20, recent)).toBe(10);
+ });
+
+ it("never shrinks below the minimum", () => {
+ const recent = Array.from({ length: 10 }, () => page("error", 15_000));
+ expect(adjustCrawlWindow(6, recent)).toBe(5);
+ });
+
+ it("grows on a clean, fast batch up to the cap", () => {
+ const recent = Array.from({ length: 25 }, () => page("ok", 400));
+ expect(adjustCrawlWindow(25, recent)).toBe(30);
+ expect(adjustCrawlWindow(40, recent)).toBe(40);
+ });
+
+ it("treats heavy pages as trouble even when they respond fast", () => {
+ const recent = Array.from({ length: 25 }, () =>
+ page("ok", 300, 2 * 1024 * 1024),
+ );
+ expect(adjustCrawlWindow(25, recent)).toBe(12);
+ });
+
+ it("does not grow when some pages are heavy", () => {
+ const recent = [
+ ...Array.from({ length: 4 }, () => page("ok", 300, 1024 * 1024)),
+ ...Array.from({ length: 21 }, () => page("ok", 300)),
+ ];
+ expect(adjustCrawlWindow(25, recent)).toBe(25);
+ });
+
+ it("holds steady on a clean but slow batch", () => {
+ const recent = Array.from({ length: 25 }, () => page("ok", 5_000));
+ expect(adjustCrawlWindow(25, recent)).toBe(25);
+ });
+});
diff --git a/src/server/lib/audit/crawl-window.ts b/src/server/lib/audit/crawl-window.ts
new file mode 100644
index 0000000..59fca1e
--- /dev/null
+++ b/src/server/lib/audit/crawl-window.ts
@@ -0,0 +1,50 @@
+import type { CrawledPageResult } from "@/server/lib/audit/types";
+
+/**
+ * Rolling fetch-concurrency window for the crawl. Unlike fixed batches, a
+ * slow page only occupies one slot instead of stalling a whole batch. The
+ * window adapts to the site: it shrinks when fetches error/block/crawl
+ * slowly (politeness toward struggling or defensive sites) and grows when
+ * the site answers fast.
+ */
+export const INITIAL_CRAWL_WINDOW = 25;
+const MIN_WINDOW = 5;
+const MAX_WINDOW = 40;
+const SLOW_RESPONSE_MS = 10_000;
+const FAST_RESPONSE_MS = 1_500;
+/**
+ * Pages at/above this HTML size count as trouble: each in-flight page
+ * buffers its body, so a wide window on a heavy-page site is memory
+ * pressure the response time can't see (it's measured at headers).
+ */
+const HEAVY_PAGE_BYTES = 1024 * 1024;
+
+/**
+ * Adapt the window to the last persisted sub-batch. Shrinks on trouble
+ * (errors, blocks, very slow responses, heavy bodies), grows only on a
+ * clean and mostly-fast batch.
+ */
+export function adjustCrawlWindow(
+ windowSize: number,
+ recent: CrawledPageResult[],
+): number {
+ if (recent.length === 0) return windowSize;
+ const troubled = recent.filter(
+ (page) =>
+ page.fetchClass !== "ok" ||
+ (page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS ||
+ page.htmlBytes >= HEAVY_PAGE_BYTES,
+ ).length;
+ if (troubled * 3 >= recent.length) {
+ return Math.max(MIN_WINDOW, Math.floor(windowSize / 2));
+ }
+ const fast = recent.filter(
+ (page) =>
+ page.fetchClass === "ok" &&
+ (page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS,
+ ).length;
+ if (troubled === 0 && fast * 2 >= recent.length) {
+ return Math.min(MAX_WINDOW, windowSize + 5);
+ }
+ return windowSize;
+}
diff --git a/src/server/lib/audit/discovery.ts b/src/server/lib/audit/discovery.ts
index 68bd387..1113f1a 100644
--- a/src/server/lib/audit/discovery.ts
+++ b/src/server/lib/audit/discovery.ts
@@ -6,10 +6,21 @@ import { XMLParser } from "fast-xml-parser";
import { isSameOrigin, normalizeUrl } from "./url-utils";
const SITEMAP_FETCH_TIMEOUT_MS = 15_000;
+// robots.txt is checkpointed as durable Workflow step state (~1MiB cap, shared
+// with the rest of the step's return). RFC 9309 requires parsers to handle at
+// least 500 KiB and permits ignoring anything beyond it — Google does exactly
+// that — so this cap matches standard crawler behavior while keeping a
+// misbehaving server (e.g. HTML at /robots.txt) from blowing the step limit.
+const MAX_ROBOTS_TXT_BYTES = 500 * 1024;
const MAX_SITEMAP_DEPTH = 3;
const MAX_SITEMAP_DOCS = 300;
const SITEMAP_CONCURRENCY = 5;
const SITEMAP_RETRIES = 1;
+// Sitemap shards can legally reach 50 MB and SITEMAP_CONCURRENCY of them are
+// read at once, so unbounded reads can exhaust Worker memory. Oversized
+// shards are skipped whole — truncated XML would not parse anyway, and real
+// generators shard far below this.
+const MAX_SITEMAP_BYTES = 10 * 1024 * 1024;
const xmlParser = new XMLParser({
ignoreAttributes: false,
@@ -34,7 +45,7 @@ async function fetchRobotsTxtText(origin: string): Promise {
});
if (!response.ok) return null;
- return await response.text();
+ return (await response.text()).slice(0, MAX_ROBOTS_TXT_BYTES);
} catch (error) {
console.warn("Failed to fetch robots.txt:", error);
return null;
@@ -119,6 +130,34 @@ function isTimeoutError(error: unknown): boolean {
return "name" in error && error.name === "TimeoutError";
}
+/** Read a response body up to maxBytes; null when the body exceeds it. */
+async function readBodyCapped(
+ response: Response,
+ maxBytes: number,
+): Promise {
+ if (!response.body) return "";
+ const reader = response.body.getReader();
+ const chunks: Uint8Array[] = [];
+ let total = 0;
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ total += value.byteLength;
+ if (total > maxBytes) {
+ await reader.cancel();
+ return null;
+ }
+ chunks.push(value);
+ }
+ const joined = new Uint8Array(total);
+ let offset = 0;
+ for (const chunk of chunks) {
+ joined.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ return new TextDecoder().decode(joined);
+}
+
async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{
nestedSitemaps: string[];
pageUrls: string[];
@@ -147,8 +186,11 @@ async function fetchSitemapDocumentWithRetry(sitemapUrl: string): Promise<{
return { nestedSitemaps: [], pageUrls: [], timedOut: false };
}
- const body = await response.text();
- if (!isProbablySitemapXml(response.headers.get("content-type"), body)) {
+ const body = await readBodyCapped(response, MAX_SITEMAP_BYTES);
+ if (
+ body === null ||
+ !isProbablySitemapXml(response.headers.get("content-type"), body)
+ ) {
return { nestedSitemaps: [], pageUrls: [], timedOut: false };
}
diff --git a/src/server/lib/audit/issues/multipage.ts b/src/server/lib/audit/issues/multipage.ts
index b089e50..c5553fc 100644
--- a/src/server/lib/audit/issues/multipage.ts
+++ b/src/server/lib/audit/issues/multipage.ts
@@ -1,15 +1,15 @@
/**
- * Cross-page (multipage) issue checks.
+ * Cross-page (multipage) issue checks over the app DB's page rows:
+ * duplicates and redirect chains/loops. Pure set-queries over crawl data —
+ * no fetching, no DOM.
*
- * These run once after the crawl, against the rows the crawl wrote to D1:
- * duplicates, broken internal links, redirect chains/loops, orphan pages.
- * Pure set-queries over crawl data — no fetching, no DOM.
+ * The two link-edge checks (broken-internal-link, orphan-page) live in the
+ * audit's scratchpad Durable Object (AuditScratchpad.runFinalizeChecks),
+ * next to the link edges themselves — link rows never touch the app DB.
*/
-import { and, eq, gte, lt, ne, notExists, sql } from "drizzle-orm";
-import { alias } from "drizzle-orm/sqlite-core";
+import { eq } from "drizzle-orm";
import { db } from "@/db";
-import { auditLinks, auditPages } from "@/db/schema";
-import { normalizeUrl } from "@/server/lib/audit/url-utils";
+import { auditPages } from "@/db/schema";
import {
findDuplicates,
findRedirectChainsAndLoops,
@@ -17,13 +17,8 @@ import {
} from "@/server/lib/audit/issues/multipage-checks";
import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters";
-const BROKEN_LINK_ISSUE_CAP = 2_000;
-
export async function runMultipageChecks(input: {
auditId: string;
- startUrl: string;
- /** Orphan detection only makes sense when the crawl wasn't truncated. */
- crawlCompleted: boolean;
}): Promise {
const pages: SlimPage[] = await db
.select({
@@ -43,112 +38,5 @@ export async function runMultipageChecks(input: {
.from(auditPages)
.where(eq(auditPages.auditId, input.auditId));
- const issues: DetectedIssue[] = [
- ...findDuplicates(pages),
- ...findRedirectChainsAndLoops(pages),
- ...(await findBrokenInternalLinks(input.auditId)),
- ];
-
- if (input.crawlCompleted) {
- // Page rows store normalized URLs; normalize the start URL the same way
- // so the orphan exclusion matches.
- const normalizedStart = normalizeUrl(input.startUrl) ?? input.startUrl;
- issues.push(...(await findOrphanPages(input.auditId, normalizedStart)));
- }
-
- return issues;
-}
-
-async function findBrokenInternalLinks(
- auditId: string,
-): Promise {
- // Only flag targets we actually crawled and saw fail — never inferred from
- // absence. Blocked targets (WAF challenges) are excluded: a 403 from bot
- // protection is not evidence of a broken link.
- const rows = await db
- .select({
- sourcePageId: auditLinks.sourcePageId,
- sourceUrl: auditLinks.sourceUrl,
- targetUrl: auditLinks.targetUrl,
- targetStatus: auditPages.statusCode,
- })
- .from(auditLinks)
- .innerJoin(
- auditPages,
- and(
- eq(auditPages.auditId, auditLinks.auditId),
- eq(auditPages.url, auditLinks.targetUrl),
- ),
- )
- .where(
- and(
- eq(auditLinks.auditId, auditId),
- eq(auditLinks.isInternal, true),
- gte(auditPages.statusCode, 400),
- eq(auditPages.fetchClass, "ok"),
- ),
- )
- .limit(BROKEN_LINK_ISSUE_CAP);
-
- return rows.map((row) => ({
- issueType: "broken-internal-link" as const,
- pageId: row.sourcePageId,
- pageUrl: row.sourceUrl,
- dedupeKey: row.targetUrl,
- details: { targetUrl: row.targetUrl, targetStatus: row.targetStatus },
- }));
-}
-
-async function findOrphanPages(
- auditId: string,
- startUrl: string,
-): Promise {
- // A live 2xx page is an orphan when no OTHER crawled page links to it and
- // nothing redirects to it. Only meaningful on a completed crawl: on a
- // truncated one, "no observed inlinks" is true of nearly everything.
- // Error/redirect rows aren't orphans — they already get their own issues.
- const inlink = db
- .select({ one: sql`1` })
- .from(auditLinks)
- .where(
- and(
- eq(auditLinks.auditId, auditId),
- eq(auditLinks.isInternal, true),
- eq(auditLinks.targetUrl, auditPages.url),
- // Self-links (breadcrumbs, anchors) don't make a page reachable.
- ne(auditLinks.sourcePageId, auditPages.id),
- ),
- );
-
- const redirectSourcePages = alias(auditPages, "redirect_sources");
- const redirectSources = db
- .select({ one: sql`1` })
- .from(redirectSourcePages)
- .where(
- and(
- eq(redirectSourcePages.auditId, auditId),
- eq(redirectSourcePages.redirectUrl, auditPages.url),
- ),
- );
-
- const rows = await db
- .select({ id: auditPages.id, url: auditPages.url })
- .from(auditPages)
- .where(
- and(
- eq(auditPages.auditId, auditId),
- ne(auditPages.url, startUrl),
- eq(auditPages.fetchClass, "ok"),
- gte(auditPages.statusCode, 200),
- lt(auditPages.statusCode, 300),
- notExists(inlink),
- notExists(redirectSources),
- ),
- );
-
- return rows.map((row) => ({
- issueType: "orphan-page" as const,
- pageId: row.id,
- pageUrl: row.url,
- }));
+ return [...findDuplicates(pages), ...findRedirectChainsAndLoops(pages)];
}
diff --git a/src/server/lib/audit/issues/page-reporters.test.ts b/src/server/lib/audit/issues/page-reporters.test.ts
index 13f76ca..698a728 100644
--- a/src/server/lib/audit/issues/page-reporters.test.ts
+++ b/src/server/lib/audit/issues/page-reporters.test.ts
@@ -41,6 +41,7 @@ function makePage(overrides: Partial): CrawledPageResult {
wordCount: 500,
contentHash: "abc123",
isHtml: true,
+ htmlBytes: 10_000,
imagesTotal: 0,
imagesMissingAlt: 0,
images: [],
diff --git a/src/server/lib/audit/issues/page-reporters.ts b/src/server/lib/audit/issues/page-reporters.ts
index d166739..b277fae 100644
--- a/src/server/lib/audit/issues/page-reporters.ts
+++ b/src/server/lib/audit/issues/page-reporters.ts
@@ -2,7 +2,7 @@
* Per-page issue reporters.
*
* Each reporter is a pure function over a single crawled page record —
- * DOM-free by design (cheerio runs once in crawlPage), so the engine works
+ * DOM-free by design (HTML parsing runs once in crawlPage), so the engine works
* over any crawl source that can produce a CrawledPageResult.
*
* Cross-page checks (duplicates, broken links, orphans, redirect chains)
diff --git a/src/server/lib/audit/page-analyzer.test.ts b/src/server/lib/audit/page-analyzer.test.ts
new file mode 100644
index 0000000..ccab983
--- /dev/null
+++ b/src/server/lib/audit/page-analyzer.test.ts
@@ -0,0 +1,208 @@
+/**
+ * Parity tests for the streaming (htmlparser2) page analyzer against a
+ * cheerio/DOM reference implementation — the exact logic the analyzer
+ * replaced. Cheerio stays as a devDependency for this test only.
+ */
+import * as cheerio from "cheerio";
+import { describe, expect, it } from "vitest";
+import { analyzeHtml } from "@/server/lib/audit/page-analyzer";
+import { normalizeUrl, isSameOrigin } from "@/server/lib/audit/url-utils";
+import type { PageAnalysis, PageLink } from "@/server/lib/audit/types";
+
+/** The previous cheerio implementation, verbatim (minus passthrough fields). */
+function analyzeHtmlWithCheerio(html: string, pageUrl: string): PageAnalysis {
+ const $ = cheerio.load(html);
+
+ const title = $("title").first().text().trim();
+ const metaDescription =
+ $('meta[name="description"]').first().attr("content")?.trim() ?? "";
+ const canonical = $('link[rel="canonical"]').first().attr("href") ?? null;
+ const robotsMeta = $('meta[name="robots"]').first().attr("content") ?? null;
+ const ogTitle =
+ $('meta[property="og:title"]').first().attr("content") ?? null;
+ const ogDescription =
+ $('meta[property="og:description"]').first().attr("content") ?? null;
+ const ogImage =
+ $('meta[property="og:image"]').first().attr("content") ?? null;
+
+ const h1s: string[] = [];
+ $("h1").each((_, el) => {
+ h1s.push($(el).text().trim());
+ });
+
+ const headingOrder: number[] = [];
+ $("h1, h2, h3, h4, h5, h6").each((_, el) => {
+ const tag =
+ "tagName" in el && typeof el.tagName === "string"
+ ? el.tagName.toLowerCase()
+ : null;
+ if (tag) {
+ const level = parseInt(tag.charAt(1), 10);
+ if (!isNaN(level)) headingOrder.push(level);
+ }
+ });
+
+ const bodyClone = $("body").clone();
+ bodyClone.find("script, style, noscript, svg").remove();
+ const bodyText = bodyClone.text().replace(/\s+/g, " ").trim();
+ const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
+
+ const images: Array<{ src: string | null; alt: string | null }> = [];
+ $("img").each((_, el) => {
+ images.push({
+ src: $(el).attr("src") ?? null,
+ alt: $(el).attr("alt") ?? null,
+ });
+ });
+
+ const linksByTarget = new Map();
+ $("a[href]").each((_, el) => {
+ const href = $(el).attr("href");
+ if (!href) return;
+ if (/^(javascript:|mailto:|tel:|#)/.test(href)) return;
+ const resolved = normalizeUrl(href, pageUrl);
+ if (!resolved) return;
+ if (linksByTarget.has(resolved)) return;
+ const anchor = $(el).text().replace(/\s+/g, " ").trim().slice(0, 200);
+ const rel = $(el).attr("rel")?.toLowerCase() ?? "";
+ linksByTarget.set(resolved, {
+ targetUrl: resolved,
+ anchor: anchor || null,
+ isInternal: isSameOrigin(resolved, pageUrl),
+ isNofollow: rel.split(/\s+/).includes("nofollow"),
+ });
+ });
+
+ let hasStructuredData = false;
+ $('script[type="application/ld+json"]').each(() => {
+ hasStructuredData = true;
+ });
+
+ const hreflangTags: string[] = [];
+ $('link[rel="alternate"][hreflang]').each((_, el) => {
+ const hreflang = $(el).attr("hreflang");
+ if (hreflang) hreflangTags.push(hreflang);
+ });
+
+ return {
+ url: pageUrl,
+ statusCode: 200,
+ redirectUrl: null,
+ responseTimeMs: 0,
+ title,
+ metaDescription,
+ canonical,
+ robotsMeta,
+ ogTitle,
+ ogDescription,
+ ogImage,
+ h1s,
+ headingOrder,
+ wordCount,
+ bodyText,
+ images,
+ links: Array.from(linksByTarget.values()),
+ hasStructuredData,
+ hreflangTags,
+ };
+}
+
+const PAGE_URL = "https://example.com/blog/post";
+
+function expectParity(html: string) {
+ const streamed = analyzeHtml(html, PAGE_URL, 200, 0);
+ const reference = analyzeHtmlWithCheerio(html, PAGE_URL);
+ expect(streamed).toEqual(reference);
+}
+
+describe("analyzeHtml parity with the DOM reference", () => {
+ it("matches on a full, well-formed document", () => {
+ expectParity(`
+
+ The Title
+
+
+
+
+
+
+
+
+
+
+ Main Heading
+ Sub Deep Sub 2
+ Some visible body text with bold words here.
+
+
+ icon svg text
+ noscript text
+
+
+
+
+ Relative link
+ Duplicate target
+ External
+ Mail
+ Fragment
+ JS
+
+ `);
+ });
+
+ it("matches on documents with no head, body, or title", () => {
+ expectParity(
+ `Bare fragment Just text and a link .
`,
+ );
+ });
+
+ it("matches on an empty document", () => {
+ expectParity("");
+ });
+
+ it("matches on a document with only a head", () => {
+ expectParity(
+ `Head only `,
+ );
+ });
+
+ it("matches with duplicate metas and titles (first wins)", () => {
+ expectParity(`
+ First Second
+
+
+
+ text
`);
+ });
+
+ it("matches on unclosed and misnested tags", () => {
+ expectParity(`
+ Unclosed heading
+ Paragraph bold both italic?
+ first second
+
trailing text`);
+ });
+
+ it("matches on entity-heavy content", () => {
+ expectParity(`
A & B <C>
+
Café "menu"
+
1 < 2 && 3 > 2
+
Query & anchor `);
+ });
+
+ it("matches heading order across nesting", () => {
+ expectParity(`
three
one four `);
+ });
+
+ it("matches word counts with whitespace-heavy markup", () => {
+ expectParity(`
+
+ one
+ two three
+
+
+ `);
+ });
+});
diff --git a/src/server/lib/audit/page-analyzer.ts b/src/server/lib/audit/page-analyzer.ts
index 8c424c8..5abc9d3 100644
--- a/src/server/lib/audit/page-analyzer.ts
+++ b/src/server/lib/audit/page-analyzer.ts
@@ -1,14 +1,38 @@
/**
- * HTML page analyzer using cheerio.
+ * HTML page analyzer using htmlparser2's streaming tokenizer.
*
- * Extracts SEO-relevant data from a page's HTML:
- * title, meta description, headings, images, links, canonical, OG tags,
- * structured data, robots meta, word count, hreflang.
+ * Extracts SEO-relevant data from a page's HTML: title, meta description,
+ * headings, images, links, canonical, OG tags, structured data, robots meta,
+ * word count, hreflang.
+ *
+ * Deliberately NOT a DOM parser: the previous cheerio implementation built a
+ * full DOM (~5-10x the HTML's size) per page, and with 25 concurrent parses
+ * on a 128MB isolate that was the audit engine's dominant OOM cause. The
+ * tokenizer keeps only the accumulated text and extracted fields in memory.
*/
-import * as cheerio from "cheerio";
+import { Parser } from "htmlparser2";
import { normalizeUrl, isSameOrigin } from "./url-utils";
import type { PageAnalysis, PageLink } from "./types";
+const SKIPPED_LINK_PROTOCOLS = /^(javascript:|mailto:|tel:|#)/;
+/** Subtrees whose text is not visible content. */
+const NON_CONTENT_TAGS = new Set(["script", "style", "noscript", "svg"]);
+const HEADING_LEVELS: Record
= {
+ h1: 1,
+ h2: 2,
+ h3: 3,
+ h4: 4,
+ h5: 5,
+ h6: 6,
+};
+const MAX_ANCHOR_CHARS = 200;
+
+interface OpenAnchor {
+ href: string;
+ rel: string;
+ text: string[];
+}
+
/**
* Analyze an HTML string and extract all SEO-relevant data.
*/
@@ -19,102 +43,195 @@ export function analyzeHtml(
responseTimeMs: number,
redirectUrl: string | null = null,
): PageAnalysis {
- const $ = cheerio.load(html);
+ let title: string | null = null;
+ let titleDepth = 0;
+ let titleDone = false;
+ // parse5 (the old DOM path) treats content as raw text when
+ // scripting is enabled; skip element extraction inside it to match.
+ let noscriptDepth = 0;
+ let metaDescription: string | null = null;
+ let canonical: string | null = null;
+ let robotsMeta: string | null = null;
+ let ogTitle: string | null = null;
+ let ogDescription: string | null = null;
+ let ogImage: string | null = null;
+ let hasStructuredData = false;
+ const hreflangTags: string[] = [];
- const title = $("title").first().text().trim();
-
- const metaDescription =
- $('meta[name="description"]').first().attr("content")?.trim() ?? "";
-
- const canonical = $('link[rel="canonical"]').first().attr("href") ?? null;
-
- const robotsMeta = $('meta[name="robots"]').first().attr("content") ?? null;
-
- // --- Open Graph ---
- const ogTitle =
- $('meta[property="og:title"]').first().attr("content") ?? null;
- const ogDescription =
- $('meta[property="og:description"]').first().attr("content") ?? null;
- const ogImage =
- $('meta[property="og:image"]').first().attr("content") ?? null;
-
- // --- Headings ---
const h1s: string[] = [];
- $("h1").each((_, el) => {
- h1s.push($(el).text().trim());
- });
-
const headingOrder: number[] = [];
- $("h1, h2, h3, h4, h5, h6").each((_, el) => {
- const tag =
- "tagName" in el && typeof el.tagName === "string"
- ? el.tagName.toLowerCase()
- : null;
- if (tag) {
- const level = parseInt(tag.charAt(1), 10);
- if (!isNaN(level)) headingOrder.push(level);
- }
- });
-
- // --- Word count (visible text in body) ---
- // Remove script/style/noscript tags, then count words in remaining text
- const bodyClone = $("body").clone();
- bodyClone.find("script, style, noscript, svg").remove();
- const bodyText = bodyClone.text().replace(/\s+/g, " ").trim();
- const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
+ let openH1: string[] | null = null;
const images: Array<{ src: string | null; alt: string | null }> = [];
- $("img").each((_, el) => {
- images.push({
- src: $(el).attr("src") ?? null,
- alt: $(el).attr("alt") ?? null,
- });
- });
-
- // --- Links (deduped by target URL; first anchor wins) ---
const linksByTarget = new Map();
+ let openAnchor: OpenAnchor | null = null;
- $("a[href]").each((_, el) => {
- const href = $(el).attr("href");
- if (!href) return;
+ // Visible text: prefer text inside an explicit ; when the document
+ // never opens one (fragments), fall back to all non-head text. Both
+ // exclude NON_CONTENT_TAGS subtrees.
+ let suppressDepth = 0;
+ let bodyDepth = 0;
+ let headDepth = 0;
+ let sawBody = false;
+ const bodyParts: string[] = [];
+ const fallbackParts: string[] = [];
- // Skip javascript:, mailto:, tel:, #anchors
- if (/^(javascript:|mailto:|tel:|#)/.test(href)) return;
+ const handleMetaTag = (attribs: Record) => {
+ const content = attribs["content"];
+ if (attribs["name"] === "description") {
+ metaDescription ??= content?.trim() ?? "";
+ } else if (attribs["name"] === "robots") {
+ robotsMeta ??= content ?? null;
+ } else if (attribs["property"] === "og:title") {
+ ogTitle ??= content ?? null;
+ } else if (attribs["property"] === "og:description") {
+ ogDescription ??= content ?? null;
+ } else if (attribs["property"] === "og:image") {
+ ogImage ??= content ?? null;
+ }
+ };
+ const handleLinkTag = (attribs: Record) => {
+ if (attribs["rel"] === "canonical") {
+ canonical ??= attribs["href"] ?? null;
+ } else if (attribs["rel"] === "alternate" && attribs["hreflang"]) {
+ hreflangTags.push(attribs["hreflang"]);
+ }
+ };
+
+ const closeAnchor = () => {
+ if (!openAnchor) return;
+ const { href, rel, text } = openAnchor;
+ openAnchor = null;
const resolved = normalizeUrl(href, pageUrl);
- if (!resolved) return;
- if (linksByTarget.has(resolved)) return;
-
- const anchor = $(el).text().replace(/\s+/g, " ").trim().slice(0, 200);
- const rel = $(el).attr("rel")?.toLowerCase() ?? "";
+ if (!resolved || linksByTarget.has(resolved)) return;
+ const anchor = text
+ .join("")
+ .replace(/\s+/g, " ")
+ .trim()
+ .slice(0, MAX_ANCHOR_CHARS);
linksByTarget.set(resolved, {
targetUrl: resolved,
anchor: anchor || null,
isInternal: isSameOrigin(resolved, pageUrl),
isNofollow: rel.split(/\s+/).includes("nofollow"),
});
- });
- const links = Array.from(linksByTarget.values());
+ };
- // --- Structured data (JSON-LD) ---
- let hasStructuredData = false;
- $('script[type="application/ld+json"]').each(() => {
- hasStructuredData = true;
- });
+ const parser = new Parser(
+ {
+ onopentag(name, attribs) {
+ if (NON_CONTENT_TAGS.has(name)) {
+ suppressDepth += 1;
+ }
+ if (name === "noscript") noscriptDepth += 1;
+ if (noscriptDepth > 0) return;
+ switch (name) {
+ case "title":
+ // Ignore inside — only the document title counts.
+ if (!titleDone && suppressDepth === 0) {
+ titleDepth += 1;
+ if (title === null) title = "";
+ }
+ break;
+ case "head":
+ headDepth += 1;
+ break;
+ case "body":
+ bodyDepth += 1;
+ sawBody = true;
+ break;
+ case "meta":
+ handleMetaTag(attribs);
+ break;
+ case "link":
+ handleLinkTag(attribs);
+ break;
+ case "img":
+ images.push({
+ src: attribs["src"] ?? null,
+ alt: "alt" in attribs ? attribs["alt"] : null,
+ });
+ break;
+ case "script":
+ if (attribs["type"] === "application/ld+json") {
+ hasStructuredData = true;
+ }
+ break;
+ case "a": {
+ // HTML forbids nested ; browsers implicitly close the open
+ // one, and the tokenizer has no tree correction, so mirror that.
+ closeAnchor();
+ const href = attribs["href"];
+ if (href && !SKIPPED_LINK_PROTOCOLS.test(href)) {
+ openAnchor = {
+ href,
+ rel: attribs["rel"]?.toLowerCase() ?? "",
+ text: [],
+ };
+ }
+ break;
+ }
+ }
+ const headingLevel = HEADING_LEVELS[name];
+ if (headingLevel !== undefined) {
+ headingOrder.push(headingLevel);
+ if (headingLevel === 1 && openH1 === null) openH1 = [];
+ }
+ },
+ ontext(text) {
+ if (suppressDepth > 0) return;
+ if (titleDepth > 0) {
+ if (title !== null) title += text;
+ return;
+ }
+ if (openH1) openH1.push(text);
+ if (openAnchor) openAnchor.text.push(text);
+ if (bodyDepth > 0) {
+ bodyParts.push(text);
+ } else if (headDepth === 0) {
+ fallbackParts.push(text);
+ }
+ },
+ onclosetag(name) {
+ if (NON_CONTENT_TAGS.has(name) && suppressDepth > 0) {
+ suppressDepth -= 1;
+ }
+ if (name === "noscript" && noscriptDepth > 0) {
+ noscriptDepth -= 1;
+ return;
+ }
+ if (noscriptDepth > 0) return;
+ if (name === "title" && titleDepth > 0) {
+ titleDepth -= 1;
+ if (titleDepth === 0) titleDone = true;
+ }
+ if (name === "head" && headDepth > 0) headDepth -= 1;
+ if (name === "body" && bodyDepth > 0) bodyDepth -= 1;
+ if (name === "a") closeAnchor();
+ if (name === "h1" && openH1) {
+ h1s.push(openH1.join("").trim());
+ openH1 = null;
+ }
+ },
+ },
+ // Defaults (non-XML mode): lowercased tag/attribute names, decoded
+ // entities — matching what the DOM-based implementation saw.
+ );
+ parser.write(html);
+ parser.end();
- const hreflangTags: string[] = [];
- $('link[rel="alternate"][hreflang]').each((_, el) => {
- const hreflang = $(el).attr("hreflang");
- if (hreflang) hreflangTags.push(hreflang);
- });
+ const rawText = (sawBody ? bodyParts : fallbackParts).join("");
+ const bodyText = rawText.replace(/\s+/g, " ").trim();
+ const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
return {
url: pageUrl,
statusCode,
redirectUrl,
responseTimeMs,
- title,
- metaDescription,
+ title: (title ?? "").trim(),
+ metaDescription: metaDescription ?? "",
canonical,
robotsMeta,
ogTitle,
@@ -125,7 +242,7 @@ export function analyzeHtml(
wordCount,
bodyText,
images,
- links,
+ links: Array.from(linksByTarget.values()),
hasStructuredData,
hreflangTags,
};
diff --git a/src/server/lib/audit/types.ts b/src/server/lib/audit/types.ts
index e7ca402..2fbe499 100644
--- a/src/server/lib/audit/types.ts
+++ b/src/server/lib/audit/types.ts
@@ -50,7 +50,7 @@ export interface PageLink {
isNofollow: boolean;
}
-/** Data extracted from a single page via cheerio. */
+/** Data extracted from a single page's HTML. */
export interface PageAnalysis {
url: string;
statusCode: number;
@@ -106,8 +106,9 @@ export interface LighthouseResult {
}
/**
- * Full result of crawling one page. Persisted to D1 inside the crawl-batch
- * step; never accumulated in memory or returned as durable step state.
+ * Full result of crawling one page. Persisted to the app DB inside the
+ * crawl-chunk step; never accumulated in memory or returned as durable
+ * step state.
*/
export interface CrawledPageResult {
id: string;
@@ -139,6 +140,12 @@ export interface CrawledPageResult {
* checked; a PDF must not). Transient — not persisted.
*/
isHtml: boolean;
+ /**
+ * HTML size read for this page (approximate; capped at MAX_HTML_BYTES).
+ * Transient — feeds the crawl window's memory-pressure signal, since
+ * response time is measured at headers and says nothing about body size.
+ */
+ htmlBytes: number;
imagesTotal: number;
imagesMissingAlt: number;
images: Array<{ src: string | null; alt: string | null }>;
@@ -151,18 +158,3 @@ export interface CrawledPageResult {
crawlDepth: number | null;
inSitemap: boolean;
}
-
-/**
- * Slim per-page summary returned as durable step state from a crawl batch.
- * Keep this small: full page data lives in D1, not in Workflow step state.
- */
-export interface StepPageSummary {
- id: string;
- url: string;
- statusCode: number;
- fetchClass: PageFetchClass;
- redirectUrl: string | null;
- title: string;
- /** Normalized same-origin link targets, for frontier expansion. */
- internalLinks: string[];
-}
diff --git a/src/server/lib/audit/url-policy.test.ts b/src/server/lib/audit/url-policy.test.ts
index f41fbae..c2d46a7 100644
--- a/src/server/lib/audit/url-policy.test.ts
+++ b/src/server/lib/audit/url-policy.test.ts
@@ -1,6 +1,9 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { AppError } from "@/server/lib/errors";
-import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
+import {
+ normalizeAndValidateStartUrl,
+ resolveStartUrlRedirects,
+} from "@/server/lib/audit/url-policy";
describe("normalizeAndValidateStartUrl", () => {
beforeEach(() => {
@@ -48,3 +51,86 @@ describe("normalizeAndValidateStartUrl", () => {
} satisfies Partial);
});
});
+
+const dnsOk = () =>
+ new Response(JSON.stringify({ Status: 0, Answer: [] }), {
+ status: 200,
+ headers: { "content-type": "application/dns-json" },
+ });
+const redirect = (location: string) =>
+ new Response(null, { status: 301, headers: { location } });
+
+describe("resolveStartUrlRedirects", () => {
+ beforeEach(() => {
+ vi.stubGlobal("fetch", vi.fn());
+ });
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ /** Route probe fetches by URL; DoH lookups always resolve clean. */
+ function stubFetch(routes: Record Response>) {
+ vi.mocked(fetch).mockImplementation((input) => {
+ const url = String(input instanceof Request ? input.url : input);
+ if (url.includes("dns-query")) return Promise.resolve(dnsOk());
+ const route = routes[url];
+ return route
+ ? Promise.resolve(route())
+ : Promise.resolve(new Response(null, { status: 200 }));
+ });
+ }
+
+ it("follows a cross-domain redirect to the real origin", async () => {
+ stubFetch({
+ "https://example.net/": () => redirect("https://example.com/"),
+ });
+ await expect(
+ resolveStartUrlRedirects("https://example.net/"),
+ ).resolves.toBe("https://example.com/");
+ });
+
+ it("follows an apex-to-www redirect chain", async () => {
+ stubFetch({
+ "https://example.com/": () => redirect("https://www.example.com/"),
+ });
+ await expect(
+ resolveStartUrlRedirects("https://example.com/"),
+ ).resolves.toBe("https://www.example.com/");
+ });
+
+ it("returns the original URL when the site does not redirect", async () => {
+ stubFetch({});
+ await expect(
+ resolveStartUrlRedirects("https://example.com/"),
+ ).resolves.toBe("https://example.com/");
+ });
+
+ it("returns the last URL when the probe fails", async () => {
+ vi.mocked(fetch).mockRejectedValue(new Error("network down"));
+ await expect(
+ resolveStartUrlRedirects("https://example.com/"),
+ ).resolves.toBe("https://example.com/");
+ });
+
+ it("stops after the hop limit on a redirect loop", async () => {
+ stubFetch({
+ "https://a.example/": () => redirect("https://b.example/"),
+ "https://b.example/": () => redirect("https://a.example/"),
+ });
+ await expect(
+ resolveStartUrlRedirects("https://a.example/"),
+ ).resolves.toMatch(/^https:\/\/(a|b)\.example\/$/);
+ });
+
+ it("rejects redirects into blocked targets", async () => {
+ stubFetch({
+ "https://example.com/": () => redirect("http://192.168.0.10/"),
+ });
+ await expect(
+ resolveStartUrlRedirects("https://example.com/"),
+ ).rejects.toMatchObject({
+ code: "CRAWL_TARGET_BLOCKED",
+ } satisfies Partial);
+ });
+});
diff --git a/src/server/lib/audit/url-policy.ts b/src/server/lib/audit/url-policy.ts
index 0c386fd..c2fb5d3 100644
--- a/src/server/lib/audit/url-policy.ts
+++ b/src/server/lib/audit/url-policy.ts
@@ -238,3 +238,48 @@ export async function normalizeAndValidateStartUrl(
parsed.hash = "";
return parsed.toString();
}
+
+const START_URL_REDIRECT_HOPS = 5;
+const START_URL_PROBE_TIMEOUT_MS = 10_000;
+
+/**
+ * Follow redirects on the audit's start URL so the audit anchors to the
+ * site's real origin. Without this, auditing a domain that 301s elsewhere
+ * (…net -> …com, apex -> www) dead-ends after one page: the redirect target
+ * is a different origin, so the same-origin crawl policy can't follow it.
+ *
+ * Every hop re-runs the full start-URL validation (SSRF, blocked hosts), so
+ * a redirect can't smuggle the audit somewhere the user couldn't have
+ * pointed it directly. Probe failures (timeouts, HEAD rejected) fall back
+ * to the last validated URL — the crawl records the real fetch result.
+ */
+export async function resolveStartUrlRedirects(
+ startUrl: string,
+): Promise {
+ let current = startUrl;
+ for (let hop = 0; hop < START_URL_REDIRECT_HOPS; hop++) {
+ let response: Response;
+ try {
+ response = await fetch(current, {
+ method: "HEAD",
+ redirect: "manual",
+ headers: { "User-Agent": "OpenSEO-Audit/1.0" },
+ signal: AbortSignal.timeout(START_URL_PROBE_TIMEOUT_MS),
+ });
+ } catch {
+ return current;
+ }
+ if (response.status < 300 || response.status >= 400) return current;
+ const location = response.headers.get("location");
+ if (!location) return current;
+
+ let next: URL;
+ try {
+ next = new URL(location, current);
+ } catch {
+ return current;
+ }
+ current = await normalizeAndValidateStartUrl(next.toString());
+ }
+ return current;
+}
diff --git a/src/server/mcp/tools/site-audit-tools.ts b/src/server/mcp/tools/site-audit-tools.ts
index 488bc78..37b2cf4 100644
--- a/src/server/mcp/tools/site-audit-tools.ts
+++ b/src/server/mcp/tools/site-audit-tools.ts
@@ -128,7 +128,7 @@ export const runSiteAuditTool = {
});
return mcpResponse({
- text: `Audit ${auditId} started for ${args.url}. Poll get_audit_status until it completes, then call get_audit_issues for the prioritized issue report.`,
+ text: `Audit ${auditId} started for ${args.url}. Poll get_audit_status until it finishes, then call get_audit_issues for the prioritized issue report (even a failed audit keeps results for every page it crawled).`,
meta: buildProjectMeta(
context,
args.projectId,
@@ -177,8 +177,16 @@ export const getAuditStatusTool = {
status.lighthouseTotal > 0
? `, lighthouse ${status.lighthouseCompleted + status.lighthouseFailed}/${status.lighthouseTotal}`
: "";
+ // Failed audits keep partial results — point agents at them instead of
+ // letting a mid-crawl death read as "no data".
+ const nextStep =
+ status.status === "completed"
+ ? " Call get_audit_issues for the issue report."
+ : status.status === "failed" && status.pagesCrawled > 0
+ ? ` The audit stopped early but kept results for the ${status.pagesCrawled} pages it crawled — call get_audit_issues for the partial issue report.`
+ : "";
return mcpResponse({
- text: `Audit ${status.id} (${status.startUrl}): ${status.status} — phase ${status.currentPhase}, ${status.pagesCrawled}/${status.pagesTotal} pages${lighthouseNote}.${status.status === "completed" ? " Call get_audit_issues for the issue report." : ""}`,
+ text: `Audit ${status.id} (${status.startUrl}): ${status.status} — phase ${status.currentPhase}, ${status.pagesCrawled}/${status.pagesTotal} pages${lighthouseNote}.${nextStep}`,
meta: buildProjectMeta(
context,
args.projectId,
diff --git a/src/server/workflows/SiteAuditWorkflow.ts b/src/server/workflows/SiteAuditWorkflow.ts
index b6fe898..35b1ef1 100644
--- a/src/server/workflows/SiteAuditWorkflow.ts
+++ b/src/server/workflows/SiteAuditWorkflow.ts
@@ -12,10 +12,12 @@ import {
import { withPgClient } from "@/db";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
+import { classifyAuditError } from "@/server/lib/audit/audit-errors";
import type { AuditConfig } from "@/server/lib/audit/types";
import { captureServerError, captureServerEvent } from "@/server/lib/posthog";
import { runAuditPhases } from "@/server/workflows/siteAuditWorkflowPhases";
import { pgStep } from "@/server/workflows/pgStep";
+import { DB_STEP } from "@/server/workflows/auditStepConfigs";
interface AuditParams {
auditId: string;
@@ -44,7 +46,7 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint {
// Inside a step so the D1 read is retried and replay-cached; a bare
// read here would re-execute on every replay and a transient failure
// would kill the instance before the catch below exists.
- await pgStep(step, "validate-context", undefined, async () => {
+ await pgStep(step, "validate-context", DB_STEP, async () => {
const audit = await AuditRepository.getAuditForWorkflow(
auditId,
event.instanceId,
@@ -86,13 +88,17 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint {
project_id: projectId,
});
}
- await pgStep(step, "mark-failed", undefined, async () => {
- await AuditRepository.failAudit(auditId, event.instanceId);
-
- const latestAudit = await AuditRepository.getAuditForWorkflow(
+ const errorInfo = classifyAuditError(error);
+ await pgStep(step, "mark-failed", DB_STEP, async () => {
+ // Read the phase before failAudit stamps currentPhase = "failed".
+ const runningAudit = await AuditRepository.getAuditForWorkflow(
auditId,
event.instanceId,
);
+ await AuditRepository.failAudit(auditId, event.instanceId, {
+ ...errorInfo,
+ failedPhase: runningAudit?.currentPhase ?? null,
+ });
await captureServerEvent({
distinctId: billingCustomer.userId,
@@ -101,8 +107,10 @@ export class SiteAuditWorkflow extends WorkflowEntrypoint {
properties: {
project_id: projectId,
status: "failed",
- pages_crawled: latestAudit?.pagesCrawled,
- pages_total: latestAudit?.pagesTotal,
+ error_code: errorInfo.errorCode,
+ failed_phase: runningAudit?.currentPhase,
+ pages_crawled: runningAudit?.pagesCrawled,
+ pages_total: runningAudit?.pagesTotal,
run_lighthouse: config.lighthouseStrategy !== "none",
},
});
diff --git a/src/server/workflows/auditStepConfigs.ts b/src/server/workflows/auditStepConfigs.ts
new file mode 100644
index 0000000..2386f53
--- /dev/null
+++ b/src/server/workflows/auditStepConfigs.ts
@@ -0,0 +1,40 @@
+import type { WorkflowStepConfig } from "cloudflare:workers";
+
+/**
+ * Explicit step configs for the site-audit workflow.
+ *
+ * Without these, steps inherit the platform default (10-minute timeout,
+ * multiple retries with backoff) — production audits against slow/hostile
+ * sites burned ~65 minutes replaying a doomed step before erroring. Every
+ * step now has a timeout sized to its real work and a small retry budget.
+ */
+
+/** Robots + sitemap walk (bounded at 300 docs / 15s per fetch internally). */
+export const DISCOVERY_STEP: WorkflowStepConfig = {
+ retries: { limit: 2, delay: "5 seconds", backoff: "exponential" },
+ timeout: "4 minutes",
+};
+
+/** One crawl chunk: ~90s soft crawl deadline + persistence headroom. */
+export const CRAWL_CHUNK_STEP: WorkflowStepConfig = {
+ retries: { limit: 1, delay: "10 seconds", backoff: "constant" },
+ timeout: "5 minutes",
+};
+
+/** One Lighthouse batch: 20 DataForSEO calls, each with internal retries. */
+export const LIGHTHOUSE_BATCH_STEP: WorkflowStepConfig = {
+ retries: { limit: 1, delay: "10 seconds", backoff: "constant" },
+ timeout: "5 minutes",
+};
+
+/** Small DB-only steps (validate, select sample, finalize, mark-failed). */
+export const DB_STEP: WorkflowStepConfig = {
+ retries: { limit: 3, delay: "5 seconds", backoff: "exponential" },
+ timeout: "2 minutes",
+};
+
+/** Cross-page checks read every page row of the audit — allow more time. */
+export const MULTIPAGE_CHECKS_STEP: WorkflowStepConfig = {
+ retries: { limit: 2, delay: "5 seconds", backoff: "exponential" },
+ timeout: "5 minutes",
+};
diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.ts
index ff08033..cf46382 100644
--- a/src/server/workflows/site-audit-workflow-helpers.ts
+++ b/src/server/workflows/site-audit-workflow-helpers.ts
@@ -104,9 +104,8 @@ export async function crawlPage(
const contentType = response.headers.get("content-type") ?? "";
const isHtml = contentType.includes("text/html");
- // Large pages make Cheerio disproportionately expensive and can exhaust a
- // crawl step's CPU or isolate memory. The first 2 MiB still contains the
- // SEO metadata and navigation needed by the audit in normal documents.
+ // Cap what we read: the first 2 MiB still contains the SEO metadata and
+ // navigation needed by the audit in normal documents.
const body = isHtml ? await readTextUpTo(response, MAX_HTML_BYTES) : "";
const fetchClass = classifyFetch(
statusCode,
@@ -128,10 +127,10 @@ export async function crawlPage(
});
}
- // Dynamic import keeps cheerio (page-analyzer's HTML parser) out of the
- // worker's startup module graph: SiteAuditWorkflow is re-exported from
- // src/server.ts, so a static import would evaluate cheerio in every
- // isolate's baseline heap, not just when an audit actually crawls.
+ // Dynamic import keeps the HTML parser out of the worker's startup
+ // module graph: SiteAuditWorkflow is re-exported from src/server.ts, so
+ // a static import would evaluate it in every isolate's baseline heap,
+ // not just when an audit actually crawls.
const { analyzeHtml } = await import("@/server/lib/audit/page-analyzer");
const analysis = analyzeHtml(body, url, statusCode, responseTimeMs);
const robotsDirectives = [analysis.robotsMeta, xRobotsTag]
@@ -171,6 +170,7 @@ export async function crawlPage(
? await sha256Hex(analysis.bodyText)
: null,
isHtml: true,
+ htmlBytes: body.length,
imagesTotal: analysis.images.length,
// Only a truly absent alt attribute counts: alt="" is the correct
// markup for decorative images.
@@ -270,6 +270,7 @@ function emptyPageResult(input: {
wordCount: 0,
contentHash: null,
isHtml: false,
+ htmlBytes: 0,
imagesTotal: 0,
imagesMissingAlt: 0,
images: [],
diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.ts
index 08b9871..0be7859 100644
--- a/src/server/workflows/siteAuditWorkflowCrawl.ts
+++ b/src/server/workflows/siteAuditWorkflowCrawl.ts
@@ -1,59 +1,77 @@
import type { WorkflowStep } from "cloudflare:workers";
import type { RobotsResult } from "@/server/lib/audit/discovery";
-import type { StepPageSummary } from "@/server/lib/audit/types";
-import { isSameOrigin, normalizeUrl } from "@/server/lib/audit/url-utils";
+import type { CrawledPageResult } from "@/server/lib/audit/types";
+import { isSameOrigin } from "@/server/lib/audit/url-utils";
import { isCrawlableUrl } from "@/server/lib/audit/url-policy";
import { deterministicAuditRowId } from "@/server/lib/audit/ids";
import { runPageReporters } from "@/server/lib/audit/issues/page-reporters";
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
+import {
+ getAuditScratchpad,
+ type ClaimedUrl,
+ type FrontierStats,
+ type ScratchpadLinkRow,
+} from "@/server/features/audit/AuditScratchpad";
import { AuditProgressKV } from "@/server/lib/audit/progress-kv";
+import {
+ adjustCrawlWindow,
+ INITIAL_CRAWL_WINDOW,
+} from "@/server/lib/audit/crawl-window";
import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";
import { pgStep } from "@/server/workflows/pgStep";
+import { CRAWL_CHUNK_STEP } from "@/server/workflows/auditStepConfigs";
-const CRAWL_CONCURRENCY = 25;
-// Keep durable step state under the ~1MiB limit: full link lists live in D1;
-// the step return only carries new-to-the-frontier targets, deduped across
-// the batch and capped.
-const MAX_FRONTIER_LINKS_PER_BATCH = 2_000;
-const MAX_SUMMARY_TITLE_CHARS = 300;
+/**
+ * The crawl runs in chunks: each chunk is one durable step that leases up to
+ * CHUNK_TARGET_PAGES URLs from the scratchpad DO, crawls them with a rolling
+ * concurrency window, and persists results incrementally (full page rows to
+ * the app DB, links/mirror/frontier updates to the DO). Step returns carry
+ * only counters, so step state stays tiny regardless of site size and the
+ * workflow heap stays O(one batch).
+ */
+const CHUNK_TARGET_PAGES = 200;
+/** Stop launching new fetches after this long; leftover leases are released. */
+const CHUNK_SOFT_DEADLINE_MS = 90_000;
+/** Crawled pages are persisted in sub-batches of this size. */
+const PERSIST_BATCH_SIZE = 25;
+
+/**
+ * Mega-menu/footer-heavy sites can carry 1000+ links per page; cap what we
+ * record so a 10k-page crawl can't produce tens of millions of link rows.
+ */
+const MAX_STORED_LINKS_PER_PAGE = 500;
+/**
+ * Cap newly discovered URLs sent to the scratchpad per persist sub-batch.
+ * Serialized RPC messages are limited to 32 MiB; a crawler-trap page family
+ * (faceted nav, calendars) can emit tens of thousands of unique URLs per
+ * page. Dropped URLs are usually re-discovered from later pages, and a site
+ * generating this many is past maxPages anyway.
+ */
+const MAX_DISCOVERED_PER_BATCH = 20_000;
+const MAX_PROGRESS_TITLE_CHARS = 300;
function shouldQueueCrawlLink(
link: string,
origin: string,
robots: RobotsResult,
- visited: Set,
- queued: Set,
): boolean {
return (
- isSameOrigin(link, origin) &&
- isCrawlableUrl(link) &&
- robots.isAllowed(link) &&
- !visited.has(link) &&
- !queued.has(link)
+ isSameOrigin(link, origin) && isCrawlableUrl(link) && robots.isAllowed(link)
);
}
-interface QueueEntry {
- url: string;
- /** Clicks from the start URL; null when only reachable via sitemap. */
- depth: number | null;
-}
-
type CrawlPhaseParams = {
auditId: string;
workflowInstanceId: string;
origin: string;
- startUrl: string;
maxPages: number;
robots: RobotsResult;
- sitemapUrls: string[];
+ /** Frontier size after discovery seeding (from the discover-urls step). */
+ seededCount: number;
};
-/** What later phases need per page — no link lists (those stay in D1). */
-export type CrawledPageSummary = Omit;
-
export type CrawlPhaseResult = {
- pages: CrawledPageSummary[];
+ pagesCrawled: number;
/** True when the frontier was exhausted before hitting maxPages. */
completed: boolean;
};
@@ -62,282 +80,219 @@ export async function runCrawlPhase(
step: WorkflowStep,
params: CrawlPhaseParams,
): Promise {
- const {
- auditId,
- workflowInstanceId,
- origin,
- startUrl,
- maxPages,
- robots,
- sitemapUrls,
- } = params;
- const visited = new Set();
- const queued = new Set();
- // Link-discovered URLs crawl first (BFS from the start URL); sitemap-only
- // URLs drain last so link discovery isn't starved of page budget and
- // orphan detection stays meaningful.
- const linkQueue: QueueEntry[] = [];
- const sitemapQueue: QueueEntry[] = [];
- const sitemapSet = new Set();
- const summaries: CrawledPageSummary[] = [];
+ let chunkNo = 0;
+ let attemptedTotal = 0;
+ let pending = params.seededCount;
+ let zeroProgressChunks = 0;
- const normalizedStart = normalizeUrl(startUrl) ?? startUrl;
- if (
- robots.isAllowed(normalizedStart) &&
- isSameOrigin(normalizedStart, origin)
- ) {
- linkQueue.push({ url: normalizedStart, depth: 0 });
- queued.add(normalizedStart);
- }
-
- for (const sitemapUrl of sitemapUrls) {
- const normalized = normalizeUrl(sitemapUrl);
- if (!normalized) continue;
- sitemapSet.add(normalized);
- if (!shouldQueueCrawlLink(normalized, origin, robots, visited, queued)) {
- continue;
- }
- sitemapQueue.push({ url: normalized, depth: null });
- queued.add(normalized);
- }
-
- let crawlBatchIndex = 0;
- while (
- (linkQueue.length > 0 || sitemapQueue.length > 0) &&
- summaries.length < maxPages
- ) {
- const batchEntries = selectNextCrawlBatch({
- linkQueue,
- sitemapQueue,
- queued,
- visited,
- robots,
- remaining: maxPages - summaries.length,
- });
- if (batchEntries.length === 0) continue;
-
- crawlBatchIndex += 1;
- const crawledBatch = await runCrawlBatch(step, {
- crawlBatchIndex,
- auditId,
- batchEntries,
- sitemapSet,
- visited,
- queued,
- });
- // Keep only the slim summary in memory: at 10k pages, retaining link
- // lists for the whole crawl would not fit in the 128MB Worker heap.
- summaries.push(
- ...crawledBatch.map(({ internalLinks: _links, ...summary }) => summary),
- );
-
- enqueueDiscoveredLinks({
- crawledBatch,
- batchEntries,
- linkQueue,
- queued,
- visited,
- origin,
- robots,
- });
- await persistCrawlProgress({
+ while (pending > 0 && attemptedTotal < params.maxPages) {
+ chunkNo += 1;
+ const result = await pgStep(
step,
- crawlBatchIndex,
- auditId,
- workflowInstanceId,
- crawledBatch,
- pagesCrawled: summaries.length,
- visitedCount: visited.size,
- queueLength: linkQueue.length + sitemapQueue.length,
- maxPages,
- });
+ `crawl-chunk-${chunkNo}`,
+ CRAWL_CHUNK_STEP,
+ () =>
+ runCrawlChunk({
+ ...params,
+ chunkNo,
+ attemptedBefore: attemptedTotal,
+ }),
+ );
+ // Apply the chunk's counters even when it did no new work (a retried
+ // chunk whose earlier attempt persisted everything reports 0 attempts
+ // with up-to-date scratchpad totals) — finalize must not see stale ones.
+ attemptedTotal = result.attempted;
+ pending = result.pending;
+ // One zero-attempt chunk is normal (retry of a completed chunk number);
+ // two in a row means the frontier is unservable — stop with what we
+ // have instead of spinning forever.
+ zeroProgressChunks =
+ result.attemptedInChunk === 0 ? zeroProgressChunks + 1 : 0;
+ if (zeroProgressChunks >= 2) break;
}
+ return { pagesCrawled: attemptedTotal, completed: pending === 0 };
+}
+
+async function runCrawlChunk(
+ input: CrawlPhaseParams & { chunkNo: number; attemptedBefore: number },
+): Promise<{ attemptedInChunk: number; attempted: number; pending: number }> {
+ const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =
+ input;
+ const scratchpad = getAuditScratchpad(auditId);
+
+ const claimLimit = Math.min(
+ CHUNK_TARGET_PAGES,
+ maxPages - input.attemptedBefore,
+ );
+ const claimed = await scratchpad.claimChunk(chunkNo, claimLimit);
+ if (claimed.length === 0) {
+ const stats = await scratchpad.getStats();
+ return {
+ attemptedInChunk: 0,
+ attempted: stats.attempted,
+ pending: stats.pending,
+ };
+ }
+
+ const depthByUrl = new Map(claimed.map((entry) => [entry.url, entry.depth]));
+ const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS;
+
+ let windowSize = INITIAL_CRAWL_WINDOW;
+ let nextIndex = 0;
+ let attemptedInChunk = 0;
+ const inFlight = new Set>();
+ let batch: CrawledPageResult[] = [];
+ // Persistence runs concurrently with fetching (pipelined) but sequentially
+ // with itself, so DB write pressure stays bounded at one batch at a time.
+ let persistChain: Promise = Promise.resolve();
+
+ const flush = () => {
+ if (batch.length === 0) return;
+ const pages = batch;
+ batch = [];
+ windowSize = adjustCrawlWindow(windowSize, pages);
+ persistChain = persistChain.then(() =>
+ persistCrawledPages({
+ auditId,
+ workflowInstanceId,
+ origin,
+ robots,
+ scratchpad,
+ pages,
+ depthByUrl,
+ maxPages,
+ }),
+ );
+ };
+
+ const launch = (entry: ClaimedUrl) => {
+ const promise = crawlPage(entry.url, entry.depth, entry.inSitemap)
+ .then((page) => {
+ attemptedInChunk += 1;
+ batch.push(page);
+ if (batch.length >= PERSIST_BATCH_SIZE) flush();
+ })
+ .finally(() => {
+ inFlight.delete(promise);
+ });
+ inFlight.add(promise);
+ };
+
+ while (true) {
+ while (
+ inFlight.size < windowSize &&
+ nextIndex < claimed.length &&
+ Date.now() < deadlineAt
+ ) {
+ launch(claimed[nextIndex]);
+ nextIndex += 1;
+ }
+ if (inFlight.size === 0) break;
+ await Promise.race(inFlight);
+ }
+ flush();
+ await persistChain;
+
+ // Leases we never launched (soft deadline) go back to the queue.
+ const unattempted = claimed.slice(nextIndex).map((entry) => entry.url);
+ if (unattempted.length > 0) {
+ await scratchpad.releaseUrls(unattempted);
+ }
+
+ // Progress counters are written per persisted sub-batch (in
+ // persistCrawledPages), so a chunk that dies mid-way underreports by at
+ // most one sub-batch, not a whole chunk.
+ const stats = await scratchpad.getStats();
return {
- pages: summaries,
- completed: linkQueue.length === 0 && sitemapQueue.length === 0,
+ attemptedInChunk,
+ attempted: stats.attempted,
+ pending: stats.pending,
};
}
-function selectNextCrawlBatch(params: {
- linkQueue: QueueEntry[];
- sitemapQueue: QueueEntry[];
- queued: Set;
- visited: Set;
- robots: RobotsResult;
- remaining: number;
-}) {
- const { linkQueue, sitemapQueue, queued, visited, robots, remaining } =
- params;
- const batchSize = Math.min(CRAWL_CONCURRENCY, remaining);
- const batchEntries: QueueEntry[] = [];
-
- while (
- (linkQueue.length > 0 || sitemapQueue.length > 0) &&
- batchEntries.length < batchSize
- ) {
- const entry = (linkQueue.length > 0 ? linkQueue : sitemapQueue).shift()!;
- queued.delete(entry.url);
- if (visited.has(entry.url)) continue;
- if (!robots.isAllowed(entry.url)) continue;
- visited.add(entry.url);
- batchEntries.push(entry);
- }
-
- return batchEntries;
-}
-
-async function runCrawlBatch(
- step: WorkflowStep,
- input: {
- crawlBatchIndex: number;
- auditId: string;
- batchEntries: QueueEntry[];
- sitemapSet: Set;
- visited: Set;
- queued: Set;
- },
-): Promise {
- const {
- crawlBatchIndex,
- auditId,
- batchEntries,
- sitemapSet,
- visited,
- queued,
- } = input;
- return pgStep(step, `crawl-batch-${crawlBatchIndex}`, undefined, async () => {
- const pages = await Promise.all(
- batchEntries.map((entry) =>
- crawlPage(entry.url, entry.depth, sitemapSet.has(entry.url)),
- ),
- );
-
- // Deterministic ids keep the D1 writes idempotent across step retries.
- for (const page of pages) {
- page.id = await deterministicAuditRowId(auditId, page.url);
- }
-
- const issues = pages.flatMap((page) => runPageReporters(page));
- await AuditRepository.insertCrawledBatch(auditId, pages, issues);
-
- // Frontier candidates only: drop targets already visited/queued and
- // dedupe across the batch, so the step return stays far under the
- // ~1MiB durable-state limit even on mega-menu sites.
- const seenTargets = new Set();
- return pages.map((page) => {
- const internalLinks: string[] = [];
- for (const link of page.links) {
- if (!link.isInternal) continue;
- if (seenTargets.size >= MAX_FRONTIER_LINKS_PER_BATCH) break;
- if (visited.has(link.targetUrl) || queued.has(link.targetUrl)) continue;
- if (seenTargets.has(link.targetUrl)) continue;
- seenTargets.add(link.targetUrl);
- internalLinks.push(link.targetUrl);
- }
- return {
- id: page.id,
- url: page.url,
- statusCode: page.statusCode,
- fetchClass: page.fetchClass,
- redirectUrl: page.redirectUrl,
- title: page.title.slice(0, MAX_SUMMARY_TITLE_CHARS),
- internalLinks,
- };
- });
- });
-}
-
-function enqueueDiscoveredLinks(params: {
- crawledBatch: StepPageSummary[];
- batchEntries: QueueEntry[];
- linkQueue: QueueEntry[];
- queued: Set;
- visited: Set;
+async function persistCrawledPages(input: {
+ auditId: string;
+ workflowInstanceId: string;
origin: string;
robots: RobotsResult;
-}) {
- const {
- crawledBatch,
- batchEntries,
- linkQueue,
- queued,
- visited,
- origin,
- robots,
- } = params;
- const depthByUrl = new Map(
- batchEntries.map((entry) => [entry.url, entry.depth]),
- );
+ scratchpad: ReturnType;
+ pages: CrawledPageResult[];
+ depthByUrl: Map;
+ maxPages: number;
+}): Promise {
+ const { auditId, origin, robots, scratchpad, pages, depthByUrl } = input;
- for (const pageResult of crawledBatch) {
- const pageDepth = depthByUrl.get(pageResult.url) ?? null;
+ // Deterministic ids keep every write idempotent across step retries.
+ for (const page of pages) {
+ page.id = await deterministicAuditRowId(auditId, page.url);
+ }
+ const issues = pages.flatMap((page) => runPageReporters(page));
+ await AuditRepository.insertCrawledBatch(auditId, pages, issues);
+
+ const links: ScratchpadLinkRow[] = [];
+ const discovered = new Map();
+ for (const page of pages) {
+ const pageDepth = depthByUrl.get(page.url) ?? null;
const childDepth = pageDepth === null ? null : pageDepth + 1;
- for (const link of pageResult.internalLinks) {
- if (!shouldQueueCrawlLink(link, origin, robots, visited, queued)) {
- continue;
+ let storedForPage = 0;
+ for (const link of page.links) {
+ if (!link.isInternal) continue;
+ if (storedForPage < MAX_STORED_LINKS_PER_PAGE) {
+ storedForPage += 1;
+ links.push({
+ sourcePageId: page.id,
+ sourceUrl: page.url,
+ targetUrl: link.targetUrl,
+ anchor: link.anchor,
+ isNofollow: link.isNofollow,
+ });
+ }
+ if (
+ discovered.size < MAX_DISCOVERED_PER_BATCH &&
+ !discovered.has(link.targetUrl) &&
+ shouldQueueCrawlLink(link.targetUrl, origin, robots)
+ ) {
+ discovered.set(link.targetUrl, childDepth);
}
- linkQueue.push({ url: link, depth: childDepth });
- queued.add(link);
}
// Redirect targets continue the same navigation path: same depth.
- const redirectTarget = pageResult.redirectUrl;
if (
- redirectTarget &&
- shouldQueueCrawlLink(redirectTarget, origin, robots, visited, queued)
+ page.redirectUrl &&
+ !discovered.has(page.redirectUrl) &&
+ shouldQueueCrawlLink(page.redirectUrl, origin, robots)
) {
- linkQueue.push({ url: redirectTarget, depth: pageDepth });
- queued.add(redirectTarget);
+ discovered.set(page.redirectUrl, pageDepth);
}
}
-}
-async function persistCrawlProgress(params: {
- step: WorkflowStep;
- crawlBatchIndex: number;
- auditId: string;
- workflowInstanceId: string;
- crawledBatch: StepPageSummary[];
- pagesCrawled: number;
- visitedCount: number;
- queueLength: number;
- maxPages: number;
-}) {
- const {
- step,
- crawlBatchIndex,
+ const stats = await scratchpad.recordBatch({
+ crawledUrls: pages.map((page) => page.url),
+ pages: pages.map((page) => ({
+ pageId: page.id,
+ url: page.url,
+ statusCode: page.statusCode,
+ fetchClass: page.fetchClass,
+ redirectUrl: page.redirectUrl,
+ })),
+ links,
+ discovered: Array.from(discovered, ([url, depth]) => ({ url, depth })),
+ });
+
+ await AuditRepository.updateAuditProgress(auditId, input.workflowInstanceId, {
+ pagesCrawled: stats.attempted,
+ pagesTotal: Math.min(stats.seen, input.maxPages),
+ });
+ await AuditProgressKV.pushCrawledUrls(
auditId,
- workflowInstanceId,
- crawledBatch,
- pagesCrawled,
- visitedCount,
- queueLength,
- maxPages,
- } = params;
- // KV push + D1 progress in one step — merging them halves the per-batch
- // step count against the ~1k step budget. The D1 update is idempotent; the
- // KV push can duplicate entries on a partial retry, which is acceptable for
- // an ephemeral progress feed (capped list, short TTL).
- await pgStep(
- step,
- `progress-batch-${crawlBatchIndex}`,
- undefined,
- async () => {
- await AuditProgressKV.pushCrawledUrls(
- auditId,
- crawledBatch.map((pageResult) => ({
- url: pageResult.url,
- statusCode: pageResult.statusCode,
- title: pageResult.title,
- crawledAt: Date.now(),
- })),
- );
- await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
- pagesCrawled,
- pagesTotal: Math.min(visitedCount + queueLength, maxPages),
- });
- },
+ pages.map((page) => ({
+ url: page.url,
+ statusCode: page.statusCode,
+ title: page.title.slice(0, MAX_PROGRESS_TITLE_CHARS),
+ crawledAt: Date.now(),
+ })),
);
+ return stats;
}
diff --git a/src/server/workflows/siteAuditWorkflowPhases.ts b/src/server/workflows/siteAuditWorkflowPhases.ts
index 1946802..00808ac 100644
--- a/src/server/workflows/siteAuditWorkflowPhases.ts
+++ b/src/server/workflows/siteAuditWorkflowPhases.ts
@@ -5,36 +5,34 @@ import {
fetchAndStoreLighthouseResult,
selectLighthouseSample,
} from "@/server/lib/audit/lighthouse";
-import { getOrigin } from "@/server/lib/audit/url-utils";
+import {
+ getOrigin,
+ isSameOrigin,
+ normalizeUrl,
+} from "@/server/lib/audit/url-utils";
+import { isCrawlableUrl } from "@/server/lib/audit/url-policy";
import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
+import { getAuditScratchpad } from "@/server/features/audit/AuditScratchpad";
import { AuditProgressKV } from "@/server/lib/audit/progress-kv";
import { runMultipageChecks } from "@/server/lib/audit/issues/multipage";
+import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters";
import type { AuditConfig } from "@/server/lib/audit/types";
import { captureServerEvent } from "@/server/lib/posthog";
import {
runCrawlPhase,
- type CrawledPageSummary,
type CrawlPhaseResult,
} from "@/server/workflows/siteAuditWorkflowCrawl";
import { pgStep } from "@/server/workflows/pgStep";
+import {
+ DB_STEP,
+ DISCOVERY_STEP,
+ LIGHTHOUSE_BATCH_STEP,
+ MULTIPAGE_CHECKS_STEP,
+} from "@/server/workflows/auditStepConfigs";
const LIGHTHOUSE_URL_BATCH_SIZE = 10;
-
-// Workflows rejects step outputs over 1MiB; keep the sitemap seed list well
-// under that. The crawl visits at most maxPages URLs, so extra seeds are moot.
-const SITEMAP_SEED_BYTE_BUDGET = 768 * 1024;
-
-function capSitemapSeeds(urls: string[], maxPages: number): string[] {
- const seeds: string[] = [];
- let bytes = 0;
- for (const url of urls) {
- if (seeds.length >= maxPages) break;
- bytes += url.length + 3; // JSON quotes + comma
- if (bytes > SITEMAP_SEED_BYTE_BUDGET) break;
- seeds.push(url);
- }
- return seeds;
-}
+/** Frontier seeds per scratchpad RPC call. */
+const SEED_RPC_BATCH = 2_000;
type AuditPhasesParams = {
auditId: string;
@@ -60,13 +58,13 @@ export async function runAuditPhases(
const origin = getOrigin(startUrl);
const maxPages = config.maxPages;
- const discovery = await runDiscoveryPhase(
- step,
+ const discovery = await runDiscoveryPhase(step, {
auditId,
workflowInstanceId,
origin,
+ startUrl,
maxPages,
- );
+ });
// Parsed outside the step from checkpointed text, so replays see the exact
// robots rules the original run used (a live re-fetch could differ and
// desync the frontier from already-persisted crawl batches).
@@ -75,10 +73,9 @@ export async function runAuditPhases(
auditId,
workflowInstanceId,
origin,
- startUrl,
maxPages,
robots,
- sitemapUrls: discovery.sitemapUrls,
+ seededCount: discovery.seededCount,
});
await runLighthousePhase(step, {
auditId,
@@ -87,7 +84,6 @@ export async function runAuditPhases(
projectId,
startUrl,
config,
- pages: crawl.pages,
});
await finalizeAudit({
step,
@@ -103,21 +99,61 @@ export async function runAuditPhases(
async function runDiscoveryPhase(
step: WorkflowStep,
- auditId: string,
- workflowInstanceId: string,
- origin: string,
- maxPages: number,
+ input: {
+ auditId: string;
+ workflowInstanceId: string;
+ origin: string;
+ startUrl: string;
+ maxPages: number;
+ },
) {
- return pgStep(step, "discover-urls", undefined, async () => {
+ const { auditId, workflowInstanceId, origin, startUrl, maxPages } = input;
+ // "-v2": the checkpoint shape changed (seeds now live in the scratchpad DO
+ // instead of the step return). A pre-refactor instance replayed under this
+ // code must re-run discovery — resuming from the old cached {sitemapUrls}
+ // shape would leave the scratchpad empty and finalize a zero-page audit.
+ return pgStep(step, "discover-urls-v2", DISCOVERY_STEP, async () => {
const result = await discoverUrls(origin, maxPages);
+ const robots = parseRobotsTxt(origin, result.robotsText);
+ const scratchpad = getAuditScratchpad(auditId);
+
+ // Seeds go straight into the scratchpad frontier — nothing large is
+ // returned as step state (an uncapped seed list used to blow the ~1MiB
+ // step-output limit on big sitemaps).
+ let seededCount = 0;
+ const normalizedStart = normalizeUrl(startUrl) ?? startUrl;
+ if (
+ robots.isAllowed(normalizedStart) &&
+ isSameOrigin(normalizedStart, origin)
+ ) {
+ await scratchpad.seedStart(normalizedStart);
+ seededCount += 1;
+ }
+
+ // The start URL is deliberately not excluded here: seedSitemapUrls
+ // upserts, so a start URL that also appears in the sitemap keeps its
+ // link-queue position but gains the in-sitemap flag.
+ const seen = new Set();
+ const seeds: string[] = [];
+ for (const url of result.urls) {
+ const normalized = normalizeUrl(url);
+ if (!normalized || seen.has(normalized)) continue;
+ seen.add(normalized);
+ if (!isSameOrigin(normalized, origin)) continue;
+ if (!isCrawlableUrl(normalized)) continue;
+ if (!robots.isAllowed(normalized)) continue;
+ seeds.push(normalized);
+ }
+ for (let i = 0; i < seeds.length; i += SEED_RPC_BATCH) {
+ await scratchpad.seedSitemapUrls(seeds.slice(i, i + SEED_RPC_BATCH));
+ }
+ seededCount += seeds.filter((seed) => seed !== normalizedStart).length;
+
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
- pagesTotal: Math.min(result.urls.length + 1, maxPages),
+ pagesTotal: Math.min(seededCount, maxPages),
currentPhase: "crawling",
});
- return {
- sitemapUrls: capSitemapSeeds(result.urls, maxPages),
- robotsText: result.robotsText,
- };
+ return { robotsText: result.robotsText, seededCount };
});
}
@@ -128,7 +164,6 @@ type LighthousePhaseParams = {
projectId: string;
startUrl: string;
config: AuditConfig;
- pages: CrawledPageSummary[];
};
async function runLighthousePhase(
@@ -142,7 +177,6 @@ async function runLighthousePhase(
projectId,
startUrl,
config,
- pages,
} = params;
if (config.lighthouseStrategy === "none") return;
@@ -150,7 +184,6 @@ async function runLighthousePhase(
step,
auditId,
workflowInstanceId,
- pages,
startUrl,
strategy: config.lighthouseStrategy,
});
@@ -165,12 +198,12 @@ async function runLighthousePhase(
const priorCompleted = completedChecks;
const priorFailed = failedChecks;
- // Fetch, store (R2 + D1) and update progress inside one step. The step
- // returns only counts; full results live in D1.
+ // Fetch, store (R2 + DB) and update progress inside one step. The step
+ // returns only counts; full results live in the DB.
const counts = await pgStep(
step,
`lighthouse-batch-${lighthouseBatchIndex}`,
- undefined,
+ LIGHTHOUSE_BATCH_STEP,
async () => {
const perUrlResults = await Promise.all(
batch.map(async ({ url, pageId }) => {
@@ -217,14 +250,22 @@ async function selectLighthousePages(params: {
step: WorkflowStep;
auditId: string;
workflowInstanceId: string;
- pages: CrawledPageSummary[];
startUrl: string;
strategy: AuditConfig["lighthouseStrategy"];
}) {
- const { step, auditId, workflowInstanceId, pages, startUrl, strategy } =
- params;
- return pgStep(step, "select-lighthouse-sample", undefined, async () => {
- const sample = selectLighthouseSample(pages, startUrl, strategy);
+ const { step, auditId, workflowInstanceId, startUrl, strategy } = params;
+ return pgStep(step, "select-lighthouse-sample", DB_STEP, async () => {
+ // Crawled pages come from the DB — the crawl phase no longer holds a
+ // whole-crawl page list in memory.
+ const crawledPages = await AuditRepository.getPagesForAudit(auditId);
+ const sample = selectLighthouseSample(
+ crawledPages.map((page) => ({
+ url: page.url,
+ statusCode: page.statusCode ?? 0,
+ })),
+ startUrl,
+ strategy,
+ );
const selectedUrls = new Set(sample);
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
@@ -233,7 +274,7 @@ async function selectLighthousePages(params: {
lighthouseCompleted: 0,
lighthouseFailed: 0,
});
- return pages.flatMap((page) =>
+ return crawledPages.flatMap((page) =>
selectedUrls.has(page.url) ? [{ url: page.url, pageId: page.id }] : [],
);
});
@@ -260,37 +301,34 @@ async function finalizeAudit(args: {
crawl,
} = args;
- await pgStep(step, "multipage-checks", undefined, async () => {
+ await pgStep(step, "multipage-checks", MULTIPAGE_CHECKS_STEP, async () => {
await AuditRepository.updateAuditProgress(auditId, workflowInstanceId, {
currentPhase: "finalizing",
});
- // Integrity guard: pages are persisted inside crawl-batch steps. If the
- // crawl claims pages but D1 has none (e.g. an instance started under the
- // pre-incremental-persistence code was replayed under this code), fail
- // loudly instead of completing with an empty audit.
+ // Integrity guard: pages are persisted inside crawl-chunk steps. If the
+ // crawl claims pages but the DB has none, fail loudly instead of
+ // completing with an empty audit.
if (
- crawl.pages.length > 0 &&
+ crawl.pagesCrawled > 0 &&
!(await AuditRepository.hasPagesForAudit(auditId))
) {
throw new Error(
- `Audit ${auditId}: crawl reported ${crawl.pages.length} pages but none were persisted`,
+ `Audit ${auditId}: crawl reported ${crawl.pagesCrawled} pages but none were persisted`,
);
}
- const issues = await runMultipageChecks({
- auditId,
- startUrl,
- crawlCompleted: crawl.completed,
- });
+ const issues = await runMultipageChecks({ auditId });
+ issues.push(...(await runScratchpadLinkChecks(auditId, startUrl, crawl)));
await AuditRepository.insertIssues(auditId, issues);
return { issueCount: issues.length };
});
- await pgStep(step, "finalize", undefined, async () => {
+ await pgStep(step, "finalize", DB_STEP, async () => {
+ const blockedPages = await AuditRepository.countBlockedPages(auditId);
await AuditRepository.completeAudit(auditId, workflowInstanceId, {
- pagesCrawled: crawl.pages.length,
- pagesTotal: crawl.pages.length,
+ pagesCrawled: crawl.pagesCrawled,
+ pagesTotal: crawl.pagesCrawled,
});
await captureServerEvent({
distinctId: billingCustomer.userId,
@@ -299,15 +337,49 @@ async function finalizeAudit(args: {
properties: {
project_id: projectId,
status: "completed",
- pages_crawled: crawl.pages.length,
- pages_total: crawl.pages.length,
+ pages_crawled: crawl.pagesCrawled,
+ pages_total: crawl.pagesCrawled,
crawl_completed: crawl.completed,
- pages_blocked: crawl.pages.filter(
- (page) => page.fetchClass === "blocked",
- ).length,
+ pages_blocked: blockedPages,
run_lighthouse: config.lighthouseStrategy !== "none",
},
});
await AuditProgressKV.clear(auditId);
+ // Crawl scratch state (frontier, links, mirror) is no longer needed.
+ await getAuditScratchpad(auditId).destroy();
});
}
+
+/**
+ * The two finalize checks that need link edges run as SQL inside the
+ * audit's scratchpad DO; map their rows onto DetectedIssue.
+ */
+async function runScratchpadLinkChecks(
+ auditId: string,
+ startUrl: string,
+ crawl: CrawlPhaseResult,
+): Promise {
+ const scratchpad = getAuditScratchpad(auditId);
+ const { brokenLinks, orphanPages } = await scratchpad.runFinalizeChecks({
+ // Page rows store normalized URLs; normalize the start URL the same way
+ // so the orphan exclusion matches.
+ startUrl: normalizeUrl(startUrl) ?? startUrl,
+ // Orphan detection only makes sense when the crawl wasn't truncated.
+ crawlCompleted: crawl.completed,
+ });
+
+ return [
+ ...brokenLinks.map((row) => ({
+ issueType: "broken-internal-link" as const,
+ pageId: row.sourcePageId,
+ pageUrl: row.sourceUrl,
+ dedupeKey: row.targetUrl,
+ details: { targetUrl: row.targetUrl, targetStatus: row.targetStatus },
+ })),
+ ...orphanPages.map((row) => ({
+ issueType: "orphan-page" as const,
+ pageId: row.pageId,
+ pageUrl: row.url,
+ })),
+ ];
+}
diff --git a/wrangler.jsonc b/wrangler.jsonc
index ddbb675..16013dd 100644
--- a/wrangler.jsonc
+++ b/wrangler.jsonc
@@ -33,7 +33,9 @@
],
// Durable Object backing the onboarding strategy chat (Agents SDK
// AIChatAgent). One instance per project; messages persist in the DO's
- // SQLite. SQLite-backed classes must be declared in `migrations` below.
+ // SQLite. SQLite-backed classes must be declared in `migrations` below for
+ // the wrangler/workerd surfaces (local dev, Docker self-host); Alchemy
+ // deploys derive SQLite backing on their own.
"durable_objects": {
"bindings": [
{
@@ -46,6 +48,13 @@
"name": "SAM_CHAT",
"class_name": "SamChatAgent",
},
+ // Per-audit crawl scratchpad: frontier, link edges, and a slim page
+ // mirror in the DO's SQLite. Destroyed at finalize; self-cleans via
+ // alarm if the audit dies.
+ {
+ "name": "AUDIT_SCRATCHPAD",
+ "class_name": "AuditScratchpad",
+ },
],
},
"migrations": [
@@ -57,6 +66,10 @@
"tag": "v2",
"new_sqlite_classes": ["SamChatAgent"],
},
+ {
+ "tag": "v3",
+ "new_sqlite_classes": ["AuditScratchpad"],
+ },
],
"triggers": {
"crons": ["*/15 * * * *"],