Site audit: DataForSEO OnPage fallback for bot-blocked crawls

Explicit, user-triggered button ("Get report from DataForSEO") shown when
the native crawler is blocked — never an automatic retry. DataForSEO's
OnPage API crawls from its own infrastructure with JS rendering, which
clears blocks a plain fetch() from our server can't.

- audits.crawlSource ("native" | "dataforseo") + dataforseoTaskId columns.
- dataforseo/onpage.ts: task_post (JS rendering + store_raw_html) / summary
  polling / pages listing / raw_html retrieval. Only task_post is billed
  (~$0.00125/page); the rest are free reads of already-billed results, per
  DataForSEO's pricing docs.
- DataForSeoAuditService: hard quota of 5 runs per project per calendar
  month, enforced server-side via the activity log (audit.dataforseo_report)
  before any DataForSEO spend — a rejected 6th run never reaches the API.
- Reuses the native pipeline instead of duplicating it: extracted
  buildAnalyzedPageResult() out of crawlPage() so both sources feed the same
  analyzeHtml -> runPageReporters -> runMultipageChecks -> auditPages/
  auditIssues path. No Cloudflare Workflow backs these audits; the existing
  getAuditStatus poll (already running every 3s while "running") drives an
  advance step each call instead.
- Known gap: broken-internal-link and orphan-page checks are native-only
  (they read the crawl's link graph from the AuditScratchpad Durable Object,
  which only the native crawl populates).
- UI: page-limit picker (25-500) + live quota display on the existing
  "blocked" screen.

Migration: drizzle/0046_*, drizzle-pg/0024_*.

tsc / oxlint / knip clean. New onpage.test.ts (7) + DataForSeoAuditService
.test.ts (5, including the quota-rejection path); full suite otherwise
unchanged (1195 pass, pre-existing samSkills Windows-CRLF failure only).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

drizzle-pg/0024_heavy_arachne.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "audits" ADD COLUMN "crawl_source" text DEFAULT 'native' NOT NULL;--> statement-breakpoint
+ALTER TABLE "audits" ADD COLUMN "dataforseo_task_id" text;

drizzle-pg/meta/_journal.json

@@ -169,6 +169,13 @@
       "when": 1788983569510,
       "tag": "0023_clever_prowler",
       "breakpoints": true
+    },
+    {
+      "idx": 24,
+      "version": "7",
+      "when": 1789226958095,
+      "tag": "0024_heavy_arachne",
+      "breakpoints": true
     }
   ]
 }

drizzle/0046_wooden_khan.sql

@@ -0,0 +1,2 @@
+ALTER TABLE `audits` ADD `crawl_source` text DEFAULT 'native' NOT NULL;--> statement-breakpoint
+ALTER TABLE `audits` ADD `dataforseo_task_id` text;

drizzle/meta/_journal.json

@@ -323,6 +323,13 @@
       "when": 1788983568067,
       "tag": "0045_noisy_ghost_rider",
       "breakpoints": true
+    },
+    {
+      "idx": 46,
+      "version": "6",
+      "when": 1789226956645,
+      "tag": "0046_wooden_khan",
+      "breakpoints": true
     }
   ]
 }

src/client/features/audit/results/DataForSeoFallbackCard.tsx

@@ -0,0 +1,100 @@
+import { useMutation, useQuery } from "@tanstack/react-query";
+import { AlertCircle } from "lucide-react";
+import { useState } from "react";
+import { toast } from "sonner";
+import {
+  getDataForSeoAuditQuota,
+  startDataForSeoAudit,
+} from "@/serverFunctions/audit";
+
+const PAGE_LIMIT_OPTIONS = [25, 50, 100, 250, 500] as const;
+
+function errorText(error: unknown, fallback: string) {
+  return error instanceof Error && error.message ? error.message : fallback;
+}
+
+/**
+ * Shown in place of (or alongside) the native-crawler blocked message. An
+ * explicit, user-triggered fallback — never automatic — that re-crawls via
+ * DataForSEO's OnPage API, which runs from DataForSEO's own infrastructure
+ * and can render JavaScript, so it gets past blocks our server's plain
+ * fetch() can't. Quota-limited per project per month.
+ */
+export function DataForSeoFallbackCard({
+  projectId,
+  startUrl,
+  onAuditStarted,
+}: {
+  projectId: string;
+  startUrl: string;
+  onAuditStarted: (auditId: string) => void;
+}) {
+  const [maxCrawlPages, setMaxCrawlPages] = useState<number>(100);
+
+  const quotaQuery = useQuery({
+    queryKey: ["dataforseo-audit-quota", projectId],
+    queryFn: () => getDataForSeoAuditQuota({ data: { projectId } }),
+  });
+
+  const startMutation = useMutation({
+    mutationFn: () =>
+      startDataForSeoAudit({ data: { projectId, startUrl, maxCrawlPages } }),
+    onSuccess: (result) => onAuditStarted(result.auditId),
+    onError: (error) =>
+      toast.error(errorText(error, "We couldn't start the DataForSEO audit.")),
+  });
+
+  const quota = quotaQuery.data;
+  const outOfQuota = quota !== undefined && quota.remaining <= 0;
+
+  return (
+    <div className="alert alert-warning items-start">
+      <AlertCircle className="size-5" />
+      <div className="space-y-3">
+        <div className="space-y-1">
+          <p className="font-medium">
+            Site audit couldn't fully crawl this website.
+          </p>
+          <p>
+            This site's bot protection blocked our crawler. You can retry with
+            DataForSEO's crawler, which runs from its own infrastructure and can
+            render JavaScript.
+          </p>
+        </div>
+
+        <div className="flex flex-wrap items-center gap-2">
+          <select
+            className="select select-bordered select-sm"
+            value={maxCrawlPages}
+            onChange={(event) => setMaxCrawlPages(Number(event.target.value))}
+            disabled={startMutation.isPending}
+            aria-label="Pages to crawl"
+          >
+            {PAGE_LIMIT_OPTIONS.map((option) => (
+              <option key={option} value={option}>
+                {option} pages
+              </option>
+            ))}
+          </select>
+          <button
+            type="button"
+            className="btn btn-primary btn-sm"
+            disabled={startMutation.isPending || outOfQuota}
+            onClick={() => startMutation.mutate()}
+          >
+            {startMutation.isPending
+              ? "Starting…"
+              : "Get report from DataForSEO"}
+          </button>
+          {quota && (
+            <span className="text-xs text-base-content/60">
+              {outOfQuota
+                ? "5/5 used this month — resets next month"
+                : `${quota.used}/${quota.limit} used this month`}
+            </span>
+          )}
+        </div>
+      </div>
+    </div>
+  );
+}

src/db/audit.schema.ts

@@ -28,6 +28,14 @@ export const audits = sqliteTable(
       .notNull()
       .default("running"),
     workflowInstanceId: text("workflow_instance_id"),
+    // "native" = our own crawler (SiteAuditWorkflow). "dataforseo" = crawled
+    // by DataForSEO's OnPage API when the target blocked the native crawler;
+    // no Cloudflare Workflow backs these, so workflowInstanceId is a
+    // synthetic placeholder rather than a real instance id.
+    crawlSource: text("crawl_source", { enum: ["native", "dataforseo"] })
+      .notNull()
+      .default("native"),
+    dataforseoTaskId: text("dataforseo_task_id"),
     // JSON config: { maxPages, lighthouseStrategy }
     config: text("config").notNull().default("{}"),
     // Progress & summary

src/db/pg/audit.schema.ts

@@ -35,6 +35,14 @@ export const audits = pgTable(
       .notNull()
       .default("running"),
     workflowInstanceId: text("workflow_instance_id"),
+    // "native" = our own crawler (SiteAuditWorkflow). "dataforseo" = crawled
+    // by DataForSEO's OnPage API when the target blocked the native crawler;
+    // no Cloudflare Workflow backs these, so workflowInstanceId is a
+    // synthetic placeholder rather than a real instance id.
+    crawlSource: text("crawl_source", { enum: ["native", "dataforseo"] })
+      .notNull()
+      .default("native"),
+    dataforseoTaskId: text("dataforseo_task_id"),
     // JSON config: { maxPages, lighthouseStrategy }
     config: text("config").notNull().default("{}"),
     // Progress & summary

src/routes/_project/p/$projectId/audit/index.tsx

@@ -9,6 +9,7 @@ import {
 } from "@/serverFunctions/audit";
 import { auditSearchSchema } from "@/types/schemas/audit";
 import { LaunchView } from "@/client/features/audit/launch/LaunchView";
+import { DataForSeoFallbackCard } from "@/client/features/audit/results/DataForSeoFallbackCard";
 import { ResultsView } from "@/client/features/audit/results/ResultsView";
 import {
   extractHostname,
@@ -57,6 +58,7 @@ function SiteAuditPage() {
       tab={tab}
       onBack={() => setSearchParams({ auditId: undefined })}
       onTabChange={(nextTab) => setSearchParams({ tab: nextTab })}
+      onAuditStarted={(id) => setSearchParams({ auditId: id })}
     />
   );
 }
@@ -67,12 +69,14 @@ function AuditDetail({
   tab,
   onBack,
   onTabChange,
+  onAuditStarted,
 }: {
   projectId: string;
   auditId: string;
   tab: string;
   onBack: () => void;
   onTabChange: (tab: "issues" | "pages" | "performance") => void;
+  onAuditStarted: (auditId: string) => void;
 }) {
   const statusQuery = useQuery({
     queryKey: ["audit-status", projectId, auditId],
@@ -161,40 +165,12 @@ function AuditDetail({
           />
         )}
 
-        {showSupportCta && (
+        {showSupportCta && status && (
-          <div
+          <DataForSeoFallbackCard
-            className={isFailed ? "alert alert-error" : "alert alert-warning"}
+            projectId={projectId}
-          >
+            startUrl={status.startUrl}
-            <AlertCircle className="size-5" />
+            onAuditStarted={onAuditStarted}
-            <div className="space-y-1">
+          />
-              <p className="font-medium">
-                Site audit couldn't fully crawl this website.
-              </p>
-              <p>
-                Sorry! This site's bot protection blocked our crawler. We don't
-                have a workaround for this yet. Desktop crawlers run from your
-                own machine and usually get past it: try{" "}
-                <a
-                  className="link link-primary"
-                  href="https://github.com/PhialsBasement/LibreCrawl"
-                  target="_blank"
-                  rel="noreferrer"
-                >
-                  LibreCrawl
-                </a>{" "}
-                (free, open source) or{" "}
-                <a
-                  className="link link-primary"
-                  href="https://www.screamingfrog.co.uk/seo-spider/"
-                  target="_blank"
-                  rel="noreferrer"
-                >
-                  Screaming Frog
-                </a>{" "}
-                (free up to 500 URLs).
-              </p>
-            </div>
-          </div>
         )}
 
         {failedWithResults && (

src/server/features/activity/ActivityRepository.ts

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { and, desc, eq, lt } from "drizzle-orm";
+import { and, count, desc, eq, gte, lt } from "drizzle-orm";
 import { db } from "@/db";
 import { activityLog } from "@/db/schema";
 import type { EnsuredUserContext } from "@/middleware/ensure-user/types";
@@ -12,6 +12,7 @@ export const ACTIVITY_ACTIONS = [
   "project.restore",
   "project.domain_set",
   "audit.start",
+  "audit.dataforseo_report",
   "team.user_created",
   "team.user_removed",
   "team.password_reset",
@@ -85,6 +86,30 @@ async function list(input: ListInput) {
     .limit(input.limit);
 }
 
+// Usage-quota check: how many times has this action already been recorded
+// against one target (e.g. a project) since a cutoff (e.g. the start of the
+// current month)? Used to cap paid/limited actions like the DataForSEO audit
+// fallback.
+async function countSince(input: {
+  organizationId: string;
+  action: string;
+  targetId: string;
+  since: Date;
+}): Promise<number> {
+  const [row] = await db
+    .select({ value: count() })
+    .from(activityLog)
+    .where(
+      and(
+        eq(activityLog.organizationId, input.organizationId),
+        eq(activityLog.action, input.action),
+        eq(activityLog.targetId, input.targetId),
+        gte(activityLog.createdAt, input.since),
+      ),
+    );
+  return row?.value ?? 0;
+}
+
 // Distinct actors seen in this org's log, for the filter dropdown.
 async function listActors(organizationId: string) {
   return db
@@ -97,4 +122,9 @@ async function listActors(organizationId: string) {
     .orderBy(activityLog.actorEmail);
 }
 
-export const ActivityRepository = { record, list, listActors } as const;
+export const ActivityRepository = {
+  record,
+  list,
+  listActors,
+  countSince,
+} as const;

src/server/features/audit/repositories/AuditRepository.ts

@@ -32,7 +32,12 @@ async function createAudit(data: {
   config: AuditConfig;
   pagesTotal: number;
   lighthouseTotal: number;
+  // DataForSEO-sourced audits (the bot-block fallback) have no Cloudflare
+  // Workflow behind them and start straight into crawling, not discovery.
+  crawlSource?: "native" | "dataforseo";
+  dataforseoTaskId?: string;
 }) {
+  const crawlSource = data.crawlSource ?? "native";
   await db.insert(audits).values({
     id: data.id,
     projectId: data.projectId,
@@ -43,7 +48,9 @@ async function createAudit(data: {
     status: "running",
     pagesTotal: data.pagesTotal,
     lighthouseTotal: data.lighthouseTotal,
-    currentPhase: "discovery",
+    currentPhase: crawlSource === "dataforseo" ? "crawling" : "discovery",
+    crawlSource,
+    dataforseoTaskId: data.dataforseoTaskId,
   });
 }
 

src/server/features/audit/services/AuditService.ts

@@ -24,6 +24,11 @@ import {
   resolveStartUrlRedirects,
 } from "@/server/lib/audit/url-policy";
 import { reconcileRunningAudit } from "@/server/features/audit/services/auditReconciler";
+import {
+  advanceDataForSeoAudit,
+  getDataForSeoAuditQuota,
+  startDataForSeoAudit,
+} from "@/server/features/audit/services/DataForSeoAuditService";
 import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
 
 // Plan-tier limits are the abuse bound in hosted mode: free accounts get small
@@ -141,10 +146,17 @@ async function getStatus(auditId: string, projectId: string) {
   if (!audit)
     throw new AppError("NOT_FOUND", "Audit not found in this project.");
 
-  // Self-heal audits whose workflow died without reaching the mark-failed
+  if (audit.status === "running" && audit.crawlSource === "dataforseo") {
-  // step (instance terminated/errored, instance expired from retention, ...).
+    // No Cloudflare Workflow backs this audit, so the poll itself has to
-  // Without this they stay "running" forever and hold capacity.
+    // drive it forward — the same status poll the client already runs every
-  if (audit.status === "running") {
+    // few seconds while "running" doubles as the work loop.
+    await advanceDataForSeoAudit({ auditId, projectId });
+    audit =
+      (await AuditRepository.getAuditForProject(auditId, projectId)) ?? audit;
+  } else if (audit.status === "running") {
+    // Self-heal audits whose workflow died without reaching the mark-failed
+    // step (instance terminated/errored, instance expired from retention, ...).
+    // Without this they stay "running" forever and hold capacity.
     const reconciled = await reconcileRunningAudit(audit);
     if (reconciled) {
       audit =
@@ -284,4 +296,6 @@ export const AuditService = {
   getResults,
   getHistory,
   remove,
+  startDataForSeoAudit,
+  getDataForSeoAuditQuota,
 } as const;

src/server/features/audit/services/DataForSeoAuditService.test.ts

@@ -0,0 +1,109 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+// The service transitively reaches cloudflare:workers via multipage.ts's own
+// @/db import (only exercised by advanceDataForSeoAudit's finalize path,
+// which these tests don't reach — the mock just lets the module load).
+vi.mock("cloudflare:workers", () => ({ env: {} }));
+
+const activityRepositoryMock = vi.hoisted(() => ({
+  countSince: vi.fn(),
+  record: vi.fn(),
+}));
+vi.mock("@/server/features/activity/ActivityRepository", () => ({
+  ActivityRepository: activityRepositoryMock,
+}));
+
+const auditRepositoryMock = vi.hoisted(() => ({
+  createAudit: vi.fn(),
+}));
+vi.mock("@/server/features/audit/repositories/AuditRepository", () => ({
+  AuditRepository: auditRepositoryMock,
+}));
+
+const taskPostMock = vi.hoisted(() => vi.fn());
+vi.mock("@/server/lib/dataforseo", () => ({
+  createDataforseoClient: () => ({ onPage: { taskPost: taskPostMock } }),
+}));
+
+import {
+  getDataForSeoAuditQuota,
+  startDataForSeoAudit,
+} from "@/server/features/audit/services/DataForSeoAuditService";
+
+const billingCustomer = {
+  userId: "u1",
+  userEmail: "a@example.com",
+  organizationId: "org1",
+};
+
+beforeEach(() => {
+  activityRepositoryMock.countSince.mockResolvedValue(0);
+  taskPostMock.mockResolvedValue({ taskId: "task-1" });
+});
+
+describe("getDataForSeoAuditQuota", () => {
+  it("reports remaining runs against the monthly limit", async () => {
+    activityRepositoryMock.countSince.mockResolvedValue(3);
+    await expect(
+      getDataForSeoAuditQuota({ organizationId: "org1", projectId: "p1" }),
+    ).resolves.toEqual({ used: 3, limit: 5, remaining: 2 });
+  });
+
+  it("never reports negative remaining runs", async () => {
+    activityRepositoryMock.countSince.mockResolvedValue(9);
+    await expect(
+      getDataForSeoAuditQuota({ organizationId: "org1", projectId: "p1" }),
+    ).resolves.toEqual({ used: 9, limit: 5, remaining: 0 });
+  });
+});
+
+describe("startDataForSeoAudit", () => {
+  const input = {
+    projectId: "p1",
+    startedByUserId: "u1",
+    startUrl: "https://example.com/",
+    maxCrawlPages: 100,
+  };
+
+  it("rejects the 6th run this month without spending anything", async () => {
+    activityRepositoryMock.countSince.mockResolvedValue(5);
+
+    await expect(startDataForSeoAudit(billingCustomer, input)).rejects.toThrow(
+      /used all 5/i,
+    );
+
+    expect(taskPostMock).not.toHaveBeenCalled();
+    expect(auditRepositoryMock.createAudit).not.toHaveBeenCalled();
+    expect(activityRepositoryMock.record).not.toHaveBeenCalled();
+  });
+
+  it("submits the crawl, creates the audit, and records the quota usage when under the limit", async () => {
+    const { auditId } = await startDataForSeoAudit(billingCustomer, input);
+
+    expect(auditId).toBeTruthy();
+    expect(taskPostMock).toHaveBeenCalledWith({
+      target: "example.com",
+      maxCrawlPages: 100,
+    });
+    expect(auditRepositoryMock.createAudit).toHaveBeenCalledWith(
+      expect.objectContaining({
+        crawlSource: "dataforseo",
+        dataforseoTaskId: "task-1",
+        projectId: "p1",
+      }),
+    );
+    expect(activityRepositoryMock.record).toHaveBeenCalledWith(
+      expect.objectContaining({ action: "audit.dataforseo_report" }),
+    );
+  });
+
+  it("strips protocol and www from the start URL for DataForSEO's target param", async () => {
+    await startDataForSeoAudit(billingCustomer, {
+      ...input,
+      startUrl: "https://www.example.com/some/page",
+    });
+    expect(taskPostMock).toHaveBeenCalledWith(
+      expect.objectContaining({ target: "example.com" }),
+    );
+  });
+});

src/server/features/audit/services/DataForSeoAuditService.ts

@@ -0,0 +1,285 @@
+import { randomUUID } from "node:crypto";
+import type { BillingCustomerContext } from "@/server/billing/subscription";
+import { createDataforseoClient } from "@/server/lib/dataforseo";
+import {
+  getOnPageCrawlStatus,
+  getOnPageRawHtml,
+  listOnPagePages,
+} from "@/server/lib/dataforseo/onpage";
+import { runMultipageChecks } from "@/server/lib/audit/issues/multipage";
+import { runPageReporters } from "@/server/lib/audit/issues/page-reporters";
+import type { PageFetchClass } from "@/server/lib/audit/types";
+import { buildAnalyzedPageResult } from "@/server/workflows/site-audit-workflow-helpers";
+import { AuditRepository } from "@/server/features/audit/repositories/AuditRepository";
+import { ActivityRepository } from "@/server/features/activity/ActivityRepository";
+import { AppError } from "@/server/lib/errors";
+
+// Fallback audit path for sites that block our own crawler: DataForSEO's
+// OnPage API crawls from its own infrastructure (optionally rendering JS), so
+// it gets past blocks a plain fetch() from our server can't. Capped at
+// MONTHLY_QUOTA runs per project per calendar month — this is a paid,
+// explicitly user-triggered action (a button, never an automatic fallback).
+//
+// Known gap vs. a native audit: broken-internal-link and orphan-page issues
+// are not detected here. Those checks read the crawl's link graph out of the
+// AuditScratchpad Durable Object, which only the native crawl populates.
+// Every other per-page and cross-page check runs identically (same
+// analyzeHtml/runPageReporters/runMultipageChecks the native path uses).
+
+const MONTHLY_QUOTA = 5;
+const DATAFORSEO_AUDIT_ACTION = "audit.dataforseo_report";
+// Pages ingested (raw_html fetch + analyze + persist) per advanceDataForSeoAudit
+// call, so one poll stays well inside a single request's time budget.
+const INGEST_BATCH_SIZE = 15;
+
+function currentMonthStart(): Date {
+  const now = new Date();
+  return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
+}
+
+function extractBareDomain(startUrl: string): string {
+  // DataForSEO's `target` param wants a bare domain — no protocol, no www.
+  const hostname = new URL(startUrl).hostname;
+  return hostname.replace(/^www\./, "");
+}
+
+export async function getDataForSeoAuditQuota(input: {
+  organizationId: string;
+  projectId: string;
+}): Promise<{ used: number; limit: number; remaining: number }> {
+  const used = await ActivityRepository.countSince({
+    organizationId: input.organizationId,
+    action: DATAFORSEO_AUDIT_ACTION,
+    targetId: input.projectId,
+    since: currentMonthStart(),
+  });
+  return {
+    used,
+    limit: MONTHLY_QUOTA,
+    remaining: Math.max(0, MONTHLY_QUOTA - used),
+  };
+}
+
+export async function startDataForSeoAudit(
+  billingCustomer: BillingCustomerContext,
+  input: {
+    projectId: string;
+    startedByUserId: string;
+    startUrl: string;
+    maxCrawlPages: number;
+  },
+): Promise<{ auditId: string }> {
+  const quota = await getDataForSeoAuditQuota({
+    organizationId: billingCustomer.organizationId,
+    projectId: input.projectId,
+  });
+  if (quota.remaining <= 0) {
+    throw new AppError(
+      "RATE_LIMITED",
+      `This project has used all ${MONTHLY_QUOTA} DataForSEO audits available this month. It resets at the start of next month.`,
+    );
+  }
+
+  const dataforseo = createDataforseoClient(billingCustomer);
+  const { taskId } = await dataforseo.onPage.taskPost({
+    target: extractBareDomain(input.startUrl),
+    maxCrawlPages: input.maxCrawlPages,
+  });
+
+  const auditId = randomUUID();
+  // No Cloudflare Workflow backs this audit; the id only needs to be stable
+  // and unique so the repository's workflowInstanceId-guarded updates work.
+  const workflowInstanceId = `dataforseo-${auditId}`;
+
+  await AuditRepository.createAudit({
+    id: auditId,
+    projectId: input.projectId,
+    startedByUserId: input.startedByUserId,
+    startUrl: input.startUrl,
+    workflowInstanceId,
+    config: { maxPages: input.maxCrawlPages, lighthouseStrategy: "none" },
+    pagesTotal: input.maxCrawlPages,
+    lighthouseTotal: 0,
+    crawlSource: "dataforseo",
+    dataforseoTaskId: taskId,
+  });
+
+  // Counts toward the monthly quota immediately (the DataForSEO spend is
+  // already committed at this point, regardless of how the crawl finishes).
+  await ActivityRepository.record({
+    context: {
+      userId: billingCustomer.userId,
+      userEmail: billingCustomer.userEmail,
+      organizationId: billingCustomer.organizationId,
+    },
+    action: DATAFORSEO_AUDIT_ACTION,
+    targetType: "project",
+    targetId: input.projectId,
+    targetLabel: input.startUrl,
+    metadata: { maxCrawlPages: input.maxCrawlPages },
+  });
+
+  return { auditId };
+}
+
+type AdvanceResult = {
+  status: "running" | "completed" | "failed";
+  pagesCrawled: number;
+  pagesTotal: number;
+};
+
+/**
+ * Do one bounded unit of work on a DataForSEO-sourced audit and report
+ * progress. Meant to be called repeatedly by the same polling loop the native
+ * audit's progress UI already uses, until status is "completed" or "failed".
+ */
+export async function advanceDataForSeoAudit(input: {
+  auditId: string;
+  projectId: string;
+}): Promise<AdvanceResult> {
+  const audit = await AuditRepository.getAuditForProject(
+    input.auditId,
+    input.projectId,
+  );
+  if (!audit || audit.crawlSource !== "dataforseo" || !audit.dataforseoTaskId) {
+    throw new AppError("NOT_FOUND", "DataForSEO audit not found.");
+  }
+  if (audit.status !== "running") {
+    return {
+      status: audit.status,
+      pagesCrawled: audit.pagesCrawled,
+      pagesTotal: audit.pagesTotal,
+    };
+  }
+
+  if (audit.currentPhase !== "analyzing") {
+    return advanceCrawlPhase(audit);
+  }
+  return advanceAnalyzePhase(audit);
+}
+
+type AuditRow = NonNullable<
+  Awaited<ReturnType<typeof AuditRepository.getAuditForProject>>
+>;
+
+// Always set by startDataForSeoAudit; a missing value means the audit row is
+// corrupt, not a normal runtime state, so this throws rather than silently
+// no-oping the guarded repository update below (an empty-string fallback
+// would never match the WHERE clause and mask the failure instead).
+function requireWorkflowInstanceId(audit: AuditRow): string {
+  if (!audit.workflowInstanceId) {
+    throw new AppError(
+      "INTERNAL_ERROR",
+      `Audit ${audit.id} is missing its workflowInstanceId.`,
+    );
+  }
+  return audit.workflowInstanceId;
+}
+
+async function advanceCrawlPhase(audit: AuditRow): Promise<AdvanceResult> {
+  const taskId = audit.dataforseoTaskId;
+  if (!taskId) {
+    throw new AppError(
+      "INTERNAL_ERROR",
+      "Audit is missing its DataForSEO task id.",
+    );
+  }
+  const workflowInstanceId = requireWorkflowInstanceId(audit);
+
+  const crawlStatus = await getOnPageCrawlStatus(taskId);
+  if (!crawlStatus) {
+    await AuditRepository.failAudit(audit.id, workflowInstanceId, {
+      errorCode: "unknown",
+      errorDetail: "DataForSEO task result expired before it was retrieved.",
+      failedPhase: audit.currentPhase,
+    });
+    return {
+      status: "failed",
+      pagesCrawled: audit.pagesCrawled,
+      pagesTotal: audit.pagesTotal,
+    };
+  }
+
+  if (crawlStatus.progress === "in_progress") {
+    await AuditRepository.updateAuditProgress(audit.id, workflowInstanceId, {
+      pagesCrawled: crawlStatus.pagesCrawled,
+    });
+    return {
+      status: "running",
+      pagesCrawled: crawlStatus.pagesCrawled,
+      pagesTotal: audit.pagesTotal,
+    };
+  }
+
+  // Crawl finished on DataForSEO's side — switch to ingesting pages. The
+  // pagesCrawled counter is repurposed as the ingest cursor for this phase.
+  await AuditRepository.updateAuditProgress(audit.id, workflowInstanceId, {
+    currentPhase: "analyzing",
+    pagesCrawled: 0,
+    pagesTotal: crawlStatus.pagesCrawled,
+  });
+  return {
+    status: "running",
+    pagesCrawled: 0,
+    pagesTotal: crawlStatus.pagesCrawled,
+  };
+}
+
+async function advanceAnalyzePhase(audit: AuditRow): Promise<AdvanceResult> {
+  const taskId = audit.dataforseoTaskId;
+  if (!taskId) {
+    throw new AppError(
+      "INTERNAL_ERROR",
+      "Audit is missing its DataForSEO task id.",
+    );
+  }
+  const workflowInstanceId = requireWorkflowInstanceId(audit);
+
+  // Free read of already-billed crawl results; re-fetched each call to avoid
+  // persisting a separate page list just for the ingest cursor.
+  const allPages = await listOnPagePages(taskId);
+  const offset = audit.pagesCrawled;
+  const batch = allPages.slice(offset, offset + INGEST_BATCH_SIZE);
+
+  if (batch.length === 0) {
+    const issues = await runMultipageChecks({ auditId: audit.id });
+    await AuditRepository.insertIssues(audit.id, issues);
+    await AuditRepository.completeAudit(audit.id, workflowInstanceId, {
+      pagesCrawled: allPages.length,
+      pagesTotal: allPages.length,
+    });
+    return {
+      status: "completed",
+      pagesCrawled: allPages.length,
+      pagesTotal: allPages.length,
+    };
+  }
+
+  const analyzedPages = await Promise.all(
+    batch.map(async (page) => {
+      const html = await getOnPageRawHtml(taskId, page.url);
+      const statusCode = page.status_code ?? 0;
+      const fetchClass: PageFetchClass = html ? "ok" : "error";
+      return buildAnalyzedPageResult({
+        url: page.url,
+        statusCode,
+        body: html ?? "",
+        fetchClass,
+        responseTimeMs: 0,
+        xRobotsTag: null,
+        headerCanonicalUrl: null,
+        crawlDepth: null,
+        inSitemap: false,
+      });
+    }),
+  );
+  const issues = analyzedPages.flatMap((page) => runPageReporters(page));
+  await AuditRepository.insertCrawledBatch(audit.id, analyzedPages, issues);
+
+  const pagesCrawled = offset + batch.length;
+  await AuditRepository.updateAuditProgress(audit.id, workflowInstanceId, {
+    pagesCrawled,
+  });
+
+  return { status: "running", pagesCrawled, pagesTotal: allPages.length };
+}

src/server/lib/dataforseo/client.ts

@@ -48,6 +48,7 @@ import {
   postRankCheckTasks,
 } from "@/server/lib/dataforseo/serp";
 import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
+import { submitOnPageTask } from "@/server/lib/dataforseo/onpage";
 import {
   fetchLlmAggregatedMetrics,
   fetchLlmCrossAggregatedMetrics,
@@ -137,6 +138,11 @@ export function createDataforseoClient(customer: BillingCustomerContext) {
     lighthouse: {
       live: meter(customer, fetchLighthouseResult),
     },
+    // task_post is where DataForSEO charges the crawl; collection (pages,
+    // raw_html, summary) is free and runs unmetered — see onpage.ts.
+    onPage: {
+      taskPost: meter(customer, submitOnPageTask, "site_audit"),
+    },
     aiSearch: {
       mentionsSearch: meter(customer, fetchLlmMentionsSearch),
       aggregatedMetrics: meter(customer, fetchLlmAggregatedMetrics),

src/server/lib/dataforseo/onpage.test.ts

@@ -0,0 +1,167 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("@/server/lib/runtime-env", () => ({
+  getRequiredEnvValue: vi.fn(async () => "test-api-key"),
+}));
+
+import {
+  getOnPageCrawlStatus,
+  getOnPageRawHtml,
+  listOnPagePages,
+  submitOnPageTask,
+} from "@/server/lib/dataforseo/onpage";
+
+function stubFetchOnce(body: unknown) {
+  vi.stubGlobal(
+    "fetch",
+    vi.fn<typeof fetch>().mockResolvedValue(Response.json(body)),
+  );
+}
+
+describe("submitOnPageTask", () => {
+  it("returns the task id and cost from a 20100 'Task Created' response", async () => {
+    stubFetchOnce({
+      status_code: 20000,
+      tasks: [
+        {
+          id: "task-123",
+          status_code: 20100,
+          status_message: "Task Created.",
+          path: ["v3", "on_page", "task_post"],
+          cost: 0.125,
+          result_count: 0,
+        },
+      ],
+    });
+
+    await expect(
+      submitOnPageTask({ target: "example.com", maxCrawlPages: 100 }),
+    ).resolves.toEqual({
+      data: { taskId: "task-123" },
+      billing: { path: ["v3", "on_page", "task_post"], costUsd: 0.125 },
+    });
+  });
+});
+
+describe("getOnPageCrawlStatus", () => {
+  it("reports in-progress crawls with their counts", async () => {
+    stubFetchOnce({
+      status_code: 20000,
+      tasks: [
+        {
+          status_code: 20000,
+          result: [
+            {
+              crawl_progress: "in_progress",
+              crawl_status: { pages_crawled: 15, pages_in_queue: 42 },
+            },
+          ],
+        },
+      ],
+    });
+
+    await expect(getOnPageCrawlStatus("task-123")).resolves.toEqual({
+      progress: "in_progress",
+      pagesCrawled: 15,
+      pagesInQueue: 42,
+    });
+  });
+
+  it("reports finished crawls", async () => {
+    stubFetchOnce({
+      status_code: 20000,
+      tasks: [
+        {
+          status_code: 20000,
+          result: [
+            {
+              crawl_progress: "finished",
+              crawl_status: { pages_crawled: 57 },
+            },
+          ],
+        },
+      ],
+    });
+
+    await expect(getOnPageCrawlStatus("task-123")).resolves.toEqual({
+      progress: "finished",
+      pagesCrawled: 57,
+    });
+  });
+});
+
+describe("listOnPagePages", () => {
+  it("returns the crawled pages", async () => {
+    stubFetchOnce({
+      status_code: 20000,
+      tasks: [
+        {
+          status_code: 20000,
+          result: [
+            {
+              items: [
+                { url: "https://example.com/", status_code: 200 },
+                { url: "https://example.com/about", status_code: 200 },
+              ],
+            },
+          ],
+        },
+      ],
+    });
+
+    await expect(listOnPagePages("task-123")).resolves.toEqual([
+      { url: "https://example.com/", status_code: 200 },
+      { url: "https://example.com/about", status_code: 200 },
+    ]);
+  });
+
+  it("stops once a page returns fewer items than the page size", async () => {
+    const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
+      Response.json({
+        status_code: 20000,
+        tasks: [
+          {
+            status_code: 20000,
+            result: [
+              { items: [{ url: "https://example.com/", status_code: 200 }] },
+            ],
+          },
+        ],
+      }),
+    );
+    vi.stubGlobal("fetch", fetchMock);
+
+    await listOnPagePages("task-123");
+    // One page came back short of the page size, so pagination stops there.
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("getOnPageRawHtml", () => {
+  it("returns the page's raw HTML", async () => {
+    stubFetchOnce({
+      status_code: 20000,
+      tasks: [
+        {
+          status_code: 20000,
+          result: [{ items: [{ html: "<html>hi</html>" }] }],
+        },
+      ],
+    });
+
+    await expect(
+      getOnPageRawHtml("task-123", "https://example.com/"),
+    ).resolves.toBe("<html>hi</html>");
+  });
+
+  it("returns null when the page has no stored HTML", async () => {
+    stubFetchOnce({
+      status_code: 20000,
+      tasks: [{ status_code: 20000, result: [{ items: [] }] }],
+    });
+
+    await expect(
+      getOnPageRawHtml("task-123", "https://example.com/"),
+    ).resolves.toBeNull();
+  });
+});

src/server/lib/dataforseo/onpage.ts

@@ -0,0 +1,148 @@
+import { z } from "zod";
+import { dataforseoGet, dataforseoPost } from "@/server/lib/dataforseo/core";
+import {
+  assertOk,
+  buildTaskBilling,
+  isRecord,
+  type DataforseoApiResponse,
+  type DataforseoTaskLike,
+} from "@/server/lib/dataforseo/envelope";
+import { AppError } from "@/server/lib/errors";
+
+// OnPage API: DataForSEO's own full-site crawler. Used as the fallback when
+// our own crawler gets blocked by a target's bot protection — DataForSEO
+// crawls from its own infrastructure, optionally rendering JavaScript, so it
+// clears blocks a plain fetch() from our server cannot.
+//
+// Pricing (checked 2026-09): ~$0.00125/page with JS rendering enabled.
+// store_raw_html adds no cost; raw_html retrieval itself is free for 7 days.
+
+const TASK_POST_PATH = "/v3/on_page/task_post";
+const PAGES_PATH = "/v3/on_page/pages";
+const RAW_HTML_PATH = "/v3/on_page/raw_html";
+const SUMMARY_PATH = "/v3/on_page/summary";
+
+const TASK_CREATED_STATUS_CODE = 20100;
+
+export async function submitOnPageTask(input: {
+  target: string;
+  maxCrawlPages: number;
+}): Promise<DataforseoApiResponse<{ taskId: string }>> {
+  const response = await dataforseoPost<DataforseoTaskLike & { id?: string }>(
+    TASK_POST_PATH,
+    [
+      {
+        target: input.target,
+        max_crawl_pages: input.maxCrawlPages,
+        enable_javascript: true,
+        store_raw_html: true,
+      },
+    ],
+    // Billed, non-idempotent: never replay a task_post.
+    { maxServerErrorRetries: 0 },
+  );
+  const task = assertOk(response, {
+    okTaskStatusCode: TASK_CREATED_STATUS_CODE,
+  });
+  const billing = buildTaskBilling(task);
+
+  if (!task.id) {
+    throw new AppError(
+      "INTERNAL_ERROR",
+      "DataForSEO task_post response is missing the task id",
+    );
+  }
+
+  return { data: { taskId: task.id }, billing };
+}
+
+const crawlStatusSchema = z.object({
+  pages_crawled: z.number().nullable().optional(),
+  pages_in_queue: z.number().nullable().optional(),
+  max_crawl_pages: z.number().nullable().optional(),
+});
+
+type OnPageCrawlStatus =
+  | { progress: "in_progress"; pagesCrawled: number; pagesInQueue: number }
+  | { progress: "finished"; pagesCrawled: number };
+
+/**
+ * Poll the crawl's progress. Free (no billing envelope) — this is a status
+ * read, not a new task. Returns null once the task has fully expired from
+ * DataForSEO's side (after their retention window).
+ */
+export async function getOnPageCrawlStatus(
+  taskId: string,
+): Promise<OnPageCrawlStatus | null> {
+  const response = await dataforseoGet(
+    `${SUMMARY_PATH}/${encodeURIComponent(taskId)}`,
+  );
+  const task = assertOk(response);
+  const first = task.result?.[0];
+  if (!isRecord(first)) return null;
+
+  const statusParsed = crawlStatusSchema.safeParse(first.crawl_status);
+  const status = statusParsed.success ? statusParsed.data : {};
+  const pagesCrawled = status.pages_crawled ?? 0;
+
+  if (first.crawl_progress === "finished") {
+    return { progress: "finished", pagesCrawled };
+  }
+  return {
+    progress: "in_progress",
+    pagesCrawled,
+    pagesInQueue: status.pages_in_queue ?? 0,
+  };
+}
+
+const onPagePageItemSchema = z.object({
+  url: z.string(),
+  status_code: z.number().nullable().optional(),
+});
+
+type OnPagePageItem = z.infer<typeof onPagePageItemSchema>;
+
+const PAGES_PAGE_SIZE = 1000;
+
+/** All crawled page URLs + status codes for a finished task. Free (a read of
+ *  already-billed crawl results, not a new task). Paginates internally. */
+export async function listOnPagePages(
+  taskId: string,
+): Promise<OnPagePageItem[]> {
+  const items: OnPagePageItem[] = [];
+  for (let offset = 0; ; offset += PAGES_PAGE_SIZE) {
+    const response = await dataforseoPost(PAGES_PATH, [
+      { id: taskId, limit: PAGES_PAGE_SIZE, offset },
+    ]);
+    const task = assertOk(response);
+    const first = task.result?.[0];
+    const rawItems = isRecord(first) ? first.items : [];
+    const parsed = z.array(onPagePageItemSchema).safeParse(rawItems ?? []);
+    if (!parsed.success) {
+      console.error(
+        "dataforseo.onpage.pages.invalid-payload",
+        parsed.error.issues.slice(0, 5),
+      );
+      break;
+    }
+    items.push(...parsed.data);
+    if (parsed.data.length < PAGES_PAGE_SIZE) break;
+  }
+  return items;
+}
+
+/** One page's raw HTML. Free for 7 days after the crawl (requires
+ *  store_raw_html:true on the original task_post). Null if unavailable. */
+export async function getOnPageRawHtml(
+  taskId: string,
+  url: string,
+): Promise<string | null> {
+  const response = await dataforseoPost(RAW_HTML_PATH, [{ id: taskId, url }]);
+  const task = assertOk(response);
+  const first = task.result?.[0];
+  if (!isRecord(first)) return null;
+  const parsed = z
+    .object({ items: z.array(z.object({ html: z.string() })).optional() })
+    .safeParse(first);
+  return parsed.success ? (parsed.data.items?.[0]?.html ?? null) : null;
+}

src/server/workflows/site-audit-workflow-helpers.ts

@@ -126,64 +126,17 @@ export async function crawlPage(
       });
     }
 
-    // Dynamic import keeps the HTML parser out of the worker's startup
+    return await buildAnalyzedPageResult({
-    // module graph: SiteAuditWorkflow is re-exported from src/server.ts, so
-    // a static import would evaluate it in every isolate's baseline heap,
-    // not just when an audit actually crawls.
-    const { analyzeHtml } = await import("@/server/lib/audit/page-analyzer");
-    const analysis = analyzeHtml(body, url, statusCode, responseTimeMs);
-    const robotsDirectives = [analysis.robotsMeta, xRobotsTag]
-      .filter(Boolean)
-      .join(",")
-      .toLowerCase();
-    const isIndexable = !robotsDirectives.includes("noindex");
-    const headingCount = (level: number) =>
-      analysis.headingOrder.filter((h) => h === level).length;
-
-    return {
-      id: crypto.randomUUID(),
       url,
       statusCode,
+      body,
       fetchClass,
-      redirectUrl: null,
+      responseTimeMs,
-      title: analysis.title,
-      metaDescription: analysis.metaDescription,
-      canonicalUrl: analysis.canonical
-        ? (normalizeUrl(analysis.canonical, url) ?? analysis.canonical)
-        : null,
-      robotsMeta: analysis.robotsMeta,
       xRobotsTag,
       headerCanonicalUrl,
-      ogTitle: analysis.ogTitle,
-      ogDescription: analysis.ogDescription,
-      ogImage: analysis.ogImage,
-      h1Count: analysis.h1s.filter((h) => h.length > 0).length,
-      h2Count: headingCount(2),
-      h3Count: headingCount(3),
-      h4Count: headingCount(4),
-      h5Count: headingCount(5),
-      h6Count: headingCount(6),
-      headingOrder: analysis.headingOrder,
-      wordCount: analysis.wordCount,
-      contentHash: analysis.bodyText
-        ? await sha256Hex(analysis.bodyText)
-        : null,
-      isHtml: true,
-      htmlBytes: body.length,
-      imagesTotal: analysis.images.length,
-      // Only a truly absent alt attribute counts: alt="" is the correct
-      // markup for decorative images.
-      imagesMissingAlt: analysis.images.filter((img) => img.alt === null)
-        .length,
-      images: analysis.images,
-      links: analysis.links,
-      hasStructuredData: analysis.hasStructuredData,
-      hreflangTags: analysis.hreflangTags,
-      isIndexable,
-      responseTimeMs,
       crawlDepth,
       inSitemap,
-    };
+    });
   } catch (error) {
     const responseTimeMs = Date.now() - startTime;
     console.warn(`Failed to crawl ${url}:`, error);
@@ -201,6 +154,108 @@ export async function crawlPage(
   }
 }
 
+/**
+ * Turn an already-fetched HTML body into a CrawledPageResult. Shared by the
+ * native crawl above (after its own fetch) and the DataForSEO OnPage path
+ * (after fetching a page's raw HTML from DataForSEO's API instead) — same
+ * analysis and issue detection run over either source, since both produce a
+ * CrawledPageResult.
+ */
+export async function buildAnalyzedPageResult(input: {
+  url: string;
+  statusCode: number;
+  body: string;
+  fetchClass: PageFetchClass;
+  responseTimeMs: number;
+  xRobotsTag: string | null;
+  headerCanonicalUrl: string | null;
+  crawlDepth: number | null;
+  inSitemap: boolean;
+}): Promise<CrawledPageResult> {
+  const {
+    url,
+    statusCode,
+    body,
+    fetchClass,
+    responseTimeMs,
+    xRobotsTag,
+    headerCanonicalUrl,
+    crawlDepth,
+    inSitemap,
+  } = input;
+
+  if (fetchClass !== "ok" || statusCode >= 400) {
+    return emptyPageResult({
+      url,
+      statusCode,
+      fetchClass,
+      redirectUrl: null,
+      responseTimeMs,
+      xRobotsTag,
+      headerCanonicalUrl,
+      crawlDepth,
+      inSitemap,
+      htmlBytes: body.length,
+    });
+  }
+
+  // Dynamic import keeps the HTML parser out of the worker's startup module
+  // graph: SiteAuditWorkflow is re-exported from src/server.ts, so a static
+  // import would evaluate it in every isolate's baseline heap, not just when
+  // an audit actually crawls.
+  const { analyzeHtml } = await import("@/server/lib/audit/page-analyzer");
+  const analysis = analyzeHtml(body, url, statusCode, responseTimeMs);
+  const robotsDirectives = [analysis.robotsMeta, xRobotsTag]
+    .filter(Boolean)
+    .join(",")
+    .toLowerCase();
+  const isIndexable = !robotsDirectives.includes("noindex");
+  const headingCount = (level: number) =>
+    analysis.headingOrder.filter((h) => h === level).length;
+
+  return {
+    id: crypto.randomUUID(),
+    url,
+    statusCode,
+    fetchClass,
+    redirectUrl: null,
+    title: analysis.title,
+    metaDescription: analysis.metaDescription,
+    canonicalUrl: analysis.canonical
+      ? (normalizeUrl(analysis.canonical, url) ?? analysis.canonical)
+      : null,
+    robotsMeta: analysis.robotsMeta,
+    xRobotsTag,
+    headerCanonicalUrl,
+    ogTitle: analysis.ogTitle,
+    ogDescription: analysis.ogDescription,
+    ogImage: analysis.ogImage,
+    h1Count: analysis.h1s.filter((h) => h.length > 0).length,
+    h2Count: headingCount(2),
+    h3Count: headingCount(3),
+    h4Count: headingCount(4),
+    h5Count: headingCount(5),
+    h6Count: headingCount(6),
+    headingOrder: analysis.headingOrder,
+    wordCount: analysis.wordCount,
+    contentHash: analysis.bodyText ? await sha256Hex(analysis.bodyText) : null,
+    isHtml: true,
+    htmlBytes: body.length,
+    imagesTotal: analysis.images.length,
+    // Only a truly absent alt attribute counts: alt="" is the correct markup
+    // for decorative images.
+    imagesMissingAlt: analysis.images.filter((img) => img.alt === null).length,
+    images: analysis.images,
+    links: analysis.links,
+    hasStructuredData: analysis.hasStructuredData,
+    hreflangTags: analysis.hreflangTags,
+    isIndexable,
+    responseTimeMs,
+    crawlDepth,
+    inSitemap,
+  };
+}
+
 async function readTextUpTo(response: Response, maxBytes: number) {
   if (!response.body) return "";
 

src/serverFunctions/audit.ts

@@ -11,7 +11,9 @@ import {
   getAuditResultsSchema,
   getAuditStatusSchema,
   getCrawlProgressSchema,
+  getDataForSeoAuditQuotaSchema,
   startAuditSchema,
+  startDataForSeoAuditSchema,
 } from "@/types/schemas/audit";
 
 export const startAudit = createServerFn({ method: "POST" })
@@ -63,6 +65,46 @@ export const getAuditStatus = createServerFn({ method: "POST" })
     return AuditService.getStatus(data.auditId, context.projectId);
   });
 
+// Fallback for sites that blocked the native crawler: an explicit,
+// user-triggered action (never automatic) that crawls via DataForSEO's
+// OnPage API instead. Quota-limited per project per month — see
+// DataForSeoAuditService.
+export const startDataForSeoAudit = createServerFn({ method: "POST" })
+  .middleware(requireProjectContext)
+  .validator(startDataForSeoAuditSchema)
+  .handler(async ({ data, context }) => {
+    const result = await AuditService.startDataForSeoAudit(context, {
+      projectId: context.projectId,
+      startedByUserId: context.userId,
+      startUrl: data.startUrl,
+      maxCrawlPages: data.maxCrawlPages,
+    });
+
+    waitUntil(
+      captureServerEvent({
+        distinctId: context.userId,
+        event: "site_audit:dataforseo_start",
+        organizationId: context.organizationId,
+        properties: {
+          project_id: context.projectId,
+          max_crawl_pages: data.maxCrawlPages,
+        },
+      }),
+    );
+
+    return result;
+  });
+
+export const getDataForSeoAuditQuota = createServerFn({ method: "POST" })
+  .middleware(requireProjectContext)
+  .validator(getDataForSeoAuditQuotaSchema)
+  .handler(async ({ context }) => {
+    return AuditService.getDataForSeoAuditQuota({
+      organizationId: context.organizationId,
+      projectId: context.projectId,
+    });
+  });
+
 export const getAuditResults = createServerFn({ method: "POST" })
   .middleware(requireProjectContext)
   .validator(getAuditResultsSchema)

src/shared/audit-limits.ts

@@ -4,3 +4,7 @@ export const MIN_AUDIT_PAGES = 10;
 export const DEFAULT_AUDIT_PAGES = 50;
 export const FREE_MAX_AUDIT_PAGES = 50;
 export const PAID_MAX_AUDIT_PAGES = 10_000;
+// The DataForSEO OnPage fallback is a paid, quota-limited action (see
+// DataForSeoAuditService) — capped well below the native ceiling so a single
+// run can't run away with spend even before the monthly-run quota kicks in.
+export const DATAFORSEO_MAX_AUDIT_PAGES = 500;

src/types/schemas/audit.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import {
+  DATAFORSEO_MAX_AUDIT_PAGES,
   DEFAULT_AUDIT_PAGES,
   MIN_AUDIT_PAGES,
   PAID_MAX_AUDIT_PAGES,
@@ -44,6 +45,20 @@ export const getCrawlProgressSchema = z.object({
   auditId: z.string().min(1),
 });
 
+export const startDataForSeoAuditSchema = z.object({
+  projectId: z.string().min(1),
+  startUrl: z.string().min(1, "URL is required").max(2048),
+  maxCrawlPages: z
+    .number()
+    .int()
+    .min(MIN_AUDIT_PAGES)
+    .max(DATAFORSEO_MAX_AUDIT_PAGES),
+});
+
+export const getDataForSeoAuditQuotaSchema = z.object({
+  projectId: z.string().min(1),
+});
+
 // ─── URL search params schema for /p/$projectId/audit ────────────────────────
 
 const auditTabs = ["issues", "pages", "performance"] as const;