6d40767d95
Explicit, user-triggered button ("Get report from DataForSEO") shown when
the native crawler is blocked — never an automatic retry. DataForSEO's
OnPage API crawls from its own infrastructure with JS rendering, which
clears blocks a plain fetch() from our server can't.
- audits.crawlSource ("native" | "dataforseo") + dataforseoTaskId columns.
- dataforseo/onpage.ts: task_post (JS rendering + store_raw_html) / summary
polling / pages listing / raw_html retrieval. Only task_post is billed
(~$0.00125/page); the rest are free reads of already-billed results, per
DataForSEO's pricing docs.
- DataForSeoAuditService: hard quota of 5 runs per project per calendar
month, enforced server-side via the activity log (audit.dataforseo_report)
before any DataForSEO spend — a rejected 6th run never reaches the API.
- Reuses the native pipeline instead of duplicating it: extracted
buildAnalyzedPageResult() out of crawlPage() so both sources feed the same
analyzeHtml -> runPageReporters -> runMultipageChecks -> auditPages/
auditIssues path. No Cloudflare Workflow backs these audits; the existing
getAuditStatus poll (already running every 3s while "running") drives an
advance step each call instead.
- Known gap: broken-internal-link and orphan-page checks are native-only
(they read the crawl's link graph from the AuditScratchpad Durable Object,
which only the native crawl populates).
- UI: page-limit picker (25-500) + live quota display on the existing
"blocked" screen.
Migration: drizzle/0046_*, drizzle-pg/0024_*.
tsc / oxlint / knip clean. New onpage.test.ts (7) + DataForSeoAuditService
.test.ts (5, including the quota-rejection path); full suite otherwise
unchanged (1195 pass, pre-existing samSkills Windows-CRLF failure only).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
168 lines
4.0 KiB
TypeScript
168 lines
4.0 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
vi.mock("@/server/lib/runtime-env", () => ({
|
|
getRequiredEnvValue: vi.fn(async () => "test-api-key"),
|
|
}));
|
|
|
|
import {
|
|
getOnPageCrawlStatus,
|
|
getOnPageRawHtml,
|
|
listOnPagePages,
|
|
submitOnPageTask,
|
|
} from "@/server/lib/dataforseo/onpage";
|
|
|
|
function stubFetchOnce(body: unknown) {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn<typeof fetch>().mockResolvedValue(Response.json(body)),
|
|
);
|
|
}
|
|
|
|
describe("submitOnPageTask", () => {
|
|
it("returns the task id and cost from a 20100 'Task Created' response", async () => {
|
|
stubFetchOnce({
|
|
status_code: 20000,
|
|
tasks: [
|
|
{
|
|
id: "task-123",
|
|
status_code: 20100,
|
|
status_message: "Task Created.",
|
|
path: ["v3", "on_page", "task_post"],
|
|
cost: 0.125,
|
|
result_count: 0,
|
|
},
|
|
],
|
|
});
|
|
|
|
await expect(
|
|
submitOnPageTask({ target: "example.com", maxCrawlPages: 100 }),
|
|
).resolves.toEqual({
|
|
data: { taskId: "task-123" },
|
|
billing: { path: ["v3", "on_page", "task_post"], costUsd: 0.125 },
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("getOnPageCrawlStatus", () => {
|
|
it("reports in-progress crawls with their counts", async () => {
|
|
stubFetchOnce({
|
|
status_code: 20000,
|
|
tasks: [
|
|
{
|
|
status_code: 20000,
|
|
result: [
|
|
{
|
|
crawl_progress: "in_progress",
|
|
crawl_status: { pages_crawled: 15, pages_in_queue: 42 },
|
|
},
|
|
],
|
|
},
|
|
],
|
|
});
|
|
|
|
await expect(getOnPageCrawlStatus("task-123")).resolves.toEqual({
|
|
progress: "in_progress",
|
|
pagesCrawled: 15,
|
|
pagesInQueue: 42,
|
|
});
|
|
});
|
|
|
|
it("reports finished crawls", async () => {
|
|
stubFetchOnce({
|
|
status_code: 20000,
|
|
tasks: [
|
|
{
|
|
status_code: 20000,
|
|
result: [
|
|
{
|
|
crawl_progress: "finished",
|
|
crawl_status: { pages_crawled: 57 },
|
|
},
|
|
],
|
|
},
|
|
],
|
|
});
|
|
|
|
await expect(getOnPageCrawlStatus("task-123")).resolves.toEqual({
|
|
progress: "finished",
|
|
pagesCrawled: 57,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("listOnPagePages", () => {
|
|
it("returns the crawled pages", async () => {
|
|
stubFetchOnce({
|
|
status_code: 20000,
|
|
tasks: [
|
|
{
|
|
status_code: 20000,
|
|
result: [
|
|
{
|
|
items: [
|
|
{ url: "https://example.com/", status_code: 200 },
|
|
{ url: "https://example.com/about", status_code: 200 },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
],
|
|
});
|
|
|
|
await expect(listOnPagePages("task-123")).resolves.toEqual([
|
|
{ url: "https://example.com/", status_code: 200 },
|
|
{ url: "https://example.com/about", status_code: 200 },
|
|
]);
|
|
});
|
|
|
|
it("stops once a page returns fewer items than the page size", async () => {
|
|
const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
|
|
Response.json({
|
|
status_code: 20000,
|
|
tasks: [
|
|
{
|
|
status_code: 20000,
|
|
result: [
|
|
{ items: [{ url: "https://example.com/", status_code: 200 }] },
|
|
],
|
|
},
|
|
],
|
|
}),
|
|
);
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
await listOnPagePages("task-123");
|
|
// One page came back short of the page size, so pagination stops there.
|
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|
|
|
|
describe("getOnPageRawHtml", () => {
|
|
it("returns the page's raw HTML", async () => {
|
|
stubFetchOnce({
|
|
status_code: 20000,
|
|
tasks: [
|
|
{
|
|
status_code: 20000,
|
|
result: [{ items: [{ html: "<html>hi</html>" }] }],
|
|
},
|
|
],
|
|
});
|
|
|
|
await expect(
|
|
getOnPageRawHtml("task-123", "https://example.com/"),
|
|
).resolves.toBe("<html>hi</html>");
|
|
});
|
|
|
|
it("returns null when the page has no stored HTML", async () => {
|
|
stubFetchOnce({
|
|
status_code: 20000,
|
|
tasks: [{ status_code: 20000, result: [{ items: [] }] }],
|
|
});
|
|
|
|
await expect(
|
|
getOnPageRawHtml("task-123", "https://example.com/"),
|
|
).resolves.toBeNull();
|
|
});
|
|
});
|