import { describe, expect, it, vi } from "vitest"; vi.mock("@/server/lib/runtime-env", () => ({ getRequiredEnvValue: vi.fn(async () => "test-api-key"), })); import { getOnPageCrawlStatus, getOnPageRawHtml, listOnPagePages, submitOnPageTask, } from "@/server/lib/dataforseo/onpage"; function stubFetchOnce(body: unknown) { vi.stubGlobal( "fetch", vi.fn().mockResolvedValue(Response.json(body)), ); } describe("submitOnPageTask", () => { it("returns the task id and cost from a 20100 'Task Created' response", async () => { stubFetchOnce({ status_code: 20000, tasks: [ { id: "task-123", status_code: 20100, status_message: "Task Created.", path: ["v3", "on_page", "task_post"], cost: 0.125, result_count: 0, }, ], }); await expect( submitOnPageTask({ target: "example.com", maxCrawlPages: 100 }), ).resolves.toEqual({ data: { taskId: "task-123" }, billing: { path: ["v3", "on_page", "task_post"], costUsd: 0.125 }, }); }); }); describe("getOnPageCrawlStatus", () => { it("reports in-progress crawls with their counts", async () => { stubFetchOnce({ status_code: 20000, tasks: [ { status_code: 20000, result: [ { crawl_progress: "in_progress", crawl_status: { pages_crawled: 15, pages_in_queue: 42 }, }, ], }, ], }); await expect(getOnPageCrawlStatus("task-123")).resolves.toEqual({ progress: "in_progress", pagesCrawled: 15, pagesInQueue: 42, }); }); it("reports finished crawls", async () => { stubFetchOnce({ status_code: 20000, tasks: [ { status_code: 20000, result: [ { crawl_progress: "finished", crawl_status: { pages_crawled: 57 }, }, ], }, ], }); await expect(getOnPageCrawlStatus("task-123")).resolves.toEqual({ progress: "finished", pagesCrawled: 57, }); }); }); describe("listOnPagePages", () => { it("returns the crawled pages", async () => { stubFetchOnce({ status_code: 20000, tasks: [ { status_code: 20000, result: [ { items: [ { url: "https://example.com/", status_code: 200 }, { url: "https://example.com/about", status_code: 200 }, ], }, ], }, ], }); await expect(listOnPagePages("task-123")).resolves.toEqual([ { url: "https://example.com/", status_code: 200 }, { url: "https://example.com/about", status_code: 200 }, ]); }); it("stops once a page returns fewer items than the page size", async () => { const fetchMock = vi.fn().mockResolvedValue( Response.json({ status_code: 20000, tasks: [ { status_code: 20000, result: [ { items: [{ url: "https://example.com/", status_code: 200 }] }, ], }, ], }), ); vi.stubGlobal("fetch", fetchMock); await listOnPagePages("task-123"); // One page came back short of the page size, so pagination stops there. expect(fetchMock).toHaveBeenCalledTimes(1); }); }); describe("getOnPageRawHtml", () => { it("returns the page's raw HTML", async () => { stubFetchOnce({ status_code: 20000, tasks: [ { status_code: 20000, result: [{ items: [{ html: "hi" }] }], }, ], }); await expect( getOnPageRawHtml("task-123", "https://example.com/"), ).resolves.toBe("hi"); }); it("returns null when the page has no stored HTML", async () => { stubFetchOnce({ status_code: 20000, tasks: [{ status_code: 20000, result: [{ items: [] }] }], }); await expect( getOnPageRawHtml("task-123", "https://example.com/"), ).resolves.toBeNull(); }); });