From 215ead81522ba8287e9b10184974d07b94032023 Mon Sep 17 00:00:00 2001 From: Ben Senescu <44480372+bensenescu@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:23:55 -0400 Subject: [PATCH] Replace dataforseo-client SDK with a thin fetch client (#532) --- knip.jsonc | 4 - package.json | 1 - pnpm-lock.yaml | 8 - scripts/dataforseo-account-usage.ts | 7 +- .../services/research/research-data.test.ts | 34 +-- src/server/lib/dataforseo/ai.ts | 176 ++++++--------- src/server/lib/dataforseo/appendix.ts | 40 +++- src/server/lib/dataforseo/backlinks.ts | 109 ++++----- src/server/lib/dataforseo/business.ts | 149 +++++++------ src/server/lib/dataforseo/client.test.ts | 7 + src/server/lib/dataforseo/client.ts | 156 +++++++------ src/server/lib/dataforseo/core.test.ts | 27 ++- src/server/lib/dataforseo/core.ts | 108 +++++---- src/server/lib/dataforseo/envelope.ts | 19 +- src/server/lib/dataforseo/google-ads.ts | 65 +++--- src/server/lib/dataforseo/index.ts | 44 ++-- src/server/lib/dataforseo/labs.ts | 209 ++++++++++++------ src/server/lib/dataforseo/lighthouse.test.ts | 82 +++---- src/server/lib/dataforseo/lighthouse.ts | 23 +- src/server/lib/dataforseo/sections.ts | 59 ----- src/server/lib/dataforseo/serp-locations.ts | 6 +- src/server/lib/dataforseo/serp.ts | 126 +++++------ src/server/lib/dataforseo/shared.ts | 8 +- vite-plugin-lean-worker-bundle.ts | 9 +- 24 files changed, 752 insertions(+), 724 deletions(-) delete mode 100644 src/server/lib/dataforseo/sections.ts diff --git a/knip.jsonc b/knip.jsonc index da0a491..a01b853 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -20,10 +20,6 @@ "src/db/pg/schema.ts", // Standalone CLI/dev scripts, invoked via package.json scripts "scripts/**", - // DataForSEO section barrel — consumed via the dynamic import in - // client.ts (loadDataforseoSections) + property access, which knip - // can't trace - "src/server/lib/dataforseo/sections.ts", ], // badseo/ is the standalone broken-SEO fixture worker (own deps/config), // like web/ it isn't part of the app's module graph. diff --git a/package.json b/package.json index fd80591..30955f1 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,6 @@ "better-auth": "^1.6.22", "cloudflare": "^5.2.0", "daisyui": "^5.5.5", - "dataforseo-client": "^2.0.19", "drizzle-orm": "^0.45.2", "fast-xml-parser": "^5.4.1", "htmlparser2": "^10.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1aff209..843b054 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -93,9 +93,6 @@ importers: daisyui: specifier: ^5.5.5 version: 5.5.19 - dataforseo-client: - specifier: ^2.0.19 - version: 2.0.19 drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@cloudflare/workers-types@4.20260702.1)(@libsql/client@0.15.15)(@opentelemetry/api@1.9.1)(kysely@0.29.2)(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(postgres@3.4.9)(sql.js@1.14.1) @@ -3607,9 +3604,6 @@ packages: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} - dataforseo-client@2.0.19: - resolution: {integrity: sha512-G1E1xI/EUxHkq1wMIZZ1eiZb7WanG1I9JKSXbIcdg7xuD8lLT0mXABrqHH+yvnjL439+GnrquD+ealzxm6uezw==} - dayjs@1.11.21: resolution: {integrity: sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==} @@ -9191,8 +9185,6 @@ snapshots: data-uri-to-buffer@4.0.1: {} - dataforseo-client@2.0.19: {} - dayjs@1.11.21: {} debug@4.4.3: diff --git a/scripts/dataforseo-account-usage.ts b/scripts/dataforseo-account-usage.ts index b6fc1da..01abf35 100644 --- a/scripts/dataforseo-account-usage.ts +++ b/scripts/dataforseo-account-usage.ts @@ -1,5 +1,4 @@ import process from "node:process"; -import type { AppendixStatisticsRatesDataInfo } from "dataforseo-client"; import { fetchUserData } from "@/server/lib/dataforseo/appendix"; import { loadLocalEnv, parseArgs } from "./cli-utils"; @@ -52,7 +51,7 @@ async function main() { } // DataForSEO groups spend under `total_` keys on each statistics -// window. Field names mirror the SDK's AppendixStatisticsRatesDataInfo. +// window. const FUNCTION_TOTALS: ReadonlyArray<{ label: string; key: string }> = [ { label: "serp", key: "total_serp" }, { label: "keywords_data", key: "total_keywords_data" }, @@ -70,7 +69,7 @@ const FUNCTION_TOTALS: ReadonlyArray<{ label: string; key: string }> = [ function printFunctionTable( heading: string, - stats: AppendixStatisticsRatesDataInfo | undefined, + stats: Record | null | undefined, ) { console.log(""); console.log(heading); @@ -104,7 +103,7 @@ function readNumber(value: unknown): number { return typeof value === "number" && Number.isFinite(value) ? value : 0; } -function formatUsd(value: number | undefined): string { +function formatUsd(value: number | null | undefined): string { if (typeof value !== "number" || !Number.isFinite(value)) return "$0.00"; return `$${value.toFixed(2)}`; } diff --git a/src/server/features/keywords/services/research/research-data.test.ts b/src/server/features/keywords/services/research/research-data.test.ts index 877d174..4da634f 100644 --- a/src/server/features/keywords/services/research/research-data.test.ts +++ b/src/server/features/keywords/services/research/research-data.test.ts @@ -4,35 +4,20 @@ vi.mock("@/server/lib/dataforseo", () => ({ createDataforseoClient: vi.fn(), })); -import { - KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo, - MonthlySearchesInfo, -} from "dataforseo-client"; +import type { AdsKeywordIdeaItem } from "@/server/lib/dataforseo"; import { mapAdsKeywordItems } from "./research-data"; -const adsItem = ( - data: ConstructorParameters< - typeof KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo - >[0], -) => new KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo(data); - describe("mapAdsKeywordItems", () => { it("maps Google Ads items to research rows without KD/intent", () => { const rows = mapAdsKeywordItems([ - adsItem({ + { keyword: "Hotel Reykjavik", search_volume: 1300, cpc: 2.54, competition: "HIGH", competition_index: 42, - monthly_searches: [ - new MonthlySearchesInfo({ - year: 2026, - month: 5, - search_volume: 1300, - }), - ], - }), + monthly_searches: [{ year: 2026, month: 5, search_volume: 1300 }], + }, ]); expect(rows).toEqual([ @@ -49,11 +34,12 @@ describe("mapAdsKeywordItems", () => { }); it("dedupes case-variant keywords and skips empty ones", () => { - const rows = mapAdsKeywordItems([ - adsItem({ keyword: "northern lights tour", search_volume: 320 }), - adsItem({ keyword: "Northern Lights Tour", search_volume: 320 }), - adsItem({ keyword: undefined }), - ]); + const items: AdsKeywordIdeaItem[] = [ + { keyword: "northern lights tour", search_volume: 320 }, + { keyword: "Northern Lights Tour", search_volume: 320 }, + { keyword: undefined }, + ]; + const rows = mapAdsKeywordItems(items); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ diff --git a/src/server/lib/dataforseo/ai.ts b/src/server/lib/dataforseo/ai.ts index 21675ea..8869896 100644 --- a/src/server/lib/dataforseo/ai.ts +++ b/src/server/lib/dataforseo/ai.ts @@ -1,18 +1,4 @@ import { z } from "zod"; -import { - AiOptimizationChatGptLlmResponsesLiveRequestInfo, - AiOptimizationClaudeLlmResponsesLiveRequestInfo, - AiOptimizationGeminiLlmResponsesLiveRequestInfo, - AiOptimizationLLmMentionsCrossAggregateMetricsTargetInfo, - AiOptimizationLLmMentionsDomainElement, - AiOptimizationLLmMentionsKeywordElement, - AiOptimizationLlmMentionsAggregatedMetricsLiveRequestInfo, - AiOptimizationLlmMentionsCrossAggregatedMetricsLiveRequestInfo, - AiOptimizationLlmMentionsSearchLiveRequestInfo, - AiOptimizationLlmMentionsTopPagesLiveRequestInfo, - type BaseAiOptimizationLLmMentionsTargetElement, - type AiOptimizationPerplexityLlmResponsesLiveRequestInfo, -} from "dataforseo-client"; import { llmAggregatedTotalSchema, llmCrossAggregatedItemSchema, @@ -27,7 +13,7 @@ import { } from "@/server/lib/dataforseoLlmSchemas"; import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification"; import { AppError } from "@/server/lib/errors"; -import { aiOptimizationApi } from "@/server/lib/dataforseo/core"; +import { dataforseoPost } from "@/server/lib/dataforseo/core"; import type { LlmPlatform, LlmTarget } from "@/server/lib/dataforseo/shared"; import { assertOk, @@ -51,14 +37,8 @@ function clampLimit(value: number, min: number, max: number): number { return Math.min(max, Math.max(min, Math.floor(value))); } -function targetList( - target: LlmTarget, -): BaseAiOptimizationLLmMentionsTargetElement[] { - return [ - "domain" in target - ? new AiOptimizationLLmMentionsDomainElement(target) - : new AiOptimizationLLmMentionsKeywordElement(target), - ]; +function targetList(target: LlmTarget): LlmTarget[] { + return [target]; } function firstResult(task: DataforseoTaskLike): Record | null { @@ -81,17 +61,19 @@ type LlmMentionsSearchInput = { export async function fetchLlmMentionsSearch( input: LlmMentionsSearchInput, ): Promise> { - const response = await aiOptimizationApi( - classifyAiSearchError, - ).llmMentionsSearchLive([ - new AiOptimizationLlmMentionsSearchLiveRequestInfo({ - target: targetList(input.target), - platform: input.platform, - location_code: input.locationCode, - language_code: input.languageCode, - limit: clampLimit(input.limit ?? 100, 1, 1000), - }), - ]); + const response = await dataforseoPost( + "/v3/ai_optimization/llm_mentions/search/live", + [ + { + target: targetList(input.target), + platform: input.platform, + location_code: input.locationCode, + language_code: input.languageCode, + limit: clampLimit(input.limit ?? 100, 1, 1000), + }, + ], + { classify: classifyAiSearchError }, + ); const task = assertOk( response, assertOptions("/v3/ai_optimization/llm_mentions/search/live"), @@ -124,17 +106,19 @@ type LlmAggregatedMetricsInput = { export async function fetchLlmAggregatedMetrics( input: LlmAggregatedMetricsInput, ): Promise> { - const response = await aiOptimizationApi( - classifyAiSearchError, - ).llmMentionsAggregatedMetricsLive([ - new AiOptimizationLlmMentionsAggregatedMetricsLiveRequestInfo({ - target: targetList(input.target), - platform: input.platform, - location_code: input.locationCode, - language_code: input.languageCode, - internal_list_limit: clampLimit(input.internalListLimit ?? 10, 1, 20), - }), - ]); + const response = await dataforseoPost( + "/v3/ai_optimization/llm_mentions/aggregated_metrics/live", + [ + { + target: targetList(input.target), + platform: input.platform, + location_code: input.locationCode, + language_code: input.languageCode, + internal_list_limit: clampLimit(input.internalListLimit ?? 10, 1, 20), + }, + ], + { classify: classifyAiSearchError }, + ); const task = assertOk( response, assertOptions("/v3/ai_optimization/llm_mentions/aggregated_metrics/live"), @@ -167,19 +151,21 @@ type LlmTopPagesInput = { export async function fetchLlmTopPages( input: LlmTopPagesInput, ): Promise> { - const response = await aiOptimizationApi( - classifyAiSearchError, - ).llmMentionsTopPagesLive([ - new AiOptimizationLlmMentionsTopPagesLiveRequestInfo({ - target: targetList(input.target), - platform: input.platform, - location_code: input.locationCode, - language_code: input.languageCode, - links_scope: "sources", - items_list_limit: clampLimit(input.itemsListLimit ?? 10, 1, 10), - internal_list_limit: 5, - }), - ]); + const response = await dataforseoPost( + "/v3/ai_optimization/llm_mentions/top_pages/live", + [ + { + target: targetList(input.target), + platform: input.platform, + location_code: input.locationCode, + language_code: input.languageCode, + links_scope: "sources", + items_list_limit: clampLimit(input.itemsListLimit ?? 10, 1, 10), + internal_list_limit: 5, + }, + ], + { classify: classifyAiSearchError }, + ); const task = assertOk( response, assertOptions("/v3/ai_optimization/llm_mentions/top_pages/live"), @@ -221,23 +207,22 @@ export async function fetchLlmCrossAggregatedMetrics( ); } - const response = await aiOptimizationApi( - classifyAiSearchError, - ).llmMentionsCrossAggregatedMetricsLive([ - new AiOptimizationLlmMentionsCrossAggregatedMetricsLiveRequestInfo({ - targets: input.groups.map( - (group) => - new AiOptimizationLLmMentionsCrossAggregateMetricsTargetInfo({ - aggregation_key: group.key, - target: targetList(group.target), - }), - ), - platform: input.platform, - location_code: input.locationCode, - language_code: input.languageCode, - internal_list_limit: clampLimit(input.internalListLimit ?? 5, 1, 10), - }), - ]); + const response = await dataforseoPost( + "/v3/ai_optimization/llm_mentions/cross_aggregated_metrics/live", + [ + { + targets: input.groups.map((group) => ({ + aggregation_key: group.key, + target: targetList(group.target), + })), + platform: input.platform, + location_code: input.locationCode, + language_code: input.languageCode, + internal_list_limit: clampLimit(input.internalListLimit ?? 5, 1, 10), + }, + ], + { classify: classifyAiSearchError }, + ); const task = assertOk( response, assertOptions( @@ -299,23 +284,6 @@ type LlmResponseRequestFields = { web_search_country_iso_code?: string; }; -function buildPerplexityLlmResponseRequest( - fields: LlmResponseRequestFields, -): AiOptimizationPerplexityLlmResponsesLiveRequestInfo { - return { - ...fields, - init(data?: unknown) { - if (isRecord(data)) Object.assign(this, data); - }, - toJSON(data?: unknown) { - return { - ...(isRecord(data) ? data : {}), - ...fields, - }; - }, - }; -} - export async function fetchLlmResponse( input: LlmResponsesInput, ): Promise> { @@ -341,25 +309,11 @@ export async function fetchLlmResponse( : {}), }; - const api = aiOptimizationApi(classifyAiSearchError); - const response = - input.modelSlug === "chat_gpt" - ? await api.chatGptLlmResponsesLive([ - new AiOptimizationChatGptLlmResponsesLiveRequestInfo(fields), - ]) - : input.modelSlug === "claude" - ? await api.claudeLlmResponsesLive([ - new AiOptimizationClaudeLlmResponsesLiveRequestInfo(fields), - ]) - : input.modelSlug === "gemini" - ? await api.geminiLlmResponsesLive([ - new AiOptimizationGeminiLlmResponsesLiveRequestInfo(fields), - ]) - : await api.perplexityLlmResponsesLive([ - // The generated Perplexity request class drops `web_search` in - // toJSON(), while the SDK method only JSON.stringify's this body. - buildPerplexityLlmResponseRequest(fields), - ]); + const response = await dataforseoPost( + `/v3/ai_optimization/${input.modelSlug}/llm_responses/live`, + [fields], + { classify: classifyAiSearchError }, + ); const task = assertOk( response, diff --git a/src/server/lib/dataforseo/appendix.ts b/src/server/lib/dataforseo/appendix.ts index fd26df3..ef932ae 100644 --- a/src/server/lib/dataforseo/appendix.ts +++ b/src/server/lib/dataforseo/appendix.ts @@ -1,6 +1,29 @@ -import type { AppendixUserDataResultInfo } from "dataforseo-client"; -import { appendixApi } from "@/server/lib/dataforseo/core"; -import { assertOk } from "@/server/lib/dataforseo/envelope"; +import { dataforseoGet } from "@/server/lib/dataforseo/core"; +import { + assertOk, + type DataforseoTaskLike, +} from "@/server/lib/dataforseo/envelope"; + +/** + * Account snapshot from the free GET /v3/appendix/user_data. Every field is + * optional on the wire; `money.statistics.day` / `.minute` group spend by + * function under `total_` keys, so those stay untyped records. + */ +interface DataforseoUserData { + login?: string | null; + timezone?: string | null; + money?: { + total?: number | null; + balance?: number | null; + statistics?: { + day?: Record | null; + minute?: Record | null; + [key: string]: unknown; + } | null; + [key: string]: unknown; + } | null; + [key: string]: unknown; +} /** * Reads account spend + balance from DataForSEO's free GET @@ -13,14 +36,11 @@ import { assertOk } from "@/server/lib/dataforseo/envelope"; * (remaining), and `money.statistics.day` / `.minute` — spend grouped by * function (serp, keywords_data, backlinks, dataforseo_labs, on_page, * business_data, …) for the rolling day / minute window. - * - * SDK types are loose (every field optional + index signatures), so callers - * must optional-chain into `.money.statistics.day`; it can be undefined. */ -export async function fetchUserData(): Promise< - AppendixUserDataResultInfo | undefined -> { - const response = await appendixApi().userData(); +export async function fetchUserData(): Promise { + const response = await dataforseoGet< + DataforseoTaskLike & { result?: DataforseoUserData[] } + >("/v3/appendix/user_data"); // Validates top-level + task status; the call is free so there is no billing // envelope to build. diff --git a/src/server/lib/dataforseo/backlinks.ts b/src/server/lib/dataforseo/backlinks.ts index 06708e1..a8001e7 100644 --- a/src/server/lib/dataforseo/backlinks.ts +++ b/src/server/lib/dataforseo/backlinks.ts @@ -1,18 +1,11 @@ import { z } from "zod"; -import { - BacklinksBacklinksLiveRequestInfo, - BacklinksDomainPagesSummaryLiveRequestInfo, - BacklinksHistoryLiveRequestInfo, - BacklinksReferringDomainsLiveRequestInfo, - BacklinksSummaryLiveRequestInfo, -} from "dataforseo-client"; import { normalizeBacklinksSpamFilterOptions, type BacklinksSpamFilterOptions, } from "@/types/schemas/backlinks"; import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification"; import { AppError } from "@/server/lib/errors"; -import { backlinksApi } from "@/server/lib/dataforseo/core"; +import { dataforseoPost } from "@/server/lib/dataforseo/core"; import { assertOk, buildTaskBilling, @@ -178,9 +171,11 @@ function combineFilters( } export async function fetchBacklinksSummary(input: BacklinksRequest) { - const response = await backlinksApi(classifyBacklinksError).summaryLive([ - new BacklinksSummaryLiveRequestInfo(buildCommonPayload(input)), - ]); + const response = await dataforseoPost( + "/v3/backlinks/summary/live", + [buildCommonPayload(input)], + { classify: classifyBacklinksError }, + ); const task = assertOk(response, assertOptions("/v3/backlinks/summary/live")); const firstResult = task.result?.[0]; @@ -216,16 +211,20 @@ export async function fetchBacklinksRows(input: BacklinksListRequest) { ? ["backlink_spam_score", "<=", spamFilterOptions.spamThreshold] : undefined, ); - const response = await backlinksApi(classifyBacklinksError).backlinksLive([ - new BacklinksBacklinksLiveRequestInfo({ - ...buildCommonPayload(input), - limit: input.limit ?? 100, - offset: input.offset, - order_by: input.orderBy ?? ["rank,desc"], - mode: input.mode, - ...(filters ? { filters } : {}), - }), - ]); + const response = await dataforseoPost( + "/v3/backlinks/backlinks/live", + [ + { + ...buildCommonPayload(input), + limit: input.limit ?? 100, + offset: input.offset, + order_by: input.orderBy ?? ["rank,desc"], + mode: input.mode, + ...(filters ? { filters } : {}), + }, + ], + { classify: classifyBacklinksError }, + ); const task = assertOk( response, assertOptions("/v3/backlinks/backlinks/live"), @@ -247,17 +246,19 @@ export async function fetchReferringDomains(input: BacklinksListRequest) { ? ["backlinks_spam_score", "<=", spamFilterOptions.spamThreshold] : undefined, ); - const response = await backlinksApi( - classifyBacklinksError, - ).referringDomainsLive([ - new BacklinksReferringDomainsLiveRequestInfo({ - ...buildCommonPayload(input), - limit: input.limit ?? 100, - offset: input.offset, - order_by: input.orderBy ?? ["backlinks,desc"], - ...(filters ? { filters } : {}), - }), - ]); + const response = await dataforseoPost( + "/v3/backlinks/referring_domains/live", + [ + { + ...buildCommonPayload(input), + limit: input.limit ?? 100, + offset: input.offset, + order_by: input.orderBy ?? ["backlinks,desc"], + ...(filters ? { filters } : {}), + }, + ], + { classify: classifyBacklinksError }, + ); const task = assertOk( response, assertOptions("/v3/backlinks/referring_domains/live"), @@ -278,17 +279,19 @@ export async function fetchReferringDomains(input: BacklinksListRequest) { export async function fetchDomainPagesSummary(input: BacklinksListRequest) { const filters = input.filters && input.filters.length > 0 ? input.filters : undefined; - const response = await backlinksApi( - classifyBacklinksError, - ).domainPagesSummaryLive([ - new BacklinksDomainPagesSummaryLiveRequestInfo({ - ...buildCommonPayload(input), - limit: input.limit ?? 100, - offset: input.offset, - order_by: input.orderBy ?? ["backlinks,desc"], - ...(filters ? { filters } : {}), - }), - ]); + const response = await dataforseoPost( + "/v3/backlinks/domain_pages_summary/live", + [ + { + ...buildCommonPayload(input), + limit: input.limit ?? 100, + offset: input.offset, + order_by: input.orderBy ?? ["backlinks,desc"], + ...(filters ? { filters } : {}), + }, + ], + { classify: classifyBacklinksError }, + ); const task = assertOk( response, assertOptions("/v3/backlinks/domain_pages_summary/live"), @@ -307,14 +310,18 @@ export async function fetchDomainPagesSummary(input: BacklinksListRequest) { } export async function fetchBacklinksHistory(input: BacklinksTimeseriesRequest) { - const response = await backlinksApi(classifyBacklinksError).historyLive([ - new BacklinksHistoryLiveRequestInfo({ - target: input.target, - date_from: input.dateFrom, - date_to: input.dateTo, - rank_scale: "one_hundred", - }), - ]); + const response = await dataforseoPost( + "/v3/backlinks/history/live", + [ + { + target: input.target, + date_from: input.dateFrom, + date_to: input.dateTo, + rank_scale: "one_hundred", + }, + ], + { classify: classifyBacklinksError }, + ); const task = assertOk(response, assertOptions("/v3/backlinks/history/live")); return { data: parseTaskItems( diff --git a/src/server/lib/dataforseo/business.ts b/src/server/lib/dataforseo/business.ts index de11515..1c90bf4 100644 --- a/src/server/lib/dataforseo/business.ts +++ b/src/server/lib/dataforseo/business.ts @@ -1,17 +1,5 @@ import { z } from "zod"; -import { - BusinessDataBusinessListingsSearchLiveRequestInfo, - BusinessDataGoogleExtendedReviewsTaskPostRequestInfo, - BusinessDataGoogleMyBusinessInfoLiveRequestInfo, - BusinessDataGoogleMyBusinessUpdatesTaskPostRequestInfo, - BusinessDataGoogleQuestionsAndAnswersLiveRequestInfo, - BusinessDataGoogleReviewsTaskPostRequestInfo, - type BusinessDataBusinessListingsSearchLiveItem, -} from "dataforseo-client"; -import { - businessDataApi, - businessDataTaskApi, -} from "@/server/lib/dataforseo/core"; +import { dataforseoGet, dataforseoPost } from "@/server/lib/dataforseo/core"; import { assertOk, buildTaskBilling, @@ -19,12 +7,19 @@ import { isRecord, isTaskInProgress, type DataforseoApiResponse, + type DataforseoItemsTask, type DataforseoResponseLike, type DataforseoTaskLike, } from "@/server/lib/dataforseo/envelope"; import { AppError } from "@/server/lib/errors"; -type BusinessListingItem = BusinessDataBusinessListingsSearchLiveItem; +// Consumers pick fields generically (pickRowFields), so listing rows stay an +// untyped record. +type BusinessListingItem = Record; + +// task_post creates a billed task. A 5xx does not prove the provider skipped +// the charge, so those posts must never be replayed. +const NO_RETRY = { maxServerErrorRetries: 0 } as const; /** * Location + language for the Google business_data endpoints. They accept @@ -53,8 +48,10 @@ export async function fetchBusinessListingsSearch(input: { limit: number; offset?: number; }): Promise> { - const response = await businessDataApi().businessListingsSearchLive([ - new BusinessDataBusinessListingsSearchLiveRequestInfo({ + const response = await dataforseoPost< + DataforseoItemsTask + >("/v3/business_data/business_listings/search/live", [ + { categories: input.categories, title: input.title, location_coordinate: input.locationCoordinate, @@ -63,7 +60,7 @@ export async function fetchBusinessListingsSearch(input: { order_by: input.orderBy, limit: input.limit, offset: input.offset, - }), + }, ]); // "No Search Results" (40501) is a valid empty result for obscure // businesses/keywords — DataForSEO still charges for it, so treat it as an @@ -106,14 +103,17 @@ export async function fetchQuestionsAnswers(input: { languageCode: string; depth: number; }): Promise[]>> { - const response = await businessDataApi().googleQuestionsAndAnswersLive([ - new BusinessDataGoogleQuestionsAndAnswersLiveRequestInfo({ - keyword: input.keyword, - location_coordinate: input.locationCoordinate, - language_code: input.languageCode, - depth: input.depth, - }), - ]); + const response = await dataforseoPost( + "/v3/business_data/google/questions_and_answers/live", + [ + { + keyword: input.keyword, + location_coordinate: input.locationCoordinate, + language_code: input.languageCode, + depth: input.depth, + }, + ], + ); // "No Search Results" (40501) is a valid empty result for obscure // businesses/keywords — DataForSEO still charges for it, so treat it as an // empty success instead of surfacing a charged-task error to the user. @@ -127,13 +127,16 @@ export async function fetchQuestionsAnswers(input: { export async function fetchMyBusinessInfo( input: { keyword: string } & BusinessLocationInput, ): Promise | null>> { - const response = await businessDataApi().googleMyBusinessInfoLive([ - new BusinessDataGoogleMyBusinessInfoLiveRequestInfo({ - keyword: input.keyword, - ...locationParams(input), - language_code: input.languageCode, - }), - ]); + const response = await dataforseoPost>( + "/v3/business_data/google/my_business_info/live", + [ + { + keyword: input.keyword, + ...locationParams(input), + language_code: input.languageCode, + }, + ], + ); // 40501 = billed empty result: a business Google has no profile for. const task = assertOk(response, { treatNoResultsAsEmpty: true }); const entry = task.result?.[0]; @@ -197,32 +200,40 @@ export async function postGoogleReviewsTask( ): Promise> { if (input.includeOtherSources) { return postedTaskId( - await businessDataTaskApi().googleExtendedReviewsTaskPost([ - new BusinessDataGoogleExtendedReviewsTaskPostRequestInfo({ + await dataforseoPost( + "/v3/business_data/google/extended_reviews/task_post", + [ + { + keyword: input.keyword, + cid: input.cid, + place_id: input.placeId, + ...locationParams(input), + language_code: input.languageCode, + depth: input.depth, + priority: TASK_PRIORITY_HIGH, + }, + ], + NO_RETRY, + ), + ); + } + return postedTaskId( + await dataforseoPost( + "/v3/business_data/google/reviews/task_post", + [ + { keyword: input.keyword, cid: input.cid, place_id: input.placeId, ...locationParams(input), language_code: input.languageCode, depth: input.depth, + sort_by: input.sortBy, priority: TASK_PRIORITY_HIGH, - }), - ]), - ); - } - return postedTaskId( - await businessDataTaskApi().googleReviewsTaskPost([ - new BusinessDataGoogleReviewsTaskPostRequestInfo({ - keyword: input.keyword, - cid: input.cid, - place_id: input.placeId, - ...locationParams(input), - language_code: input.languageCode, - depth: input.depth, - sort_by: input.sortBy, - priority: TASK_PRIORITY_HIGH, - }), - ]), + }, + ], + NO_RETRY, + ), ); } @@ -230,15 +241,19 @@ export async function postMyBusinessUpdatesTask( input: { keyword: string; depth: number } & BusinessLocationInput, ): Promise> { return postedTaskId( - await businessDataTaskApi().googleMyBusinessUpdatesTaskPost([ - new BusinessDataGoogleMyBusinessUpdatesTaskPostRequestInfo({ - keyword: input.keyword, - ...locationParams(input), - language_code: input.languageCode, - depth: input.depth, - priority: TASK_PRIORITY_HIGH, - }), - ]), + await dataforseoPost( + "/v3/business_data/google/my_business_updates/task_post", + [ + { + keyword: input.keyword, + ...locationParams(input), + language_code: input.languageCode, + depth: input.depth, + priority: TASK_PRIORITY_HIGH, + }, + ], + NO_RETRY, + ), ); } @@ -257,13 +272,9 @@ export async function fetchBusinessDataTaskResult(input: { endpoint: BusinessTaskEndpoint; taskId: string; }): Promise { - const api = businessDataApi(); - const response = - input.endpoint === "reviews" - ? await api.googleReviewsTaskGet(input.taskId) - : input.endpoint === "extended_reviews" - ? await api.googleExtendedReviewsTaskGet(input.taskId) - : await api.googleMyBusinessUpdatesTaskGet(input.taskId); + const response = await dataforseoGet( + `/v3/business_data/google/${input.endpoint}/task_get/${encodeURIComponent(input.taskId)}`, + ); const task = response?.tasks?.[0]; if (!response || response.status_code !== 20000 || !task) { @@ -306,7 +317,9 @@ type BusinessCategoryRow = { export async function fetchBusinessListingsCategories(): Promise< DataforseoApiResponse > { - const response = await businessDataApi().businessListingsCategories(); + const response = await dataforseoGet( + "/v3/business_data/business_listings/categories", + ); const task = assertOk(response); // This endpoint puts rows directly on `result` rather than `result[0].items`. const rows = (task.result ?? []).flatMap((entry) => { diff --git a/src/server/lib/dataforseo/client.test.ts b/src/server/lib/dataforseo/client.test.ts index 322ca7d..c8b22bf 100644 --- a/src/server/lib/dataforseo/client.test.ts +++ b/src/server/lib/dataforseo/client.test.ts @@ -74,6 +74,13 @@ vi.mock("@/server/lib/dataforseo/serp", () => ({ vi.mock("@/server/lib/dataforseo/business", () => ({ fetchBusinessListingsSearch: vi.fn(), fetchQuestionsAnswers: vi.fn(), + fetchMyBusinessInfo: vi.fn(), + postGoogleReviewsTask: vi.fn(), + postMyBusinessUpdatesTask: vi.fn(), +})); +vi.mock("@/server/lib/dataforseo/google-ads", () => ({ + fetchAdsKeywordIdeas: vi.fn(), + fetchAdsSearchVolume: vi.fn(), })); vi.mock("@/server/lib/dataforseo/backlinks", () => ({ fetchBacklinksSummary: vi.fn(), diff --git a/src/server/lib/dataforseo/client.ts b/src/server/lib/dataforseo/client.ts index 4a42276..e0dafa4 100644 --- a/src/server/lib/dataforseo/client.ts +++ b/src/server/lib/dataforseo/client.ts @@ -8,37 +8,62 @@ import { trackUsageCreditSpend, } from "@/server/billing/subscription"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -// Type-only namespace import: erased at compile, so the section modules (and -// the SDK they pull in) still only load through loadDataforseoSections below. -import type * as sections from "@/server/lib/dataforseo/sections"; import { DataforseoChargedTaskError, type DataforseoApiCallCost, type DataforseoApiResponse, } from "@/server/lib/dataforseo/envelope"; +import { + fetchBusinessListingsSearch, + fetchMyBusinessInfo, + fetchQuestionsAnswers, + postGoogleReviewsTask, + postMyBusinessUpdatesTask, +} from "@/server/lib/dataforseo/business"; +import { + fetchBacklinksHistory, + fetchBacklinksRows, + fetchBacklinksSummary, + fetchDomainPagesSummary, + fetchReferringDomains, +} from "@/server/lib/dataforseo/backlinks"; +import { + fetchDomainRankOverview, + fetchKeywordIdeas, + fetchKeywordOverview, + fetchKeywordSuggestions, + fetchRankedKeywords, + fetchRelatedKeywords, + fetchRelevantPages, + fetchSerpCompetitors, +} from "@/server/lib/dataforseo/labs"; +import { + fetchAdsKeywordIdeas, + fetchAdsSearchVolume, +} from "@/server/lib/dataforseo/google-ads"; +import { + fetchLiveSerp, + fetchLocalSerp, + fetchRankCheckSerp, + postRankCheckTasks, +} from "@/server/lib/dataforseo/serp"; +import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse"; +import { + fetchLlmAggregatedMetrics, + fetchLlmCrossAggregatedMetrics, + fetchLlmMentionsSearch, + fetchLlmResponse, + fetchLlmTopPages, +} from "@/server/lib/dataforseo/ai"; import { isHostedServerAuthMode } from "@/server/lib/runtime-env"; import { AppError } from "@/server/lib/errors"; export { mapDataforseoPathToCreditFeature }; -/** The section-fetcher barrel (sections.ts), as a type for `meter` pickers. */ -export type DataforseoSections = typeof sections; - -let sectionsPromise: Promise | undefined; - -/** Single lazy boundary for the DataForSEO subtree: the section fetchers and - * the ~3 MB dataforseo-client SDK they statically import stay out of the - * eager isolate startup graph and load once, on the first API call. */ -export function loadDataforseoSections(): Promise { - return (sectionsPromise ??= import("@/server/lib/dataforseo/sections")); -} - /** * Wraps a section fetcher with billing metering. Each entry on the client is - * `meter(customer, (s) => s.fetchX, defaultFeature?)`, which returns a function - * with the fetcher's own input type and resolves to its unwrapped `.data`. The - * picker indirection (rather than the fetcher itself) keeps the section - * modules behind loadDataforseoSections. + * `meter(customer, fetchX, defaultFeature?)`, which returns a function with + * the fetcher's own input type and resolves to its unwrapped `.data`. * * `defaultFeature` is the fallback credit feature; a caller can override it per * call by passing `creditFeature` in the input (e.g. an MCP tool attributing @@ -47,15 +72,13 @@ export function loadDataforseoSections(): Promise { */ function meter( customer: BillingCustomerContext, - pick: ( - sections: DataforseoSections, - ) => (input: I) => Promise>, + fetcher: (input: I) => Promise>, defaultFeature?: CreditFeature, ): (input: I & { creditFeature?: CreditFeature }) => Promise { return (input) => meterDataforseoCall( customer, - async () => pick(await loadDataforseoSections())(input), + () => fetcher(input), input.creditFeature ?? defaultFeature, ); } @@ -65,88 +88,61 @@ export function createDataforseoClient(customer: BillingCustomerContext) { business: { businessListings: meter( customer, - (s) => s.fetchBusinessListingsSearch, - "local_seo", - ), - questionsAnswers: meter( - customer, - (s) => s.fetchQuestionsAnswers, - "local_seo", - ), - myBusinessInfo: meter( - customer, - (s) => s.fetchMyBusinessInfo, + fetchBusinessListingsSearch, "local_seo", ), + questionsAnswers: meter(customer, fetchQuestionsAnswers, "local_seo"), + myBusinessInfo: meter(customer, fetchMyBusinessInfo, "local_seo"), // task_post is where DataForSEO charges; collection runs unmetered // through fetchBusinessDataTaskResult (see index.ts). - reviewsTaskPost: meter( - customer, - (s) => s.postGoogleReviewsTask, - "local_seo", - ), - updatesTaskPost: meter( - customer, - (s) => s.postMyBusinessUpdatesTask, - "local_seo", - ), + reviewsTaskPost: meter(customer, postGoogleReviewsTask, "local_seo"), + updatesTaskPost: meter(customer, postMyBusinessUpdatesTask, "local_seo"), }, backlinks: { - summary: meter(customer, (s) => s.fetchBacklinksSummary), - rows: meter(customer, (s) => s.fetchBacklinksRows), - referringDomains: meter(customer, (s) => s.fetchReferringDomains), - domainPages: meter(customer, (s) => s.fetchDomainPagesSummary), - history: meter(customer, (s) => s.fetchBacklinksHistory), + summary: meter(customer, fetchBacklinksSummary), + rows: meter(customer, fetchBacklinksRows), + referringDomains: meter(customer, fetchReferringDomains), + domainPages: meter(customer, fetchDomainPagesSummary), + history: meter(customer, fetchBacklinksHistory), }, keywords: { - related: meter(customer, (s) => s.fetchRelatedKeywords), - suggestions: meter(customer, (s) => s.fetchKeywordSuggestions), - ideas: meter(customer, (s) => s.fetchKeywordIdeas), + related: meter(customer, fetchRelatedKeywords), + suggestions: meter(customer, fetchKeywordSuggestions), + ideas: meter(customer, fetchKeywordIdeas), // Google Ads endpoints for countries Labs doesn't support. - adsIdeas: meter(customer, (s) => s.fetchAdsKeywordIdeas), - adsSearchVolume: meter(customer, (s) => s.fetchAdsSearchVolume), + adsIdeas: meter(customer, fetchAdsKeywordIdeas), + adsSearchVolume: meter(customer, fetchAdsSearchVolume), }, domain: { - rankOverview: meter(customer, (s) => s.fetchDomainRankOverview), - rankedKeywords: meter(customer, (s) => s.fetchRankedKeywords), - relevantPages: meter(customer, (s) => s.fetchRelevantPages), + rankOverview: meter(customer, fetchDomainRankOverview), + rankedKeywords: meter(customer, fetchRankedKeywords), + relevantPages: meter(customer, fetchRelevantPages), }, serp: { - live: meter(customer, (s) => s.fetchLiveSerp), - rankCheck: meter(customer, (s) => s.fetchRankCheckSerp, "rank_tracking"), + live: meter(customer, fetchLiveSerp), + rankCheck: meter(customer, fetchRankCheckSerp, "rank_tracking"), // Posts up to 100 queued rank check tasks; one metered charge covers the // whole batch (DataForSEO bills task_post at post time, collection is // free). - rankCheckTaskPost: meter( - customer, - (s) => s.postRankCheckTasks, - "rank_tracking", - ), - local: meter(customer, (s) => s.fetchLocalSerp, "local_seo"), + rankCheckTaskPost: meter(customer, postRankCheckTasks, "rank_tracking"), + local: meter(customer, fetchLocalSerp, "local_seo"), }, labs: { // Callers (e.g. the keyword-metrics MCP tool) can attribute the spend to // their own feature by passing `creditFeature` in the input; defaults to // rank_tracking when omitted. - keywordOverview: meter( - customer, - (s) => s.fetchKeywordOverview, - "rank_tracking", - ), - serpCompetitors: meter(customer, (s) => s.fetchSerpCompetitors), + keywordOverview: meter(customer, fetchKeywordOverview, "rank_tracking"), + serpCompetitors: meter(customer, fetchSerpCompetitors), }, lighthouse: { - live: meter(customer, (s) => s.fetchLighthouseResult), + live: meter(customer, fetchLighthouseResult), }, aiSearch: { - mentionsSearch: meter(customer, (s) => s.fetchLlmMentionsSearch), - aggregatedMetrics: meter(customer, (s) => s.fetchLlmAggregatedMetrics), - topPages: meter(customer, (s) => s.fetchLlmTopPages), - crossAggregatedMetrics: meter( - customer, - (s) => s.fetchLlmCrossAggregatedMetrics, - ), - llmResponse: meter(customer, (s) => s.fetchLlmResponse), + mentionsSearch: meter(customer, fetchLlmMentionsSearch), + aggregatedMetrics: meter(customer, fetchLlmAggregatedMetrics), + topPages: meter(customer, fetchLlmTopPages), + crossAggregatedMetrics: meter(customer, fetchLlmCrossAggregatedMetrics), + llmResponse: meter(customer, fetchLlmResponse), }, } as const; } diff --git a/src/server/lib/dataforseo/core.test.ts b/src/server/lib/dataforseo/core.test.ts index f6af940..5d231a1 100644 --- a/src/server/lib/dataforseo/core.test.ts +++ b/src/server/lib/dataforseo/core.test.ts @@ -1,25 +1,32 @@ import { afterEach, describe, expect, it, vi } from "vitest"; vi.mock("@/server/lib/runtime-env", () => ({ - getRequiredEnvValue: vi.fn().mockResolvedValue("encoded-credentials"), + getRequiredEnvValue: vi.fn(async () => "encoded-credentials"), })); -import { onPageApi } from "@/server/lib/dataforseo/core"; +import { dataforseoPost } from "@/server/lib/dataforseo/core"; afterEach(() => { vi.unstubAllGlobals(); }); -describe("DataForSEO OnPage transport", () => { - it("does not retry a Lighthouse HTTP 5xx response", async () => { +describe("DataForSEO transport", () => { + it("retries a transient 5xx on idempotent reads and returns the parsed envelope", async () => { const fetchMock = vi - .fn() - .mockResolvedValue(new Response("upstream failure", { status: 503 })); + .fn() + .mockResolvedValueOnce(new Response("upstream failure", { status: 503 })) + .mockResolvedValueOnce(Response.json({ status_code: 20000, tasks: [] })); vi.stubGlobal("fetch", fetchMock); - await expect(onPageApi().lighthouseLiveJson([])).rejects.toMatchObject({ - code: "UPSTREAM_UNAVAILABLE", - }); - expect(fetchMock).toHaveBeenCalledOnce(); + await expect( + dataforseoPost("/v3/backlinks/summary/live", []), + ).resolves.toEqual({ status_code: 20000, tasks: [] }); + expect(fetchMock).toHaveBeenCalledTimes(2); + + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe("https://api.dataforseo.com/v3/backlinks/summary/live"); + expect(new Headers(init?.headers).get("Authorization")).toBe( + "Basic encoded-credentials", + ); }); }); diff --git a/src/server/lib/dataforseo/core.ts b/src/server/lib/dataforseo/core.ts index 4d7271a..6aa38c9 100644 --- a/src/server/lib/dataforseo/core.ts +++ b/src/server/lib/dataforseo/core.ts @@ -1,16 +1,12 @@ -import { - AiOptimizationApi, - AppendixApi, - BacklinksApi, - BusinessDataApi, - DataforseoLabsApi, - KeywordsDataApi, - OnPageApi, - SerpApi, -} from "dataforseo-client"; import { AppError } from "@/server/lib/errors"; import { getRequiredEnvValue } from "@/server/lib/runtime-env"; import type { ErrorCode } from "@/shared/error-codes"; +// Type-only: erased at compile, so no runtime cycle with envelope.ts (which +// imports DataforseoErrorClassifier from here the same way). +import type { + DataforseoResponseLike, + DataforseoTaskLike, +} from "@/server/lib/dataforseo/envelope"; const API_BASE = "https://api.dataforseo.com"; const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600; @@ -60,10 +56,10 @@ function formatDataforseoRequestPath(url: RequestInfo): string { } /** - * The single authenticated `fetch` used by every DataForSEO SDK call. Throws on - * non-2xx so the SDK's own `ApiException` path never fires; task-level failures - * (which return HTTP 200) are handled downstream by {@link assertOk}. An - * optional classifier maps recognised HTTP failures to product errors. + * The single authenticated `fetch` used by every DataForSEO call. Throws on + * non-2xx; task-level failures (which return HTTP 200) are handled downstream + * by {@link assertOk}. An optional classifier maps recognised HTTP failures to + * product errors. */ function createAuthenticatedFetch( classify?: DataforseoErrorClassifier, @@ -119,30 +115,64 @@ function createAuthenticatedFetch( }; } -function http( - classify?: DataforseoErrorClassifier, - maxServerErrorRetries = DATAFORSEO_MAX_RETRIES, -) { - return { fetch: createAuthenticatedFetch(classify, maxServerErrorRetries) }; +type DataforseoRequestOptions = { + /** Maps a recognised access / billing HTTP failure to a product error. */ + classify?: DataforseoErrorClassifier; + /** + * Set 0 for billed, non-idempotent calls (business task_post, Lighthouse): + * a 5xx does not prove the provider skipped the charge, so those must never + * be replayed. Defaults to retrying idempotent reads on transient 5xx. + */ + maxServerErrorRetries?: number; +}; + +async function requestDataforseo( + method: "GET" | "POST", + path: string, + body: unknown, + options: DataforseoRequestOptions, +): Promise | null> { + const doFetch = createAuthenticatedFetch( + options.classify, + options.maxServerErrorRetries, + ); + const response = await doFetch(`${API_BASE}${path}`, { + method, + headers: { + Accept: "application/json", + ...(method === "POST" ? { "Content-Type": "application/json" } : {}), + }, + body: method === "POST" ? JSON.stringify(body) : undefined, + }); + const text = await response.text(); + if (text === "") return null; + // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the task type is the caller's claim about the payload; billing metadata and item fields are validated downstream (envelope.ts + section Zod schemas) + return JSON.parse(text) as DataforseoResponseLike; } -// Per-section API factories. Each is created per-request so the auth secret is -// read lazily (it lives in the Worker env, not in module scope). -export const labsApi = () => new DataforseoLabsApi(API_BASE, http()); -export const keywordsDataApi = () => new KeywordsDataApi(API_BASE, http()); -export const serpApi = () => new SerpApi(API_BASE, http()); -export const businessDataApi = () => new BusinessDataApi(API_BASE, http()); -// task_post creates a billed task. A 5xx does not prove the provider skipped -// the charge, so this client must not replay it (same rule as Lighthouse). -export const businessDataTaskApi = () => - new BusinessDataApi(API_BASE, http(undefined, 0)); -// Lighthouse live is a billed, non-idempotent POST. A 5xx does not prove the -// provider skipped the charge, so this client must not replay it. -export const onPageApi = () => new OnPageApi(API_BASE, http(undefined, 0)); -// Account/appendix data (spend, balance, rates). userData() is FREE ($0) and -// read-only — do NOT wire it through metering. -export const appendixApi = () => new AppendixApi(API_BASE, http()); -export const backlinksApi = (classify?: DataforseoErrorClassifier) => - new BacklinksApi(API_BASE, http(classify)); -export const aiOptimizationApi = (classify?: DataforseoErrorClassifier) => - new AiOptimizationApi(API_BASE, http(classify)); +/** + * POST `tasks` (the standard array-of-task-payloads body) to a DataForSEO + * endpoint and return the parsed response envelope. The task type parameter is + * the caller's claim about the payload shape — fields we act on are validated + * downstream (billing metadata in envelope.ts, items via the section fetchers' + * Zod schemas). Auth is read per-call from the Worker env. + */ +export function dataforseoPost< + TTask extends DataforseoTaskLike = DataforseoTaskLike, +>( + path: string, + tasks: unknown[], + options: DataforseoRequestOptions = {}, +): Promise | null> { + return requestDataforseo("POST", path, tasks, options); +} + +/** GET a DataForSEO endpoint (task_get collection, appendix/locations data). */ +export function dataforseoGet< + TTask extends DataforseoTaskLike = DataforseoTaskLike, +>( + path: string, + options: DataforseoRequestOptions = {}, +): Promise | null> { + return requestDataforseo("GET", path, undefined, options); +} diff --git a/src/server/lib/dataforseo/envelope.ts b/src/server/lib/dataforseo/envelope.ts index 216e94c..e1ddf93 100644 --- a/src/server/lib/dataforseo/envelope.ts +++ b/src/server/lib/dataforseo/envelope.ts @@ -42,8 +42,8 @@ export class DataforseoChargedTaskError extends AppError { } } -// The SDK types cost / path / result_count as optional with no runtime -// validation, so this is the one guard that guarantees we can bill a call. +// cost / path / result_count arrive from the wire untyped and optional, so +// this is the one guard that guarantees we can bill a call. const billingMetadataSchema = z.object({ path: z.array(z.string()), cost: z.number(), @@ -67,6 +67,21 @@ export interface DataforseoResponseLike { [key: string]: unknown; } +/** `task.result[0]` entry carrying an `items` list — the common live-endpoint + * shape. The index signature covers per-endpoint extras (`check_url`, …). */ +export interface DataforseoItemsResult { + items?: TItem[] | null; + total_count?: number | null; + [key: string]: unknown; +} + +/** Task whose `result` entries follow the `items` shape. Item types are the + * caller's claim about the payload (as the SDK's were); fields we act on are + * Zod-validated by the section fetchers. */ +export interface DataforseoItemsTask extends DataforseoTaskLike { + result?: DataforseoItemsResult[]; +} + function tryBuildTaskBilling(task: unknown): DataforseoApiCallCost | null { const parsed = billingMetadataSchema.safeParse(task); if (!parsed.success) return null; diff --git a/src/server/lib/dataforseo/google-ads.ts b/src/server/lib/dataforseo/google-ads.ts index 7433796..fb3d19a 100644 --- a/src/server/lib/dataforseo/google-ads.ts +++ b/src/server/lib/dataforseo/google-ads.ts @@ -1,26 +1,31 @@ -import { - KeywordsDataGoogleAdsKeywordsForKeywordsLiveRequestInfo, - KeywordsDataGoogleAdsSearchVolumeLiveRequestInfo, - type KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo, - type KeywordsDataGoogleAdsSearchVolumeLiveResultInfo, -} from "dataforseo-client"; -import { keywordsDataApi } from "@/server/lib/dataforseo/core"; +import { dataforseoPost } from "@/server/lib/dataforseo/core"; +import type { LabsMonthlySearch } from "@/server/lib/dataforseo/labs"; import { assertOk, buildTaskBilling, type DataforseoApiResponse, + type DataforseoTaskLike, } from "@/server/lib/dataforseo/envelope"; // Google Ads keyword data for countries DataForSEO Labs doesn't cover (see // specs/0004-keyword-data-source-routing.md). Flat-priced per request; items // carry volume / CPC / competition but no keyword difficulty or intent. -export type AdsKeywordItem = KeywordsDataGoogleAdsSearchVolumeLiveResultInfo; -export type AdsKeywordIdeaItem = - KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo; +export interface AdsKeywordItem { + keyword?: string | null; + search_volume?: number | null; + cpc?: number | null; + /** "LOW" | "MEDIUM" | "HIGH" bucket (Labs reports a 0-1 ratio instead). */ + competition?: string | null; + /** 0-100 competition scale; the app stores a 0-1 ratio. */ + competition_index?: number | null; + monthly_searches?: LabsMonthlySearch[] | null; + [key: string]: unknown; +} +export type AdsKeywordIdeaItem = AdsKeywordItem; -type KeywordsDataResult = { result?: T[] }; +type KeywordsDataTask = DataforseoTaskLike & { result?: T[] }; -function taskItems(task: KeywordsDataResult): T[] { +function taskItems(task: KeywordsDataTask): T[] { // keywords_data tasks return keyword items directly in `result` (no nested // `items` wrapper like Labs). return task.result ?? []; @@ -40,13 +45,16 @@ export async function fetchAdsSearchVolume(input: { const locationParams = input.locationName ? { location_name: input.locationName } : { location_code: input.locationCode }; - const response = await keywordsDataApi().googleAdsSearchVolumeLive([ - new KeywordsDataGoogleAdsSearchVolumeLiveRequestInfo({ - keywords: input.keywords, - ...locationParams, - language_code: input.languageCode, - }), - ]); + const response = await dataforseoPost>( + "/v3/keywords_data/google_ads/search_volume/live", + [ + { + keywords: input.keywords, + ...locationParams, + language_code: input.languageCode, + }, + ], + ); const task = assertOk(response); return { data: taskItems(task), @@ -60,14 +68,17 @@ export async function fetchAdsKeywordIdeas(input: { languageCode: string; limit: number; }): Promise> { - const response = await keywordsDataApi().googleAdsKeywordsForKeywordsLive([ - new KeywordsDataGoogleAdsKeywordsForKeywordsLiveRequestInfo({ - keywords: [input.keyword], - location_code: input.locationCode, - language_code: input.languageCode, - sort_by: "search_volume", - }), - ]); + const response = await dataforseoPost>( + "/v3/keywords_data/google_ads/keywords_for_keywords/live", + [ + { + keywords: [input.keyword], + location_code: input.locationCode, + language_code: input.languageCode, + sort_by: "search_volume", + }, + ], + ); const task = assertOk(response); // The endpoint has no limit parameter (it can return thousands of // suggestions for one flat fee); truncate to what the caller asked for. diff --git a/src/server/lib/dataforseo/index.ts b/src/server/lib/dataforseo/index.ts index 70625cb..2123ad0 100644 --- a/src/server/lib/dataforseo/index.ts +++ b/src/server/lib/dataforseo/index.ts @@ -1,15 +1,7 @@ // Public surface of the DataForSEO integration. Internals live in the -// per-section files (labs / serp / business / backlinks / ai / lighthouse), -// which sit behind the single dynamic import in client.ts so the ~3 MB SDK -// loads lazily; everything funnels through envelope.ts (status + billing) and -// is metered in client.ts. Runtime values re-exported here must be SDK-free -// (shared.ts) or lazy — a static value re-export from a section file would -// drag the SDK back into the eager isolate startup graph. - -import { - loadDataforseoSections, - type DataforseoSections, -} from "@/server/lib/dataforseo/client"; +// per-section files (labs / serp / business / backlinks / ai / lighthouse); +// everything funnels through envelope.ts (status + billing) and is metered in +// client.ts. export { createDataforseoClient } from "@/server/lib/dataforseo/client"; @@ -28,26 +20,16 @@ export { export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget"; -/** Lazy wrappers for the section fetchers called outside the metered client. - * Task collection is free at DataForSEO (the task was charged at task_post), so - * routing these through the metering seam would charge the customer twice. */ -export const fetchRankCheckTaskResult: DataforseoSections["fetchRankCheckTaskResult"] = - async (input) => - (await loadDataforseoSections()).fetchRankCheckTaskResult(input); - -export const fetchBusinessDataTaskResult: DataforseoSections["fetchBusinessDataTaskResult"] = - async (input) => - (await loadDataforseoSections()).fetchBusinessDataTaskResult(input); - -/** Free ($0) at DataForSEO, so it skips the metered client entirely — a - * zero-credit org can still list categories. */ -export const fetchBusinessListingsCategories: DataforseoSections["fetchBusinessListingsCategories"] = - async () => - (await loadDataforseoSections()).fetchBusinessListingsCategories(); - -export type { - BusinessTaskEndpoint, - BusinessTaskOutcome, +// Section fetchers called outside the metered client. Task collection is free +// at DataForSEO (the task was charged at task_post), so routing these through +// the metering seam would charge the customer twice; business categories are +// free ($0), so a zero-credit org can still list them. +export { fetchRankCheckTaskResult } from "@/server/lib/dataforseo/serp"; +export { + fetchBusinessDataTaskResult, + fetchBusinessListingsCategories, + type BusinessTaskEndpoint, + type BusinessTaskOutcome, } from "@/server/lib/dataforseo/business"; export type { diff --git a/src/server/lib/dataforseo/labs.ts b/src/server/lib/dataforseo/labs.ts index 895527b..c3c5512 100644 --- a/src/server/lib/dataforseo/labs.ts +++ b/src/server/lib/dataforseo/labs.ts @@ -1,36 +1,84 @@ import { z } from "zod"; -import { - DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo, - DataforseoLabsGoogleKeywordIdeasLiveRequestInfo, - DataforseoLabsGoogleKeywordOverviewLiveRequestInfo, - DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo, - DataforseoLabsGoogleRankedKeywordsLiveRequestInfo, - DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo, - DataforseoLabsGoogleRelevantPagesLiveRequestInfo, - DataforseoLabsGoogleSerpCompetitorsLiveRequestInfo, - type DataforseoLabsDomainRankOverviewLiveItem, - type DataforseoLabsGoogleKeywordOverviewLiveItem, - type DataforseoLabsRelatedKeywordsLiveItem, - type DataforseoLabsRelevantPagesLiveItem, - type DataforseoLabsSerpCompetitorsLiveItem, - type KeywordDataInfo, -} from "dataforseo-client"; -import { labsApi } from "@/server/lib/dataforseo/core"; +import { dataforseoPost } from "@/server/lib/dataforseo/core"; import { assertOk, buildTaskBilling, parseTaskItems, type DataforseoApiResponse, + type DataforseoItemsTask, } from "@/server/lib/dataforseo/envelope"; -// SDK item models are 1:1 supersets of what we need, so we expose them directly -// under the names the rest of the app already uses (no hand-written Zod). -export type LabsKeywordDataItem = KeywordDataInfo; -type RelatedKeywordItem = DataforseoLabsRelatedKeywordsLiveItem; -type DomainMetricsItem = DataforseoLabsDomainRankOverviewLiveItem; -export type RelevantPagesItem = DataforseoLabsRelevantPagesLiveItem; -export type KeywordOverviewItem = DataforseoLabsGoogleKeywordOverviewLiveItem; -type SerpCompetitorItem = DataforseoLabsSerpCompetitorsLiveItem; +// Labs payload types: the fields the app reads, typed honestly (the wire nulls +// any of them); the index signature carries everything else through untyped, +// like the SDK's item models did. These are claims about the payload, not +// validation — fields that must hold get a Zod schema (see below). + +export interface LabsMonthlySearch { + year?: number | null; + month?: number | null; + search_volume?: number | null; + [key: string]: unknown; +} + +export interface LabsKeywordInfo { + search_volume?: number | null; + cpc?: number | null; + /** 0-1 paid-competition ratio (Google Ads reports a 0-100 index instead). */ + competition?: number | null; + competition_level?: string | null; + monthly_searches?: LabsMonthlySearch[] | null; + [key: string]: unknown; +} + +export interface LabsKeywordDataItem { + keyword?: string | null; + keyword_info?: LabsKeywordInfo | null; + keyword_info_normalized_with_clickstream?: LabsKeywordInfo | null; + keyword_properties?: { + keyword_difficulty?: number | null; + [key: string]: unknown; + } | null; + search_intent_info?: { + main_intent?: string | null; + [key: string]: unknown; + } | null; + [key: string]: unknown; +} + +/** keyword_overview items share the keyword-data field surface. */ +export type KeywordOverviewItem = LabsKeywordDataItem; + +/** related_keywords wraps the keyword payload one level deeper. */ +type RelatedKeywordItem = { + keyword_data?: LabsKeywordDataItem | null; + [key: string]: unknown; +}; + +type LabsMetricsBlock = { + organic?: { etv?: number | null; count?: number | null } | null; + [key: string]: unknown; +}; + +type DomainMetricsItem = { + metrics?: LabsMetricsBlock | null; + [key: string]: unknown; +}; + +export interface RelevantPagesItem { + page_address?: string | null; + metrics?: LabsMetricsBlock | null; + [key: string]: unknown; +} + +type SerpCompetitorItem = { + domain?: string | null; + avg_position?: number | null; + median_position?: number | null; + visibility?: number | null; + etv?: number | null; + keywords_count?: number | null; + [key: string]: unknown; +}; // Ranked keywords is the one Labs endpoint the SDK types loosely: its // `ranked_serp_element.serp_item` is the base element item, so the url / etv / @@ -104,8 +152,10 @@ export async function fetchRelatedKeywords(input: { depth?: number; includeClickstreamData?: boolean; }): Promise> { - const response = await labsApi().googleRelatedKeywordsLive([ - new DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo({ + const response = await dataforseoPost< + DataforseoItemsTask + >("/v3/dataforseo_labs/google/related_keywords/live", [ + { keyword: input.keyword, location_code: input.locationCode, language_code: input.languageCode, @@ -115,7 +165,7 @@ export async function fetchRelatedKeywords(input: { // opt-in — see specs/0004-keyword-data-source-routing.md. include_clickstream_data: input.includeClickstreamData ?? false, include_serp_info: false, - }), + }, ]); const task = assertOk(response); return { @@ -131,8 +181,10 @@ export async function fetchKeywordSuggestions(input: { limit: number; includeClickstreamData?: boolean; }): Promise> { - const response = await labsApi().googleKeywordSuggestionsLive([ - new DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo({ + const response = await dataforseoPost< + DataforseoItemsTask + >("/v3/dataforseo_labs/google/keyword_suggestions/live", [ + { keyword: input.keyword, location_code: input.locationCode, language_code: input.languageCode, @@ -142,7 +194,7 @@ export async function fetchKeywordSuggestions(input: { include_seed_keyword: true, ignore_synonyms: false, exact_match: false, - }), + }, ]); const task = assertOk(response); return { @@ -158,8 +210,10 @@ export async function fetchKeywordIdeas(input: { limit: number; includeClickstreamData?: boolean; }): Promise> { - const response = await labsApi().googleKeywordIdeasLive([ - new DataforseoLabsGoogleKeywordIdeasLiveRequestInfo({ + const response = await dataforseoPost< + DataforseoItemsTask + >("/v3/dataforseo_labs/google/keyword_ideas/live", [ + { keywords: [input.keyword], location_code: input.locationCode, language_code: input.languageCode, @@ -168,7 +222,7 @@ export async function fetchKeywordIdeas(input: { include_serp_info: false, ignore_synonyms: false, closely_variants: false, - }), + }, ]); const task = assertOk(response); return { @@ -182,14 +236,17 @@ export async function fetchDomainRankOverview(input: { locationCode: number; languageCode: string; }): Promise> { - const response = await labsApi().googleDomainRankOverviewLive([ - new DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo({ - target: input.target, - location_code: input.locationCode, - language_code: input.languageCode, - limit: 1, - }), - ]); + const response = await dataforseoPost>( + "/v3/dataforseo_labs/google/domain_rank_overview/live", + [ + { + target: input.target, + location_code: input.locationCode, + language_code: input.languageCode, + limit: 1, + }, + ], + ); const task = assertOk(response); return { data: task.result?.[0]?.items ?? [], @@ -215,18 +272,21 @@ export async function fetchRankedKeywords(input: { // Note: ranked_keywords has no include_subdomains parameter — a domain // target always covers the hostname plus its subdomains. Narrower scopes // are expressed through `filters` (see researchScopeFilters.ts). - const response = await labsApi().googleRankedKeywordsLive([ - new DataforseoLabsGoogleRankedKeywordsLiveRequestInfo({ - target: input.target, - location_code: input.locationCode, - language_code: input.languageCode, - limit: input.limit, - offset: input.offset, - order_by: input.orderBy, - filters: input.filters, - item_types: input.itemTypes, - }), - ]); + const response = await dataforseoPost>( + "/v3/dataforseo_labs/google/ranked_keywords/live", + [ + { + target: input.target, + location_code: input.locationCode, + language_code: input.languageCode, + limit: input.limit, + offset: input.offset, + order_by: input.orderBy, + filters: input.filters, + item_types: input.itemTypes, + }, + ], + ); const task = assertOk(response); return { data: { @@ -255,17 +315,20 @@ export async function fetchRelevantPages(input: { orderBy?: string[]; filters?: unknown[]; }): Promise> { - const response = await labsApi().googleRelevantPagesLive([ - new DataforseoLabsGoogleRelevantPagesLiveRequestInfo({ - target: input.target, - location_code: input.locationCode, - language_code: input.languageCode, - limit: input.limit, - offset: input.offset, - order_by: input.orderBy, - filters: input.filters, - }), - ]); + const response = await dataforseoPost>( + "/v3/dataforseo_labs/google/relevant_pages/live", + [ + { + target: input.target, + location_code: input.locationCode, + language_code: input.languageCode, + limit: input.limit, + offset: input.offset, + order_by: input.orderBy, + filters: input.filters, + }, + ], + ); const task = assertOk(response); return { data: { @@ -282,13 +345,15 @@ export async function fetchKeywordOverview(input: { languageCode: string; includeClickstreamData?: boolean; }): Promise> { - const response = await labsApi().googleKeywordOverviewLive([ - new DataforseoLabsGoogleKeywordOverviewLiveRequestInfo({ + const response = await dataforseoPost< + DataforseoItemsTask + >("/v3/dataforseo_labs/google/keyword_overview/live", [ + { keywords: input.keywords, location_code: input.locationCode, language_code: input.languageCode, include_clickstream_data: input.includeClickstreamData ?? false, - }), + }, ]); const task = assertOk(response); return { @@ -306,8 +371,10 @@ export async function fetchSerpCompetitors(input: { limit: number; offset?: number; }): Promise> { - const response = await labsApi().googleSerpCompetitorsLive([ - new DataforseoLabsGoogleSerpCompetitorsLiveRequestInfo({ + const response = await dataforseoPost< + DataforseoItemsTask + >("/v3/dataforseo_labs/google/serp_competitors/live", [ + { keywords: input.keywords, location_code: input.locationCode, language_code: input.languageCode, @@ -315,7 +382,7 @@ export async function fetchSerpCompetitors(input: { include_subdomains: input.includeSubdomains, limit: input.limit, offset: input.offset, - }), + }, ]); const task = assertOk(response); return { diff --git a/src/server/lib/dataforseo/lighthouse.test.ts b/src/server/lib/dataforseo/lighthouse.test.ts index 201faed..3a1ab29 100644 --- a/src/server/lib/dataforseo/lighthouse.test.ts +++ b/src/server/lib/dataforseo/lighthouse.test.ts @@ -1,51 +1,42 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -const { onPageApiMock, lighthouseLiveJson } = vi.hoisted(() => ({ - onPageApiMock: vi.fn(), - lighthouseLiveJson: vi.fn(), -})); - -vi.mock("dataforseo-client", () => ({ - OnPageLighthouseLiveJsonRequestInfo: class { - constructor(public input: unknown) {} - }, -})); - -vi.mock("@/server/lib/dataforseo/core", () => ({ - onPageApi: onPageApiMock, +vi.mock("@/server/lib/runtime-env", () => ({ + getRequiredEnvValue: vi.fn(async () => "test-api-key"), })); import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope"; import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse"; -beforeEach(() => { - vi.clearAllMocks(); - onPageApiMock.mockReturnValue({ lighthouseLiveJson }); +afterEach(() => { + vi.unstubAllGlobals(); }); describe("fetchLighthouseResult", () => { it("carries billing metadata when parsing fails after a billed success", async () => { - lighthouseLiveJson.mockResolvedValue({ - status_code: 20000, - status_message: "Ok.", - tasks: [ - { - id: "task-1", - status_code: 20000, - status_message: "Ok.", - path: ["v3", "on_page", "lighthouse", "live", "json"], - cost: 0.00425, - result: [ - { - requestedUrl: "https://example.com/", - finalUrl: "https://example.com/", - categories: {}, - audits: {}, - }, - ], - }, - ], - }); + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + status_code: 20000, + status_message: "Ok.", + tasks: [ + { + id: "task-1", + status_code: 20000, + status_message: "Ok.", + path: ["v3", "on_page", "lighthouse", "live", "json"], + cost: 0.00425, + result: [ + { + requestedUrl: "https://example.com/", + finalUrl: "https://example.com/", + categories: {}, + audits: {}, + }, + ], + }, + ], + }), + ); + vi.stubGlobal("fetch", fetchMock); const rejection = fetchLighthouseResult({ url: "https://example.com/", @@ -60,4 +51,19 @@ describe("fetchLighthouseResult", () => { }, }); }); + + it("does not retry an HTTP 5xx (the provider may have charged the task)", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(new Response("upstream failure", { status: 503 })); + vi.stubGlobal("fetch", fetchMock); + + await expect( + fetchLighthouseResult({ + url: "https://example.com/", + strategy: "mobile", + }), + ).rejects.toMatchObject({ code: "UPSTREAM_UNAVAILABLE" }); + expect(fetchMock).toHaveBeenCalledOnce(); + }); }); diff --git a/src/server/lib/dataforseo/lighthouse.ts b/src/server/lib/dataforseo/lighthouse.ts index 3181e83..9a92521 100644 --- a/src/server/lib/dataforseo/lighthouse.ts +++ b/src/server/lib/dataforseo/lighthouse.ts @@ -1,11 +1,10 @@ -import { OnPageLighthouseLiveJsonRequestInfo } from "dataforseo-client"; import { parseDataforseoLighthousePayload, requestCategories, type LighthouseStrategy, } from "@/server/lib/dataforseoLighthousePayload"; import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload"; -import { onPageApi } from "@/server/lib/dataforseo/core"; +import { dataforseoPost } from "@/server/lib/dataforseo/core"; import { assertOk, buildTaskBilling, @@ -17,13 +16,19 @@ export async function fetchLighthouseResult(input: { url: string; strategy: LighthouseStrategy; }): Promise> { - const response = await onPageApi().lighthouseLiveJson([ - new OnPageLighthouseLiveJsonRequestInfo({ - url: input.url, - for_mobile: input.strategy === "mobile", - categories: [...requestCategories], - }), - ]); + const response = await dataforseoPost( + "/v3/on_page/lighthouse/live/json", + [ + { + url: input.url, + for_mobile: input.strategy === "mobile", + categories: [...requestCategories], + }, + ], + // Billed, non-idempotent POST: a 5xx does not prove the provider skipped + // the charge, so never replay it. + { maxServerErrorRetries: 0 }, + ); // Build the metering envelope before parsing. The provider has already // charged a successful task, so a malformed payload must carry its billing diff --git a/src/server/lib/dataforseo/sections.ts b/src/server/lib/dataforseo/sections.ts deleted file mode 100644 index fe8846d..0000000 --- a/src/server/lib/dataforseo/sections.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Root of the lazily loaded DataForSEO subtree. The section fetchers — and -// the ~3 MB dataforseo-client SDK they statically import — are reached only -// through the single dynamic import in client.ts (loadDataforseoSections), so -// the whole subtree lands in one lazy chunk outside the eager isolate startup -// graph. Never import this barrel or a section file statically from eager -// server code; the leanWorkerBundle vite plugin fails the build if the SDK -// re-enters the eager graph. SDK-free values live in shared.ts instead. - -export { - fetchBusinessDataTaskResult, - fetchBusinessListingsCategories, - fetchBusinessListingsSearch, - fetchMyBusinessInfo, - fetchQuestionsAnswers, - postGoogleReviewsTask, - postMyBusinessUpdatesTask, -} from "@/server/lib/dataforseo/business"; - -export { - fetchBacklinksHistory, - fetchBacklinksRows, - fetchBacklinksSummary, - fetchDomainPagesSummary, - fetchReferringDomains, -} from "@/server/lib/dataforseo/backlinks"; - -export { - fetchDomainRankOverview, - fetchKeywordIdeas, - fetchKeywordOverview, - fetchKeywordSuggestions, - fetchRankedKeywords, - fetchRelatedKeywords, - fetchRelevantPages, - fetchSerpCompetitors, -} from "@/server/lib/dataforseo/labs"; - -export { - fetchAdsKeywordIdeas, - fetchAdsSearchVolume, -} from "@/server/lib/dataforseo/google-ads"; - -export { - fetchLiveSerp, - fetchLocalSerp, - fetchRankCheckSerp, - fetchRankCheckTaskResult, - postRankCheckTasks, -} from "@/server/lib/dataforseo/serp"; - -export { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse"; - -export { - fetchLlmAggregatedMetrics, - fetchLlmCrossAggregatedMetrics, - fetchLlmMentionsSearch, - fetchLlmResponse, - fetchLlmTopPages, -} from "@/server/lib/dataforseo/ai"; diff --git a/src/server/lib/dataforseo/serp-locations.ts b/src/server/lib/dataforseo/serp-locations.ts index 422dcb0..0bdcd61 100644 --- a/src/server/lib/dataforseo/serp-locations.ts +++ b/src/server/lib/dataforseo/serp-locations.ts @@ -1,6 +1,6 @@ import { env } from "cloudflare:workers"; import { z } from "zod"; -import { serpApi } from "@/server/lib/dataforseo/core"; +import { dataforseoGet } from "@/server/lib/dataforseo/core"; import { assertOk } from "@/server/lib/dataforseo/envelope"; import { formatLocationLabel } from "@/shared/keyword-locations"; @@ -95,7 +95,9 @@ function fillFromOrigin(iso: string): Promise { } async function fetchFromDataforseo(iso: string): Promise { - const response = await serpApi().googleLocationsCountry(iso); + const response = await dataforseoGet( + `/v3/serp/google/locations/${encodeURIComponent(iso)}`, + ); const task = assertOk(response); return (task.result ?? []) .map((item) => locationItemSchema.safeParse(item)) diff --git a/src/server/lib/dataforseo/serp.ts b/src/server/lib/dataforseo/serp.ts index 239b2c6..a89c146 100644 --- a/src/server/lib/dataforseo/serp.ts +++ b/src/server/lib/dataforseo/serp.ts @@ -1,12 +1,5 @@ import { z } from "zod"; -import { - SerpApiStopCrawlOnMatchInfo, - SerpGoogleLocalFinderLiveAdvancedRequestInfo, - SerpGoogleMapsLiveAdvancedRequestInfo, - SerpGoogleOrganicLiveAdvancedRequestInfo, - SerpGoogleOrganicTaskPostRequestInfo, -} from "dataforseo-client"; -import { serpApi } from "@/server/lib/dataforseo/core"; +import { dataforseoGet, dataforseoPost } from "@/server/lib/dataforseo/core"; import { MAX_TASKS_PER_POST } from "@/server/lib/dataforseo/shared"; import { assertOk, @@ -15,6 +8,8 @@ import { isTaskInProgress, parseTaskItems, type DataforseoApiResponse, + type DataforseoItemsTask, + type DataforseoTaskLike, } from "@/server/lib/dataforseo/envelope"; import { AppError } from "@/server/lib/errors"; @@ -34,10 +29,7 @@ function clampSerpDepth(depth: number): number { function stopCrawlOnTarget(targetDomain: string) { return { stop_crawl_on_match: [ - new SerpApiStopCrawlOnMatchInfo({ - match_value: targetDomain, - match_type: "with_subdomains", - }), + { match_value: targetDomain, match_type: "with_subdomains" }, ], find_targets_in: ["organic"], }; @@ -87,16 +79,19 @@ export async function fetchLiveSerp(input: { locationCode: number; languageCode: string; }): Promise> { - const response = await serpApi().googleOrganicLiveAdvanced([ - new SerpGoogleOrganicLiveAdvancedRequestInfo({ - keyword: input.keyword, - location_code: input.locationCode, - language_code: input.languageCode, - device: "desktop", - os: "windows", - depth: 100, - }), - ]); + const response = await dataforseoPost( + "/v3/serp/google/organic/live/advanced", + [ + { + keyword: input.keyword, + location_code: input.locationCode, + language_code: input.languageCode, + device: "desktop", + os: "windows", + depth: 100, + }, + ], + ); const task = assertOk(response); return { data: parseTaskItems( @@ -155,17 +150,20 @@ export async function fetchRankCheckSerp(input: { const locationParams = input.locationName ? { location_name: input.locationName } : { location_code: input.locationCode }; - const response = await serpApi().googleOrganicLiveAdvanced([ - new SerpGoogleOrganicLiveAdvancedRequestInfo({ - keyword: input.keyword, - ...locationParams, - language_code: input.languageCode, - device: input.device, - os: input.device === "desktop" ? "windows" : "android", - depth, - ...stopCrawlOnTarget(input.targetDomain), - }), - ]); + const response = await dataforseoPost( + "/v3/serp/google/organic/live/advanced", + [ + { + keyword: input.keyword, + ...locationParams, + language_code: input.languageCode, + device: input.device, + os: input.device === "desktop" ? "windows" : "android", + depth, + ...stopCrawlOnTarget(input.targetDomain), + }, + ], + ); // "No Search Results" (40501) is valid for obscure/new keywords — treat as an // empty result set rather than failing the whole rank-tracking run. @@ -217,26 +215,26 @@ export async function postRankCheckTasks(input: { const locationParams = input.locationName ? { location_name: input.locationName } : { location_code: input.locationCode }; - const response = await serpApi().googleOrganicTaskPost( - input.tasks.map( - (task) => - new SerpGoogleOrganicTaskPostRequestInfo({ - keyword: task.keyword, - ...locationParams, - language_code: input.languageCode, - device: task.device, - os: task.device === "desktop" ? "windows" : "android", - depth, - // Queued tasks are billed provisionally at full depth at post time; - // task_get later reports the reduced actual cost when the crawl - // stopped early. We meter customers on the post-time amount — - // collection-time metering is a possible future optimization. - ...stopCrawlOnTarget(input.targetDomain), - // Echoed back on the response entry and task_get; used to map a - // DataForSEO task id back to our keyword without relying on order. - tag: `${task.keywordId}:${task.device}`, - }), - ), + const response = await dataforseoPost< + DataforseoTaskLike & { id?: string; data?: Record } + >( + "/v3/serp/google/organic/task_post", + input.tasks.map((task) => ({ + keyword: task.keyword, + ...locationParams, + language_code: input.languageCode, + device: task.device, + os: task.device === "desktop" ? "windows" : "android", + depth, + // Queued tasks are billed provisionally at full depth at post time; + // task_get later reports the reduced actual cost when the crawl + // stopped early. We meter customers on the post-time amount — + // collection-time metering is a possible future optimization. + ...stopCrawlOnTarget(input.targetDomain), + // Echoed back on the response entry and task_get; used to map a + // DataForSEO task id back to our keyword without relying on order. + tag: `${task.keywordId}:${task.device}`, + })), ); if (!response || response.status_code !== 20000) { @@ -296,7 +294,9 @@ export async function fetchRankCheckTaskResult(input: { keyword: string; targetDomain: string; }): Promise { - const response = await serpApi().googleOrganicTaskGetAdvanced(input.taskId); + const response = await dataforseoGet( + `/v3/serp/google/organic/task_get/advanced/${encodeURIComponent(input.taskId)}`, + ); const task = response?.tasks?.[0]; if (!response || response.status_code !== 20000 || !task) { throw new AppError( @@ -344,11 +344,11 @@ export async function fetchLocalSerp(input: { }): Promise[]>> { const os = input.device === "desktop" ? "windows" : "android"; - // Maps and Local Finder return different SDK item models; both carry an index - // signature, so the typed items assign cleanly to the generic row shape. if (input.searchType === "maps") { - const response = await serpApi().googleMapsLiveAdvanced([ - new SerpGoogleMapsLiveAdvancedRequestInfo({ + const response = await dataforseoPost< + DataforseoItemsTask> + >("/v3/serp/google/maps/live/advanced", [ + { keyword: input.keyword, location_coordinate: input.locationCoordinate, language_code: input.languageCode, @@ -356,7 +356,7 @@ export async function fetchLocalSerp(input: { os, depth: input.depth, search_places: input.searchPlaces, - }), + }, ]); // 40501 = billed empty SERP; DataForSEO returns it for some coordinate-only // Maps and Local Finder queries (both paths below opt in). @@ -367,15 +367,17 @@ export async function fetchLocalSerp(input: { }; } - const response = await serpApi().googleLocalFinderLiveAdvanced([ - new SerpGoogleLocalFinderLiveAdvancedRequestInfo({ + const response = await dataforseoPost< + DataforseoItemsTask> + >("/v3/serp/google/local_finder/live/advanced", [ + { keyword: input.keyword, location_coordinate: input.locationCoordinate, language_code: input.languageCode, device: input.device, os, depth: input.depth, - }), + }, ]); const task = assertOk(response, { treatNoResultsAsEmpty: true }); return { diff --git a/src/server/lib/dataforseo/shared.ts b/src/server/lib/dataforseo/shared.ts index 63de3d4..79d9225 100644 --- a/src/server/lib/dataforseo/shared.ts +++ b/src/server/lib/dataforseo/shared.ts @@ -1,8 +1,6 @@ -// SDK-free constants and target builders shared between eager server code -// (features, workflows, MCP tools) and the lazily loaded section fetchers. -// Keep this module free of dataforseo-client and section-file imports — -// anything imported from here must be safe to evaluate in the eager isolate -// startup graph. +// Constants and target builders shared between server code (features, +// workflows, MCP tools) and the section fetchers. Keep this module free of +// section-file imports so both sides can import it without cycles. // ChatGPT mention/response data is only available for US/en per DataForSEO docs. export const CHATGPT_LOCATION_CODE = 2840; diff --git a/vite-plugin-lean-worker-bundle.ts b/vite-plugin-lean-worker-bundle.ts index 271fa8d..3b965aa 100644 --- a/vite-plugin-lean-worker-bundle.ts +++ b/vite-plugin-lean-worker-bundle.ts @@ -15,16 +15,9 @@ const WORKERS_AI_PROVIDER_STUB = fileURLToPath( * startup (production OOM bursts trace back to baseline heap, not leaks), so * each of these is either loaded lazily behind a dynamic import or stubbed * out. `generateBundle` below fails the build if one sneaks back in via a - * static import chain — e.g. an eager `import { fetchLiveSerp } from - * "@/server/lib/dataforseo/serp"` instead of going through the metered client. + * static import chain. */ const EAGER_DENYLIST: Array<{ pattern: RegExp; expected: string }> = [ - { - pattern: /node_modules\/dataforseo-client\//, - expected: - "lazy-loaded behind loadDataforseoSections() — eager code must go " + - "through the metered client or src/server/lib/dataforseo/shared.ts", - }, { pattern: /node_modules\/autumn-js\//, expected: