Replace dataforseo-client SDK with a thin fetch client (#532)

This commit is contained in:
Ben Senescu 2026-08-25 22:23:55 -04:00 committed by GitHub
parent ac7ebfe13a
commit 215ead8152
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 752 additions and 724 deletions

View File

@ -20,10 +20,6 @@
"src/db/pg/schema.ts", "src/db/pg/schema.ts",
// Standalone CLI/dev scripts, invoked via package.json scripts // Standalone CLI/dev scripts, invoked via package.json scripts
"scripts/**", "scripts/**",
// DataForSEO section barrel consumed via the dynamic import in
// client.ts (loadDataforseoSections) + property access, which knip
// can't trace
"src/server/lib/dataforseo/sections.ts",
], ],
// badseo/ is the standalone broken-SEO fixture worker (own deps/config), // badseo/ is the standalone broken-SEO fixture worker (own deps/config),
// like web/ it isn't part of the app's module graph. // like web/ it isn't part of the app's module graph.

View File

@ -98,7 +98,6 @@
"better-auth": "^1.6.22", "better-auth": "^1.6.22",
"cloudflare": "^5.2.0", "cloudflare": "^5.2.0",
"daisyui": "^5.5.5", "daisyui": "^5.5.5",
"dataforseo-client": "^2.0.19",
"drizzle-orm": "^0.45.2", "drizzle-orm": "^0.45.2",
"fast-xml-parser": "^5.4.1", "fast-xml-parser": "^5.4.1",
"htmlparser2": "^10.1.0", "htmlparser2": "^10.1.0",

8
pnpm-lock.yaml generated
View File

@ -93,9 +93,6 @@ importers:
daisyui: daisyui:
specifier: ^5.5.5 specifier: ^5.5.5
version: 5.5.19 version: 5.5.19
dataforseo-client:
specifier: ^2.0.19
version: 2.0.19
drizzle-orm: drizzle-orm:
specifier: ^0.45.2 specifier: ^0.45.2
version: 0.45.2(@cloudflare/workers-types@4.20260702.1)(@libsql/client@0.15.15)(@opentelemetry/api@1.9.1)(kysely@0.29.2)(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(postgres@3.4.9)(sql.js@1.14.1) version: 0.45.2(@cloudflare/workers-types@4.20260702.1)(@libsql/client@0.15.15)(@opentelemetry/api@1.9.1)(kysely@0.29.2)(mysql2@3.22.6(@types/node@22.19.11))(pg@8.22.0)(postgres@3.4.9)(sql.js@1.14.1)
@ -3607,9 +3604,6 @@ packages:
resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
engines: {node: '>= 12'} engines: {node: '>= 12'}
dataforseo-client@2.0.19:
resolution: {integrity: sha512-G1E1xI/EUxHkq1wMIZZ1eiZb7WanG1I9JKSXbIcdg7xuD8lLT0mXABrqHH+yvnjL439+GnrquD+ealzxm6uezw==}
dayjs@1.11.21: dayjs@1.11.21:
resolution: {integrity: sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==} resolution: {integrity: sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==}
@ -9191,8 +9185,6 @@ snapshots:
data-uri-to-buffer@4.0.1: {} data-uri-to-buffer@4.0.1: {}
dataforseo-client@2.0.19: {}
dayjs@1.11.21: {} dayjs@1.11.21: {}
debug@4.4.3: debug@4.4.3:

View File

@ -1,5 +1,4 @@
import process from "node:process"; import process from "node:process";
import type { AppendixStatisticsRatesDataInfo } from "dataforseo-client";
import { fetchUserData } from "@/server/lib/dataforseo/appendix"; import { fetchUserData } from "@/server/lib/dataforseo/appendix";
import { loadLocalEnv, parseArgs } from "./cli-utils"; import { loadLocalEnv, parseArgs } from "./cli-utils";
@ -52,7 +51,7 @@ async function main() {
} }
// DataForSEO groups spend under `total_<function>` keys on each statistics // DataForSEO groups spend under `total_<function>` keys on each statistics
// window. Field names mirror the SDK's AppendixStatisticsRatesDataInfo. // window.
const FUNCTION_TOTALS: ReadonlyArray<{ label: string; key: string }> = [ const FUNCTION_TOTALS: ReadonlyArray<{ label: string; key: string }> = [
{ label: "serp", key: "total_serp" }, { label: "serp", key: "total_serp" },
{ label: "keywords_data", key: "total_keywords_data" }, { label: "keywords_data", key: "total_keywords_data" },
@ -70,7 +69,7 @@ const FUNCTION_TOTALS: ReadonlyArray<{ label: string; key: string }> = [
function printFunctionTable( function printFunctionTable(
heading: string, heading: string,
stats: AppendixStatisticsRatesDataInfo | undefined, stats: Record<string, unknown> | null | undefined,
) { ) {
console.log(""); console.log("");
console.log(heading); console.log(heading);
@ -104,7 +103,7 @@ function readNumber(value: unknown): number {
return typeof value === "number" && Number.isFinite(value) ? value : 0; return typeof value === "number" && Number.isFinite(value) ? value : 0;
} }
function formatUsd(value: number | undefined): string { function formatUsd(value: number | null | undefined): string {
if (typeof value !== "number" || !Number.isFinite(value)) return "$0.00"; if (typeof value !== "number" || !Number.isFinite(value)) return "$0.00";
return `$${value.toFixed(2)}`; return `$${value.toFixed(2)}`;
} }

View File

@ -4,35 +4,20 @@ vi.mock("@/server/lib/dataforseo", () => ({
createDataforseoClient: vi.fn(), createDataforseoClient: vi.fn(),
})); }));
import { import type { AdsKeywordIdeaItem } from "@/server/lib/dataforseo";
KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo,
MonthlySearchesInfo,
} from "dataforseo-client";
import { mapAdsKeywordItems } from "./research-data"; import { mapAdsKeywordItems } from "./research-data";
const adsItem = (
data: ConstructorParameters<
typeof KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo
>[0],
) => new KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo(data);
describe("mapAdsKeywordItems", () => { describe("mapAdsKeywordItems", () => {
it("maps Google Ads items to research rows without KD/intent", () => { it("maps Google Ads items to research rows without KD/intent", () => {
const rows = mapAdsKeywordItems([ const rows = mapAdsKeywordItems([
adsItem({ {
keyword: "Hotel Reykjavik", keyword: "Hotel Reykjavik",
search_volume: 1300, search_volume: 1300,
cpc: 2.54, cpc: 2.54,
competition: "HIGH", competition: "HIGH",
competition_index: 42, competition_index: 42,
monthly_searches: [ monthly_searches: [{ year: 2026, month: 5, search_volume: 1300 }],
new MonthlySearchesInfo({ },
year: 2026,
month: 5,
search_volume: 1300,
}),
],
}),
]); ]);
expect(rows).toEqual([ expect(rows).toEqual([
@ -49,11 +34,12 @@ describe("mapAdsKeywordItems", () => {
}); });
it("dedupes case-variant keywords and skips empty ones", () => { it("dedupes case-variant keywords and skips empty ones", () => {
const rows = mapAdsKeywordItems([ const items: AdsKeywordIdeaItem[] = [
adsItem({ keyword: "northern lights tour", search_volume: 320 }), { keyword: "northern lights tour", search_volume: 320 },
adsItem({ keyword: "Northern Lights Tour", search_volume: 320 }), { keyword: "Northern Lights Tour", search_volume: 320 },
adsItem({ keyword: undefined }), { keyword: undefined },
]); ];
const rows = mapAdsKeywordItems(items);
expect(rows).toHaveLength(1); expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ expect(rows[0]).toMatchObject({

View File

@ -1,18 +1,4 @@
import { z } from "zod"; import { z } from "zod";
import {
AiOptimizationChatGptLlmResponsesLiveRequestInfo,
AiOptimizationClaudeLlmResponsesLiveRequestInfo,
AiOptimizationGeminiLlmResponsesLiveRequestInfo,
AiOptimizationLLmMentionsCrossAggregateMetricsTargetInfo,
AiOptimizationLLmMentionsDomainElement,
AiOptimizationLLmMentionsKeywordElement,
AiOptimizationLlmMentionsAggregatedMetricsLiveRequestInfo,
AiOptimizationLlmMentionsCrossAggregatedMetricsLiveRequestInfo,
AiOptimizationLlmMentionsSearchLiveRequestInfo,
AiOptimizationLlmMentionsTopPagesLiveRequestInfo,
type BaseAiOptimizationLLmMentionsTargetElement,
type AiOptimizationPerplexityLlmResponsesLiveRequestInfo,
} from "dataforseo-client";
import { import {
llmAggregatedTotalSchema, llmAggregatedTotalSchema,
llmCrossAggregatedItemSchema, llmCrossAggregatedItemSchema,
@ -27,7 +13,7 @@ import {
} from "@/server/lib/dataforseoLlmSchemas"; } from "@/server/lib/dataforseoLlmSchemas";
import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification"; import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification";
import { AppError } from "@/server/lib/errors"; import { AppError } from "@/server/lib/errors";
import { aiOptimizationApi } from "@/server/lib/dataforseo/core"; import { dataforseoPost } from "@/server/lib/dataforseo/core";
import type { LlmPlatform, LlmTarget } from "@/server/lib/dataforseo/shared"; import type { LlmPlatform, LlmTarget } from "@/server/lib/dataforseo/shared";
import { import {
assertOk, assertOk,
@ -51,14 +37,8 @@ function clampLimit(value: number, min: number, max: number): number {
return Math.min(max, Math.max(min, Math.floor(value))); return Math.min(max, Math.max(min, Math.floor(value)));
} }
function targetList( function targetList(target: LlmTarget): LlmTarget[] {
target: LlmTarget, return [target];
): BaseAiOptimizationLLmMentionsTargetElement[] {
return [
"domain" in target
? new AiOptimizationLLmMentionsDomainElement(target)
: new AiOptimizationLLmMentionsKeywordElement(target),
];
} }
function firstResult(task: DataforseoTaskLike): Record<string, unknown> | null { function firstResult(task: DataforseoTaskLike): Record<string, unknown> | null {
@ -81,17 +61,19 @@ type LlmMentionsSearchInput = {
export async function fetchLlmMentionsSearch( export async function fetchLlmMentionsSearch(
input: LlmMentionsSearchInput, input: LlmMentionsSearchInput,
): Promise<DataforseoApiResponse<LlmMentionItem[]>> { ): Promise<DataforseoApiResponse<LlmMentionItem[]>> {
const response = await aiOptimizationApi( const response = await dataforseoPost(
classifyAiSearchError, "/v3/ai_optimization/llm_mentions/search/live",
).llmMentionsSearchLive([ [
new AiOptimizationLlmMentionsSearchLiveRequestInfo({ {
target: targetList(input.target), target: targetList(input.target),
platform: input.platform, platform: input.platform,
location_code: input.locationCode, location_code: input.locationCode,
language_code: input.languageCode, language_code: input.languageCode,
limit: clampLimit(input.limit ?? 100, 1, 1000), limit: clampLimit(input.limit ?? 100, 1, 1000),
}), },
]); ],
{ classify: classifyAiSearchError },
);
const task = assertOk( const task = assertOk(
response, response,
assertOptions("/v3/ai_optimization/llm_mentions/search/live"), assertOptions("/v3/ai_optimization/llm_mentions/search/live"),
@ -124,17 +106,19 @@ type LlmAggregatedMetricsInput = {
export async function fetchLlmAggregatedMetrics( export async function fetchLlmAggregatedMetrics(
input: LlmAggregatedMetricsInput, input: LlmAggregatedMetricsInput,
): Promise<DataforseoApiResponse<LlmAggregatedTotal>> { ): Promise<DataforseoApiResponse<LlmAggregatedTotal>> {
const response = await aiOptimizationApi( const response = await dataforseoPost(
classifyAiSearchError, "/v3/ai_optimization/llm_mentions/aggregated_metrics/live",
).llmMentionsAggregatedMetricsLive([ [
new AiOptimizationLlmMentionsAggregatedMetricsLiveRequestInfo({ {
target: targetList(input.target), target: targetList(input.target),
platform: input.platform, platform: input.platform,
location_code: input.locationCode, location_code: input.locationCode,
language_code: input.languageCode, language_code: input.languageCode,
internal_list_limit: clampLimit(input.internalListLimit ?? 10, 1, 20), internal_list_limit: clampLimit(input.internalListLimit ?? 10, 1, 20),
}), },
]); ],
{ classify: classifyAiSearchError },
);
const task = assertOk( const task = assertOk(
response, response,
assertOptions("/v3/ai_optimization/llm_mentions/aggregated_metrics/live"), assertOptions("/v3/ai_optimization/llm_mentions/aggregated_metrics/live"),
@ -167,19 +151,21 @@ type LlmTopPagesInput = {
export async function fetchLlmTopPages( export async function fetchLlmTopPages(
input: LlmTopPagesInput, input: LlmTopPagesInput,
): Promise<DataforseoApiResponse<LlmTopPagesItem[]>> { ): Promise<DataforseoApiResponse<LlmTopPagesItem[]>> {
const response = await aiOptimizationApi( const response = await dataforseoPost(
classifyAiSearchError, "/v3/ai_optimization/llm_mentions/top_pages/live",
).llmMentionsTopPagesLive([ [
new AiOptimizationLlmMentionsTopPagesLiveRequestInfo({ {
target: targetList(input.target), target: targetList(input.target),
platform: input.platform, platform: input.platform,
location_code: input.locationCode, location_code: input.locationCode,
language_code: input.languageCode, language_code: input.languageCode,
links_scope: "sources", links_scope: "sources",
items_list_limit: clampLimit(input.itemsListLimit ?? 10, 1, 10), items_list_limit: clampLimit(input.itemsListLimit ?? 10, 1, 10),
internal_list_limit: 5, internal_list_limit: 5,
}), },
]); ],
{ classify: classifyAiSearchError },
);
const task = assertOk( const task = assertOk(
response, response,
assertOptions("/v3/ai_optimization/llm_mentions/top_pages/live"), assertOptions("/v3/ai_optimization/llm_mentions/top_pages/live"),
@ -221,23 +207,22 @@ export async function fetchLlmCrossAggregatedMetrics(
); );
} }
const response = await aiOptimizationApi( const response = await dataforseoPost(
classifyAiSearchError, "/v3/ai_optimization/llm_mentions/cross_aggregated_metrics/live",
).llmMentionsCrossAggregatedMetricsLive([ [
new AiOptimizationLlmMentionsCrossAggregatedMetricsLiveRequestInfo({ {
targets: input.groups.map( targets: input.groups.map((group) => ({
(group) => aggregation_key: group.key,
new AiOptimizationLLmMentionsCrossAggregateMetricsTargetInfo({ target: targetList(group.target),
aggregation_key: group.key, })),
target: targetList(group.target), platform: input.platform,
}), location_code: input.locationCode,
), language_code: input.languageCode,
platform: input.platform, internal_list_limit: clampLimit(input.internalListLimit ?? 5, 1, 10),
location_code: input.locationCode, },
language_code: input.languageCode, ],
internal_list_limit: clampLimit(input.internalListLimit ?? 5, 1, 10), { classify: classifyAiSearchError },
}), );
]);
const task = assertOk( const task = assertOk(
response, response,
assertOptions( assertOptions(
@ -299,23 +284,6 @@ type LlmResponseRequestFields = {
web_search_country_iso_code?: string; web_search_country_iso_code?: string;
}; };
function buildPerplexityLlmResponseRequest(
fields: LlmResponseRequestFields,
): AiOptimizationPerplexityLlmResponsesLiveRequestInfo {
return {
...fields,
init(data?: unknown) {
if (isRecord(data)) Object.assign(this, data);
},
toJSON(data?: unknown) {
return {
...(isRecord(data) ? data : {}),
...fields,
};
},
};
}
export async function fetchLlmResponse( export async function fetchLlmResponse(
input: LlmResponsesInput, input: LlmResponsesInput,
): Promise<DataforseoApiResponse<LlmResponseResult>> { ): Promise<DataforseoApiResponse<LlmResponseResult>> {
@ -341,25 +309,11 @@ export async function fetchLlmResponse(
: {}), : {}),
}; };
const api = aiOptimizationApi(classifyAiSearchError); const response = await dataforseoPost(
const response = `/v3/ai_optimization/${input.modelSlug}/llm_responses/live`,
input.modelSlug === "chat_gpt" [fields],
? await api.chatGptLlmResponsesLive([ { classify: classifyAiSearchError },
new AiOptimizationChatGptLlmResponsesLiveRequestInfo(fields), );
])
: input.modelSlug === "claude"
? await api.claudeLlmResponsesLive([
new AiOptimizationClaudeLlmResponsesLiveRequestInfo(fields),
])
: input.modelSlug === "gemini"
? await api.geminiLlmResponsesLive([
new AiOptimizationGeminiLlmResponsesLiveRequestInfo(fields),
])
: await api.perplexityLlmResponsesLive([
// The generated Perplexity request class drops `web_search` in
// toJSON(), while the SDK method only JSON.stringify's this body.
buildPerplexityLlmResponseRequest(fields),
]);
const task = assertOk( const task = assertOk(
response, response,

View File

@ -1,6 +1,29 @@
import type { AppendixUserDataResultInfo } from "dataforseo-client"; import { dataforseoGet } from "@/server/lib/dataforseo/core";
import { appendixApi } from "@/server/lib/dataforseo/core"; import {
import { assertOk } from "@/server/lib/dataforseo/envelope"; assertOk,
type DataforseoTaskLike,
} from "@/server/lib/dataforseo/envelope";
/**
* Account snapshot from the free GET /v3/appendix/user_data. Every field is
* optional on the wire; `money.statistics.day` / `.minute` group spend by
* function under `total_<function>` keys, so those stay untyped records.
*/
interface DataforseoUserData {
login?: string | null;
timezone?: string | null;
money?: {
total?: number | null;
balance?: number | null;
statistics?: {
day?: Record<string, unknown> | null;
minute?: Record<string, unknown> | null;
[key: string]: unknown;
} | null;
[key: string]: unknown;
} | null;
[key: string]: unknown;
}
/** /**
* Reads account spend + balance from DataForSEO's free GET * Reads account spend + balance from DataForSEO's free GET
@ -13,14 +36,11 @@ import { assertOk } from "@/server/lib/dataforseo/envelope";
* (remaining), and `money.statistics.day` / `.minute` spend grouped by * (remaining), and `money.statistics.day` / `.minute` spend grouped by
* function (serp, keywords_data, backlinks, dataforseo_labs, on_page, * function (serp, keywords_data, backlinks, dataforseo_labs, on_page,
* business_data, ) for the rolling day / minute window. * business_data, ) for the rolling day / minute window.
*
* SDK types are loose (every field optional + index signatures), so callers
* must optional-chain into `.money.statistics.day`; it can be undefined.
*/ */
export async function fetchUserData(): Promise< export async function fetchUserData(): Promise<DataforseoUserData | undefined> {
AppendixUserDataResultInfo | undefined const response = await dataforseoGet<
> { DataforseoTaskLike & { result?: DataforseoUserData[] }
const response = await appendixApi().userData(); >("/v3/appendix/user_data");
// Validates top-level + task status; the call is free so there is no billing // Validates top-level + task status; the call is free so there is no billing
// envelope to build. // envelope to build.

View File

@ -1,18 +1,11 @@
import { z } from "zod"; import { z } from "zod";
import {
BacklinksBacklinksLiveRequestInfo,
BacklinksDomainPagesSummaryLiveRequestInfo,
BacklinksHistoryLiveRequestInfo,
BacklinksReferringDomainsLiveRequestInfo,
BacklinksSummaryLiveRequestInfo,
} from "dataforseo-client";
import { import {
normalizeBacklinksSpamFilterOptions, normalizeBacklinksSpamFilterOptions,
type BacklinksSpamFilterOptions, type BacklinksSpamFilterOptions,
} from "@/types/schemas/backlinks"; } from "@/types/schemas/backlinks";
import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification"; import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification";
import { AppError } from "@/server/lib/errors"; import { AppError } from "@/server/lib/errors";
import { backlinksApi } from "@/server/lib/dataforseo/core"; import { dataforseoPost } from "@/server/lib/dataforseo/core";
import { import {
assertOk, assertOk,
buildTaskBilling, buildTaskBilling,
@ -178,9 +171,11 @@ function combineFilters(
} }
export async function fetchBacklinksSummary(input: BacklinksRequest) { export async function fetchBacklinksSummary(input: BacklinksRequest) {
const response = await backlinksApi(classifyBacklinksError).summaryLive([ const response = await dataforseoPost(
new BacklinksSummaryLiveRequestInfo(buildCommonPayload(input)), "/v3/backlinks/summary/live",
]); [buildCommonPayload(input)],
{ classify: classifyBacklinksError },
);
const task = assertOk(response, assertOptions("/v3/backlinks/summary/live")); const task = assertOk(response, assertOptions("/v3/backlinks/summary/live"));
const firstResult = task.result?.[0]; const firstResult = task.result?.[0];
@ -216,16 +211,20 @@ export async function fetchBacklinksRows(input: BacklinksListRequest) {
? ["backlink_spam_score", "<=", spamFilterOptions.spamThreshold] ? ["backlink_spam_score", "<=", spamFilterOptions.spamThreshold]
: undefined, : undefined,
); );
const response = await backlinksApi(classifyBacklinksError).backlinksLive([ const response = await dataforseoPost(
new BacklinksBacklinksLiveRequestInfo({ "/v3/backlinks/backlinks/live",
...buildCommonPayload(input), [
limit: input.limit ?? 100, {
offset: input.offset, ...buildCommonPayload(input),
order_by: input.orderBy ?? ["rank,desc"], limit: input.limit ?? 100,
mode: input.mode, offset: input.offset,
...(filters ? { filters } : {}), order_by: input.orderBy ?? ["rank,desc"],
}), mode: input.mode,
]); ...(filters ? { filters } : {}),
},
],
{ classify: classifyBacklinksError },
);
const task = assertOk( const task = assertOk(
response, response,
assertOptions("/v3/backlinks/backlinks/live"), assertOptions("/v3/backlinks/backlinks/live"),
@ -247,17 +246,19 @@ export async function fetchReferringDomains(input: BacklinksListRequest) {
? ["backlinks_spam_score", "<=", spamFilterOptions.spamThreshold] ? ["backlinks_spam_score", "<=", spamFilterOptions.spamThreshold]
: undefined, : undefined,
); );
const response = await backlinksApi( const response = await dataforseoPost(
classifyBacklinksError, "/v3/backlinks/referring_domains/live",
).referringDomainsLive([ [
new BacklinksReferringDomainsLiveRequestInfo({ {
...buildCommonPayload(input), ...buildCommonPayload(input),
limit: input.limit ?? 100, limit: input.limit ?? 100,
offset: input.offset, offset: input.offset,
order_by: input.orderBy ?? ["backlinks,desc"], order_by: input.orderBy ?? ["backlinks,desc"],
...(filters ? { filters } : {}), ...(filters ? { filters } : {}),
}), },
]); ],
{ classify: classifyBacklinksError },
);
const task = assertOk( const task = assertOk(
response, response,
assertOptions("/v3/backlinks/referring_domains/live"), assertOptions("/v3/backlinks/referring_domains/live"),
@ -278,17 +279,19 @@ export async function fetchReferringDomains(input: BacklinksListRequest) {
export async function fetchDomainPagesSummary(input: BacklinksListRequest) { export async function fetchDomainPagesSummary(input: BacklinksListRequest) {
const filters = const filters =
input.filters && input.filters.length > 0 ? input.filters : undefined; input.filters && input.filters.length > 0 ? input.filters : undefined;
const response = await backlinksApi( const response = await dataforseoPost(
classifyBacklinksError, "/v3/backlinks/domain_pages_summary/live",
).domainPagesSummaryLive([ [
new BacklinksDomainPagesSummaryLiveRequestInfo({ {
...buildCommonPayload(input), ...buildCommonPayload(input),
limit: input.limit ?? 100, limit: input.limit ?? 100,
offset: input.offset, offset: input.offset,
order_by: input.orderBy ?? ["backlinks,desc"], order_by: input.orderBy ?? ["backlinks,desc"],
...(filters ? { filters } : {}), ...(filters ? { filters } : {}),
}), },
]); ],
{ classify: classifyBacklinksError },
);
const task = assertOk( const task = assertOk(
response, response,
assertOptions("/v3/backlinks/domain_pages_summary/live"), assertOptions("/v3/backlinks/domain_pages_summary/live"),
@ -307,14 +310,18 @@ export async function fetchDomainPagesSummary(input: BacklinksListRequest) {
} }
export async function fetchBacklinksHistory(input: BacklinksTimeseriesRequest) { export async function fetchBacklinksHistory(input: BacklinksTimeseriesRequest) {
const response = await backlinksApi(classifyBacklinksError).historyLive([ const response = await dataforseoPost(
new BacklinksHistoryLiveRequestInfo({ "/v3/backlinks/history/live",
target: input.target, [
date_from: input.dateFrom, {
date_to: input.dateTo, target: input.target,
rank_scale: "one_hundred", date_from: input.dateFrom,
}), date_to: input.dateTo,
]); rank_scale: "one_hundred",
},
],
{ classify: classifyBacklinksError },
);
const task = assertOk(response, assertOptions("/v3/backlinks/history/live")); const task = assertOk(response, assertOptions("/v3/backlinks/history/live"));
return { return {
data: parseTaskItems( data: parseTaskItems(

View File

@ -1,17 +1,5 @@
import { z } from "zod"; import { z } from "zod";
import { import { dataforseoGet, dataforseoPost } from "@/server/lib/dataforseo/core";
BusinessDataBusinessListingsSearchLiveRequestInfo,
BusinessDataGoogleExtendedReviewsTaskPostRequestInfo,
BusinessDataGoogleMyBusinessInfoLiveRequestInfo,
BusinessDataGoogleMyBusinessUpdatesTaskPostRequestInfo,
BusinessDataGoogleQuestionsAndAnswersLiveRequestInfo,
BusinessDataGoogleReviewsTaskPostRequestInfo,
type BusinessDataBusinessListingsSearchLiveItem,
} from "dataforseo-client";
import {
businessDataApi,
businessDataTaskApi,
} from "@/server/lib/dataforseo/core";
import { import {
assertOk, assertOk,
buildTaskBilling, buildTaskBilling,
@ -19,12 +7,19 @@ import {
isRecord, isRecord,
isTaskInProgress, isTaskInProgress,
type DataforseoApiResponse, type DataforseoApiResponse,
type DataforseoItemsTask,
type DataforseoResponseLike, type DataforseoResponseLike,
type DataforseoTaskLike, type DataforseoTaskLike,
} from "@/server/lib/dataforseo/envelope"; } from "@/server/lib/dataforseo/envelope";
import { AppError } from "@/server/lib/errors"; import { AppError } from "@/server/lib/errors";
type BusinessListingItem = BusinessDataBusinessListingsSearchLiveItem; // Consumers pick fields generically (pickRowFields), so listing rows stay an
// untyped record.
type BusinessListingItem = Record<string, unknown>;
// task_post creates a billed task. A 5xx does not prove the provider skipped
// the charge, so those posts must never be replayed.
const NO_RETRY = { maxServerErrorRetries: 0 } as const;
/** /**
* Location + language for the Google business_data endpoints. They accept * Location + language for the Google business_data endpoints. They accept
@ -53,8 +48,10 @@ export async function fetchBusinessListingsSearch(input: {
limit: number; limit: number;
offset?: number; offset?: number;
}): Promise<DataforseoApiResponse<BusinessListingItem[]>> { }): Promise<DataforseoApiResponse<BusinessListingItem[]>> {
const response = await businessDataApi().businessListingsSearchLive([ const response = await dataforseoPost<
new BusinessDataBusinessListingsSearchLiveRequestInfo({ DataforseoItemsTask<BusinessListingItem>
>("/v3/business_data/business_listings/search/live", [
{
categories: input.categories, categories: input.categories,
title: input.title, title: input.title,
location_coordinate: input.locationCoordinate, location_coordinate: input.locationCoordinate,
@ -63,7 +60,7 @@ export async function fetchBusinessListingsSearch(input: {
order_by: input.orderBy, order_by: input.orderBy,
limit: input.limit, limit: input.limit,
offset: input.offset, offset: input.offset,
}), },
]); ]);
// "No Search Results" (40501) is a valid empty result for obscure // "No Search Results" (40501) is a valid empty result for obscure
// businesses/keywords — DataForSEO still charges for it, so treat it as an // businesses/keywords — DataForSEO still charges for it, so treat it as an
@ -106,14 +103,17 @@ export async function fetchQuestionsAnswers(input: {
languageCode: string; languageCode: string;
depth: number; depth: number;
}): Promise<DataforseoApiResponse<Record<string, unknown>[]>> { }): Promise<DataforseoApiResponse<Record<string, unknown>[]>> {
const response = await businessDataApi().googleQuestionsAndAnswersLive([ const response = await dataforseoPost(
new BusinessDataGoogleQuestionsAndAnswersLiveRequestInfo({ "/v3/business_data/google/questions_and_answers/live",
keyword: input.keyword, [
location_coordinate: input.locationCoordinate, {
language_code: input.languageCode, keyword: input.keyword,
depth: input.depth, location_coordinate: input.locationCoordinate,
}), language_code: input.languageCode,
]); depth: input.depth,
},
],
);
// "No Search Results" (40501) is a valid empty result for obscure // "No Search Results" (40501) is a valid empty result for obscure
// businesses/keywords — DataForSEO still charges for it, so treat it as an // businesses/keywords — DataForSEO still charges for it, so treat it as an
// empty success instead of surfacing a charged-task error to the user. // empty success instead of surfacing a charged-task error to the user.
@ -127,13 +127,16 @@ export async function fetchQuestionsAnswers(input: {
export async function fetchMyBusinessInfo( export async function fetchMyBusinessInfo(
input: { keyword: string } & BusinessLocationInput, input: { keyword: string } & BusinessLocationInput,
): Promise<DataforseoApiResponse<Record<string, unknown> | null>> { ): Promise<DataforseoApiResponse<Record<string, unknown> | null>> {
const response = await businessDataApi().googleMyBusinessInfoLive([ const response = await dataforseoPost<DataforseoItemsTask<unknown>>(
new BusinessDataGoogleMyBusinessInfoLiveRequestInfo({ "/v3/business_data/google/my_business_info/live",
keyword: input.keyword, [
...locationParams(input), {
language_code: input.languageCode, keyword: input.keyword,
}), ...locationParams(input),
]); language_code: input.languageCode,
},
],
);
// 40501 = billed empty result: a business Google has no profile for. // 40501 = billed empty result: a business Google has no profile for.
const task = assertOk(response, { treatNoResultsAsEmpty: true }); const task = assertOk(response, { treatNoResultsAsEmpty: true });
const entry = task.result?.[0]; const entry = task.result?.[0];
@ -197,32 +200,40 @@ export async function postGoogleReviewsTask(
): Promise<DataforseoApiResponse<string>> { ): Promise<DataforseoApiResponse<string>> {
if (input.includeOtherSources) { if (input.includeOtherSources) {
return postedTaskId( return postedTaskId(
await businessDataTaskApi().googleExtendedReviewsTaskPost([ await dataforseoPost<DataforseoTaskLike & { id?: string }>(
new BusinessDataGoogleExtendedReviewsTaskPostRequestInfo({ "/v3/business_data/google/extended_reviews/task_post",
[
{
keyword: input.keyword,
cid: input.cid,
place_id: input.placeId,
...locationParams(input),
language_code: input.languageCode,
depth: input.depth,
priority: TASK_PRIORITY_HIGH,
},
],
NO_RETRY,
),
);
}
return postedTaskId(
await dataforseoPost<DataforseoTaskLike & { id?: string }>(
"/v3/business_data/google/reviews/task_post",
[
{
keyword: input.keyword, keyword: input.keyword,
cid: input.cid, cid: input.cid,
place_id: input.placeId, place_id: input.placeId,
...locationParams(input), ...locationParams(input),
language_code: input.languageCode, language_code: input.languageCode,
depth: input.depth, depth: input.depth,
sort_by: input.sortBy,
priority: TASK_PRIORITY_HIGH, priority: TASK_PRIORITY_HIGH,
}), },
]), ],
); NO_RETRY,
} ),
return postedTaskId(
await businessDataTaskApi().googleReviewsTaskPost([
new BusinessDataGoogleReviewsTaskPostRequestInfo({
keyword: input.keyword,
cid: input.cid,
place_id: input.placeId,
...locationParams(input),
language_code: input.languageCode,
depth: input.depth,
sort_by: input.sortBy,
priority: TASK_PRIORITY_HIGH,
}),
]),
); );
} }
@ -230,15 +241,19 @@ export async function postMyBusinessUpdatesTask(
input: { keyword: string; depth: number } & BusinessLocationInput, input: { keyword: string; depth: number } & BusinessLocationInput,
): Promise<DataforseoApiResponse<string>> { ): Promise<DataforseoApiResponse<string>> {
return postedTaskId( return postedTaskId(
await businessDataTaskApi().googleMyBusinessUpdatesTaskPost([ await dataforseoPost<DataforseoTaskLike & { id?: string }>(
new BusinessDataGoogleMyBusinessUpdatesTaskPostRequestInfo({ "/v3/business_data/google/my_business_updates/task_post",
keyword: input.keyword, [
...locationParams(input), {
language_code: input.languageCode, keyword: input.keyword,
depth: input.depth, ...locationParams(input),
priority: TASK_PRIORITY_HIGH, language_code: input.languageCode,
}), depth: input.depth,
]), priority: TASK_PRIORITY_HIGH,
},
],
NO_RETRY,
),
); );
} }
@ -257,13 +272,9 @@ export async function fetchBusinessDataTaskResult(input: {
endpoint: BusinessTaskEndpoint; endpoint: BusinessTaskEndpoint;
taskId: string; taskId: string;
}): Promise<BusinessTaskOutcome> { }): Promise<BusinessTaskOutcome> {
const api = businessDataApi(); const response = await dataforseoGet(
const response = `/v3/business_data/google/${input.endpoint}/task_get/${encodeURIComponent(input.taskId)}`,
input.endpoint === "reviews" );
? await api.googleReviewsTaskGet(input.taskId)
: input.endpoint === "extended_reviews"
? await api.googleExtendedReviewsTaskGet(input.taskId)
: await api.googleMyBusinessUpdatesTaskGet(input.taskId);
const task = response?.tasks?.[0]; const task = response?.tasks?.[0];
if (!response || response.status_code !== 20000 || !task) { if (!response || response.status_code !== 20000 || !task) {
@ -306,7 +317,9 @@ type BusinessCategoryRow = {
export async function fetchBusinessListingsCategories(): Promise< export async function fetchBusinessListingsCategories(): Promise<
DataforseoApiResponse<BusinessCategoryRow[]> DataforseoApiResponse<BusinessCategoryRow[]>
> { > {
const response = await businessDataApi().businessListingsCategories(); const response = await dataforseoGet(
"/v3/business_data/business_listings/categories",
);
const task = assertOk(response); const task = assertOk(response);
// This endpoint puts rows directly on `result` rather than `result[0].items`. // This endpoint puts rows directly on `result` rather than `result[0].items`.
const rows = (task.result ?? []).flatMap((entry) => { const rows = (task.result ?? []).flatMap((entry) => {

View File

@ -74,6 +74,13 @@ vi.mock("@/server/lib/dataforseo/serp", () => ({
vi.mock("@/server/lib/dataforseo/business", () => ({ vi.mock("@/server/lib/dataforseo/business", () => ({
fetchBusinessListingsSearch: vi.fn(), fetchBusinessListingsSearch: vi.fn(),
fetchQuestionsAnswers: vi.fn(), fetchQuestionsAnswers: vi.fn(),
fetchMyBusinessInfo: vi.fn(),
postGoogleReviewsTask: vi.fn(),
postMyBusinessUpdatesTask: vi.fn(),
}));
vi.mock("@/server/lib/dataforseo/google-ads", () => ({
fetchAdsKeywordIdeas: vi.fn(),
fetchAdsSearchVolume: vi.fn(),
})); }));
vi.mock("@/server/lib/dataforseo/backlinks", () => ({ vi.mock("@/server/lib/dataforseo/backlinks", () => ({
fetchBacklinksSummary: vi.fn(), fetchBacklinksSummary: vi.fn(),

View File

@ -8,37 +8,62 @@ import {
trackUsageCreditSpend, trackUsageCreditSpend,
} from "@/server/billing/subscription"; } from "@/server/billing/subscription";
import type { BillingCustomerContext } from "@/server/billing/subscription"; import type { BillingCustomerContext } from "@/server/billing/subscription";
// Type-only namespace import: erased at compile, so the section modules (and
// the SDK they pull in) still only load through loadDataforseoSections below.
import type * as sections from "@/server/lib/dataforseo/sections";
import { import {
DataforseoChargedTaskError, DataforseoChargedTaskError,
type DataforseoApiCallCost, type DataforseoApiCallCost,
type DataforseoApiResponse, type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope"; } from "@/server/lib/dataforseo/envelope";
import {
fetchBusinessListingsSearch,
fetchMyBusinessInfo,
fetchQuestionsAnswers,
postGoogleReviewsTask,
postMyBusinessUpdatesTask,
} from "@/server/lib/dataforseo/business";
import {
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
fetchDomainPagesSummary,
fetchReferringDomains,
} from "@/server/lib/dataforseo/backlinks";
import {
fetchDomainRankOverview,
fetchKeywordIdeas,
fetchKeywordOverview,
fetchKeywordSuggestions,
fetchRankedKeywords,
fetchRelatedKeywords,
fetchRelevantPages,
fetchSerpCompetitors,
} from "@/server/lib/dataforseo/labs";
import {
fetchAdsKeywordIdeas,
fetchAdsSearchVolume,
} from "@/server/lib/dataforseo/google-ads";
import {
fetchLiveSerp,
fetchLocalSerp,
fetchRankCheckSerp,
postRankCheckTasks,
} from "@/server/lib/dataforseo/serp";
import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
import {
fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";
import { isHostedServerAuthMode } from "@/server/lib/runtime-env"; import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
import { AppError } from "@/server/lib/errors"; import { AppError } from "@/server/lib/errors";
export { mapDataforseoPathToCreditFeature }; export { mapDataforseoPathToCreditFeature };
/** The section-fetcher barrel (sections.ts), as a type for `meter` pickers. */
export type DataforseoSections = typeof sections;
let sectionsPromise: Promise<DataforseoSections> | undefined;
/** Single lazy boundary for the DataForSEO subtree: the section fetchers and
* the ~3 MB dataforseo-client SDK they statically import stay out of the
* eager isolate startup graph and load once, on the first API call. */
export function loadDataforseoSections(): Promise<DataforseoSections> {
return (sectionsPromise ??= import("@/server/lib/dataforseo/sections"));
}
/** /**
* Wraps a section fetcher with billing metering. Each entry on the client is * Wraps a section fetcher with billing metering. Each entry on the client is
* `meter(customer, (s) => s.fetchX, defaultFeature?)`, which returns a function * `meter(customer, fetchX, defaultFeature?)`, which returns a function with
* with the fetcher's own input type and resolves to its unwrapped `.data`. The * the fetcher's own input type and resolves to its unwrapped `.data`.
* picker indirection (rather than the fetcher itself) keeps the section
* modules behind loadDataforseoSections.
* *
* `defaultFeature` is the fallback credit feature; a caller can override it per * `defaultFeature` is the fallback credit feature; a caller can override it per
* call by passing `creditFeature` in the input (e.g. an MCP tool attributing * call by passing `creditFeature` in the input (e.g. an MCP tool attributing
@ -47,15 +72,13 @@ export function loadDataforseoSections(): Promise<DataforseoSections> {
*/ */
function meter<I, T>( function meter<I, T>(
customer: BillingCustomerContext, customer: BillingCustomerContext,
pick: ( fetcher: (input: I) => Promise<DataforseoApiResponse<T>>,
sections: DataforseoSections,
) => (input: I) => Promise<DataforseoApiResponse<T>>,
defaultFeature?: CreditFeature, defaultFeature?: CreditFeature,
): (input: I & { creditFeature?: CreditFeature }) => Promise<T> { ): (input: I & { creditFeature?: CreditFeature }) => Promise<T> {
return (input) => return (input) =>
meterDataforseoCall( meterDataforseoCall(
customer, customer,
async () => pick(await loadDataforseoSections())(input), () => fetcher(input),
input.creditFeature ?? defaultFeature, input.creditFeature ?? defaultFeature,
); );
} }
@ -65,88 +88,61 @@ export function createDataforseoClient(customer: BillingCustomerContext) {
business: { business: {
businessListings: meter( businessListings: meter(
customer, customer,
(s) => s.fetchBusinessListingsSearch, fetchBusinessListingsSearch,
"local_seo",
),
questionsAnswers: meter(
customer,
(s) => s.fetchQuestionsAnswers,
"local_seo",
),
myBusinessInfo: meter(
customer,
(s) => s.fetchMyBusinessInfo,
"local_seo", "local_seo",
), ),
questionsAnswers: meter(customer, fetchQuestionsAnswers, "local_seo"),
myBusinessInfo: meter(customer, fetchMyBusinessInfo, "local_seo"),
// task_post is where DataForSEO charges; collection runs unmetered // task_post is where DataForSEO charges; collection runs unmetered
// through fetchBusinessDataTaskResult (see index.ts). // through fetchBusinessDataTaskResult (see index.ts).
reviewsTaskPost: meter( reviewsTaskPost: meter(customer, postGoogleReviewsTask, "local_seo"),
customer, updatesTaskPost: meter(customer, postMyBusinessUpdatesTask, "local_seo"),
(s) => s.postGoogleReviewsTask,
"local_seo",
),
updatesTaskPost: meter(
customer,
(s) => s.postMyBusinessUpdatesTask,
"local_seo",
),
}, },
backlinks: { backlinks: {
summary: meter(customer, (s) => s.fetchBacklinksSummary), summary: meter(customer, fetchBacklinksSummary),
rows: meter(customer, (s) => s.fetchBacklinksRows), rows: meter(customer, fetchBacklinksRows),
referringDomains: meter(customer, (s) => s.fetchReferringDomains), referringDomains: meter(customer, fetchReferringDomains),
domainPages: meter(customer, (s) => s.fetchDomainPagesSummary), domainPages: meter(customer, fetchDomainPagesSummary),
history: meter(customer, (s) => s.fetchBacklinksHistory), history: meter(customer, fetchBacklinksHistory),
}, },
keywords: { keywords: {
related: meter(customer, (s) => s.fetchRelatedKeywords), related: meter(customer, fetchRelatedKeywords),
suggestions: meter(customer, (s) => s.fetchKeywordSuggestions), suggestions: meter(customer, fetchKeywordSuggestions),
ideas: meter(customer, (s) => s.fetchKeywordIdeas), ideas: meter(customer, fetchKeywordIdeas),
// Google Ads endpoints for countries Labs doesn't support. // Google Ads endpoints for countries Labs doesn't support.
adsIdeas: meter(customer, (s) => s.fetchAdsKeywordIdeas), adsIdeas: meter(customer, fetchAdsKeywordIdeas),
adsSearchVolume: meter(customer, (s) => s.fetchAdsSearchVolume), adsSearchVolume: meter(customer, fetchAdsSearchVolume),
}, },
domain: { domain: {
rankOverview: meter(customer, (s) => s.fetchDomainRankOverview), rankOverview: meter(customer, fetchDomainRankOverview),
rankedKeywords: meter(customer, (s) => s.fetchRankedKeywords), rankedKeywords: meter(customer, fetchRankedKeywords),
relevantPages: meter(customer, (s) => s.fetchRelevantPages), relevantPages: meter(customer, fetchRelevantPages),
}, },
serp: { serp: {
live: meter(customer, (s) => s.fetchLiveSerp), live: meter(customer, fetchLiveSerp),
rankCheck: meter(customer, (s) => s.fetchRankCheckSerp, "rank_tracking"), rankCheck: meter(customer, fetchRankCheckSerp, "rank_tracking"),
// Posts up to 100 queued rank check tasks; one metered charge covers the // Posts up to 100 queued rank check tasks; one metered charge covers the
// whole batch (DataForSEO bills task_post at post time, collection is // whole batch (DataForSEO bills task_post at post time, collection is
// free). // free).
rankCheckTaskPost: meter( rankCheckTaskPost: meter(customer, postRankCheckTasks, "rank_tracking"),
customer, local: meter(customer, fetchLocalSerp, "local_seo"),
(s) => s.postRankCheckTasks,
"rank_tracking",
),
local: meter(customer, (s) => s.fetchLocalSerp, "local_seo"),
}, },
labs: { labs: {
// Callers (e.g. the keyword-metrics MCP tool) can attribute the spend to // Callers (e.g. the keyword-metrics MCP tool) can attribute the spend to
// their own feature by passing `creditFeature` in the input; defaults to // their own feature by passing `creditFeature` in the input; defaults to
// rank_tracking when omitted. // rank_tracking when omitted.
keywordOverview: meter( keywordOverview: meter(customer, fetchKeywordOverview, "rank_tracking"),
customer, serpCompetitors: meter(customer, fetchSerpCompetitors),
(s) => s.fetchKeywordOverview,
"rank_tracking",
),
serpCompetitors: meter(customer, (s) => s.fetchSerpCompetitors),
}, },
lighthouse: { lighthouse: {
live: meter(customer, (s) => s.fetchLighthouseResult), live: meter(customer, fetchLighthouseResult),
}, },
aiSearch: { aiSearch: {
mentionsSearch: meter(customer, (s) => s.fetchLlmMentionsSearch), mentionsSearch: meter(customer, fetchLlmMentionsSearch),
aggregatedMetrics: meter(customer, (s) => s.fetchLlmAggregatedMetrics), aggregatedMetrics: meter(customer, fetchLlmAggregatedMetrics),
topPages: meter(customer, (s) => s.fetchLlmTopPages), topPages: meter(customer, fetchLlmTopPages),
crossAggregatedMetrics: meter( crossAggregatedMetrics: meter(customer, fetchLlmCrossAggregatedMetrics),
customer, llmResponse: meter(customer, fetchLlmResponse),
(s) => s.fetchLlmCrossAggregatedMetrics,
),
llmResponse: meter(customer, (s) => s.fetchLlmResponse),
}, },
} as const; } as const;
} }

View File

@ -1,25 +1,32 @@
import { afterEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("@/server/lib/runtime-env", () => ({ vi.mock("@/server/lib/runtime-env", () => ({
getRequiredEnvValue: vi.fn().mockResolvedValue("encoded-credentials"), getRequiredEnvValue: vi.fn(async () => "encoded-credentials"),
})); }));
import { onPageApi } from "@/server/lib/dataforseo/core"; import { dataforseoPost } from "@/server/lib/dataforseo/core";
afterEach(() => { afterEach(() => {
vi.unstubAllGlobals(); vi.unstubAllGlobals();
}); });
describe("DataForSEO OnPage transport", () => { describe("DataForSEO transport", () => {
it("does not retry a Lighthouse HTTP 5xx response", async () => { it("retries a transient 5xx on idempotent reads and returns the parsed envelope", async () => {
const fetchMock = vi const fetchMock = vi
.fn() .fn<typeof fetch>()
.mockResolvedValue(new Response("upstream failure", { status: 503 })); .mockResolvedValueOnce(new Response("upstream failure", { status: 503 }))
.mockResolvedValueOnce(Response.json({ status_code: 20000, tasks: [] }));
vi.stubGlobal("fetch", fetchMock); vi.stubGlobal("fetch", fetchMock);
await expect(onPageApi().lighthouseLiveJson([])).rejects.toMatchObject({ await expect(
code: "UPSTREAM_UNAVAILABLE", dataforseoPost("/v3/backlinks/summary/live", []),
}); ).resolves.toEqual({ status_code: 20000, tasks: [] });
expect(fetchMock).toHaveBeenCalledOnce(); expect(fetchMock).toHaveBeenCalledTimes(2);
const [url, init] = fetchMock.mock.calls[0];
expect(url).toBe("https://api.dataforseo.com/v3/backlinks/summary/live");
expect(new Headers(init?.headers).get("Authorization")).toBe(
"Basic encoded-credentials",
);
}); });
}); });

View File

@ -1,16 +1,12 @@
import {
AiOptimizationApi,
AppendixApi,
BacklinksApi,
BusinessDataApi,
DataforseoLabsApi,
KeywordsDataApi,
OnPageApi,
SerpApi,
} from "dataforseo-client";
import { AppError } from "@/server/lib/errors"; import { AppError } from "@/server/lib/errors";
import { getRequiredEnvValue } from "@/server/lib/runtime-env"; import { getRequiredEnvValue } from "@/server/lib/runtime-env";
import type { ErrorCode } from "@/shared/error-codes"; import type { ErrorCode } from "@/shared/error-codes";
// Type-only: erased at compile, so no runtime cycle with envelope.ts (which
// imports DataforseoErrorClassifier from here the same way).
import type {
DataforseoResponseLike,
DataforseoTaskLike,
} from "@/server/lib/dataforseo/envelope";
const API_BASE = "https://api.dataforseo.com"; const API_BASE = "https://api.dataforseo.com";
const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600; const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600;
@ -60,10 +56,10 @@ function formatDataforseoRequestPath(url: RequestInfo): string {
} }
/** /**
* The single authenticated `fetch` used by every DataForSEO SDK call. Throws on * The single authenticated `fetch` used by every DataForSEO call. Throws on
* non-2xx so the SDK's own `ApiException` path never fires; task-level failures * non-2xx; task-level failures (which return HTTP 200) are handled downstream
* (which return HTTP 200) are handled downstream by {@link assertOk}. An * by {@link assertOk}. An optional classifier maps recognised HTTP failures to
* optional classifier maps recognised HTTP failures to product errors. * product errors.
*/ */
function createAuthenticatedFetch( function createAuthenticatedFetch(
classify?: DataforseoErrorClassifier, classify?: DataforseoErrorClassifier,
@ -119,30 +115,64 @@ function createAuthenticatedFetch(
}; };
} }
function http( type DataforseoRequestOptions = {
classify?: DataforseoErrorClassifier, /** Maps a recognised access / billing HTTP failure to a product error. */
maxServerErrorRetries = DATAFORSEO_MAX_RETRIES, classify?: DataforseoErrorClassifier;
) { /**
return { fetch: createAuthenticatedFetch(classify, maxServerErrorRetries) }; * Set 0 for billed, non-idempotent calls (business task_post, Lighthouse):
* a 5xx does not prove the provider skipped the charge, so those must never
* be replayed. Defaults to retrying idempotent reads on transient 5xx.
*/
maxServerErrorRetries?: number;
};
async function requestDataforseo<TTask extends DataforseoTaskLike>(
method: "GET" | "POST",
path: string,
body: unknown,
options: DataforseoRequestOptions,
): Promise<DataforseoResponseLike<TTask> | null> {
const doFetch = createAuthenticatedFetch(
options.classify,
options.maxServerErrorRetries,
);
const response = await doFetch(`${API_BASE}${path}`, {
method,
headers: {
Accept: "application/json",
...(method === "POST" ? { "Content-Type": "application/json" } : {}),
},
body: method === "POST" ? JSON.stringify(body) : undefined,
});
const text = await response.text();
if (text === "") return null;
// oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the task type is the caller's claim about the payload; billing metadata and item fields are validated downstream (envelope.ts + section Zod schemas)
return JSON.parse(text) as DataforseoResponseLike<TTask>;
} }
// Per-section API factories. Each is created per-request so the auth secret is /**
// read lazily (it lives in the Worker env, not in module scope). * POST `tasks` (the standard array-of-task-payloads body) to a DataForSEO
export const labsApi = () => new DataforseoLabsApi(API_BASE, http()); * endpoint and return the parsed response envelope. The task type parameter is
export const keywordsDataApi = () => new KeywordsDataApi(API_BASE, http()); * the caller's claim about the payload shape fields we act on are validated
export const serpApi = () => new SerpApi(API_BASE, http()); * downstream (billing metadata in envelope.ts, items via the section fetchers'
export const businessDataApi = () => new BusinessDataApi(API_BASE, http()); * Zod schemas). Auth is read per-call from the Worker env.
// task_post creates a billed task. A 5xx does not prove the provider skipped */
// the charge, so this client must not replay it (same rule as Lighthouse). export function dataforseoPost<
export const businessDataTaskApi = () => TTask extends DataforseoTaskLike = DataforseoTaskLike,
new BusinessDataApi(API_BASE, http(undefined, 0)); >(
// Lighthouse live is a billed, non-idempotent POST. A 5xx does not prove the path: string,
// provider skipped the charge, so this client must not replay it. tasks: unknown[],
export const onPageApi = () => new OnPageApi(API_BASE, http(undefined, 0)); options: DataforseoRequestOptions = {},
// Account/appendix data (spend, balance, rates). userData() is FREE ($0) and ): Promise<DataforseoResponseLike<TTask> | null> {
// read-only — do NOT wire it through metering. return requestDataforseo("POST", path, tasks, options);
export const appendixApi = () => new AppendixApi(API_BASE, http()); }
export const backlinksApi = (classify?: DataforseoErrorClassifier) =>
new BacklinksApi(API_BASE, http(classify)); /** GET a DataForSEO endpoint (task_get collection, appendix/locations data). */
export const aiOptimizationApi = (classify?: DataforseoErrorClassifier) => export function dataforseoGet<
new AiOptimizationApi(API_BASE, http(classify)); TTask extends DataforseoTaskLike = DataforseoTaskLike,
>(
path: string,
options: DataforseoRequestOptions = {},
): Promise<DataforseoResponseLike<TTask> | null> {
return requestDataforseo("GET", path, undefined, options);
}

View File

@ -42,8 +42,8 @@ export class DataforseoChargedTaskError extends AppError {
} }
} }
// The SDK types cost / path / result_count as optional with no runtime // cost / path / result_count arrive from the wire untyped and optional, so
// validation, so this is the one guard that guarantees we can bill a call. // this is the one guard that guarantees we can bill a call.
const billingMetadataSchema = z.object({ const billingMetadataSchema = z.object({
path: z.array(z.string()), path: z.array(z.string()),
cost: z.number(), cost: z.number(),
@ -67,6 +67,21 @@ export interface DataforseoResponseLike<T extends DataforseoTaskLike> {
[key: string]: unknown; [key: string]: unknown;
} }
/** `task.result[0]` entry carrying an `items` list the common live-endpoint
* shape. The index signature covers per-endpoint extras (`check_url`, ). */
export interface DataforseoItemsResult<TItem> {
items?: TItem[] | null;
total_count?: number | null;
[key: string]: unknown;
}
/** Task whose `result` entries follow the `items` shape. Item types are the
* caller's claim about the payload (as the SDK's were); fields we act on are
* Zod-validated by the section fetchers. */
export interface DataforseoItemsTask<TItem> extends DataforseoTaskLike {
result?: DataforseoItemsResult<TItem>[];
}
function tryBuildTaskBilling(task: unknown): DataforseoApiCallCost | null { function tryBuildTaskBilling(task: unknown): DataforseoApiCallCost | null {
const parsed = billingMetadataSchema.safeParse(task); const parsed = billingMetadataSchema.safeParse(task);
if (!parsed.success) return null; if (!parsed.success) return null;

View File

@ -1,26 +1,31 @@
import { import { dataforseoPost } from "@/server/lib/dataforseo/core";
KeywordsDataGoogleAdsKeywordsForKeywordsLiveRequestInfo, import type { LabsMonthlySearch } from "@/server/lib/dataforseo/labs";
KeywordsDataGoogleAdsSearchVolumeLiveRequestInfo,
type KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo,
type KeywordsDataGoogleAdsSearchVolumeLiveResultInfo,
} from "dataforseo-client";
import { keywordsDataApi } from "@/server/lib/dataforseo/core";
import { import {
assertOk, assertOk,
buildTaskBilling, buildTaskBilling,
type DataforseoApiResponse, type DataforseoApiResponse,
type DataforseoTaskLike,
} from "@/server/lib/dataforseo/envelope"; } from "@/server/lib/dataforseo/envelope";
// Google Ads keyword data for countries DataForSEO Labs doesn't cover (see // Google Ads keyword data for countries DataForSEO Labs doesn't cover (see
// specs/0004-keyword-data-source-routing.md). Flat-priced per request; items // specs/0004-keyword-data-source-routing.md). Flat-priced per request; items
// carry volume / CPC / competition but no keyword difficulty or intent. // carry volume / CPC / competition but no keyword difficulty or intent.
export type AdsKeywordItem = KeywordsDataGoogleAdsSearchVolumeLiveResultInfo; export interface AdsKeywordItem {
export type AdsKeywordIdeaItem = keyword?: string | null;
KeywordsDataGoogleAdsKeywordsForKeywordsLiveResultInfo; search_volume?: number | null;
cpc?: number | null;
/** "LOW" | "MEDIUM" | "HIGH" bucket (Labs reports a 0-1 ratio instead). */
competition?: string | null;
/** 0-100 competition scale; the app stores a 0-1 ratio. */
competition_index?: number | null;
monthly_searches?: LabsMonthlySearch[] | null;
[key: string]: unknown;
}
export type AdsKeywordIdeaItem = AdsKeywordItem;
type KeywordsDataResult<T> = { result?: T[] }; type KeywordsDataTask<T> = DataforseoTaskLike & { result?: T[] };
function taskItems<T>(task: KeywordsDataResult<T>): T[] { function taskItems<T>(task: KeywordsDataTask<T>): T[] {
// keywords_data tasks return keyword items directly in `result` (no nested // keywords_data tasks return keyword items directly in `result` (no nested
// `items` wrapper like Labs). // `items` wrapper like Labs).
return task.result ?? []; return task.result ?? [];
@ -40,13 +45,16 @@ export async function fetchAdsSearchVolume(input: {
const locationParams = input.locationName const locationParams = input.locationName
? { location_name: input.locationName } ? { location_name: input.locationName }
: { location_code: input.locationCode }; : { location_code: input.locationCode };
const response = await keywordsDataApi().googleAdsSearchVolumeLive([ const response = await dataforseoPost<KeywordsDataTask<AdsKeywordItem>>(
new KeywordsDataGoogleAdsSearchVolumeLiveRequestInfo({ "/v3/keywords_data/google_ads/search_volume/live",
keywords: input.keywords, [
...locationParams, {
language_code: input.languageCode, keywords: input.keywords,
}), ...locationParams,
]); language_code: input.languageCode,
},
],
);
const task = assertOk(response); const task = assertOk(response);
return { return {
data: taskItems(task), data: taskItems(task),
@ -60,14 +68,17 @@ export async function fetchAdsKeywordIdeas(input: {
languageCode: string; languageCode: string;
limit: number; limit: number;
}): Promise<DataforseoApiResponse<AdsKeywordIdeaItem[]>> { }): Promise<DataforseoApiResponse<AdsKeywordIdeaItem[]>> {
const response = await keywordsDataApi().googleAdsKeywordsForKeywordsLive([ const response = await dataforseoPost<KeywordsDataTask<AdsKeywordIdeaItem>>(
new KeywordsDataGoogleAdsKeywordsForKeywordsLiveRequestInfo({ "/v3/keywords_data/google_ads/keywords_for_keywords/live",
keywords: [input.keyword], [
location_code: input.locationCode, {
language_code: input.languageCode, keywords: [input.keyword],
sort_by: "search_volume", location_code: input.locationCode,
}), language_code: input.languageCode,
]); sort_by: "search_volume",
},
],
);
const task = assertOk(response); const task = assertOk(response);
// The endpoint has no limit parameter (it can return thousands of // The endpoint has no limit parameter (it can return thousands of
// suggestions for one flat fee); truncate to what the caller asked for. // suggestions for one flat fee); truncate to what the caller asked for.

View File

@ -1,15 +1,7 @@
// Public surface of the DataForSEO integration. Internals live in the // Public surface of the DataForSEO integration. Internals live in the
// per-section files (labs / serp / business / backlinks / ai / lighthouse), // per-section files (labs / serp / business / backlinks / ai / lighthouse);
// which sit behind the single dynamic import in client.ts so the ~3 MB SDK // everything funnels through envelope.ts (status + billing) and is metered in
// loads lazily; everything funnels through envelope.ts (status + billing) and // client.ts.
// is metered in client.ts. Runtime values re-exported here must be SDK-free
// (shared.ts) or lazy — a static value re-export from a section file would
// drag the SDK back into the eager isolate startup graph.
import {
loadDataforseoSections,
type DataforseoSections,
} from "@/server/lib/dataforseo/client";
export { createDataforseoClient } from "@/server/lib/dataforseo/client"; export { createDataforseoClient } from "@/server/lib/dataforseo/client";
@ -28,26 +20,16 @@ export {
export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget"; export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
/** Lazy wrappers for the section fetchers called outside the metered client. // Section fetchers called outside the metered client. Task collection is free
* Task collection is free at DataForSEO (the task was charged at task_post), so // at DataForSEO (the task was charged at task_post), so routing these through
* routing these through the metering seam would charge the customer twice. */ // the metering seam would charge the customer twice; business categories are
export const fetchRankCheckTaskResult: DataforseoSections["fetchRankCheckTaskResult"] = // free ($0), so a zero-credit org can still list them.
async (input) => export { fetchRankCheckTaskResult } from "@/server/lib/dataforseo/serp";
(await loadDataforseoSections()).fetchRankCheckTaskResult(input); export {
fetchBusinessDataTaskResult,
export const fetchBusinessDataTaskResult: DataforseoSections["fetchBusinessDataTaskResult"] = fetchBusinessListingsCategories,
async (input) => type BusinessTaskEndpoint,
(await loadDataforseoSections()).fetchBusinessDataTaskResult(input); type BusinessTaskOutcome,
/** Free ($0) at DataForSEO, so it skips the metered client entirely a
* zero-credit org can still list categories. */
export const fetchBusinessListingsCategories: DataforseoSections["fetchBusinessListingsCategories"] =
async () =>
(await loadDataforseoSections()).fetchBusinessListingsCategories();
export type {
BusinessTaskEndpoint,
BusinessTaskOutcome,
} from "@/server/lib/dataforseo/business"; } from "@/server/lib/dataforseo/business";
export type { export type {

View File

@ -1,36 +1,84 @@
import { z } from "zod"; import { z } from "zod";
import { import { dataforseoPost } from "@/server/lib/dataforseo/core";
DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo,
DataforseoLabsGoogleKeywordIdeasLiveRequestInfo,
DataforseoLabsGoogleKeywordOverviewLiveRequestInfo,
DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo,
DataforseoLabsGoogleRankedKeywordsLiveRequestInfo,
DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo,
DataforseoLabsGoogleRelevantPagesLiveRequestInfo,
DataforseoLabsGoogleSerpCompetitorsLiveRequestInfo,
type DataforseoLabsDomainRankOverviewLiveItem,
type DataforseoLabsGoogleKeywordOverviewLiveItem,
type DataforseoLabsRelatedKeywordsLiveItem,
type DataforseoLabsRelevantPagesLiveItem,
type DataforseoLabsSerpCompetitorsLiveItem,
type KeywordDataInfo,
} from "dataforseo-client";
import { labsApi } from "@/server/lib/dataforseo/core";
import { import {
assertOk, assertOk,
buildTaskBilling, buildTaskBilling,
parseTaskItems, parseTaskItems,
type DataforseoApiResponse, type DataforseoApiResponse,
type DataforseoItemsTask,
} from "@/server/lib/dataforseo/envelope"; } from "@/server/lib/dataforseo/envelope";
// SDK item models are 1:1 supersets of what we need, so we expose them directly // Labs payload types: the fields the app reads, typed honestly (the wire nulls
// under the names the rest of the app already uses (no hand-written Zod). // any of them); the index signature carries everything else through untyped,
export type LabsKeywordDataItem = KeywordDataInfo; // like the SDK's item models did. These are claims about the payload, not
type RelatedKeywordItem = DataforseoLabsRelatedKeywordsLiveItem; // validation — fields that must hold get a Zod schema (see below).
type DomainMetricsItem = DataforseoLabsDomainRankOverviewLiveItem;
export type RelevantPagesItem = DataforseoLabsRelevantPagesLiveItem; export interface LabsMonthlySearch {
export type KeywordOverviewItem = DataforseoLabsGoogleKeywordOverviewLiveItem; year?: number | null;
type SerpCompetitorItem = DataforseoLabsSerpCompetitorsLiveItem; month?: number | null;
search_volume?: number | null;
[key: string]: unknown;
}
export interface LabsKeywordInfo {
search_volume?: number | null;
cpc?: number | null;
/** 0-1 paid-competition ratio (Google Ads reports a 0-100 index instead). */
competition?: number | null;
competition_level?: string | null;
monthly_searches?: LabsMonthlySearch[] | null;
[key: string]: unknown;
}
export interface LabsKeywordDataItem {
keyword?: string | null;
keyword_info?: LabsKeywordInfo | null;
keyword_info_normalized_with_clickstream?: LabsKeywordInfo | null;
keyword_properties?: {
keyword_difficulty?: number | null;
[key: string]: unknown;
} | null;
search_intent_info?: {
main_intent?: string | null;
[key: string]: unknown;
} | null;
[key: string]: unknown;
}
/** keyword_overview items share the keyword-data field surface. */
export type KeywordOverviewItem = LabsKeywordDataItem;
/** related_keywords wraps the keyword payload one level deeper. */
type RelatedKeywordItem = {
keyword_data?: LabsKeywordDataItem | null;
[key: string]: unknown;
};
type LabsMetricsBlock = {
organic?: { etv?: number | null; count?: number | null } | null;
[key: string]: unknown;
};
type DomainMetricsItem = {
metrics?: LabsMetricsBlock | null;
[key: string]: unknown;
};
export interface RelevantPagesItem {
page_address?: string | null;
metrics?: LabsMetricsBlock | null;
[key: string]: unknown;
}
type SerpCompetitorItem = {
domain?: string | null;
avg_position?: number | null;
median_position?: number | null;
visibility?: number | null;
etv?: number | null;
keywords_count?: number | null;
[key: string]: unknown;
};
// Ranked keywords is the one Labs endpoint the SDK types loosely: its // Ranked keywords is the one Labs endpoint the SDK types loosely: its
// `ranked_serp_element.serp_item` is the base element item, so the url / etv / // `ranked_serp_element.serp_item` is the base element item, so the url / etv /
@ -104,8 +152,10 @@ export async function fetchRelatedKeywords(input: {
depth?: number; depth?: number;
includeClickstreamData?: boolean; includeClickstreamData?: boolean;
}): Promise<DataforseoApiResponse<RelatedKeywordItem[]>> { }): Promise<DataforseoApiResponse<RelatedKeywordItem[]>> {
const response = await labsApi().googleRelatedKeywordsLive([ const response = await dataforseoPost<
new DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo({ DataforseoItemsTask<RelatedKeywordItem>
>("/v3/dataforseo_labs/google/related_keywords/live", [
{
keyword: input.keyword, keyword: input.keyword,
location_code: input.locationCode, location_code: input.locationCode,
language_code: input.languageCode, language_code: input.languageCode,
@ -115,7 +165,7 @@ export async function fetchRelatedKeywords(input: {
// opt-in — see specs/0004-keyword-data-source-routing.md. // opt-in — see specs/0004-keyword-data-source-routing.md.
include_clickstream_data: input.includeClickstreamData ?? false, include_clickstream_data: input.includeClickstreamData ?? false,
include_serp_info: false, include_serp_info: false,
}), },
]); ]);
const task = assertOk(response); const task = assertOk(response);
return { return {
@ -131,8 +181,10 @@ export async function fetchKeywordSuggestions(input: {
limit: number; limit: number;
includeClickstreamData?: boolean; includeClickstreamData?: boolean;
}): Promise<DataforseoApiResponse<LabsKeywordDataItem[]>> { }): Promise<DataforseoApiResponse<LabsKeywordDataItem[]>> {
const response = await labsApi().googleKeywordSuggestionsLive([ const response = await dataforseoPost<
new DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo({ DataforseoItemsTask<LabsKeywordDataItem>
>("/v3/dataforseo_labs/google/keyword_suggestions/live", [
{
keyword: input.keyword, keyword: input.keyword,
location_code: input.locationCode, location_code: input.locationCode,
language_code: input.languageCode, language_code: input.languageCode,
@ -142,7 +194,7 @@ export async function fetchKeywordSuggestions(input: {
include_seed_keyword: true, include_seed_keyword: true,
ignore_synonyms: false, ignore_synonyms: false,
exact_match: false, exact_match: false,
}), },
]); ]);
const task = assertOk(response); const task = assertOk(response);
return { return {
@ -158,8 +210,10 @@ export async function fetchKeywordIdeas(input: {
limit: number; limit: number;
includeClickstreamData?: boolean; includeClickstreamData?: boolean;
}): Promise<DataforseoApiResponse<LabsKeywordDataItem[]>> { }): Promise<DataforseoApiResponse<LabsKeywordDataItem[]>> {
const response = await labsApi().googleKeywordIdeasLive([ const response = await dataforseoPost<
new DataforseoLabsGoogleKeywordIdeasLiveRequestInfo({ DataforseoItemsTask<LabsKeywordDataItem>
>("/v3/dataforseo_labs/google/keyword_ideas/live", [
{
keywords: [input.keyword], keywords: [input.keyword],
location_code: input.locationCode, location_code: input.locationCode,
language_code: input.languageCode, language_code: input.languageCode,
@ -168,7 +222,7 @@ export async function fetchKeywordIdeas(input: {
include_serp_info: false, include_serp_info: false,
ignore_synonyms: false, ignore_synonyms: false,
closely_variants: false, closely_variants: false,
}), },
]); ]);
const task = assertOk(response); const task = assertOk(response);
return { return {
@ -182,14 +236,17 @@ export async function fetchDomainRankOverview(input: {
locationCode: number; locationCode: number;
languageCode: string; languageCode: string;
}): Promise<DataforseoApiResponse<DomainMetricsItem[]>> { }): Promise<DataforseoApiResponse<DomainMetricsItem[]>> {
const response = await labsApi().googleDomainRankOverviewLive([ const response = await dataforseoPost<DataforseoItemsTask<DomainMetricsItem>>(
new DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo({ "/v3/dataforseo_labs/google/domain_rank_overview/live",
target: input.target, [
location_code: input.locationCode, {
language_code: input.languageCode, target: input.target,
limit: 1, location_code: input.locationCode,
}), language_code: input.languageCode,
]); limit: 1,
},
],
);
const task = assertOk(response); const task = assertOk(response);
return { return {
data: task.result?.[0]?.items ?? [], data: task.result?.[0]?.items ?? [],
@ -215,18 +272,21 @@ export async function fetchRankedKeywords(input: {
// Note: ranked_keywords has no include_subdomains parameter — a domain // Note: ranked_keywords has no include_subdomains parameter — a domain
// target always covers the hostname plus its subdomains. Narrower scopes // target always covers the hostname plus its subdomains. Narrower scopes
// are expressed through `filters` (see researchScopeFilters.ts). // are expressed through `filters` (see researchScopeFilters.ts).
const response = await labsApi().googleRankedKeywordsLive([ const response = await dataforseoPost<DataforseoItemsTask<unknown>>(
new DataforseoLabsGoogleRankedKeywordsLiveRequestInfo({ "/v3/dataforseo_labs/google/ranked_keywords/live",
target: input.target, [
location_code: input.locationCode, {
language_code: input.languageCode, target: input.target,
limit: input.limit, location_code: input.locationCode,
offset: input.offset, language_code: input.languageCode,
order_by: input.orderBy, limit: input.limit,
filters: input.filters, offset: input.offset,
item_types: input.itemTypes, order_by: input.orderBy,
}), filters: input.filters,
]); item_types: input.itemTypes,
},
],
);
const task = assertOk(response); const task = assertOk(response);
return { return {
data: { data: {
@ -255,17 +315,20 @@ export async function fetchRelevantPages(input: {
orderBy?: string[]; orderBy?: string[];
filters?: unknown[]; filters?: unknown[];
}): Promise<DataforseoApiResponse<RelevantPagesPage>> { }): Promise<DataforseoApiResponse<RelevantPagesPage>> {
const response = await labsApi().googleRelevantPagesLive([ const response = await dataforseoPost<DataforseoItemsTask<RelevantPagesItem>>(
new DataforseoLabsGoogleRelevantPagesLiveRequestInfo({ "/v3/dataforseo_labs/google/relevant_pages/live",
target: input.target, [
location_code: input.locationCode, {
language_code: input.languageCode, target: input.target,
limit: input.limit, location_code: input.locationCode,
offset: input.offset, language_code: input.languageCode,
order_by: input.orderBy, limit: input.limit,
filters: input.filters, offset: input.offset,
}), order_by: input.orderBy,
]); filters: input.filters,
},
],
);
const task = assertOk(response); const task = assertOk(response);
return { return {
data: { data: {
@ -282,13 +345,15 @@ export async function fetchKeywordOverview(input: {
languageCode: string; languageCode: string;
includeClickstreamData?: boolean; includeClickstreamData?: boolean;
}): Promise<DataforseoApiResponse<KeywordOverviewItem[]>> { }): Promise<DataforseoApiResponse<KeywordOverviewItem[]>> {
const response = await labsApi().googleKeywordOverviewLive([ const response = await dataforseoPost<
new DataforseoLabsGoogleKeywordOverviewLiveRequestInfo({ DataforseoItemsTask<KeywordOverviewItem>
>("/v3/dataforseo_labs/google/keyword_overview/live", [
{
keywords: input.keywords, keywords: input.keywords,
location_code: input.locationCode, location_code: input.locationCode,
language_code: input.languageCode, language_code: input.languageCode,
include_clickstream_data: input.includeClickstreamData ?? false, include_clickstream_data: input.includeClickstreamData ?? false,
}), },
]); ]);
const task = assertOk(response); const task = assertOk(response);
return { return {
@ -306,8 +371,10 @@ export async function fetchSerpCompetitors(input: {
limit: number; limit: number;
offset?: number; offset?: number;
}): Promise<DataforseoApiResponse<SerpCompetitorItem[]>> { }): Promise<DataforseoApiResponse<SerpCompetitorItem[]>> {
const response = await labsApi().googleSerpCompetitorsLive([ const response = await dataforseoPost<
new DataforseoLabsGoogleSerpCompetitorsLiveRequestInfo({ DataforseoItemsTask<SerpCompetitorItem>
>("/v3/dataforseo_labs/google/serp_competitors/live", [
{
keywords: input.keywords, keywords: input.keywords,
location_code: input.locationCode, location_code: input.locationCode,
language_code: input.languageCode, language_code: input.languageCode,
@ -315,7 +382,7 @@ export async function fetchSerpCompetitors(input: {
include_subdomains: input.includeSubdomains, include_subdomains: input.includeSubdomains,
limit: input.limit, limit: input.limit,
offset: input.offset, offset: input.offset,
}), },
]); ]);
const task = assertOk(response); const task = assertOk(response);
return { return {

View File

@ -1,51 +1,42 @@
import { beforeEach, describe, expect, it, vi } from "vitest"; import { afterEach, describe, expect, it, vi } from "vitest";
const { onPageApiMock, lighthouseLiveJson } = vi.hoisted(() => ({ vi.mock("@/server/lib/runtime-env", () => ({
onPageApiMock: vi.fn(), getRequiredEnvValue: vi.fn(async () => "test-api-key"),
lighthouseLiveJson: vi.fn(),
}));
vi.mock("dataforseo-client", () => ({
OnPageLighthouseLiveJsonRequestInfo: class {
constructor(public input: unknown) {}
},
}));
vi.mock("@/server/lib/dataforseo/core", () => ({
onPageApi: onPageApiMock,
})); }));
import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope"; import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";
import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse"; import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
beforeEach(() => { afterEach(() => {
vi.clearAllMocks(); vi.unstubAllGlobals();
onPageApiMock.mockReturnValue({ lighthouseLiveJson });
}); });
describe("fetchLighthouseResult", () => { describe("fetchLighthouseResult", () => {
it("carries billing metadata when parsing fails after a billed success", async () => { it("carries billing metadata when parsing fails after a billed success", async () => {
lighthouseLiveJson.mockResolvedValue({ const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
status_code: 20000, Response.json({
status_message: "Ok.", status_code: 20000,
tasks: [ status_message: "Ok.",
{ tasks: [
id: "task-1", {
status_code: 20000, id: "task-1",
status_message: "Ok.", status_code: 20000,
path: ["v3", "on_page", "lighthouse", "live", "json"], status_message: "Ok.",
cost: 0.00425, path: ["v3", "on_page", "lighthouse", "live", "json"],
result: [ cost: 0.00425,
{ result: [
requestedUrl: "https://example.com/", {
finalUrl: "https://example.com/", requestedUrl: "https://example.com/",
categories: {}, finalUrl: "https://example.com/",
audits: {}, categories: {},
}, audits: {},
], },
}, ],
], },
}); ],
}),
);
vi.stubGlobal("fetch", fetchMock);
const rejection = fetchLighthouseResult({ const rejection = fetchLighthouseResult({
url: "https://example.com/", url: "https://example.com/",
@ -60,4 +51,19 @@ describe("fetchLighthouseResult", () => {
}, },
}); });
}); });
it("does not retry an HTTP 5xx (the provider may have charged the task)", async () => {
const fetchMock = vi
.fn<typeof fetch>()
.mockResolvedValue(new Response("upstream failure", { status: 503 }));
vi.stubGlobal("fetch", fetchMock);
await expect(
fetchLighthouseResult({
url: "https://example.com/",
strategy: "mobile",
}),
).rejects.toMatchObject({ code: "UPSTREAM_UNAVAILABLE" });
expect(fetchMock).toHaveBeenCalledOnce();
});
}); });

View File

@ -1,11 +1,10 @@
import { OnPageLighthouseLiveJsonRequestInfo } from "dataforseo-client";
import { import {
parseDataforseoLighthousePayload, parseDataforseoLighthousePayload,
requestCategories, requestCategories,
type LighthouseStrategy, type LighthouseStrategy,
} from "@/server/lib/dataforseoLighthousePayload"; } from "@/server/lib/dataforseoLighthousePayload";
import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload"; import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload";
import { onPageApi } from "@/server/lib/dataforseo/core"; import { dataforseoPost } from "@/server/lib/dataforseo/core";
import { import {
assertOk, assertOk,
buildTaskBilling, buildTaskBilling,
@ -17,13 +16,19 @@ export async function fetchLighthouseResult(input: {
url: string; url: string;
strategy: LighthouseStrategy; strategy: LighthouseStrategy;
}): Promise<DataforseoApiResponse<StoredLighthousePayload>> { }): Promise<DataforseoApiResponse<StoredLighthousePayload>> {
const response = await onPageApi().lighthouseLiveJson([ const response = await dataforseoPost(
new OnPageLighthouseLiveJsonRequestInfo({ "/v3/on_page/lighthouse/live/json",
url: input.url, [
for_mobile: input.strategy === "mobile", {
categories: [...requestCategories], url: input.url,
}), for_mobile: input.strategy === "mobile",
]); categories: [...requestCategories],
},
],
// Billed, non-idempotent POST: a 5xx does not prove the provider skipped
// the charge, so never replay it.
{ maxServerErrorRetries: 0 },
);
// Build the metering envelope before parsing. The provider has already // Build the metering envelope before parsing. The provider has already
// charged a successful task, so a malformed payload must carry its billing // charged a successful task, so a malformed payload must carry its billing

View File

@ -1,59 +0,0 @@
// Root of the lazily loaded DataForSEO subtree. The section fetchers — and
// the ~3 MB dataforseo-client SDK they statically import — are reached only
// through the single dynamic import in client.ts (loadDataforseoSections), so
// the whole subtree lands in one lazy chunk outside the eager isolate startup
// graph. Never import this barrel or a section file statically from eager
// server code; the leanWorkerBundle vite plugin fails the build if the SDK
// re-enters the eager graph. SDK-free values live in shared.ts instead.
export {
fetchBusinessDataTaskResult,
fetchBusinessListingsCategories,
fetchBusinessListingsSearch,
fetchMyBusinessInfo,
fetchQuestionsAnswers,
postGoogleReviewsTask,
postMyBusinessUpdatesTask,
} from "@/server/lib/dataforseo/business";
export {
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
fetchDomainPagesSummary,
fetchReferringDomains,
} from "@/server/lib/dataforseo/backlinks";
export {
fetchDomainRankOverview,
fetchKeywordIdeas,
fetchKeywordOverview,
fetchKeywordSuggestions,
fetchRankedKeywords,
fetchRelatedKeywords,
fetchRelevantPages,
fetchSerpCompetitors,
} from "@/server/lib/dataforseo/labs";
export {
fetchAdsKeywordIdeas,
fetchAdsSearchVolume,
} from "@/server/lib/dataforseo/google-ads";
export {
fetchLiveSerp,
fetchLocalSerp,
fetchRankCheckSerp,
fetchRankCheckTaskResult,
postRankCheckTasks,
} from "@/server/lib/dataforseo/serp";
export { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
export {
fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";

View File

@ -1,6 +1,6 @@
import { env } from "cloudflare:workers"; import { env } from "cloudflare:workers";
import { z } from "zod"; import { z } from "zod";
import { serpApi } from "@/server/lib/dataforseo/core"; import { dataforseoGet } from "@/server/lib/dataforseo/core";
import { assertOk } from "@/server/lib/dataforseo/envelope"; import { assertOk } from "@/server/lib/dataforseo/envelope";
import { formatLocationLabel } from "@/shared/keyword-locations"; import { formatLocationLabel } from "@/shared/keyword-locations";
@ -95,7 +95,9 @@ function fillFromOrigin(iso: string): Promise<SerpLocationResult[]> {
} }
async function fetchFromDataforseo(iso: string): Promise<SerpLocationResult[]> { async function fetchFromDataforseo(iso: string): Promise<SerpLocationResult[]> {
const response = await serpApi().googleLocationsCountry(iso); const response = await dataforseoGet(
`/v3/serp/google/locations/${encodeURIComponent(iso)}`,
);
const task = assertOk(response); const task = assertOk(response);
return (task.result ?? []) return (task.result ?? [])
.map((item) => locationItemSchema.safeParse(item)) .map((item) => locationItemSchema.safeParse(item))

View File

@ -1,12 +1,5 @@
import { z } from "zod"; import { z } from "zod";
import { import { dataforseoGet, dataforseoPost } from "@/server/lib/dataforseo/core";
SerpApiStopCrawlOnMatchInfo,
SerpGoogleLocalFinderLiveAdvancedRequestInfo,
SerpGoogleMapsLiveAdvancedRequestInfo,
SerpGoogleOrganicLiveAdvancedRequestInfo,
SerpGoogleOrganicTaskPostRequestInfo,
} from "dataforseo-client";
import { serpApi } from "@/server/lib/dataforseo/core";
import { MAX_TASKS_PER_POST } from "@/server/lib/dataforseo/shared"; import { MAX_TASKS_PER_POST } from "@/server/lib/dataforseo/shared";
import { import {
assertOk, assertOk,
@ -15,6 +8,8 @@ import {
isTaskInProgress, isTaskInProgress,
parseTaskItems, parseTaskItems,
type DataforseoApiResponse, type DataforseoApiResponse,
type DataforseoItemsTask,
type DataforseoTaskLike,
} from "@/server/lib/dataforseo/envelope"; } from "@/server/lib/dataforseo/envelope";
import { AppError } from "@/server/lib/errors"; import { AppError } from "@/server/lib/errors";
@ -34,10 +29,7 @@ function clampSerpDepth(depth: number): number {
function stopCrawlOnTarget(targetDomain: string) { function stopCrawlOnTarget(targetDomain: string) {
return { return {
stop_crawl_on_match: [ stop_crawl_on_match: [
new SerpApiStopCrawlOnMatchInfo({ { match_value: targetDomain, match_type: "with_subdomains" },
match_value: targetDomain,
match_type: "with_subdomains",
}),
], ],
find_targets_in: ["organic"], find_targets_in: ["organic"],
}; };
@ -87,16 +79,19 @@ export async function fetchLiveSerp(input: {
locationCode: number; locationCode: number;
languageCode: string; languageCode: string;
}): Promise<DataforseoApiResponse<SerpLiveItem[]>> { }): Promise<DataforseoApiResponse<SerpLiveItem[]>> {
const response = await serpApi().googleOrganicLiveAdvanced([ const response = await dataforseoPost(
new SerpGoogleOrganicLiveAdvancedRequestInfo({ "/v3/serp/google/organic/live/advanced",
keyword: input.keyword, [
location_code: input.locationCode, {
language_code: input.languageCode, keyword: input.keyword,
device: "desktop", location_code: input.locationCode,
os: "windows", language_code: input.languageCode,
depth: 100, device: "desktop",
}), os: "windows",
]); depth: 100,
},
],
);
const task = assertOk(response); const task = assertOk(response);
return { return {
data: parseTaskItems( data: parseTaskItems(
@ -155,17 +150,20 @@ export async function fetchRankCheckSerp(input: {
const locationParams = input.locationName const locationParams = input.locationName
? { location_name: input.locationName } ? { location_name: input.locationName }
: { location_code: input.locationCode }; : { location_code: input.locationCode };
const response = await serpApi().googleOrganicLiveAdvanced([ const response = await dataforseoPost(
new SerpGoogleOrganicLiveAdvancedRequestInfo({ "/v3/serp/google/organic/live/advanced",
keyword: input.keyword, [
...locationParams, {
language_code: input.languageCode, keyword: input.keyword,
device: input.device, ...locationParams,
os: input.device === "desktop" ? "windows" : "android", language_code: input.languageCode,
depth, device: input.device,
...stopCrawlOnTarget(input.targetDomain), os: input.device === "desktop" ? "windows" : "android",
}), depth,
]); ...stopCrawlOnTarget(input.targetDomain),
},
],
);
// "No Search Results" (40501) is valid for obscure/new keywords — treat as an // "No Search Results" (40501) is valid for obscure/new keywords — treat as an
// empty result set rather than failing the whole rank-tracking run. // empty result set rather than failing the whole rank-tracking run.
@ -217,26 +215,26 @@ export async function postRankCheckTasks(input: {
const locationParams = input.locationName const locationParams = input.locationName
? { location_name: input.locationName } ? { location_name: input.locationName }
: { location_code: input.locationCode }; : { location_code: input.locationCode };
const response = await serpApi().googleOrganicTaskPost( const response = await dataforseoPost<
input.tasks.map( DataforseoTaskLike & { id?: string; data?: Record<string, unknown> }
(task) => >(
new SerpGoogleOrganicTaskPostRequestInfo({ "/v3/serp/google/organic/task_post",
keyword: task.keyword, input.tasks.map((task) => ({
...locationParams, keyword: task.keyword,
language_code: input.languageCode, ...locationParams,
device: task.device, language_code: input.languageCode,
os: task.device === "desktop" ? "windows" : "android", device: task.device,
depth, os: task.device === "desktop" ? "windows" : "android",
// Queued tasks are billed provisionally at full depth at post time; depth,
// task_get later reports the reduced actual cost when the crawl // Queued tasks are billed provisionally at full depth at post time;
// stopped early. We meter customers on the post-time amount — // task_get later reports the reduced actual cost when the crawl
// collection-time metering is a possible future optimization. // stopped early. We meter customers on the post-time amount —
...stopCrawlOnTarget(input.targetDomain), // collection-time metering is a possible future optimization.
// Echoed back on the response entry and task_get; used to map a ...stopCrawlOnTarget(input.targetDomain),
// DataForSEO task id back to our keyword without relying on order. // Echoed back on the response entry and task_get; used to map a
tag: `${task.keywordId}:${task.device}`, // DataForSEO task id back to our keyword without relying on order.
}), tag: `${task.keywordId}:${task.device}`,
), })),
); );
if (!response || response.status_code !== 20000) { if (!response || response.status_code !== 20000) {
@ -296,7 +294,9 @@ export async function fetchRankCheckTaskResult(input: {
keyword: string; keyword: string;
targetDomain: string; targetDomain: string;
}): Promise<RankCheckTaskOutcome> { }): Promise<RankCheckTaskOutcome> {
const response = await serpApi().googleOrganicTaskGetAdvanced(input.taskId); const response = await dataforseoGet(
`/v3/serp/google/organic/task_get/advanced/${encodeURIComponent(input.taskId)}`,
);
const task = response?.tasks?.[0]; const task = response?.tasks?.[0];
if (!response || response.status_code !== 20000 || !task) { if (!response || response.status_code !== 20000 || !task) {
throw new AppError( throw new AppError(
@ -344,11 +344,11 @@ export async function fetchLocalSerp(input: {
}): Promise<DataforseoApiResponse<Record<string, unknown>[]>> { }): Promise<DataforseoApiResponse<Record<string, unknown>[]>> {
const os = input.device === "desktop" ? "windows" : "android"; const os = input.device === "desktop" ? "windows" : "android";
// Maps and Local Finder return different SDK item models; both carry an index
// signature, so the typed items assign cleanly to the generic row shape.
if (input.searchType === "maps") { if (input.searchType === "maps") {
const response = await serpApi().googleMapsLiveAdvanced([ const response = await dataforseoPost<
new SerpGoogleMapsLiveAdvancedRequestInfo({ DataforseoItemsTask<Record<string, unknown>>
>("/v3/serp/google/maps/live/advanced", [
{
keyword: input.keyword, keyword: input.keyword,
location_coordinate: input.locationCoordinate, location_coordinate: input.locationCoordinate,
language_code: input.languageCode, language_code: input.languageCode,
@ -356,7 +356,7 @@ export async function fetchLocalSerp(input: {
os, os,
depth: input.depth, depth: input.depth,
search_places: input.searchPlaces, search_places: input.searchPlaces,
}), },
]); ]);
// 40501 = billed empty SERP; DataForSEO returns it for some coordinate-only // 40501 = billed empty SERP; DataForSEO returns it for some coordinate-only
// Maps and Local Finder queries (both paths below opt in). // Maps and Local Finder queries (both paths below opt in).
@ -367,15 +367,17 @@ export async function fetchLocalSerp(input: {
}; };
} }
const response = await serpApi().googleLocalFinderLiveAdvanced([ const response = await dataforseoPost<
new SerpGoogleLocalFinderLiveAdvancedRequestInfo({ DataforseoItemsTask<Record<string, unknown>>
>("/v3/serp/google/local_finder/live/advanced", [
{
keyword: input.keyword, keyword: input.keyword,
location_coordinate: input.locationCoordinate, location_coordinate: input.locationCoordinate,
language_code: input.languageCode, language_code: input.languageCode,
device: input.device, device: input.device,
os, os,
depth: input.depth, depth: input.depth,
}), },
]); ]);
const task = assertOk(response, { treatNoResultsAsEmpty: true }); const task = assertOk(response, { treatNoResultsAsEmpty: true });
return { return {

View File

@ -1,8 +1,6 @@
// SDK-free constants and target builders shared between eager server code // Constants and target builders shared between server code (features,
// (features, workflows, MCP tools) and the lazily loaded section fetchers. // workflows, MCP tools) and the section fetchers. Keep this module free of
// Keep this module free of dataforseo-client and section-file imports — // section-file imports so both sides can import it without cycles.
// anything imported from here must be safe to evaluate in the eager isolate
// startup graph.
// ChatGPT mention/response data is only available for US/en per DataForSEO docs. // ChatGPT mention/response data is only available for US/en per DataForSEO docs.
export const CHATGPT_LOCATION_CODE = 2840; export const CHATGPT_LOCATION_CODE = 2840;

View File

@ -15,16 +15,9 @@ const WORKERS_AI_PROVIDER_STUB = fileURLToPath(
* startup (production OOM bursts trace back to baseline heap, not leaks), so * startup (production OOM bursts trace back to baseline heap, not leaks), so
* each of these is either loaded lazily behind a dynamic import or stubbed * each of these is either loaded lazily behind a dynamic import or stubbed
* out. `generateBundle` below fails the build if one sneaks back in via a * out. `generateBundle` below fails the build if one sneaks back in via a
* static import chain e.g. an eager `import { fetchLiveSerp } from * static import chain.
* "@/server/lib/dataforseo/serp"` instead of going through the metered client.
*/ */
const EAGER_DENYLIST: Array<{ pattern: RegExp; expected: string }> = [ const EAGER_DENYLIST: Array<{ pattern: RegExp; expected: string }> = [
{
pattern: /node_modules\/dataforseo-client\//,
expected:
"lazy-loaded behind loadDataforseoSections() — eager code must go " +
"through the metered client or src/server/lib/dataforseo/shared.ts",
},
{ {
pattern: /node_modules\/autumn-js\//, pattern: /node_modules\/autumn-js\//,
expected: expected: