admin/metatron-open-seo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: unify dataforseo sdk usage (#231)

Browse Source
This commit is contained in:
Ben Senescu 2026-06-01 09:48:11 -04:00 committed by GitHub
parent 06e15dc70f
commit b64ef56b4d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
45 changed files with 2440 additions and 2872 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scripts/brand-lookup-cost-profile.ts
View File

@ -3,11 +3,11 @@ import {
buildLlmTarget,
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
fetchLlmAggregatedMetricsRaw,
fetchLlmMentionsSearchRaw,
fetchLlmTopPagesRaw,
fetchLlmAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmTopPages,
type LlmPlatform,
} from "@/server/lib/dataforseoLlm";
} from "@/server/lib/dataforseo/ai";
import { applyBillingMarkupUsd } from "@/shared/billing";
import { loadLocalEnv, parseArgs } from "./cli-utils";
@ -65,7 +65,7 @@ async function main() {
const languageCode =
platform === "chat_gpt" ? CHATGPT_LANGUAGE_CODE : userLanguageCode;
const aggregated = await fetchLlmAggregatedMetricsRaw({
const aggregated = await fetchLlmAggregatedMetrics({
target: llmTarget,
platform,
locationCode,
@ -74,7 +74,7 @@ async function main() {
});
calls.push(toRecord(platform, "aggregated_metrics", aggregated.billing));
const topPages = await fetchLlmTopPagesRaw({
const topPages = await fetchLlmTopPages({
target: llmTarget,
platform,
locationCode,
@ -83,7 +83,7 @@ async function main() {
});
calls.push(toRecord(platform, "top_pages", topPages.billing));
const mentions = await fetchLlmMentionsSearchRaw({
const mentions = await fetchLlmMentionsSearch({
target: llmTarget,
platform,
locationCode,
@ -133,7 +133,6 @@ type CallRecord = {
platform: LlmPlatform;
endpoint: string;
path: string;
resultCount: number | null;
rawUsd: number;
billedUsd: number;
};
@ -148,13 +147,12 @@ type RunSummary = {
function toRecord(
platform: LlmPlatform,
endpoint: string,
billing: { costUsd: number; path: string[]; resultCount: number | null },
billing: { costUsd: number; path: string[] },
): CallRecord {
return {
platform,
endpoint,
path: billing.path.join("/"),
resultCount: billing.resultCount,
rawUsd: round(billing.costUsd),
billedUsd: applyBillingMarkupUsd(billing.costUsd),
};

15
scripts/seed-rank-tracking.ts
View File

@ -19,11 +19,7 @@ import {
DataforseoLabsGoogleRankedKeywordsLiveRequestInfo,
} from "dataforseo-client";
import * as schema from "../src/db/schema";
import {
domainRankedKeywordItemSchema,
type DomainRankedKeywordItem,
} from "../src/server/lib/dataforseoSchemas";
import { z } from "zod";
import type { DomainRankedKeywordItem } from "../src/server/lib/dataforseo";
import { loadLocalEnv, parseArgs } from "./cli-utils";
loadLocalEnv();
@ -207,14 +203,7 @@ async function fetchRankedKeywords(
);
}
const rawItems = task.result?.[0]?.items ?? [];
const parsed = z.array(domainRankedKeywordItemSchema).safeParse(rawItems);
if (!parsed.success) {
console.error("Schema validation issues:", parsed.error.issues.slice(0, 3));
throw new Error("DataForSEO response failed schema validation");
}
return parsed.data;
return task.result?.[0]?.items ?? [];
}
// ---------------------------------------------------------------------------

4
src/server/features/ai-search/services/brandLookup.ts
View File

@ -1,13 +1,13 @@
import { waitUntil } from "cloudflare:workers";
import { sortBy } from "remeda";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import {
buildLlmTarget,
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
type LlmPlatform,
} from "@/server/lib/dataforseoLlm";
} from "@/server/lib/dataforseo";
import type {
LlmAggregatedTotal,
LlmMentionItem,

2
src/server/features/ai-search/services/promptExplorer.ts
View File

@ -1,6 +1,6 @@
import { waitUntil } from "cloudflare:workers";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import type { LlmResponseResult } from "@/server/lib/dataforseoLlmSchemas";
import { AppError } from "@/server/lib/errors";
import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";

7
src/server/features/backlinks/services/BacklinksService.billing.test.ts
View File

@ -15,11 +15,8 @@ vi.mock("@/server/lib/r2-cache", () => ({
setCached: vi.fn(async () => undefined),
}));
vi.mock("@/server/lib/dataforseoBacklinks", () => ({
vi.mock("@/server/lib/dataforseo", () => ({
normalizeBacklinksTarget: vi.fn(),
}));
vi.mock("@/server/lib/dataforseoClient", () => ({
createDataforseoClient: vi.fn(() => ({
backlinks: {
summary: backlinksSummaryMock,
@ -31,7 +28,7 @@ vi.mock("@/server/lib/dataforseoClient", () => ({
})),
}));
import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinks";
import { normalizeBacklinksTarget } from "@/server/lib/dataforseo";
import { createBacklinksService } from "./BacklinksService";
const billingCustomer = {

2
src/server/features/backlinks/services/BacklinksService.ts
View File

@ -1,5 +1,5 @@
import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinks";
import { normalizeBacklinksTarget } from "@/server/lib/dataforseo";
import {
normalizeBacklinksSpamFilterOptions,
type BacklinksSpamFilterOptions,

32
src/server/features/backlinks/services/backlinksServiceData.ts
View File

@ -1,14 +1,14 @@
import { z } from "zod";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import {
type fetchBacklinksHistoryRaw,
type fetchBacklinksRowsRaw,
type fetchBacklinksSummaryRaw,
type fetchDomainPagesSummaryRaw,
type fetchReferringDomainsRaw,
createDataforseoClient,
normalizeBacklinksTarget,
} from "@/server/lib/dataforseoBacklinks";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
type BacklinksHistoryItem,
type BacklinksItem,
type BacklinksSummaryItem,
type DomainPageSummaryItem,
type ReferringDomainItem,
} from "@/server/lib/dataforseo";
import {
normalizeBacklinksSpamFilterOptions,
type BacklinksSpamFilterOptions,
@ -204,9 +204,9 @@ function buildBacklinksDateRange(now: Date): BacklinksDateRange {
function buildOverviewResult(args: {
normalizedTarget: ReturnType<typeof normalizeBacklinksTarget>;
now: Date;
summary: Awaited<ReturnType<typeof fetchBacklinksSummaryRaw>>["data"];
backlinks: Awaited<ReturnType<typeof fetchBacklinksRowsRaw>>["data"];
history: Awaited<ReturnType<typeof fetchBacklinksHistoryRaw>>["data"];
summary: BacklinksSummaryItem;
backlinks: BacklinksItem[];
history: BacklinksHistoryItem[];
}): BacklinksOverviewResult {
const historyRows = args.history
.map((item) => ({
@ -277,9 +277,7 @@ function normalizeHistoryDate(value: string | null | undefined) {
return value ? value.slice(0, 10) : null;
}
function mapBacklinksRows(
rows: Awaited<ReturnType<typeof fetchBacklinksRowsRaw>>["data"],
) {
function mapBacklinksRows(rows: BacklinksItem[]) {
return rows.map((item) => ({
domainFrom: item.domain_from ?? null,
urlFrom: item.url_from ?? null,
@ -300,9 +298,7 @@ function mapBacklinksRows(
}));
}
function mapReferringDomainsRows(
rows: Awaited<ReturnType<typeof fetchReferringDomainsRaw>>["data"],
) {
function mapReferringDomainsRows(rows: ReferringDomainItem[]) {
return rows.map((item) => ({
domain: item.domain ?? null,
backlinks: item.backlinks ?? null,
@ -315,9 +311,7 @@ function mapReferringDomainsRows(
}));
}
function mapTopPagesRows(
rows: Awaited<ReturnType<typeof fetchDomainPagesSummaryRaw>>["data"],
) {
function mapTopPagesRows(rows: DomainPageSummaryItem[]) {
return rows.map((item) => ({
page: item.page ?? item.url ?? null,
backlinks: item.backlinks ?? null,

2
src/server/features/domain/services/DomainService.ts
View File

@ -1,7 +1,7 @@
import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
import { z } from "zod";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import { normalizeDomainInput } from "@/server/lib/domainUtils";
import { mapKeywordItem } from "@/server/features/domain/services/domainKeywordMapper";
import { getKeywordsPage } from "@/server/features/domain/services/domainKeywordsPage";

2
src/server/features/domain/services/domainKeywordsPage.ts
View File

@ -1,6 +1,6 @@
import { z } from "zod";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
import { normalizeDomainInput } from "@/server/lib/domainUtils";
import { mapKeywordItem } from "@/server/features/domain/services/domainKeywordMapper";

2
src/server/features/domain/services/domainPagesPage.ts
View File

@ -1,6 +1,6 @@
import { z } from "zod";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
import { normalizeDomainInput, toRelativePath } from "@/server/lib/domainUtils";
import type { RelevantPagesItem } from "@/server/lib/dataforseo";

49
src/server/features/keywords/services/research/research-data.ts
View File

@ -1,6 +1,6 @@
import { type LabsKeywordDataItem } from "@/server/lib/dataforseoClient";
import { type LabsKeywordDataItem } from "@/server/lib/dataforseo";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import {
normalizeIntent,
normalizeKeyword,
@ -37,8 +37,8 @@ function mapKeywordDataItems(items: LabsKeywordDataItem[]): EnrichedKeyword[] {
keyword: normalized,
searchVolume: keywordInfo?.search_volume ?? null,
trend: (keywordInfo?.monthly_searches ?? []).map((entry) => ({
year: entry.year,
month: entry.month,
year: entry.year ?? 0,
month: entry.month ?? 0,
searchVolume: entry.search_volume ?? 0,
})),
cpc: item.keyword_info?.cpc ?? null,
@ -63,40 +63,13 @@ async function fetchRelatedRows(
depth: 3,
});
const rows: EnrichedKeyword[] = [];
const seen = new Set<string>();
for (const item of items) {
const keywordData = item.keyword_data;
const keyword = keywordData.keyword;
if (!keyword) continue;
const normalized = normalizeKeyword(keyword);
if (seen.has(normalized)) continue;
seen.add(normalized);
const keywordInfo = keywordData.keyword_info_normalized_with_clickstream
?.search_volume
? keywordData.keyword_info_normalized_with_clickstream
: keywordData.keyword_info;
rows.push({
keyword: normalized,
searchVolume: keywordInfo?.search_volume ?? null,
trend: (keywordInfo?.monthly_searches ?? []).map((entry) => ({
year: entry.year,
month: entry.month,
searchVolume: entry.search_volume ?? 0,
})),
cpc: keywordData.keyword_info?.cpc ?? null,
competition: keywordData.keyword_info?.competition ?? null,
keywordDifficulty:
keywordData.keyword_properties?.keyword_difficulty ?? null,
intent: normalizeIntent(keywordData.search_intent_info?.main_intent),
});
}
return rows;
// Related items wrap the keyword payload one level deeper; unwrap and reuse
// the same mapper as suggestions/ideas.
return mapKeywordDataItems(
items
.map((item) => item.keyword_data)
.filter((data): data is NonNullable<typeof data> => data != null),
);
}
export async function fetchResearchRowsBySource(

4
src/server/features/keywords/services/research/serp.ts
View File

@ -1,9 +1,9 @@
import { type SerpLiveItem } from "@/server/lib/dataforseoClient";
import { type SerpLiveItem } from "@/server/lib/dataforseo";
import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
import type { SerpResultItem } from "@/types/keywords";
import { z } from "zod";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import { normalizeKeyword } from "./helpers";
const SERP_CACHE_TTL_SECONDS = 12 * 60 * 60;

3
src/server/features/rank-tracking/services/RankTrackingService.ts
View File

@ -1,6 +1,6 @@
import { env } from "cloudflare:workers";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository";
import { AppError } from "@/server/lib/errors";
import type {
@ -270,6 +270,7 @@ async function refreshKeywordMetrics(
}
>();
for (const item of items) {
if (!item.keyword) continue;
metricsMap.set(item.keyword.toLowerCase(), {
searchVolume: item.keyword_info?.search_volume ?? null,
keywordDifficulty: item.keyword_properties?.keyword_difficulty ?? null,

2
src/server/lib/audit/lighthouse.ts
View File

@ -1,6 +1,6 @@
import { detectUrlTemplate } from "./url-utils";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import type { LighthouseResult, LighthouseStrategy } from "./types";
import { putTextToR2 } from "@/server/lib/r2";

134
src/server/lib/dataforseo.test.ts
View File

@ -1,134 +0,0 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("cloudflare:workers", () => ({
env: {
DATAFORSEO_API_KEY: "encoded-key",
},
}));
describe("DataForSEO raw wrappers", () => {
beforeEach(() => {
vi.resetModules();
vi.restoreAllMocks();
});
it("uses the live endpoint for Google Business Q&A", async () => {
const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({
status_code: 20000,
tasks: [
{
status_code: 20000,
path: [
"v3",
"business_data",
"google",
"questions_and_answers",
"live",
],
cost: 0.0006,
result_count: 1,
result: [
{
items: [
{
question_text: "Do you offer indoor storage?",
answer_text: "Yes.",
},
],
},
],
},
],
}),
);
vi.stubGlobal("fetch", fetchMock);
const { fetchBusinessQuestionsAnswersRaw } = await import("./dataforseo");
const result = await fetchBusinessQuestionsAnswersRaw({
keyword: "Acme Storage",
locationCoordinate: "33.1234568,-84.9876543,5000",
languageCode: "en",
depth: 20,
});
expect(
fetchMock.mock.calls.map(([url]) =>
typeof url === "string" || url instanceof URL
? url.toString()
: url.url,
),
).toEqual([
"https://api.dataforseo.com/v3/business_data/google/questions_and_answers/live",
]);
expect(result.data).toEqual([
{
question_text: "Do you offer indoor storage?",
answer_text: "Yes.",
},
]);
expect(result.billing).toEqual({
path: ["v3", "business_data", "google", "questions_and_answers", "live"],
costUsd: 0.0006,
resultCount: 1,
});
});
it("does not send location_name for keyword search volume", async () => {
const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({
status_code: 20000,
tasks: [
{
status_code: 20000,
path: [
"v3",
"keywords_data",
"google_ads",
"search_volume",
"live",
],
cost: 0.0001,
result_count: 1,
result: [
{
items: [
{
keyword: "storage units",
location_code: 2840,
language_code: "en",
search_volume: 1000,
},
],
},
],
},
],
}),
);
vi.stubGlobal("fetch", fetchMock);
const { fetchKeywordSearchVolumeRaw } = await import("./dataforseo");
await fetchKeywordSearchVolumeRaw({
keywords: ["storage units"],
locationCode: 2840,
languageCode: "en",
});
const init = fetchMock.mock.calls[0]?.[1];
expect(typeof init?.body).toBe("string");
const body = init?.body;
if (typeof body !== "string") {
throw new Error("Expected DataForSEO request body to be a string");
}
const payload = JSON.parse(body) as unknown;
expect(payload).toEqual([
{
keywords: ["storage units"],
location_code: 2840,
language_code: "en",
},
]);
expect(JSON.stringify(payload)).not.toContain("location_name");
});
});

798
src/server/lib/dataforseo.ts
View File

@ -1,798 +0,0 @@
/* eslint-disable max-lines */
import {
DataforseoLabsApi,
DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo,
DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo,
DataforseoLabsGoogleKeywordIdeasLiveRequestInfo,
DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo,
DataforseoLabsGoogleRankedKeywordsLiveRequestInfo,
DataforseoLabsGoogleRelevantPagesLiveRequestInfo,
} from "dataforseo-client";
import { env } from "cloudflare:workers";
import { z } from "zod";
import {
DataforseoChargedTaskError,
type DataforseoApiCallCost,
type DataforseoApiResponse,
} from "@/server/lib/dataforseoCost";
import { AppError } from "@/server/lib/errors";
import {
dataforseoResponseSchema,
domainMetricsItemSchema,
domainRankedKeywordItemSchema,
keywordOverviewItemSchema,
labsKeywordDataItemSchema,
parseTaskItems,
relatedKeywordItemSchema,
relevantPagesItemSchema,
serpSnapshotItemSchema,
type DataforseoTask,
type DomainMetricsItem,
type DomainRankedKeywordItem,
type KeywordOverviewItem,
type LabsKeywordDataItem,
type RelatedKeywordItem,
type RelevantPagesItem,
type SerpLiveItem,
successfulDataforseoTaskSchema,
} from "@/server/lib/dataforseoSchemas";
export type {
DomainRankedKeywordItem,
LabsKeywordDataItem,
RelevantPagesItem,
SerpLiveItem,
} from "@/server/lib/dataforseoSchemas";
// ---------------------------------------------------------------------------
// SDK client factories (lazily created per-request using the env secret)
// ---------------------------------------------------------------------------
function createAuthenticatedFetch() {
return async (url: RequestInfo, init?: RequestInit): Promise<Response> => {
const headers = new Headers(init?.headers);
headers.set("Authorization", `Basic ${env.DATAFORSEO_API_KEY}`);
const newInit: RequestInit = {
...init,
headers,
};
const response = await fetch(url, newInit);
if (!response.ok) {
const rawText = await response.text();
const path = formatDataforseoRequestPath(url);
const err = new AppError(
response.status === 429 ? "RATE_LIMITED" : "INTERNAL_ERROR",
`DataForSEO HTTP ${response.status} on ${path}`,
{
provider: "dataforseo",
providerStatus: String(response.status),
providerPath: path,
responseBody: formatDataforseoErrorPayload(rawText),
},
);
err.name = "DataForSEOHttpError";
throw err;
}
return response;
};
}
const API_BASE = "https://api.dataforseo.com";
const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600;
function getLabsApi() {
return new DataforseoLabsApi(API_BASE, { fetch: createAuthenticatedFetch() });
}
function formatDataforseoRequestPath(url: RequestInfo): string {
const rawUrl = typeof url === "string" ? url : url.url;
try {
return new URL(rawUrl).pathname;
} catch {
return rawUrl;
}
}
async function postDataforseo(
path: string,
payload: unknown,
): Promise<unknown> {
const authenticatedFetch = createAuthenticatedFetch();
const response = await authenticatedFetch(`${API_BASE}${path}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(payload),
});
const rawText = await response.text();
try {
return JSON.parse(rawText);
} catch {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned a non-JSON response. Response: ${formatDataforseoErrorPayload(rawText)}`,
);
}
}
// ---------------------------------------------------------------------------
// Response helpers
// ---------------------------------------------------------------------------
/**
* Validate that the top-level response and first task both succeeded.
* Throws a descriptive error on failure. Returns the first task.
*/
type DataforseoTaskLike = {
id?: string;
status_code?: number;
status_message?: string;
path?: string[];
cost?: number;
result_count?: number | null;
data?: unknown;
result?: DataforseoTask["result"];
};
const dataforseoGenericItemSchema = z.record(z.string(), z.unknown());
const dataforseoGenericResultSchema = z
.object({
total_count: z.number().nullable().optional(),
count: z.number().nullable().optional(),
offset: z.number().nullable().optional(),
items: z.array(dataforseoGenericItemSchema).nullable().optional(),
items_without_answers: z
.array(dataforseoGenericItemSchema)
.nullable()
.optional(),
})
.passthrough();
type GenericDataforseoItem = z.infer<typeof dataforseoGenericItemSchema>;
const dataforseoChargedTaskSchema = z.object({
path: z.array(z.string()),
cost: z.number(),
result_count: z.number().nullable().optional(),
});
function formatDataforseoErrorPayload(value: unknown): string {
const text =
typeof value === "string"
? value
: (() => {
try {
return JSON.stringify(value);
} catch {
return String(value);
}
})();
return text.length > MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH
? `${text.slice(0, MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH)}... [truncated]`
: text;
}
function compactObject(
value: Record<string, unknown>,
): Record<string, unknown> {
return Object.fromEntries(
Object.entries(value).filter(([, entry]) => entry !== undefined),
);
}
function extractGenericItems(task: DataforseoTask): GenericDataforseoItem[] {
const result = task.result ?? [];
const resultItems = z.array(dataforseoGenericResultSchema).parse(result);
const resultHasNestedItems = resultItems.some((item) =>
Object.hasOwn(item, "items"),
);
return resultHasNestedItems
? resultItems.flatMap((item) => [
...(item.items ?? []),
...(item.items_without_answers ?? []),
])
: z.array(dataforseoGenericItemSchema).parse(result);
}
async function postGenericItems(
path: string,
payload: Record<string, unknown>,
): Promise<DataforseoApiResponse<GenericDataforseoItem[]>> {
const responseRaw = await postDataforseo(path, [compactObject(payload)]);
const response = dataforseoResponseSchema.parse(responseRaw);
const task = assertOk(response);
return {
data: extractGenericItems(task),
billing: buildTaskBilling(task),
};
}
function getTaskDebugPayload(task: DataforseoTaskLike) {
return {
id: task.id ?? null,
status_code: task.status_code ?? null,
status_message: task.status_message ?? null,
path: task.path ?? null,
cost: task.cost ?? null,
result_count: task.result_count ?? null,
data: task.data ?? null,
result_length: Array.isArray(task.result) ? task.result.length : null,
result_preview: Array.isArray(task.result)
? (task.result[0] ?? null)
: null,
};
}
function assertOk<T extends DataforseoTaskLike>(
response: {
status_code?: number;
status_message?: string;
tasks?: T[];
} | null,
): DataforseoTask {
if (!response) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO returned an empty response",
);
}
if (response.status_code !== 20000) {
throw new AppError(
"INTERNAL_ERROR",
response.status_message || "DataForSEO request failed",
);
}
const task = response.tasks?.[0];
if (!task) {
throw new AppError("INTERNAL_ERROR", "DataForSEO response missing task");
}
if (task.status_code !== 20000) {
const chargedTask = dataforseoChargedTaskSchema.safeParse(task);
if (chargedTask.success) {
throw new DataforseoChargedTaskError(
task.status_message || "DataForSEO task failed",
buildTaskBilling(chargedTask.data),
);
}
throw new AppError(
"INTERNAL_ERROR",
task.status_message || "DataForSEO task failed",
);
}
const parsedTask = successfulDataforseoTaskSchema.safeParse(task);
if (!parsedTask.success) {
const issueSummary = parsedTask.error.issues
.slice(0, 5)
.map((issue) => {
const path = issue.path.length > 0 ? issue.path.join(".") : "task";
return `${path}: ${issue.message}`;
})
.join("; ");
const responseSummary = formatDataforseoErrorPayload({
status_code: response.status_code ?? null,
status_message: response.status_message ?? null,
task: getTaskDebugPayload(task),
});
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO task missing billing metadata (${issueSummary}). Response: ${responseSummary}`,
);
}
return parsedTask.data;
}
function buildTaskBilling(task: {
path: string[];
cost: number;
result_count?: number | null;
}): DataforseoApiCallCost {
return {
path: task.path,
costUsd: task.cost,
resultCount: task.result_count ?? null,
};
}
// ---------------------------------------------------------------------------
// DataForSEO Labs API wrappers
// ---------------------------------------------------------------------------
export async function fetchRelatedKeywordsRaw(
keyword: string,
locationCode: number,
languageCode: string,
limit: number,
depth: number = 3,
): Promise<DataforseoApiResponse<RelatedKeywordItem[]>> {
const api = getLabsApi();
const req = new DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo({
keyword,
location_code: locationCode,
language_code: languageCode,
limit,
depth,
include_clickstream_data: true,
include_serp_info: false,
});
const endpoint = "google-related-keywords-live";
const response = await api.googleRelatedKeywordsLive([req]);
const task = assertOk(response);
const data = parseTaskItems(endpoint, task, relatedKeywordItemSchema);
return {
data,
billing: buildTaskBilling(task),
};
}
export async function fetchKeywordSuggestionsRaw(
keyword: string,
locationCode: number,
languageCode: string,
limit: number,
): Promise<DataforseoApiResponse<LabsKeywordDataItem[]>> {
const api = getLabsApi();
const req = new DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo({
keyword,
location_code: locationCode,
language_code: languageCode,
limit,
include_clickstream_data: true,
include_serp_info: false,
include_seed_keyword: true,
ignore_synonyms: false,
exact_match: false,
});
const endpoint = "google-keyword-suggestions-live";
const response = await api.googleKeywordSuggestionsLive([req]);
const task = assertOk(response);
const data = parseTaskItems(endpoint, task, labsKeywordDataItemSchema);
return {
data,
billing: buildTaskBilling(task),
};
}
export async function fetchKeywordIdeasRaw(
keyword: string,
locationCode: number,
languageCode: string,
limit: number,
): Promise<DataforseoApiResponse<LabsKeywordDataItem[]>> {
const api = getLabsApi();
const req = new DataforseoLabsGoogleKeywordIdeasLiveRequestInfo({
keywords: [keyword],
location_code: locationCode,
language_code: languageCode,
limit,
include_clickstream_data: true,
include_serp_info: false,
ignore_synonyms: false,
closely_variants: false,
});
const endpoint = "google-keyword-ideas-live";
const response = await api.googleKeywordIdeasLive([req]);
const task = assertOk(response);
const data = parseTaskItems(endpoint, task, labsKeywordDataItemSchema);
return {
data,
billing: buildTaskBilling(task),
};
}
// ---------------------------------------------------------------------------
// Domain API wrappers
// ---------------------------------------------------------------------------
export async function fetchDomainRankOverviewRaw(
target: string,
locationCode: number,
languageCode: string,
): Promise<DataforseoApiResponse<DomainMetricsItem[]>> {
const api = getLabsApi();
const req = new DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo({
target,
location_code: locationCode,
language_code: languageCode,
limit: 1,
});
const endpoint = "google-domain-rank-overview-live";
const response = await api.googleDomainRankOverviewLive([req]);
const task = assertOk(response);
const data = parseTaskItems(endpoint, task, domainMetricsItemSchema);
return {
data,
billing: buildTaskBilling(task),
};
}
type RankedKeywordsPage = {
items: DomainRankedKeywordItem[];
totalCount: number | null;
};
export async function fetchRankedKeywordsRaw(input: {
target: string;
locationCode: number;
languageCode: string;
limit: number;
offset?: number;
orderBy?: string[];
filters?: unknown[];
itemTypes?: DataforseoLabsItemType[];
includeSubdomains?: boolean;
}): Promise<DataforseoApiResponse<RankedKeywordsPage>> {
const api = getLabsApi();
const req = new DataforseoLabsGoogleRankedKeywordsLiveRequestInfo({
target: input.target,
location_code: input.locationCode,
language_code: input.languageCode,
limit: input.limit,
offset: input.offset,
order_by: input.orderBy,
filters: input.filters,
item_types: input.itemTypes,
include_subdomains: input.includeSubdomains,
});
const endpoint = "google-ranked-keywords-live";
const response = await api.googleRankedKeywordsLive([req]);
const task = assertOk(response);
const items = parseTaskItems(endpoint, task, domainRankedKeywordItemSchema);
const rawTotal = (task.result?.[0] as Record<string, unknown> | undefined)
?.total_count;
const totalCount = typeof rawTotal === "number" ? rawTotal : null;
return {
data: { items, totalCount },
billing: buildTaskBilling(task),
};
}
type RelevantPagesPage = {
items: RelevantPagesItem[];
totalCount: number | null;
};
export async function fetchRelevantPagesRaw(input: {
target: string;
locationCode: number;
languageCode: string;
limit: number;
offset?: number;
orderBy?: string[];
filters?: unknown[];
}): Promise<DataforseoApiResponse<RelevantPagesPage>> {
const api = getLabsApi();
const req = new DataforseoLabsGoogleRelevantPagesLiveRequestInfo({
target: input.target,
location_code: input.locationCode,
language_code: input.languageCode,
limit: input.limit,
offset: input.offset,
order_by: input.orderBy,
filters: input.filters,
});
const endpoint = "google-relevant-pages-live";
const response = await api.googleRelevantPagesLive([req]);
const task = assertOk(response);
const items = parseTaskItems(endpoint, task, relevantPagesItemSchema);
const rawTotal = (task.result?.[0] as Record<string, unknown> | undefined)
?.total_count;
const totalCount = typeof rawTotal === "number" ? rawTotal : null;
return {
data: { items, totalCount },
billing: buildTaskBilling(task),
};
}
// ---------------------------------------------------------------------------
// SERP Analysis API wrapper (Google Organic Live)
// ---------------------------------------------------------------------------
export async function fetchLiveSerpItemsRaw(
keyword: string,
locationCode: number,
languageCode: string,
): Promise<DataforseoApiResponse<SerpLiveItem[]>> {
const responseRaw = await postDataforseo(
"/v3/serp/google/organic/live/advanced",
[
{
keyword,
location_code: locationCode,
language_code: languageCode,
device: "desktop",
os: "windows",
depth: 100,
},
],
);
const response = dataforseoResponseSchema.parse(responseRaw);
const endpoint = "google-organic-live-advanced";
const task = assertOk(response);
const data = parseTaskItems(endpoint, task, serpSnapshotItemSchema);
return {
data,
billing: buildTaskBilling(task),
};
}
export async function fetchLocalSerpItemsRaw(input: {
keyword: string;
locationCoordinate?: string;
languageCode: string;
searchType: "maps" | "local_finder";
device: "desktop" | "mobile";
depth: number;
searchPlaces?: boolean;
}): Promise<DataforseoApiResponse<GenericDataforseoItem[]>> {
const path =
input.searchType === "maps"
? "/v3/serp/google/maps/live/advanced"
: "/v3/serp/google/local_finder/live/advanced";
return postGenericItems(path, {
keyword: input.keyword,
location_coordinate: input.locationCoordinate,
language_code: input.languageCode,
device: input.device,
os: input.device === "desktop" ? "windows" : "android",
depth: input.depth,
search_places: input.searchPlaces,
});
}
// ---------------------------------------------------------------------------
// SERP Rank Check API wrapper (Google Organic Live with target matching)
// ---------------------------------------------------------------------------
export interface RankCheckResult {
keywordId: string;
keyword: string;
position: number | null;
url: string | null;
serpFeatures: string[];
}
export async function fetchRankCheckSerpRaw(input: {
keyword: string;
keywordId: string;
locationCode: number;
languageCode: string;
device: "desktop" | "mobile";
targetDomain: string;
depth: number;
}): Promise<DataforseoApiResponse<RankCheckResult>> {
const depth = Math.min(100, Math.max(10, input.depth));
const responseRaw = await postDataforseo(
"/v3/serp/google/organic/live/advanced",
[
{
keyword: input.keyword,
location_code: input.locationCode,
language_code: input.languageCode,
device: input.device,
os: input.device === "desktop" ? "windows" : "android",
depth,
},
],
);
const response = dataforseoResponseSchema.parse(responseRaw);
if (response.status_code !== 20000) {
throw new AppError(
"INTERNAL_ERROR",
response.status_message || "DataForSEO request failed",
);
}
const task = response.tasks?.[0];
if (!task) {
throw new AppError("INTERNAL_ERROR", "DataForSEO response missing task");
}
// "No Search Results" (40501) is valid for obscure/new keywords —
// treat as empty result set rather than failing the entire run.
const isNoResults =
task.status_code === 40501 ||
task.status_message?.toLowerCase().includes("no search results");
if (task.status_code !== 20000 && !isNoResults) {
throw new AppError(
"INTERNAL_ERROR",
task.status_message || "DataForSEO task failed",
);
}
const parsedTask = successfulDataforseoTaskSchema.safeParse(task);
if (!parsedTask.success) {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO rank check task missing billing metadata`,
);
}
const items = z
.array(serpSnapshotItemSchema)
.parse(parsedTask.data.result?.[0]?.items ?? []);
const target = input.targetDomain.toLowerCase();
const organicMatch = items.find((item) => {
if (item.type !== "organic" || item.domain == null) return false;
const d = item.domain.toLowerCase();
return d === target || d.endsWith(`.${target}`);
});
return {
data: {
keywordId: input.keywordId,
keyword: input.keyword,
position: organicMatch
? (organicMatch.rank_absolute ?? organicMatch.rank_group ?? null)
: null,
url: organicMatch?.url ?? null,
serpFeatures: [
...new Set(items.map((item) => item.type).filter(Boolean)),
],
},
billing: buildTaskBilling(parsedTask.data),
};
}
// ---------------------------------------------------------------------------
// DataForSEO Labs — Keyword Overview (batch up to 700 keywords)
// ---------------------------------------------------------------------------
export async function fetchKeywordOverviewRaw(
keywords: string[],
locationCode: number,
languageCode: string,
): Promise<DataforseoApiResponse<KeywordOverviewItem[]>> {
const responseRaw = await postDataforseo(
"/v3/dataforseo_labs/google/keyword_overview/live",
[
{
keywords,
location_code: locationCode,
language_code: languageCode,
},
],
);
const response = dataforseoResponseSchema.parse(responseRaw);
if (response.status_code !== 20000) {
throw new AppError(
"INTERNAL_ERROR",
response.status_message || "DataForSEO keyword overview request failed",
);
}
const task = response.tasks?.[0];
if (!task) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO keyword overview response missing task",
);
}
if (task.status_code !== 20000) {
throw new AppError(
"INTERNAL_ERROR",
task.status_message || "DataForSEO keyword overview task failed",
);
}
const parsedTask = successfulDataforseoTaskSchema.safeParse(task);
if (!parsedTask.success) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO keyword overview task missing billing metadata",
);
}
const data = parseTaskItems(
"google-keyword-overview-live",
parsedTask.data,
keywordOverviewItemSchema,
);
return {
data,
billing: buildTaskBilling(parsedTask.data),
};
}
export async function fetchBusinessListingsSearchRaw(input: {
categories?: string[];
title?: string;
locationCoordinate: string;
orderBy?: string[];
limit: number;
}): Promise<DataforseoApiResponse<GenericDataforseoItem[]>> {
return postGenericItems("/v3/business_data/business_listings/search/live", {
categories: input.categories,
title: input.title,
location_coordinate: input.locationCoordinate,
order_by: input.orderBy,
limit: input.limit,
});
}
export async function fetchBusinessQuestionsAnswersRaw(input: {
keyword: string;
locationCoordinate: string;
languageCode: string;
depth: number;
}): Promise<DataforseoApiResponse<GenericDataforseoItem[]>> {
return postGenericItems(
"/v3/business_data/google/questions_and_answers/live",
{
keyword: input.keyword,
location_coordinate: input.locationCoordinate,
language_code: input.languageCode,
depth: input.depth,
},
);
}
export async function fetchKeywordSearchVolumeRaw(input: {
keywords: string[];
locationCode?: number;
languageCode?: string;
}): Promise<DataforseoApiResponse<GenericDataforseoItem[]>> {
return postGenericItems("/v3/keywords_data/google_ads/search_volume/live", {
keywords: input.keywords,
location_code: input.locationCode,
language_code: input.languageCode,
});
}
export type DataforseoLabsItemType =
| "organic"
| "paid"
| "featured_snippet"
| "local_pack"
| "ai_overview_reference";
export async function fetchSerpCompetitorsRaw(input: {
keywords: string[];
locationCode: number;
languageCode: string;
itemTypes?: DataforseoLabsItemType[];
includeSubdomains?: boolean;
limit: number;
offset?: number;
}): Promise<DataforseoApiResponse<GenericDataforseoItem[]>> {
return postGenericItems("/v3/dataforseo_labs/google/serp_competitors/live", {
keywords: input.keywords,
location_code: input.locationCode,
language_code: input.languageCode,
item_types: input.itemTypes,
include_subdomains: input.includeSubdomains,
limit: input.limit,
offset: input.offset,
});
}

333
src/server/lib/dataforseo/ai.ts Normal file
View File

@ -0,0 +1,333 @@
import { z } from "zod";
import {
AiOptimizationChatGptLlmResponsesLiveRequestInfo,
AiOptimizationClaudeLlmResponsesLiveRequestInfo,
AiOptimizationGeminiLlmResponsesLiveRequestInfo,
AiOptimizationLLmMentionsDomainElement,
AiOptimizationLLmMentionsKeywordElement,
AiOptimizationLlmMentionsAggregatedMetricsLiveRequestInfo,
AiOptimizationLlmMentionsSearchLiveRequestInfo,
AiOptimizationLlmMentionsTopPagesLiveRequestInfo,
type BaseAiOptimizationLLmMentionsTargetElement,
type AiOptimizationPerplexityLlmResponsesLiveRequestInfo,
} from "dataforseo-client";
import {
llmAggregatedTotalSchema,
llmMentionItemSchema,
llmResponseResultSchema,
llmTopPagesItemSchema,
type LlmAggregatedTotal,
type LlmMentionItem,
type LlmResponseResult,
type LlmTopPagesItem,
} from "@/server/lib/dataforseoLlmSchemas";
import { createDataforseoAccessClassifier } from "@/server/lib/dataforseoAccessClassification";
import { AppError } from "@/server/lib/errors";
import { aiOptimizationApi } from "@/server/lib/dataforseo/core";
import {
assertOk,
buildTaskBilling,
isRecord,
type DataforseoApiResponse,
type DataforseoTaskLike,
} from "@/server/lib/dataforseo/envelope";
// ChatGPT mention/response data is only available for US/en per DataForSEO docs.
export const CHATGPT_LOCATION_CODE = 2840;
export const CHATGPT_LANGUAGE_CODE = "en";
export type LlmPlatform = "chat_gpt" | "google";
const classifyAiSearchError = createDataforseoAccessClassifier({
pathPrefix: "/ai_optimization/",
notEnabledCode: "AI_SEARCH_NOT_ENABLED",
notEnabledMessage:
"AI Optimization is not enabled for the connected DataForSEO account",
billingIssueCode: "AI_SEARCH_BILLING_ISSUE",
billingIssueMessage:
"The connected DataForSEO account has a billing or balance issue",
});
const assertOptions = (path: string) =>
({ classify: classifyAiSearchError, classifyPath: path }) as const;
// ---------------------------------------------------------------------------
// Target builders — DataForSEO's `target` array accepts domain OR keyword
// entries. We always pass exactly one target per call.
// ---------------------------------------------------------------------------
type LlmTarget =
| {
domain: string;
include_subdomains?: boolean;
search_filter?: "include" | "exclude";
search_scope?: string[];
}
| {
keyword: string;
search_filter?: "include" | "exclude";
search_scope?: string[];
match_type?: "word_match" | "partial_match";
};
export function buildLlmTarget(input: {
type: "domain" | "keyword";
value: string;
}): LlmTarget {
if (input.type === "domain") {
return {
domain: input.value,
include_subdomains: true,
search_filter: "include",
search_scope: ["any"],
};
}
return {
keyword: input.value,
search_filter: "include",
search_scope: ["any", "brand_entities"],
match_type: "word_match",
};
}
function clampLimit(value: number, min: number, max: number): number {
return Math.min(max, Math.max(min, Math.floor(value)));
}
function targetList(
target: LlmTarget,
): BaseAiOptimizationLLmMentionsTargetElement[] {
return [
"domain" in target
? new AiOptimizationLLmMentionsDomainElement(target)
: new AiOptimizationLLmMentionsKeywordElement(target),
];
}
function firstResult(task: DataforseoTaskLike): Record<string, unknown> | null {
const first = task.result?.[0];
return isRecord(first) ? first : null;
}
// ---------------------------------------------------------------------------
// LLM Mentions Search
// ---------------------------------------------------------------------------
type LlmMentionsSearchInput = {
target: LlmTarget;
platform: LlmPlatform;
locationCode: number;
languageCode: string;
limit?: number;
};
export async function fetchLlmMentionsSearch(
input: LlmMentionsSearchInput,
): Promise<DataforseoApiResponse<LlmMentionItem[]>> {
const response = await aiOptimizationApi(
classifyAiSearchError,
).llmMentionsSearchLive([
new AiOptimizationLlmMentionsSearchLiveRequestInfo({
target: targetList(input.target),
platform: input.platform,
location_code: input.locationCode,
language_code: input.languageCode,
limit: clampLimit(input.limit ?? 100, 1, 1000),
}),
]);
const task = assertOk(
response,
assertOptions("/v3/ai_optimization/llm_mentions/search/live"),
);
const items = z
.array(llmMentionItemSchema)
.safeParse(firstResult(task)?.items ?? []);
if (!items.success) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO llm_mentions/search returned an invalid mention items shape",
);
}
return { data: items.data, billing: buildTaskBilling(task) };
}
// ---------------------------------------------------------------------------
// LLM Mentions Aggregated Metrics
// ---------------------------------------------------------------------------
type LlmAggregatedMetricsInput = {
target: LlmTarget;
platform: LlmPlatform;
locationCode: number;
languageCode: string;
internalListLimit?: number;
};
export async function fetchLlmAggregatedMetrics(
input: LlmAggregatedMetricsInput,
): Promise<DataforseoApiResponse<LlmAggregatedTotal>> {
const response = await aiOptimizationApi(
classifyAiSearchError,
).llmMentionsAggregatedMetricsLive([
new AiOptimizationLlmMentionsAggregatedMetricsLiveRequestInfo({
target: targetList(input.target),
platform: input.platform,
location_code: input.locationCode,
language_code: input.languageCode,
internal_list_limit: clampLimit(input.internalListLimit ?? 10, 1, 20),
}),
]);
const task = assertOk(
response,
assertOptions("/v3/ai_optimization/llm_mentions/aggregated_metrics/live"),
);
const total = llmAggregatedTotalSchema.safeParse(
firstResult(task)?.total ?? {},
);
if (!total.success) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO llm_mentions/aggregated_metrics returned an invalid shape",
);
}
return { data: total.data, billing: buildTaskBilling(task) };
}
// ---------------------------------------------------------------------------
// LLM Mentions Top Pages
// ---------------------------------------------------------------------------
type LlmTopPagesInput = {
target: LlmTarget;
platform: LlmPlatform;
locationCode: number;
languageCode: string;
itemsListLimit?: number;
};
export async function fetchLlmTopPages(
input: LlmTopPagesInput,
): Promise<DataforseoApiResponse<LlmTopPagesItem[]>> {
const response = await aiOptimizationApi(
classifyAiSearchError,
).llmMentionsTopPagesLive([
new AiOptimizationLlmMentionsTopPagesLiveRequestInfo({
target: targetList(input.target),
platform: input.platform,
location_code: input.locationCode,
language_code: input.languageCode,
links_scope: "sources",
items_list_limit: clampLimit(input.itemsListLimit ?? 10, 1, 10),
internal_list_limit: 5,
}),
]);
const task = assertOk(
response,
assertOptions("/v3/ai_optimization/llm_mentions/top_pages/live"),
);
const items = z
.array(llmTopPagesItemSchema)
.safeParse(firstResult(task)?.items ?? []);
if (!items.success) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO llm_mentions/top_pages returned an invalid shape",
);
}
return { data: items.data, billing: buildTaskBilling(task) };
}
// ---------------------------------------------------------------------------
// LLM Responses (per-model)
// ---------------------------------------------------------------------------
type LlmResponseModelSlug = "chat_gpt" | "claude" | "gemini" | "perplexity";
type LlmResponsesInput = {
userPrompt: string;
modelSlug: LlmResponseModelSlug;
modelName: string;
webSearch?: boolean;
maxOutputTokens?: number;
/** Two-letter ISO country code used to geolocate the web-search component. */
webSearchCountryCode?: string;
};
type LlmResponseRequestFields = {
user_prompt: string;
model_name: string;
web_search: boolean;
max_output_tokens: number;
web_search_country_iso_code?: string;
};
function buildPerplexityLlmResponseRequest(
fields: LlmResponseRequestFields,
): AiOptimizationPerplexityLlmResponsesLiveRequestInfo {
return {
...fields,
init(data?: unknown) {
if (isRecord(data)) Object.assign(this, data);
},
toJSON(data?: unknown) {
return {
...(isRecord(data) ? data : {}),
...fields,
};
},
};
}
export async function fetchLlmResponse(
input: LlmResponsesInput,
): Promise<DataforseoApiResponse<LlmResponseResult>> {
// DataForSEO's Gemini endpoint rejects `web_search_country_iso_code` with a
// 40501 "Invalid Field" error. The other three models accept it.
const supportsCountry = input.modelSlug !== "gemini";
const fields: LlmResponseRequestFields = {
user_prompt: input.userPrompt,
model_name: input.modelName,
web_search: input.webSearch ?? true,
max_output_tokens: clampLimit(input.maxOutputTokens ?? 1024, 256, 4096),
...(supportsCountry && input.webSearchCountryCode
? { web_search_country_iso_code: input.webSearchCountryCode }
: {}),
};
const api = aiOptimizationApi(classifyAiSearchError);
const response =
input.modelSlug === "chat_gpt"
? await api.chatGptLlmResponsesLive([
new AiOptimizationChatGptLlmResponsesLiveRequestInfo(fields),
])
: input.modelSlug === "claude"
? await api.claudeLlmResponsesLive([
new AiOptimizationClaudeLlmResponsesLiveRequestInfo(fields),
])
: input.modelSlug === "gemini"
? await api.geminiLlmResponsesLive([
new AiOptimizationGeminiLlmResponsesLiveRequestInfo(fields),
])
: await api.perplexityLlmResponsesLive([
// The generated Perplexity request class drops `web_search` in
// toJSON(), while the SDK method only JSON.stringify's this body.
buildPerplexityLlmResponseRequest(fields),
]);
const task = assertOk(
response,
assertOptions(`/v3/ai_optimization/${input.modelSlug}/llm_responses/live`),
);
const result = llmResponseResultSchema.safeParse(firstResult(task) ?? {});
if (!result.success) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO llm_responses returned an invalid response shape",
);
}
return { data: result.data, billing: buildTaskBilling(task) };
}

122
src/server/lib/dataforseoBacklinks.test.ts → src/server/lib/dataforseo/backlinks.test.ts
View File

@ -1,6 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { AppError } from "@/server/lib/errors";
import type * as DataforseoBacklinksSupport from "@/server/lib/dataforseoBacklinksSupport";
vi.mock("@/server/lib/runtime-env", () => ({
getRequiredEnvValue: vi.fn(async () => "test-api-key"),
@ -10,19 +9,25 @@ const { classifyBacklinksError } = vi.hoisted(() => ({
classifyBacklinksError: vi.fn(),
}));
vi.mock("@/server/lib/dataforseoBacklinksSupport", async () => {
const actual = await vi.importActual<typeof DataforseoBacklinksSupport>(
"@/server/lib/dataforseoBacklinksSupport",
);
return { ...actual, classifyBacklinksError };
});
// The classifier is built inside backlinks.ts via createDataforseoAccessClassifier;
// returning our hoisted mock lets the test drive classification.
vi.mock("@/server/lib/dataforseoAccessClassification", () => ({
createDataforseoAccessClassifier: () => classifyBacklinksError,
}));
import {
fetchBacklinksHistoryRaw,
fetchBacklinksRowsRaw,
fetchBacklinksSummaryRaw,
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
normalizeBacklinksTarget,
} from "@/server/lib/dataforseoBacklinks";
} from "@/server/lib/dataforseo/backlinks";
// A successful DataForSEO task always carries billing metadata (path + cost).
const billed = {
path: ["v3", "backlinks", "summary", "live"],
cost: 0.02,
result_count: 0,
};
describe("normalizeBacklinksTarget", () => {
it("treats explicit homepage URLs as page lookups", () => {
@ -43,14 +48,6 @@ describe("normalizeBacklinksTarget", () => {
});
});
it("keeps trailing slashes for root page URLs", () => {
expect(normalizeBacklinksTarget("https://example.com/")).toEqual({
apiTarget: "https://example.com/",
displayTarget: "https://example.com/",
scope: "page",
});
});
it("treats bare hostnames as domain lookups", () => {
expect(normalizeBacklinksTarget("Example.com")).toEqual({
apiTarget: "example.com",
@ -71,21 +68,6 @@ describe("normalizeBacklinksTarget", () => {
});
});
it("normalizes domain scope for URLs with query strings or fragments", () => {
expect(
normalizeBacklinksTarget(
"https://Example.com/pricing?utm_source=newsletter#hero",
{
scope: "domain",
},
),
).toEqual({
apiTarget: "example.com",
displayTarget: "example.com",
scope: "domain",
});
});
it("lets callers force page scope for bare hostnames", () => {
expect(normalizeBacklinksTarget("Example.com", { scope: "page" })).toEqual({
apiTarget: "https://example.com/",
@ -111,7 +93,7 @@ describe("normalizeBacklinksTarget", () => {
});
});
describe("fetchBacklinksSummaryRaw", () => {
describe("fetchBacklinksSummary", () => {
beforeEach(() => {
vi.stubGlobal("fetch", vi.fn());
});
@ -143,9 +125,7 @@ describe("fetchBacklinksSummaryRaw", () => {
});
await expect(
fetchBacklinksSummaryRaw({
target: "example.com",
}),
fetchBacklinksSummary({ target: "example.com" }),
).rejects.toMatchObject({ code: "BACKLINKS_NOT_ENABLED" });
expect(classifyBacklinksError).toHaveBeenCalledWith(
@ -165,6 +145,7 @@ describe("fetchBacklinksSummaryRaw", () => {
{
status_code: 20000,
status_message: "Ok.",
...billed,
result: [null],
},
],
@ -175,9 +156,7 @@ describe("fetchBacklinksSummaryRaw", () => {
classifyBacklinksError.mockReturnValue(null);
await expect(
fetchBacklinksSummaryRaw({
target: "not-a-real-input.example",
}),
fetchBacklinksSummary({ target: "not-a-real-input.example" }),
).resolves.toMatchObject({ data: {} });
});
@ -191,6 +170,7 @@ describe("fetchBacklinksSummaryRaw", () => {
{
status_code: 20000,
status_message: "Ok.",
...billed,
result: [],
},
],
@ -201,55 +181,37 @@ describe("fetchBacklinksSummaryRaw", () => {
classifyBacklinksError.mockReturnValue(null);
await expect(
fetchBacklinksSummaryRaw({
target: "example.com",
}),
fetchBacklinksSummary({ target: "example.com" }),
).resolves.toMatchObject({ data: {} });
});
it("treats empty backlinks rows and history results as valid empty arrays", async () => {
vi.mocked(fetch)
.mockResolvedValueOnce(
new Response(
JSON.stringify({
status_code: 20000,
status_message: "Ok.",
tasks: [
{
status_code: 20000,
status_message: "Ok.",
result: [],
},
],
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
),
)
.mockResolvedValueOnce(
new Response(
JSON.stringify({
status_code: 20000,
status_message: "Ok.",
tasks: [
{
status_code: 20000,
status_message: "Ok.",
result: [],
},
],
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
),
const emptyOk = () =>
new Response(
JSON.stringify({
status_code: 20000,
status_message: "Ok.",
tasks: [
{
status_code: 20000,
status_message: "Ok.",
...billed,
result: [],
},
],
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
);
vi.mocked(fetch)
.mockResolvedValueOnce(emptyOk())
.mockResolvedValueOnce(emptyOk());
classifyBacklinksError.mockReturnValue(null);
await expect(
fetchBacklinksRowsRaw({
target: "example.com",
}),
fetchBacklinksRows({ target: "example.com" }),
).resolves.toMatchObject({ data: [] });
await expect(
fetchBacklinksHistoryRaw({
fetchBacklinksHistory({
target: "example.com",
dateFrom: "2025-01-01",
dateTo: "2025-12-31",

280
src/server/lib/dataforseo/backlinks.ts Normal file
View File

@ -0,0 +1,280 @@
import { z } from "zod";
import {
BacklinksBacklinksLiveRequestInfo,
BacklinksDomainPagesSummaryLiveRequestInfo,
BacklinksHistoryLiveRequestInfo,
BacklinksReferringDomainsLiveRequestInfo,
BacklinksSummaryLiveRequestInfo,
} from "dataforseo-client";
import {
normalizeBacklinksSpamFilterOptions,
type BacklinksSpamFilterOptions,
} from "@/types/schemas/backlinks";
import { createDataforseoAccessClassifier } from "@/server/lib/dataforseoAccessClassification";
import { AppError } from "@/server/lib/errors";
import { backlinksApi } from "@/server/lib/dataforseo/core";
import {
assertOk,
buildTaskBilling,
parseTaskItems,
type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope";
export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
type BacklinksRequest = { target: string };
type BacklinksListRequest = BacklinksRequest &
BacklinksSpamFilterOptions & { limit?: number };
type BacklinksTimeseriesRequest = {
target: string;
dateFrom: string;
dateTo: string;
};
const classifyBacklinksError = createDataforseoAccessClassifier({
pathPrefix: "/backlinks/",
notEnabledCode: "BACKLINKS_NOT_ENABLED",
notEnabledMessage:
"Backlinks is not enabled for the connected DataForSEO account",
billingIssueCode: "BACKLINKS_BILLING_ISSUE",
billingIssueMessage:
"The connected DataForSEO account has a billing or balance issue",
});
// DataForSEO ships both the misspelled (`*_reffering_*`) and corrected keys; we
// accept both via passthrough so callers can read whichever is present.
export const backlinksSummaryItemSchema = z
.object({
target: z.string().optional(),
rank: z.number().nullable().optional(),
backlinks: z.number().nullable().optional(),
referring_pages: z.number().nullable().optional(),
referring_domains: z.number().nullable().optional(),
broken_backlinks: z.number().nullable().optional(),
broken_pages: z.number().nullable().optional(),
new_backlinks: z.number().nullable().optional(),
lost_backlinks: z.number().nullable().optional(),
new_reffering_domains: z.number().nullable().optional(),
lost_reffering_domains: z.number().nullable().optional(),
new_referring_domains: z.number().nullable().optional(),
lost_referring_domains: z.number().nullable().optional(),
backlinks_spam_score: z.number().nullable().optional(),
info: z
.object({ target_spam_score: z.number().nullable().optional() })
.passthrough()
.nullable()
.optional(),
})
.passthrough();
export const backlinksItemSchema = z
.object({
domain_from: z.string().nullable().optional(),
url_from: z.string().nullable().optional(),
url_to: z.string().nullable().optional(),
anchor: z.string().nullable().optional(),
item_type: z.string().nullable().optional(),
dofollow: z.boolean().nullable().optional(),
rank: z.number().nullable().optional(),
domain_from_rank: z.number().nullable().optional(),
page_from_rank: z.number().nullable().optional(),
backlinks_spam_score: z.number().nullable().optional(),
backlink_spam_score: z.number().nullable().optional(),
first_seen: z.string().nullable().optional(),
last_visited: z.string().nullable().optional(),
lost_date: z.string().nullable().optional(),
is_new: z.boolean().nullable().optional(),
is_lost: z.boolean().nullable().optional(),
is_broken: z.boolean().nullable().optional(),
links_count: z.number().nullable().optional(),
rel_attributes: z.array(z.string()).nullable().optional(),
attributes: z.array(z.string()).nullable().optional(),
})
.passthrough();
export const referringDomainItemSchema = z
.object({
domain: z.string().nullable().optional(),
backlinks: z.number().nullable().optional(),
referring_pages: z.number().nullable().optional(),
rank: z.number().nullable().optional(),
first_seen: z.string().nullable().optional(),
broken_backlinks: z.number().nullable().optional(),
broken_pages: z.number().nullable().optional(),
backlinks_spam_score: z.number().nullable().optional(),
target_spam_score: z.number().nullable().optional(),
})
.passthrough();
export const domainPageSummaryItemSchema = z
.object({
page: z.string().nullable().optional(),
url: z.string().nullable().optional(),
backlinks: z.number().nullable().optional(),
referring_domains: z.number().nullable().optional(),
rank: z.number().nullable().optional(),
broken_backlinks: z.number().nullable().optional(),
})
.passthrough();
export const backlinksHistoryItemSchema = z
.object({
date: z.string().nullable().optional(),
rank: z.number().nullable().optional(),
backlinks: z.number().nullable().optional(),
referring_domains: z.number().nullable().optional(),
new_backlinks: z.number().nullable().optional(),
lost_backlinks: z.number().nullable().optional(),
new_reffering_domains: z.number().nullable().optional(),
lost_reffering_domains: z.number().nullable().optional(),
new_referring_domains: z.number().nullable().optional(),
lost_referring_domains: z.number().nullable().optional(),
})
.passthrough();
function buildCommonPayload(input: BacklinksRequest) {
return {
target: input.target,
include_subdomains: true,
include_indirect_links: true,
exclude_internal_backlinks: true,
backlinks_status_type: "live",
rank_scale: "one_hundred",
};
}
const assertOptions = (path: string) =>
({ classify: classifyBacklinksError, classifyPath: path }) as const;
export async function fetchBacklinksSummary(input: BacklinksRequest) {
const response = await backlinksApi(classifyBacklinksError).summaryLive([
new BacklinksSummaryLiveRequestInfo(buildCommonPayload(input)),
]);
const task = assertOk(response, assertOptions("/v3/backlinks/summary/live"));
const firstResult = task.result?.[0];
if (firstResult) {
const parsed = backlinksSummaryItemSchema.safeParse(firstResult);
if (!parsed.success) {
console.error(
"dataforseo.backlinks-summary-live.invalid-result",
parsed.error.issues.slice(0, 5),
);
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO backlinks-summary-live returned an invalid response shape",
);
}
return {
data: parsed.data,
billing: buildTaskBilling(task),
} satisfies DataforseoApiResponse<typeof parsed.data>;
}
return {
data: {} as z.infer<typeof backlinksSummaryItemSchema>,
billing: buildTaskBilling(task),
};
}
export async function fetchBacklinksRows(input: BacklinksListRequest) {
const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input);
const filters = spamFilterOptions.hideSpam
? [["backlink_spam_score", "<=", spamFilterOptions.spamThreshold]]
: undefined;
const response = await backlinksApi(classifyBacklinksError).backlinksLive([
new BacklinksBacklinksLiveRequestInfo({
...buildCommonPayload(input),
limit: input.limit ?? 100,
order_by: ["rank,desc"],
...(filters ? { filters } : {}),
}),
]);
const task = assertOk(
response,
assertOptions("/v3/backlinks/backlinks/live"),
);
return {
data: parseTaskItems("backlinks-live", task, backlinksItemSchema),
billing: buildTaskBilling(task),
};
}
export async function fetchReferringDomains(input: BacklinksListRequest) {
const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input);
const filters = spamFilterOptions.hideSpam
? [["backlinks_spam_score", "<=", spamFilterOptions.spamThreshold]]
: undefined;
const response = await backlinksApi(
classifyBacklinksError,
).referringDomainsLive([
new BacklinksReferringDomainsLiveRequestInfo({
...buildCommonPayload(input),
limit: input.limit ?? 100,
order_by: ["backlinks,desc"],
...(filters ? { filters } : {}),
}),
]);
const task = assertOk(
response,
assertOptions("/v3/backlinks/referring_domains/live"),
);
return {
data: parseTaskItems(
"referring-domains-live",
task,
referringDomainItemSchema,
),
billing: buildTaskBilling(task),
};
}
export async function fetchDomainPagesSummary(input: BacklinksListRequest) {
const response = await backlinksApi(
classifyBacklinksError,
).domainPagesSummaryLive([
new BacklinksDomainPagesSummaryLiveRequestInfo({
...buildCommonPayload(input),
limit: input.limit ?? 100,
order_by: ["backlinks,desc"],
}),
]);
const task = assertOk(
response,
assertOptions("/v3/backlinks/domain_pages_summary/live"),
);
return {
data: parseTaskItems(
"domain-pages-summary-live",
task,
domainPageSummaryItemSchema,
),
billing: buildTaskBilling(task),
};
}
export async function fetchBacklinksHistory(input: BacklinksTimeseriesRequest) {
const response = await backlinksApi(classifyBacklinksError).historyLive([
new BacklinksHistoryLiveRequestInfo({
target: input.target,
date_from: input.dateFrom,
date_to: input.dateTo,
rank_scale: "one_hundred",
}),
]);
const task = assertOk(response, assertOptions("/v3/backlinks/history/live"));
return {
data: parseTaskItems(
"backlinks-history-live",
task,
backlinksHistoryItemSchema,
),
billing: buildTaskBilling(task),
};
}
export type BacklinksSummaryItem = z.infer<typeof backlinksSummaryItemSchema>;
export type BacklinksItem = z.infer<typeof backlinksItemSchema>;
export type ReferringDomainItem = z.infer<typeof referringDomainItemSchema>;
export type DomainPageSummaryItem = z.infer<typeof domainPageSummaryItemSchema>;
export type BacklinksHistoryItem = z.infer<typeof backlinksHistoryItemSchema>;

83
src/server/lib/dataforseo/business.ts Normal file
View File

@ -0,0 +1,83 @@
import { z } from "zod";
import {
BusinessDataBusinessListingsSearchLiveRequestInfo,
BusinessDataGoogleQuestionsAndAnswersLiveRequestInfo,
type BusinessDataBusinessListingsSearchLiveItem,
} from "dataforseo-client";
import { businessDataApi } from "@/server/lib/dataforseo/core";
import {
assertOk,
buildTaskBilling,
type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope";
type BusinessListingItem = BusinessDataBusinessListingsSearchLiveItem;
export async function fetchBusinessListingsSearch(input: {
categories?: string[];
title?: string;
locationCoordinate: string;
orderBy?: string[];
limit: number;
}): Promise<DataforseoApiResponse<BusinessListingItem[]>> {
const response = await businessDataApi().businessListingsSearchLive([
new BusinessDataBusinessListingsSearchLiveRequestInfo({
categories: input.categories,
title: input.title,
location_coordinate: input.locationCoordinate,
order_by: input.orderBy,
limit: input.limit,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}
// Q&A results carry both answered (`items`) and unanswered
// (`items_without_answers`) rows; the SDK types this result as `any`, so we
// validate a generic record shape and flatten both.
const questionsResultSchema = z
.object({
items: z.array(z.record(z.string(), z.unknown())).nullable().optional(),
items_without_answers: z
.array(z.record(z.string(), z.unknown()))
.nullable()
.optional(),
})
.passthrough();
function combinedQuestionItems(results: unknown): Record<string, unknown>[] {
const list = Array.isArray(results) ? results : [];
return list.flatMap((result) => {
const parsed = questionsResultSchema.safeParse(result ?? {});
if (!parsed.success) return [];
return [
...(parsed.data.items ?? []),
...(parsed.data.items_without_answers ?? []),
];
});
}
export async function fetchQuestionsAnswers(input: {
keyword: string;
locationCoordinate: string;
languageCode: string;
depth: number;
}): Promise<DataforseoApiResponse<Record<string, unknown>[]>> {
const response = await businessDataApi().googleQuestionsAndAnswersLive([
new BusinessDataGoogleQuestionsAndAnswersLiveRequestInfo({
keyword: input.keyword,
location_coordinate: input.locationCoordinate,
language_code: input.languageCode,
depth: input.depth,
}),
]);
const task = assertOk(response);
return {
data: combinedQuestionItems(task.result),
billing: buildTaskBilling(task),
};
}

77
src/server/lib/dataforseoClient.test.ts → src/server/lib/dataforseo/client.test.ts
View File

@ -45,45 +45,53 @@ vi.mock("@/server/lib/posthog", () => ({
captureServerEvent: vi.fn(),
}));
vi.mock("@/server/lib/dataforseo", () => ({
fetchKeywordIdeasRaw: vi.fn(),
fetchKeywordSuggestionsRaw: vi.fn(),
fetchRelatedKeywordsRaw: vi.fn(),
fetchBusinessListingsSearchRaw: vi.fn(),
fetchBusinessQuestionsAnswersRaw: vi.fn(),
fetchDomainRankOverviewRaw: vi.fn(),
fetchKeywordSearchVolumeRaw: vi.fn(),
fetchLocalSerpItemsRaw: vi.fn(),
fetchSerpCompetitorsRaw: vi.fn(),
fetchRankedKeywordsRaw: vi.fn(),
fetchLiveSerpItemsRaw: vi.fn(),
// Mock every section module the client wraps so meterDataforseoCall's
// `execute()` resolves to a controllable fixture.
vi.mock("@/server/lib/dataforseo/labs", () => ({
fetchRelatedKeywords: vi.fn(),
fetchKeywordSuggestions: vi.fn(),
fetchKeywordIdeas: vi.fn(),
fetchDomainRankOverview: vi.fn(),
fetchRankedKeywords: vi.fn(),
fetchRelevantPages: vi.fn(),
fetchKeywordOverview: vi.fn(),
fetchSerpCompetitors: vi.fn(),
}));
vi.mock("@/server/lib/dataforseoLighthouse", () => ({
fetchDataforseoLighthouseResultRaw: vi.fn(),
vi.mock("@/server/lib/dataforseo/serp", () => ({
fetchLiveSerp: vi.fn(),
fetchRankCheckSerp: vi.fn(),
fetchLocalSerp: vi.fn(),
}));
vi.mock("@/server/lib/dataforseoBacklinks", () => ({
fetchBacklinksHistoryRaw: vi.fn(),
fetchBacklinksRowsRaw: vi.fn(),
fetchBacklinksSummaryRaw: vi.fn(),
fetchDomainPagesSummaryRaw: vi.fn(),
fetchReferringDomainsRaw: vi.fn(),
vi.mock("@/server/lib/dataforseo/keywordsData", () => ({
fetchKeywordSearchVolume: vi.fn(),
}));
vi.mock("@/server/lib/dataforseoLlm", () => ({
fetchLlmResponseRaw: vi.fn(),
fetchLlmAggregatedMetricsRaw: vi.fn(),
fetchLlmMentionsSearchRaw: vi.fn(),
fetchLlmTopPagesRaw: vi.fn(),
vi.mock("@/server/lib/dataforseo/business", () => ({
fetchBusinessListingsSearch: vi.fn(),
fetchQuestionsAnswers: vi.fn(),
}));
vi.mock("@/server/lib/dataforseo/backlinks", () => ({
fetchBacklinksSummary: vi.fn(),
fetchBacklinksRows: vi.fn(),
fetchReferringDomains: vi.fn(),
fetchDomainPagesSummary: vi.fn(),
fetchBacklinksHistory: vi.fn(),
}));
vi.mock("@/server/lib/dataforseo/lighthouse", () => ({
fetchLighthouseResult: vi.fn(),
}));
vi.mock("@/server/lib/dataforseo/ai", () => ({
fetchLlmMentionsSearch: vi.fn(),
fetchLlmAggregatedMetrics: vi.fn(),
fetchLlmTopPages: vi.fn(),
fetchLlmResponse: vi.fn(),
}));
import {
createDataforseoClient,
mapDataforseoPathToCreditFeature,
} from "./dataforseoClient";
import { DataforseoChargedTaskError } from "./dataforseoCost";
import { fetchBacklinksSummaryRaw } from "./dataforseoBacklinks";
} from "@/server/lib/dataforseo/client";
import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope";
import { fetchBacklinksSummary } from "@/server/lib/dataforseo/backlinks";
const billingCustomer = {
organizationId: "org_123",
@ -113,9 +121,9 @@ function mockBalances(monthly: number, topup: number) {
}
function mockDataforseoResult(costUsd: number) {
vi.mocked(fetchBacklinksSummaryRaw).mockResolvedValue({
vi.mocked(fetchBacklinksSummary).mockResolvedValue({
data: { rank: 42 },
billing: { costUsd, path: ["backlinks", "summary"], resultCount: 1 },
billing: { costUsd, path: ["backlinks", "summary"] },
});
}
@ -238,11 +246,10 @@ describe("meterDataforseoCall with split balances", () => {
it("meters charged DataForSEO task errors before rethrowing", async () => {
setupHostedMode();
mockBalances(5000, 3000);
vi.mocked(fetchBacklinksSummaryRaw).mockRejectedValue(
vi.mocked(fetchBacklinksSummary).mockRejectedValue(
new DataforseoChargedTaskError("DataForSEO task failed", {
costUsd: RAW_COST,
path: ["v3", "backlinks", "summary", "live"],
resultCount: 0,
}),
);

261
src/server/lib/dataforseo/client.ts Normal file
View File

@ -0,0 +1,261 @@
import {
AUTUMN_SEO_DATA_BALANCE_FEATURE_ID,
AUTUMN_SEO_DATA_CREDITS_PER_USD,
AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID,
SEO_DATA_COST_MARKUP,
roundUsdForBilling,
} from "@/shared/billing";
import {
type CreditFeature,
mapDataforseoPathToCreditFeature,
} from "@/shared/billing-credit-features";
import { autumn } from "@/server/billing/autumn";
import { getOrCreateOrganizationCustomer } from "@/server/billing/subscription";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import {
fetchBusinessListingsSearch,
fetchQuestionsAnswers,
} from "@/server/lib/dataforseo/business";
import {
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
fetchDomainPagesSummary,
fetchReferringDomains,
} from "@/server/lib/dataforseo/backlinks";
import {
fetchDomainRankOverview,
fetchKeywordIdeas,
fetchKeywordOverview,
fetchKeywordSuggestions,
fetchRankedKeywords,
fetchRelatedKeywords,
fetchRelevantPages,
fetchSerpCompetitors,
} from "@/server/lib/dataforseo/labs";
import {
fetchLiveSerp,
fetchLocalSerp,
fetchRankCheckSerp,
} from "@/server/lib/dataforseo/serp";
import { fetchKeywordSearchVolume } from "@/server/lib/dataforseo/keywordsData";
import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
import {
fetchLlmAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";
import {
DataforseoChargedTaskError,
type DataforseoApiCallCost,
type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope";
import { AppError } from "@/server/lib/errors";
import { captureServerEvent } from "@/server/lib/posthog";
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
export { mapDataforseoPathToCreditFeature };
/**
* Wraps a section fetcher with billing metering. Each entry on the client is
* `meter(customer, fetcher, creditFeature?)`, which returns a function with the
* fetcher's own input type and resolves to its unwrapped `.data`.
*/
function meter<I, T>(
customer: BillingCustomerContext,
fetcher: (input: I) => Promise<DataforseoApiResponse<T>>,
creditFeature?: CreditFeature,
): (input: I) => Promise<T> {
return (input: I) =>
meterDataforseoCall(customer, () => fetcher(input), creditFeature);
}
export function createDataforseoClient(customer: BillingCustomerContext) {
return {
business: {
businessListings: meter(
customer,
fetchBusinessListingsSearch,
"local_seo",
),
questionsAnswers: meter(customer, fetchQuestionsAnswers, "local_seo"),
},
backlinks: {
summary: meter(customer, fetchBacklinksSummary),
rows: meter(customer, fetchBacklinksRows),
referringDomains: meter(customer, fetchReferringDomains),
domainPages: meter(customer, fetchDomainPagesSummary),
history: meter(customer, fetchBacklinksHistory),
},
keywords: {
related: meter(customer, fetchRelatedKeywords),
suggestions: meter(customer, fetchKeywordSuggestions),
ideas: meter(customer, fetchKeywordIdeas),
},
domain: {
rankOverview: meter(customer, fetchDomainRankOverview),
rankedKeywords: meter(customer, fetchRankedKeywords),
relevantPages: meter(customer, fetchRelevantPages),
},
serp: {
live: meter(customer, fetchLiveSerp),
rankCheck: meter(customer, fetchRankCheckSerp, "rank_tracking"),
local: meter(customer, fetchLocalSerp, "local_seo"),
},
keywordData: {
searchVolume: meter(customer, fetchKeywordSearchVolume),
},
labs: {
keywordOverview: meter(customer, fetchKeywordOverview, "rank_tracking"),
serpCompetitors: meter(customer, fetchSerpCompetitors),
},
lighthouse: {
live: meter(customer, fetchLighthouseResult),
},
aiSearch: {
mentionsSearch: meter(customer, fetchLlmMentionsSearch),
aggregatedMetrics: meter(customer, fetchLlmAggregatedMetrics),
topPages: meter(customer, fetchLlmTopPages),
llmResponse: meter(customer, fetchLlmResponse),
},
} as const;
}
async function meterDataforseoCall<T>(
customer: BillingCustomerContext,
execute: () => Promise<DataforseoApiResponse<T>>,
creditFeature?: CreditFeature,
): Promise<T> {
const isHostedMode = await isHostedServerAuthMode();
if (!isHostedMode) {
const result = await execute();
return result.data;
}
const billingCustomer = await getOrCreateOrganizationCustomer(customer);
const { monthlyRemaining } = await assertSeoDataBalanceAvailable(
billingCustomer.id,
);
let result: DataforseoApiResponse<T>;
try {
result = await execute();
} catch (error) {
if (error instanceof DataforseoChargedTaskError) {
await trackDataforseoCost({
customer,
customerId: billingCustomer.id,
billing: error.billing,
monthlyRemaining,
creditFeature,
});
}
throw error;
}
await trackDataforseoCost({
customer,
customerId: billingCustomer.id,
billing: result.billing,
monthlyRemaining,
creditFeature,
});
return result.data;
}
async function assertSeoDataBalanceAvailable(customerId: string) {
const [monthlyCheck, topupCheck] = await Promise.all([
autumn.check({
customerId,
featureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID,
}),
autumn.check({
customerId,
featureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID,
}),
]);
const monthlyRemaining = monthlyCheck.balance?.remaining ?? 0;
const topupRemaining = topupCheck.balance?.remaining ?? 0;
if (monthlyRemaining + topupRemaining <= 0) {
throw new AppError("INSUFFICIENT_CREDITS");
}
return { monthlyRemaining };
}
async function trackDataforseoCost(args: {
customer: BillingCustomerContext;
customerId: string;
billing: DataforseoApiCallCost;
monthlyRemaining: number;
creditFeature?: CreditFeature;
}) {
const totalCostUsd = roundUsdForBilling(
args.billing.costUsd * SEO_DATA_COST_MARKUP,
);
const totalCostCredits = Math.ceil(
totalCostUsd * AUTUMN_SEO_DATA_CREDITS_PER_USD,
);
const monthlyDeduct = Math.min(args.monthlyRemaining, totalCostCredits);
const topupDeduct = totalCostCredits - monthlyDeduct;
const creditFeature =
args.creditFeature ?? mapDataforseoPathToCreditFeature(args.billing.path);
const properties = {
provider: "dataforseo",
currency: "USD",
paths: [args.billing.path.join("/")],
creditFeature,
totalCostUsd,
totalCostCredits,
fromCache: false,
};
if (monthlyDeduct > 0) {
await autumn.track({
customerId: args.customerId,
featureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID,
value: monthlyDeduct,
properties: {
...properties,
balanceFeatureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID,
},
});
}
if (topupDeduct > 0) {
await autumn.track({
customerId: args.customerId,
featureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID,
value: topupDeduct,
properties: {
...properties,
balanceFeatureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID,
},
});
}
if (totalCostCredits > 0) {
await captureServerEvent({
distinctId: args.customer.userId,
event: "usage:credits_consume",
organizationId: args.customer.organizationId,
properties: {
project_id: args.customer.projectId,
credit_feature: creditFeature,
monthly_credits: monthlyDeduct,
topup_credits: topupDeduct,
total_credits: totalCostCredits,
cost_usd: totalCostUsd,
},
});
}
}

110
src/server/lib/dataforseo/core.ts Normal file
View File

@ -0,0 +1,110 @@
import {
AiOptimizationApi,
BacklinksApi,
BusinessDataApi,
DataforseoLabsApi,
KeywordsDataApi,
OnPageApi,
SerpApi,
} from "dataforseo-client";
import { AppError } from "@/server/lib/errors";
import { getRequiredEnvValue } from "@/server/lib/runtime-env";
const API_BASE = "https://api.dataforseo.com";
const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600;
// Safety ceiling on any live call (Lighthouse is the slowest, ~tens of seconds).
const DATAFORSEO_REQUEST_TIMEOUT_MS = 60_000;
/**
* Translates a DataForSEO HTTP/task failure into a product-specific AppError
* (e.g. "backlinks not enabled", "billing issue"). Returns null when the
* failure isn't one this classifier recognises, so the caller can fall back to
* a generic error. See {@link createDataforseoAccessClassifier}.
*/
export type DataforseoErrorClassifier = (
status: number | undefined,
details: string,
path: string,
) => AppError | null;
function formatDataforseoErrorPayload(value: unknown): string {
const text =
typeof value === "string"
? value
: (() => {
try {
return JSON.stringify(value);
} catch {
return String(value);
}
})();
return text.length > MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH
? `${text.slice(0, MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH)}... [truncated]`
: text;
}
function formatDataforseoRequestPath(url: RequestInfo): string {
const rawUrl = typeof url === "string" ? url : url.url;
try {
return new URL(rawUrl).pathname;
} catch {
return rawUrl;
}
}
/**
* The single authenticated `fetch` used by every DataForSEO SDK call. Throws on
* non-2xx so the SDK's own `ApiException` path never fires; task-level failures
* (which return HTTP 200) are handled downstream by {@link assertOk}. An
* optional classifier maps recognised HTTP failures to product errors.
*/
function createAuthenticatedFetch(classify?: DataforseoErrorClassifier) {
return async (url: RequestInfo, init?: RequestInit): Promise<Response> => {
const apiKey = await getRequiredEnvValue("DATAFORSEO_API_KEY");
const headers = new Headers(init?.headers);
headers.set("Authorization", `Basic ${apiKey}`);
const response = await fetch(url, {
...init,
headers,
signal:
init?.signal ?? AbortSignal.timeout(DATAFORSEO_REQUEST_TIMEOUT_MS),
});
if (response.ok) return response;
const rawText = await response.text();
const path = formatDataforseoRequestPath(url);
const classified = classify?.(response.status, rawText, path);
if (classified) throw classified;
const error = new AppError(
response.status === 429 ? "RATE_LIMITED" : "INTERNAL_ERROR",
`DataForSEO HTTP ${response.status} on ${path}`,
{
provider: "dataforseo",
providerStatus: String(response.status),
providerPath: path,
responseBody: formatDataforseoErrorPayload(rawText),
},
);
error.name = "DataForSEOHttpError";
throw error;
};
}
function http(classify?: DataforseoErrorClassifier) {
return { fetch: createAuthenticatedFetch(classify) };
}
// Per-section API factories. Each is created per-request so the auth secret is
// read lazily (it lives in the Worker env, not in module scope).
export const labsApi = () => new DataforseoLabsApi(API_BASE, http());
export const serpApi = () => new SerpApi(API_BASE, http());
export const keywordsDataApi = () => new KeywordsDataApi(API_BASE, http());
export const businessDataApi = () => new BusinessDataApi(API_BASE, http());
export const onPageApi = () => new OnPageApi(API_BASE, http());
export const backlinksApi = (classify?: DataforseoErrorClassifier) =>
new BacklinksApi(API_BASE, http(classify));
export const aiOptimizationApi = (classify?: DataforseoErrorClassifier) =>
new AiOptimizationApi(API_BASE, http(classify));

332
src/server/lib/dataforseo/endpoints.test.ts Normal file
View File

@ -0,0 +1,332 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@/server/lib/runtime-env", () => ({
getRequiredEnvValue: vi.fn(async () => "test-api-key"),
}));
import { fetchQuestionsAnswers } from "@/server/lib/dataforseo/business";
import {
buildLlmTarget,
fetchLlmAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";
import { fetchKeywordSearchVolume } from "@/server/lib/dataforseo/keywordsData";
function parseDataforseoRequestBody(init: RequestInit | undefined): unknown {
const body = init?.body;
if (typeof body !== "string") {
throw new Error("Expected DataForSEO request body to be a string");
}
return JSON.parse(body) as unknown;
}
describe("DataForSEO SDK-backed endpoints", () => {
beforeEach(() => {
vi.restoreAllMocks();
});
it("uses the live endpoint for Google Business Q&A and returns items + billing", async () => {
const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({
status_code: 20000,
tasks: [
{
status_code: 20000,
path: [
"v3",
"business_data",
"google",
"questions_and_answers",
"live",
],
cost: 0.0006,
result_count: 1,
result: [
{
items: [
{
question_text: "Do you offer indoor storage?",
answer_text: "Yes.",
},
],
},
],
},
],
}),
);
vi.stubGlobal("fetch", fetchMock);
const result = await fetchQuestionsAnswers({
keyword: "Acme Storage",
locationCoordinate: "33.1234568,-84.9876543,5000",
languageCode: "en",
depth: 20,
});
expect(
fetchMock.mock.calls.map(([url]) =>
typeof url === "string" || url instanceof URL
? url.toString()
: url.url,
),
).toEqual([
"https://api.dataforseo.com/v3/business_data/google/questions_and_answers/live",
]);
expect(result.data).toEqual([
{ question_text: "Do you offer indoor storage?", answer_text: "Yes." },
]);
expect(result.billing).toEqual({
path: ["v3", "business_data", "google", "questions_and_answers", "live"],
costUsd: 0.0006,
});
});
it("does not send location_name for keyword search volume", async () => {
const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({
status_code: 20000,
tasks: [
{
status_code: 20000,
path: [
"v3",
"keywords_data",
"google_ads",
"search_volume",
"live",
],
cost: 0.0001,
result_count: 1,
result: [
{
keyword: "storage units",
location_code: 2840,
language_code: "en",
search_volume: 1000,
},
],
},
],
}),
);
vi.stubGlobal("fetch", fetchMock);
await fetchKeywordSearchVolume({
keywords: ["storage units"],
locationCode: 2840,
languageCode: "en",
});
const payload = parseDataforseoRequestBody(fetchMock.mock.calls[0]?.[1]);
expect(payload).toEqual([
{
keywords: ["storage units"],
location_code: 2840,
language_code: "en",
},
]);
expect(JSON.stringify(payload)).not.toContain("location_name");
});
it("serializes LLM mentions domain targets for all live endpoints", async () => {
const fetchMock = vi.fn<typeof fetch>().mockImplementation((url) => {
const path =
typeof url === "string" || url instanceof URL
? url.toString()
: url.url;
const result = path.includes("/aggregated_metrics/")
? { total: { platform: [] } }
: { items: [] };
return Promise.resolve(
Response.json({
status_code: 20000,
tasks: [
{
status_code: 20000,
path: new URL(path).pathname.split("/").filter(Boolean),
cost: 0.0001,
result_count: 1,
result: [result],
},
],
}),
);
});
vi.stubGlobal("fetch", fetchMock);
const target = buildLlmTarget({
type: "domain",
value: "example.com",
});
await fetchLlmMentionsSearch({
target,
platform: "google",
locationCode: 2840,
languageCode: "en",
});
await fetchLlmAggregatedMetrics({
target,
platform: "google",
locationCode: 2840,
languageCode: "en",
});
await fetchLlmTopPages({
target,
platform: "google",
locationCode: 2840,
languageCode: "en",
});
const expectedTarget = [
{
search_scope: ["any"],
search_filter: "include",
domain: "example.com",
include_subdomains: true,
},
];
const payloads = fetchMock.mock.calls.map(([, init]) =>
parseDataforseoRequestBody(init),
);
expect(payloads).toEqual([
[
{
target: expectedTarget,
location_code: 2840,
language_code: "en",
platform: "google",
limit: 100,
},
],
[
{
target: expectedTarget,
location_code: 2840,
language_code: "en",
platform: "google",
internal_list_limit: 10,
},
],
[
{
target: expectedTarget,
location_code: 2840,
language_code: "en",
platform: "google",
links_scope: "sources",
items_list_limit: 10,
internal_list_limit: 5,
},
],
]);
});
it("serializes LLM mentions keyword targets", async () => {
const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({
status_code: 20000,
tasks: [
{
status_code: 20000,
path: ["v3", "ai_optimization", "llm_mentions", "search", "live"],
cost: 0.0001,
result_count: 1,
result: [{ items: [] }],
},
],
}),
);
vi.stubGlobal("fetch", fetchMock);
await fetchLlmMentionsSearch({
target: buildLlmTarget({
type: "keyword",
value: "Acme Storage",
}),
platform: "chat_gpt",
locationCode: 2840,
languageCode: "en",
});
expect(parseDataforseoRequestBody(fetchMock.mock.calls[0]?.[1])).toEqual([
{
target: [
{
search_scope: ["any", "brand_entities"],
search_filter: "include",
keyword: "Acme Storage",
match_type: "word_match",
},
],
location_code: 2840,
language_code: "en",
platform: "chat_gpt",
limit: 100,
},
]);
});
it("preserves web_search for Perplexity LLM responses", async () => {
const fetchMock = vi.fn<typeof fetch>().mockResolvedValue(
Response.json({
status_code: 20000,
tasks: [
{
status_code: 20000,
path: [
"v3",
"ai_optimization",
"perplexity",
"llm_responses",
"live",
],
cost: 0.0001,
result_count: 1,
result: [
{
model_name: "sonar",
output_tokens: 12,
web_search: false,
items: [],
},
],
},
],
}),
);
vi.stubGlobal("fetch", fetchMock);
await fetchLlmResponse({
userPrompt: "What is OpenSEO?",
modelSlug: "perplexity",
modelName: "sonar",
webSearch: false,
webSearchCountryCode: "US",
});
expect(
fetchMock.mock.calls.map(([url]) =>
typeof url === "string" || url instanceof URL
? url.toString()
: url.url,
),
).toEqual([
"https://api.dataforseo.com/v3/ai_optimization/perplexity/llm_responses/live",
]);
expect(parseDataforseoRequestBody(fetchMock.mock.calls[0]?.[1])).toEqual([
{
user_prompt: "What is OpenSEO?",
model_name: "sonar",
web_search: false,
max_output_tokens: 1024,
web_search_country_iso_code: "US",
},
]);
});
});

129
src/server/lib/dataforseo/envelope.test.ts Normal file
View File

@ -0,0 +1,129 @@
import { describe, expect, it, vi } from "vitest";
import { z } from "zod";
import {
assertOk,
DataforseoChargedTaskError,
parseTaskItems,
} from "@/server/lib/dataforseo/envelope";
import { AppError } from "@/server/lib/errors";
const itemSchema = z.object({ keyword: z.string().optional() }).passthrough();
describe("parseTaskItems", () => {
it("returns [] when the result items are null", () => {
const task = { status_code: 20000, result: [{ items: null }] };
expect(parseTaskItems("x", task, itemSchema)).toEqual([]);
});
it("returns [] when there is no result", () => {
expect(parseTaskItems("x", { result: undefined }, itemSchema)).toEqual([]);
});
it("parses present items", () => {
const task = { result: [{ items: [{ keyword: "seo" }] }] };
expect(parseTaskItems("x", task, itemSchema)).toEqual([{ keyword: "seo" }]);
});
});
describe("assertOk", () => {
const okTask = {
status_code: 20000,
path: ["v3", "backlinks", "summary", "live"],
cost: 0.1,
result_count: 1,
result: [],
};
it("returns the first task on success", () => {
expect(assertOk({ status_code: 20000, tasks: [okTask] })).toBe(okTask);
});
it("throws DataforseoChargedTaskError when a charged task fails", () => {
const task = {
status_code: 40000,
status_message: "fail",
path: ["v3", "backlinks", "summary", "live"],
cost: 0.05,
result_count: 0,
};
try {
assertOk({ status_code: 20000, tasks: [task] });
throw new Error("expected assertOk to throw");
} catch (error) {
expect(error).toBeInstanceOf(DataforseoChargedTaskError);
if (error instanceof DataforseoChargedTaskError) {
expect(error.billing).toEqual({
path: task.path,
costUsd: 0.05,
});
}
}
});
it("uses the classifier for non-charged (no-cost) failures", () => {
const classify = vi.fn(() => new AppError("BACKLINKS_NOT_ENABLED", "nope"));
const task = {
status_code: 40204,
status_message: "subscription required",
};
expect(() =>
assertOk(
{ status_code: 20000, tasks: [task] },
{ classify, classifyPath: "/v3/backlinks/summary/live" },
),
).toThrow("nope");
expect(classify).toHaveBeenCalledWith(
40204,
"subscription required",
"/v3/backlinks/summary/live",
);
});
it.each([
[40204, "BACKLINKS_NOT_ENABLED"],
[403, "BACKLINKS_NOT_ENABLED"],
[40200, "BACKLINKS_BILLING_ISSUE"],
] as const)(
"uses the classifier for account failure %s before charging billed task metadata",
(status, code) => {
const classify = vi.fn(
() => new AppError(code, "Classified DataForSEO account failure"),
);
const task = {
status_code: status,
status_message: "Backlinks subscription required",
path: ["v3", "backlinks", "summary", "live"],
cost: 0.05,
result_count: 0,
};
try {
assertOk({ status_code: 20000, tasks: [task] }, { classify });
throw new Error("expected assertOk to throw");
} catch (error) {
expect(error).not.toBeInstanceOf(DataforseoChargedTaskError);
expect(error).toMatchObject({ code });
}
expect(classify).toHaveBeenCalledWith(
status,
"Backlinks subscription required",
"/v3/backlinks/summary/live",
);
},
);
it("treats 40501 as an empty success when asked", () => {
const task = {
status_code: 40501,
status_message: "No Search Results",
path: ["v3", "serp", "google", "organic", "live", "advanced"],
cost: 0.0,
};
expect(
assertOk(
{ status_code: 20000, tasks: [task] },
{ treatNoResultsAsEmpty: true },
),
).toBe(task);
});
});

171
src/server/lib/dataforseo/envelope.ts Normal file
View File

@ -0,0 +1,171 @@
import { z } from "zod";
import { AppError } from "@/server/lib/errors";
import type { DataforseoErrorClassifier } from "@/server/lib/dataforseo/core";
// ---------------------------------------------------------------------------
// Billing envelope — the load-bearing seam that carries each call's USD cost
// out to the single metering point in client.ts. Every section fetcher returns
// DataforseoApiResponse<T>; nothing else constructs a billing object.
// ---------------------------------------------------------------------------
export type DataforseoApiCallCost = {
path: string[];
costUsd: number;
};
export type DataforseoApiResponse<T> = {
data: T;
billing: DataforseoApiCallCost;
};
/**
* Thrown when a DataForSEO task fails *after* it was billed (cost + path are
* present). meterDataforseoCall catches this to charge the customer for the
* failed-but-charged call before rethrowing. Do not throw this for access /
* balance failures; classify those first even when DataForSEO includes billing
* metadata on the failed task.
*/
export class DataforseoChargedTaskError extends AppError {
constructor(
message: string,
public readonly billing: DataforseoApiCallCost,
) {
super("INTERNAL_ERROR", message);
this.name = "DataforseoChargedTaskError";
}
}
// The SDK types cost / path / result_count as optional with no runtime
// validation, so this is the one guard that guarantees we can bill a call.
const billingMetadataSchema = z.object({
path: z.array(z.string()),
cost: z.number(),
result_count: z.number().nullable().optional(),
});
export interface DataforseoTaskLike {
status_code?: number;
status_message?: string;
path?: string[];
cost?: number;
result_count?: number;
result?: unknown[];
[key: string]: unknown;
}
interface DataforseoResponseLike<T extends DataforseoTaskLike> {
status_code?: number;
status_message?: string;
tasks?: T[];
[key: string]: unknown;
}
function tryBuildTaskBilling(task: unknown): DataforseoApiCallCost | null {
const parsed = billingMetadataSchema.safeParse(task);
if (!parsed.success) return null;
return {
path: parsed.data.path,
costUsd: parsed.data.cost,
};
}
export function buildTaskBilling(
task: DataforseoTaskLike,
): DataforseoApiCallCost {
const billing = tryBuildTaskBilling(task);
if (!billing) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO task is missing billing metadata (path/cost)",
);
}
return billing;
}
type AssertOkOptions = {
/** Maps a recognised access / billing failure to a product error. */
classify?: DataforseoErrorClassifier;
/** Request path string handed to the classifier (e.g. "/v3/backlinks/summary/live"). */
classifyPath?: string;
/** Treat DataForSEO's "no search results" (40501) as an empty success. */
treatNoResultsAsEmpty?: boolean;
};
/**
* Validates that the top-level response and its first task both succeeded, and
* returns that (SDK-typed) task. The single status / billing ladder shared by
* every endpoint:
* - access / balance failure -> classified AppError
* - charged-but-failed task (cost present) -> DataforseoChargedTaskError
*/
export function assertOk<T extends DataforseoTaskLike>(
response: DataforseoResponseLike<T> | null,
options: AssertOkOptions = {},
): T {
if (!response) {
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO returned an empty response",
);
}
const { classify, classifyPath, treatNoResultsAsEmpty } = options;
if (response.status_code !== 20000) {
const message = response.status_message || "DataForSEO request failed";
throw (
classify?.(response.status_code, message, classifyPath ?? "") ??
new AppError("INTERNAL_ERROR", message)
);
}
const task = response.tasks?.[0];
if (!task) {
throw new AppError("INTERNAL_ERROR", "DataForSEO response missing task");
}
if (task.status_code !== 20000) {
const isNoResults =
task.status_code === 40501 ||
(task.status_message?.toLowerCase().includes("no search results") ??
false);
if (treatNoResultsAsEmpty && isNoResults) return task;
const message = task.status_message || "DataForSEO task failed";
const path = classifyPath ?? (task.path ? `/${task.path.join("/")}` : "");
const classified = classify?.(task.status_code, message, path);
if (classified) throw classified;
const billing = tryBuildTaskBilling(task);
if (billing) throw new DataforseoChargedTaskError(message, billing);
throw new AppError("INTERNAL_ERROR", message);
}
return task;
}
export function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
/** Reads `task.result[0].items`, validating against a Zod schema for loosely-typed endpoints. */
export function parseTaskItems<T extends z.ZodTypeAny>(
endpoint: string,
task: DataforseoTaskLike,
itemSchema: T,
): Array<z.infer<T>> {
const first = task.result?.[0];
const items = isRecord(first) ? first.items : [];
const parsed = z.array(itemSchema).safeParse(items ?? []);
if (!parsed.success) {
console.error(
`dataforseo.${endpoint}.invalid-payload`,
parsed.error.issues.slice(0, 5),
);
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${endpoint} returned an invalid response shape`,
);
}
return parsed.data;
}

33
src/server/lib/dataforseo/index.ts Normal file
View File

@ -0,0 +1,33 @@
// Public surface of the DataForSEO integration. Internals live in the
// per-section files (labs / serp / keywordsData / business / backlinks / ai /
// lighthouse); everything funnels through envelope.ts (status + billing) and is
// metered in client.ts.
export { createDataforseoClient } from "@/server/lib/dataforseo/client";
export {
type LabsKeywordDataItem,
type DomainRankedKeywordItem,
type RelevantPagesItem,
} from "@/server/lib/dataforseo/labs";
export {
type SerpLiveItem,
type RankCheckResult,
} from "@/server/lib/dataforseo/serp";
export {
normalizeBacklinksTarget,
type BacklinksSummaryItem,
type BacklinksItem,
type ReferringDomainItem,
type DomainPageSummaryItem,
type BacklinksHistoryItem,
} from "@/server/lib/dataforseo/backlinks";
export {
buildLlmTarget,
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
type LlmPlatform,
} from "@/server/lib/dataforseo/ai";

34
src/server/lib/dataforseo/keywordsData.ts Normal file
View File

@ -0,0 +1,34 @@
import {
KeywordsDataGoogleAdsSearchVolumeLiveRequestInfo,
type KeywordsDataGoogleAdsSearchVolumeLiveResultInfo,
} from "dataforseo-client";
import { keywordsDataApi } from "@/server/lib/dataforseo/core";
import {
assertOk,
buildTaskBilling,
type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope";
// The search_volume live response returns keyword rows directly as the task
// result array (no nested `items`). `competition` is a string enum here
// ("LOW"/"MEDIUM"/"HIGH"), distinct from the numeric Labs competition.
type KeywordSearchVolumeRow = KeywordsDataGoogleAdsSearchVolumeLiveResultInfo;
export async function fetchKeywordSearchVolume(input: {
keywords: string[];
locationCode?: number;
languageCode?: string;
}): Promise<DataforseoApiResponse<KeywordSearchVolumeRow[]>> {
const response = await keywordsDataApi().googleAdsSearchVolumeLive([
new KeywordsDataGoogleAdsSearchVolumeLiveRequestInfo({
keywords: input.keywords,
location_code: input.locationCode,
language_code: input.languageCode,
}),
]);
const task = assertOk(response);
return {
data: task.result ?? [],
billing: buildTaskBilling(task),
};
}

317
src/server/lib/dataforseo/labs.ts Normal file
View File

@ -0,0 +1,317 @@
import { z } from "zod";
import {
DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo,
DataforseoLabsGoogleKeywordIdeasLiveRequestInfo,
DataforseoLabsGoogleKeywordOverviewLiveRequestInfo,
DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo,
DataforseoLabsGoogleRankedKeywordsLiveRequestInfo,
DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo,
DataforseoLabsGoogleRelevantPagesLiveRequestInfo,
DataforseoLabsGoogleSerpCompetitorsLiveRequestInfo,
type DataforseoLabsDomainRankOverviewLiveItem,
type DataforseoLabsGoogleKeywordOverviewLiveItem,
type DataforseoLabsRelatedKeywordsLiveItem,
type DataforseoLabsRelevantPagesLiveItem,
type DataforseoLabsSerpCompetitorsLiveItem,
type KeywordDataInfo,
} from "dataforseo-client";
import { labsApi } from "@/server/lib/dataforseo/core";
import {
assertOk,
buildTaskBilling,
parseTaskItems,
type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope";
// SDK item models are 1:1 supersets of what we need, so we expose them directly
// under the names the rest of the app already uses (no hand-written Zod).
export type LabsKeywordDataItem = KeywordDataInfo;
type RelatedKeywordItem = DataforseoLabsRelatedKeywordsLiveItem;
type DomainMetricsItem = DataforseoLabsDomainRankOverviewLiveItem;
export type RelevantPagesItem = DataforseoLabsRelevantPagesLiveItem;
type KeywordOverviewItem = DataforseoLabsGoogleKeywordOverviewLiveItem;
type SerpCompetitorItem = DataforseoLabsSerpCompetitorsLiveItem;
// Ranked keywords is the one Labs endpoint the SDK types loosely: its
// `ranked_serp_element.serp_item` is the base element item, so the url / etv /
// rank fields we read are untyped (`any`). Keep a focused schema so the
// domain-keyword mapper stays type-safe.
const rankedSerpItemSchema = z
.object({
url: z.string().nullable().optional(),
relative_url: z.string().nullable().optional(),
rank_absolute: z.number().nullable().optional(),
etv: z.number().nullable().optional(),
})
.passthrough();
const domainRankedKeywordItemSchema = z
.object({
keyword_data: z
.object({
keyword: z.string().nullable().optional(),
keyword_info: z
.object({
search_volume: z.number().nullable().optional(),
cpc: z.number().nullable().optional(),
keyword_difficulty: z.number().nullable().optional(),
})
.passthrough()
.nullable()
.optional(),
keyword_properties: z
.object({
keyword_difficulty: z.number().nullable().optional(),
})
.passthrough()
.nullable()
.optional(),
})
.passthrough()
.nullable()
.optional(),
ranked_serp_element: z
.object({
serp_item: rankedSerpItemSchema.nullable().optional(),
url: z.string().nullable().optional(),
relative_url: z.string().nullable().optional(),
rank_absolute: z.number().nullable().optional(),
etv: z.number().nullable().optional(),
})
.passthrough()
.nullable()
.optional(),
keyword: z.string().nullable().optional(),
})
.passthrough();
export type DomainRankedKeywordItem = z.infer<
typeof domainRankedKeywordItemSchema
>;
type DataforseoLabsItemType =
| "organic"
| "paid"
| "featured_snippet"
| "local_pack"
| "ai_overview_reference";
export async function fetchRelatedKeywords(input: {
keyword: string;
locationCode: number;
languageCode: string;
limit: number;
depth?: number;
}): Promise<DataforseoApiResponse<RelatedKeywordItem[]>> {
const response = await labsApi().googleRelatedKeywordsLive([
new DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo({
keyword: input.keyword,
location_code: input.locationCode,
language_code: input.languageCode,
limit: input.limit,
depth: input.depth ?? 3,
include_clickstream_data: true,
include_serp_info: false,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}
export async function fetchKeywordSuggestions(input: {
keyword: string;
locationCode: number;
languageCode: string;
limit: number;
}): Promise<DataforseoApiResponse<LabsKeywordDataItem[]>> {
const response = await labsApi().googleKeywordSuggestionsLive([
new DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo({
keyword: input.keyword,
location_code: input.locationCode,
language_code: input.languageCode,
limit: input.limit,
include_clickstream_data: true,
include_serp_info: false,
include_seed_keyword: true,
ignore_synonyms: false,
exact_match: false,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}
export async function fetchKeywordIdeas(input: {
keyword: string;
locationCode: number;
languageCode: string;
limit: number;
}): Promise<DataforseoApiResponse<LabsKeywordDataItem[]>> {
const response = await labsApi().googleKeywordIdeasLive([
new DataforseoLabsGoogleKeywordIdeasLiveRequestInfo({
keywords: [input.keyword],
location_code: input.locationCode,
language_code: input.languageCode,
limit: input.limit,
include_clickstream_data: true,
include_serp_info: false,
ignore_synonyms: false,
closely_variants: false,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}
export async function fetchDomainRankOverview(input: {
target: string;
locationCode: number;
languageCode: string;
}): Promise<DataforseoApiResponse<DomainMetricsItem[]>> {
const response = await labsApi().googleDomainRankOverviewLive([
new DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo({
target: input.target,
location_code: input.locationCode,
language_code: input.languageCode,
limit: 1,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}
type RankedKeywordsPage = {
items: DomainRankedKeywordItem[];
totalCount: number | null;
};
export async function fetchRankedKeywords(input: {
target: string;
locationCode: number;
languageCode: string;
limit: number;
offset?: number;
orderBy?: string[];
filters?: unknown[];
itemTypes?: DataforseoLabsItemType[];
includeSubdomains?: boolean;
}): Promise<DataforseoApiResponse<RankedKeywordsPage>> {
const response = await labsApi().googleRankedKeywordsLive([
new DataforseoLabsGoogleRankedKeywordsLiveRequestInfo({
target: input.target,
location_code: input.locationCode,
language_code: input.languageCode,
limit: input.limit,
offset: input.offset,
order_by: input.orderBy,
filters: input.filters,
item_types: input.itemTypes,
include_subdomains: input.includeSubdomains,
}),
]);
const task = assertOk(response);
return {
data: {
items: parseTaskItems(
"google-ranked-keywords-live",
task,
domainRankedKeywordItemSchema,
),
totalCount: task.result?.[0]?.total_count ?? null,
},
billing: buildTaskBilling(task),
};
}
type RelevantPagesPage = {
items: RelevantPagesItem[];
totalCount: number | null;
};
export async function fetchRelevantPages(input: {
target: string;
locationCode: number;
languageCode: string;
limit: number;
offset?: number;
orderBy?: string[];
filters?: unknown[];
}): Promise<DataforseoApiResponse<RelevantPagesPage>> {
const response = await labsApi().googleRelevantPagesLive([
new DataforseoLabsGoogleRelevantPagesLiveRequestInfo({
target: input.target,
location_code: input.locationCode,
language_code: input.languageCode,
limit: input.limit,
offset: input.offset,
order_by: input.orderBy,
filters: input.filters,
}),
]);
const task = assertOk(response);
return {
data: {
items: task.result?.[0]?.items ?? [],
totalCount: task.result?.[0]?.total_count ?? null,
},
billing: buildTaskBilling(task),
};
}
export async function fetchKeywordOverview(input: {
keywords: string[];
locationCode: number;
languageCode: string;
}): Promise<DataforseoApiResponse<KeywordOverviewItem[]>> {
const response = await labsApi().googleKeywordOverviewLive([
new DataforseoLabsGoogleKeywordOverviewLiveRequestInfo({
keywords: input.keywords,
location_code: input.locationCode,
language_code: input.languageCode,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}
export async function fetchSerpCompetitors(input: {
keywords: string[];
locationCode: number;
languageCode: string;
itemTypes?: DataforseoLabsItemType[];
includeSubdomains?: boolean;
limit: number;
offset?: number;
}): Promise<DataforseoApiResponse<SerpCompetitorItem[]>> {
const response = await labsApi().googleSerpCompetitorsLive([
new DataforseoLabsGoogleSerpCompetitorsLiveRequestInfo({
keywords: input.keywords,
location_code: input.locationCode,
language_code: input.languageCode,
item_types: input.itemTypes,
include_subdomains: input.includeSubdomains,
limit: input.limit,
offset: input.offset,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}

32
src/server/lib/dataforseo/lighthouse.ts Normal file
View File

@ -0,0 +1,32 @@
import { OnPageLighthouseLiveJsonRequestInfo } from "dataforseo-client";
import {
parseDataforseoLighthousePayload,
requestCategories,
type LighthouseStrategy,
} from "@/server/lib/dataforseoLighthousePayload";
import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload";
import { onPageApi } from "@/server/lib/dataforseo/core";
import {
assertOk,
buildTaskBilling,
type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope";
export async function fetchLighthouseResult(input: {
url: string;
strategy: LighthouseStrategy;
}): Promise<DataforseoApiResponse<StoredLighthousePayload>> {
const response = await onPageApi().lighthouseLiveJson([
new OnPageLighthouseLiveJsonRequestInfo({
url: input.url,
for_mobile: input.strategy === "mobile",
categories: [...requestCategories],
}),
]);
// assertOk handles status / charged-task billing; parse extracts the scores.
const task = assertOk(response);
const data = parseDataforseoLighthousePayload(response, input);
return { data, billing: buildTaskBilling(task) };
}

188
src/server/lib/dataforseo/serp.ts Normal file
View File

@ -0,0 +1,188 @@
import { z } from "zod";
import {
SerpGoogleLocalFinderLiveAdvancedRequestInfo,
SerpGoogleMapsLiveAdvancedRequestInfo,
SerpGoogleOrganicLiveAdvancedRequestInfo,
} from "dataforseo-client";
import { serpApi } from "@/server/lib/dataforseo/core";
import {
assertOk,
buildTaskBilling,
parseTaskItems,
type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope";
// Kept as a hand-written schema: the SDK's BaseSerpApiElementItem type omits
// etv / estimated_paid_traffic_cost / backlinks_info / rank_changes, which we
// rely on. The fields survive deserialization (the SDK copies unknown keys), so
// validating here is both our type-safety guard and how we read those fields.
const serpSnapshotItemSchema = z
.object({
type: z.string(),
rank_group: z.number().nullable().optional(),
rank_absolute: z.number().nullable().optional(),
domain: z.string().nullable().optional(),
title: z.string().nullable().optional(),
url: z.string().nullable().optional(),
description: z.string().nullable().optional(),
breadcrumb: z.string().nullable().optional(),
etv: z.number().nullable().optional(),
estimated_paid_traffic_cost: z.number().nullable().optional(),
backlinks_info: z
.object({
referring_domains: z.number().nullable().optional(),
backlinks: z.number().nullable().optional(),
})
.passthrough()
.nullable()
.optional(),
rank_changes: z
.object({
previous_rank_absolute: z.number().nullable().optional(),
is_new: z.boolean().nullable().optional(),
is_up: z.boolean().nullable().optional(),
is_down: z.boolean().nullable().optional(),
})
.passthrough()
.nullable()
.optional(),
})
.passthrough();
export type SerpLiveItem = z.infer<typeof serpSnapshotItemSchema>;
export async function fetchLiveSerp(input: {
keyword: string;
locationCode: number;
languageCode: string;
}): Promise<DataforseoApiResponse<SerpLiveItem[]>> {
const response = await serpApi().googleOrganicLiveAdvanced([
new SerpGoogleOrganicLiveAdvancedRequestInfo({
keyword: input.keyword,
location_code: input.locationCode,
language_code: input.languageCode,
device: "desktop",
os: "windows",
depth: 100,
}),
]);
const task = assertOk(response);
return {
data: parseTaskItems(
"google-organic-live-advanced",
task,
serpSnapshotItemSchema,
),
billing: buildTaskBilling(task),
};
}
export interface RankCheckResult {
keywordId: string;
keyword: string;
position: number | null;
url: string | null;
serpFeatures: string[];
}
export async function fetchRankCheckSerp(input: {
keyword: string;
keywordId: string;
locationCode: number;
languageCode: string;
device: "desktop" | "mobile";
targetDomain: string;
depth: number;
}): Promise<DataforseoApiResponse<RankCheckResult>> {
const depth = Math.min(100, Math.max(10, input.depth));
const response = await serpApi().googleOrganicLiveAdvanced([
new SerpGoogleOrganicLiveAdvancedRequestInfo({
keyword: input.keyword,
location_code: input.locationCode,
language_code: input.languageCode,
device: input.device,
os: input.device === "desktop" ? "windows" : "android",
depth,
}),
]);
// "No Search Results" (40501) is valid for obscure/new keywords — treat as an
// empty result set rather than failing the whole rank-tracking run.
const task = assertOk(response, { treatNoResultsAsEmpty: true });
const items = parseTaskItems(
"google-organic-live-advanced",
task,
serpSnapshotItemSchema,
);
const target = input.targetDomain.toLowerCase();
const organicMatch = items.find((item) => {
if (item.type !== "organic" || item.domain == null) return false;
const domain = item.domain.toLowerCase();
return domain === target || domain.endsWith(`.${target}`);
});
return {
data: {
keywordId: input.keywordId,
keyword: input.keyword,
position: organicMatch
? (organicMatch.rank_absolute ?? organicMatch.rank_group ?? null)
: null,
url: organicMatch?.url ?? null,
serpFeatures: [
...new Set(items.map((item) => item.type).filter(Boolean)),
],
},
billing: buildTaskBilling(task),
};
}
export async function fetchLocalSerp(input: {
keyword: string;
locationCoordinate?: string;
languageCode: string;
searchType: "maps" | "local_finder";
device: "desktop" | "mobile";
depth: number;
searchPlaces?: boolean;
}): Promise<DataforseoApiResponse<Record<string, unknown>[]>> {
const os = input.device === "desktop" ? "windows" : "android";
// Maps and Local Finder return different SDK item models; both carry an index
// signature, so the typed items assign cleanly to the generic row shape.
if (input.searchType === "maps") {
const response = await serpApi().googleMapsLiveAdvanced([
new SerpGoogleMapsLiveAdvancedRequestInfo({
keyword: input.keyword,
location_coordinate: input.locationCoordinate,
language_code: input.languageCode,
device: input.device,
os,
depth: input.depth,
search_places: input.searchPlaces,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}
const response = await serpApi().googleLocalFinderLiveAdvanced([
new SerpGoogleLocalFinderLiveAdvancedRequestInfo({
keyword: input.keyword,
location_coordinate: input.locationCoordinate,
language_code: input.languageCode,
device: input.device,
os,
depth: input.depth,
}),
]);
const task = assertOk(response);
return {
data: task.result?.[0]?.items ?? [],
billing: buildTaskBilling(task),
};
}

292
src/server/lib/dataforseoBacklinks.ts
View File

@ -1,292 +0,0 @@
import { AppError } from "@/server/lib/errors";
import {
normalizeBacklinksSpamFilterOptions,
type BacklinksSpamFilterOptions,
} from "@/types/schemas/backlinks";
import type {
DataforseoApiCallCost,
DataforseoApiResponse,
} from "@/server/lib/dataforseoCost";
import { getRequiredEnvValue } from "@/server/lib/runtime-env";
import {
classifyBacklinksError,
type BacklinksTaskResult,
backlinksHistoryItemSchema,
backlinksItemSchema,
backlinksSummaryItemSchema,
domainPageSummaryItemSchema,
parseItems,
referringDomainItemSchema,
responseSchema,
} from "@/server/lib/dataforseoBacklinksSupport";
export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
const API_BASE = "https://api.dataforseo.com";
export type BacklinksRequest = {
target: string;
};
export type BacklinksListRequest = BacklinksRequest &
BacklinksSpamFilterOptions & {
limit?: number;
};
export type BacklinksTimeseriesRequest = {
target: string;
dateFrom: string;
dateTo: string;
};
type DataforseoTaskResponse = {
results: BacklinksTaskResult[];
billing: DataforseoApiCallCost;
};
async function createAuthenticatedFetch() {
const apiKey = await getRequiredEnvValue("DATAFORSEO_API_KEY");
return (url: RequestInfo, init?: RequestInit): Promise<Response> => {
const headers = new Headers(init?.headers);
headers.set("Authorization", `Basic ${apiKey}`);
return fetch(url, {
...init,
headers,
});
};
}
async function postBacklinks(path: string, payload: unknown) {
const authenticatedFetch = await createAuthenticatedFetch();
const response = await authenticatedFetch(`${API_BASE}${path}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(payload),
});
const rawText = await response.text();
if (!response.ok) {
const classifiedError = classifyBacklinksError(
response.status,
rawText,
path,
);
if (classifiedError) throw classifiedError;
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO HTTP ${response.status} on ${path}`,
);
}
let raw: unknown;
try {
raw = JSON.parse(rawText);
} catch {
const classifiedError = classifyBacklinksError(
response.status,
rawText,
path,
);
if (classifiedError) throw classifiedError;
console.error(
`dataforseo.${path}.non-json-response`,
rawText.slice(0, 800),
);
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned a non-JSON response`,
);
}
const parsed = responseSchema.safeParse(raw);
if (!parsed.success) {
const classifiedError = classifyBacklinksError(
response.status,
rawText,
path,
);
if (classifiedError) throw classifiedError;
console.error(
`dataforseo.${path}.invalid-top-level-shape`,
rawText.slice(0, 800),
);
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned an invalid response shape`,
);
}
const responseData = parsed.data;
if (responseData.status_code !== 20000) {
const classifiedError = classifyBacklinksError(
responseData.status_code,
`${responseData.status_message ?? ""} ${rawText}`,
path,
);
if (classifiedError) throw classifiedError;
throw new AppError(
"INTERNAL_ERROR",
responseData.status_message || "DataForSEO request failed",
);
}
const task = responseData.tasks?.[0];
if (!task) {
throw new AppError("INTERNAL_ERROR", "DataForSEO response missing task");
}
if (task.status_code !== 20000) {
const classifiedError = classifyBacklinksError(
task.status_code,
`${task.status_message ?? ""} ${rawText}`,
path,
);
if (classifiedError) throw classifiedError;
throw new AppError(
"INTERNAL_ERROR",
task.status_message || "DataForSEO task failed",
);
}
return {
results: (task.result ?? []).filter(
(r): r is BacklinksTaskResult => r != null,
),
billing: {
path: task.path ?? [],
costUsd: task.cost ?? responseData.cost ?? 0,
resultCount: task.result_count ?? null,
},
} satisfies DataforseoTaskResponse;
}
function buildCommonPayload(input: BacklinksRequest) {
return {
target: input.target,
include_subdomains: true,
include_indirect_links: true,
exclude_internal_backlinks: true,
backlinks_status_type: "live",
rank_scale: "one_hundred",
};
}
export async function fetchBacklinksSummaryRaw(input: BacklinksRequest) {
const response = await postBacklinks("/v3/backlinks/summary/live", [
buildCommonPayload(input),
]);
const firstResult = response.results[0];
const parsed = firstResult
? backlinksSummaryItemSchema.safeParse(firstResult)
: null;
if (parsed && !parsed.success) {
console.error(
"dataforseo.backlinks-summary-live.invalid-result",
parsed.error.issues.slice(0, 5),
);
throw new AppError(
"INTERNAL_ERROR",
"DataForSEO backlinks-summary-live returned an invalid response shape",
);
}
const data = parsed?.data ?? {};
return {
data,
billing: response.billing,
} satisfies DataforseoApiResponse<typeof data>;
}
export async function fetchBacklinksRowsRaw(input: BacklinksListRequest) {
const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input);
const filters = spamFilterOptions.hideSpam
? [["backlink_spam_score", "<=", spamFilterOptions.spamThreshold]]
: undefined;
const response = await postBacklinks("/v3/backlinks/backlinks/live", [
{
...buildCommonPayload(input),
limit: input.limit ?? 100,
order_by: ["rank,desc"],
...(filters ? { filters } : {}),
},
]);
const data = parseItems(
"backlinks-live",
response.results,
backlinksItemSchema,
);
return {
data,
billing: response.billing,
} satisfies DataforseoApiResponse<typeof data>;
}
export async function fetchReferringDomainsRaw(input: BacklinksListRequest) {
const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input);
const filters = spamFilterOptions.hideSpam
? [["backlinks_spam_score", "<=", spamFilterOptions.spamThreshold]]
: undefined;
const response = await postBacklinks("/v3/backlinks/referring_domains/live", [
{
...buildCommonPayload(input),
limit: input.limit ?? 100,
order_by: ["backlinks,desc"],
...(filters ? { filters } : {}),
},
]);
const data = parseItems(
"referring-domains-live",
response.results,
referringDomainItemSchema,
);
return {
data,
billing: response.billing,
} satisfies DataforseoApiResponse<typeof data>;
}
export async function fetchDomainPagesSummaryRaw(input: BacklinksListRequest) {
const response = await postBacklinks(
"/v3/backlinks/domain_pages_summary/live",
[
{
...buildCommonPayload(input),
limit: input.limit ?? 100,
order_by: ["backlinks,desc"],
},
],
);
const data = parseItems(
"domain-pages-summary-live",
response.results,
domainPageSummaryItemSchema,
);
return {
data,
billing: response.billing,
} satisfies DataforseoApiResponse<typeof data>;
}
export async function fetchBacklinksHistoryRaw(
input: BacklinksTimeseriesRequest,
) {
const response = await postBacklinks("/v3/backlinks/history/live", [
{
target: input.target,
date_from: input.dateFrom,
date_to: input.dateTo,
rank_scale: "one_hundred",
},
]);
const data = parseItems(
"backlinks-history-live",
response.results,
backlinksHistoryItemSchema,
);
return {
data,
billing: response.billing,
} satisfies DataforseoApiResponse<typeof data>;
}

157
src/server/lib/dataforseoBacklinksSupport.ts
View File

@ -1,157 +0,0 @@
import { z } from "zod";
import { createDataforseoAccessClassifier } from "@/server/lib/dataforseoAccessClassification";
import { AppError } from "@/server/lib/errors";
const taskResultSchema = z
.object({
items: z.array(z.unknown()).nullable().optional(),
})
.passthrough();
export type BacklinksTaskResult = z.infer<typeof taskResultSchema>;
const taskSchema = z
.object({
status_code: z.number().optional(),
status_message: z.string().optional(),
cost: z.number().nullable().optional(),
result_count: z.number().nullable().optional(),
path: z.array(z.string()).optional(),
result: z.array(taskResultSchema.nullable()).nullable().optional(),
})
.passthrough();
export const responseSchema = z
.object({
status_code: z.number().optional(),
status_message: z.string().optional(),
cost: z.number().nullable().optional(),
tasks: z.array(taskSchema).optional(),
})
.passthrough();
export const backlinksSummaryItemSchema = z
.object({
target: z.string().optional(),
rank: z.number().nullable().optional(),
backlinks: z.number().nullable().optional(),
referring_pages: z.number().nullable().optional(),
referring_domains: z.number().nullable().optional(),
broken_backlinks: z.number().nullable().optional(),
broken_pages: z.number().nullable().optional(),
new_backlinks: z.number().nullable().optional(),
lost_backlinks: z.number().nullable().optional(),
new_reffering_domains: z.number().nullable().optional(),
lost_reffering_domains: z.number().nullable().optional(),
new_referring_domains: z.number().nullable().optional(),
lost_referring_domains: z.number().nullable().optional(),
backlinks_spam_score: z.number().nullable().optional(),
info: z
.object({
target_spam_score: z.number().nullable().optional(),
})
.passthrough()
.nullable()
.optional(),
})
.passthrough();
export const backlinksItemSchema = z
.object({
domain_from: z.string().nullable().optional(),
url_from: z.string().nullable().optional(),
url_to: z.string().nullable().optional(),
anchor: z.string().nullable().optional(),
item_type: z.string().nullable().optional(),
dofollow: z.boolean().nullable().optional(),
rank: z.number().nullable().optional(),
domain_from_rank: z.number().nullable().optional(),
page_from_rank: z.number().nullable().optional(),
backlinks_spam_score: z.number().nullable().optional(),
backlink_spam_score: z.number().nullable().optional(),
first_seen: z.string().nullable().optional(),
last_visited: z.string().nullable().optional(),
lost_date: z.string().nullable().optional(),
is_new: z.boolean().nullable().optional(),
is_lost: z.boolean().nullable().optional(),
is_broken: z.boolean().nullable().optional(),
links_count: z.number().nullable().optional(),
rel_attributes: z.array(z.string()).nullable().optional(),
attributes: z.array(z.string()).nullable().optional(),
})
.passthrough();
export const referringDomainItemSchema = z
.object({
domain: z.string().nullable().optional(),
backlinks: z.number().nullable().optional(),
referring_pages: z.number().nullable().optional(),
rank: z.number().nullable().optional(),
first_seen: z.string().nullable().optional(),
broken_backlinks: z.number().nullable().optional(),
broken_pages: z.number().nullable().optional(),
backlinks_spam_score: z.number().nullable().optional(),
target_spam_score: z.number().nullable().optional(),
})
.passthrough();
export const domainPageSummaryItemSchema = z
.object({
page: z.string().nullable().optional(),
url: z.string().nullable().optional(),
backlinks: z.number().nullable().optional(),
referring_domains: z.number().nullable().optional(),
rank: z.number().nullable().optional(),
broken_backlinks: z.number().nullable().optional(),
})
.passthrough();
export const backlinksHistoryItemSchema = z
.object({
date: z.string().nullable().optional(),
rank: z.number().nullable().optional(),
backlinks: z.number().nullable().optional(),
referring_domains: z.number().nullable().optional(),
new_backlinks: z.number().nullable().optional(),
lost_backlinks: z.number().nullable().optional(),
new_reffering_domains: z.number().nullable().optional(),
lost_reffering_domains: z.number().nullable().optional(),
new_referring_domains: z.number().nullable().optional(),
lost_referring_domains: z.number().nullable().optional(),
})
.passthrough();
export const classifyBacklinksError = createDataforseoAccessClassifier({
pathPrefix: "/backlinks/",
notEnabledCode: "BACKLINKS_NOT_ENABLED",
notEnabledMessage:
"Backlinks is not enabled for the connected DataForSEO account",
billingIssueCode: "BACKLINKS_BILLING_ISSUE",
billingIssueMessage:
"The connected DataForSEO account has a billing or balance issue",
});
export function parseItems<T extends z.ZodTypeAny>(
endpointName: string,
results: BacklinksTaskResult[],
itemSchema: T,
): Array<z.infer<T>> {
const firstResult = results[0] ?? null;
if (firstResult == null) {
return [];
}
const parsed = z.array(itemSchema).safeParse(firstResult.items ?? []);
if (!parsed.success) {
console.error(
`dataforseo.${endpointName}.invalid-items`,
parsed.error.issues.slice(0, 5),
);
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${endpointName} returned an invalid response shape`,
);
}
return parsed.data;
}

470
src/server/lib/dataforseoClient.ts
View File

@ -1,470 +0,0 @@
/* eslint-disable max-lines, max-lines-per-function */
import {
AUTUMN_SEO_DATA_BALANCE_FEATURE_ID,
AUTUMN_SEO_DATA_CREDITS_PER_USD,
AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID,
SEO_DATA_COST_MARKUP,
roundUsdForBilling,
} from "@/shared/billing";
import {
type CreditFeature,
mapDataforseoPathToCreditFeature,
} from "@/shared/billing-credit-features";
import { autumn } from "@/server/billing/autumn";
import { getOrCreateOrganizationCustomer } from "@/server/billing/subscription";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import {
fetchKeywordIdeasRaw,
fetchKeywordOverviewRaw,
fetchKeywordSuggestionsRaw,
fetchRelatedKeywordsRaw,
fetchBusinessListingsSearchRaw,
fetchBusinessQuestionsAnswersRaw,
fetchDomainRankOverviewRaw,
fetchKeywordSearchVolumeRaw,
fetchLocalSerpItemsRaw,
fetchRankedKeywordsRaw,
fetchRelevantPagesRaw,
fetchSerpCompetitorsRaw,
type DataforseoLabsItemType,
fetchLiveSerpItemsRaw,
fetchRankCheckSerpRaw,
type LabsKeywordDataItem,
type SerpLiveItem,
} from "@/server/lib/dataforseo";
import {
fetchLlmAggregatedMetricsRaw,
fetchLlmMentionsSearchRaw,
fetchLlmResponseRaw,
fetchLlmTopPagesRaw,
type LlmAggregatedMetricsInput,
type LlmMentionsSearchInput,
type LlmResponsesInput,
type LlmTopPagesInput,
} from "@/server/lib/dataforseoLlm";
import { fetchDataforseoLighthouseResultRaw } from "@/server/lib/dataforseoLighthouse";
import type { LighthouseStrategy } from "@/server/lib/dataforseoLighthousePayload";
import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload";
import {
fetchBacklinksHistoryRaw,
fetchBacklinksRowsRaw,
fetchBacklinksSummaryRaw,
fetchDomainPagesSummaryRaw,
fetchReferringDomainsRaw,
type BacklinksListRequest,
type BacklinksRequest,
type BacklinksTimeseriesRequest,
} from "@/server/lib/dataforseoBacklinks";
import {
type DataforseoApiResponse,
type DataforseoApiCallCost,
DataforseoChargedTaskError,
} from "@/server/lib/dataforseoCost";
import { AppError } from "@/server/lib/errors";
import { captureServerEvent } from "@/server/lib/posthog";
import { isHostedServerAuthMode } from "@/server/lib/runtime-env";
export { mapDataforseoPathToCreditFeature };
export function createDataforseoClient(customer: BillingCustomerContext) {
return {
business: {
businessListings(input: {
categories?: string[];
title?: string;
locationCoordinate: string;
orderBy?: string[];
limit: number;
}) {
return meterDataforseoCall(
customer,
() => fetchBusinessListingsSearchRaw(input),
"local_seo",
);
},
questionsAnswers(input: {
keyword: string;
locationCoordinate: string;
languageCode: string;
depth: number;
}) {
return meterDataforseoCall(
customer,
() => fetchBusinessQuestionsAnswersRaw(input),
"local_seo",
);
},
},
backlinks: {
summary(input: BacklinksRequest) {
return meterDataforseoCall(customer, () =>
fetchBacklinksSummaryRaw(input),
);
},
rows(input: BacklinksListRequest) {
return meterDataforseoCall(customer, () =>
fetchBacklinksRowsRaw(input),
);
},
referringDomains(input: BacklinksListRequest) {
return meterDataforseoCall(customer, () =>
fetchReferringDomainsRaw(input),
);
},
domainPages(input: BacklinksListRequest) {
return meterDataforseoCall(customer, () =>
fetchDomainPagesSummaryRaw(input),
);
},
history(input: BacklinksTimeseriesRequest) {
return meterDataforseoCall(customer, () =>
fetchBacklinksHistoryRaw(input),
);
},
},
keywords: {
related(input: {
keyword: string;
locationCode: number;
languageCode: string;
limit: number;
depth?: number;
}) {
return meterDataforseoCall(customer, () =>
fetchRelatedKeywordsRaw(
input.keyword,
input.locationCode,
input.languageCode,
input.limit,
input.depth,
),
);
},
suggestions(input: {
keyword: string;
locationCode: number;
languageCode: string;
limit: number;
}) {
return meterDataforseoCall(customer, () =>
fetchKeywordSuggestionsRaw(
input.keyword,
input.locationCode,
input.languageCode,
input.limit,
),
);
},
ideas(input: {
keyword: string;
locationCode: number;
languageCode: string;
limit: number;
}) {
return meterDataforseoCall(customer, () =>
fetchKeywordIdeasRaw(
input.keyword,
input.locationCode,
input.languageCode,
input.limit,
),
);
},
},
domain: {
rankOverview(input: {
target: string;
locationCode: number;
languageCode: string;
}) {
return meterDataforseoCall(customer, () =>
fetchDomainRankOverviewRaw(
input.target,
input.locationCode,
input.languageCode,
),
);
},
rankedKeywords(input: {
target: string;
locationCode: number;
languageCode: string;
limit: number;
offset?: number;
orderBy?: string[];
filters?: unknown[];
itemTypes?: DataforseoLabsItemType[];
includeSubdomains?: boolean;
}) {
return meterDataforseoCall(customer, () =>
fetchRankedKeywordsRaw(input),
);
},
relevantPages(input: {
target: string;
locationCode: number;
languageCode: string;
limit: number;
offset?: number;
orderBy?: string[];
filters?: unknown[];
}) {
return meterDataforseoCall(customer, () =>
fetchRelevantPagesRaw(input),
);
},
},
serp: {
live(input: {
keyword: string;
locationCode: number;
languageCode: string;
}) {
return meterDataforseoCall(customer, () =>
fetchLiveSerpItemsRaw(
input.keyword,
input.locationCode,
input.languageCode,
),
);
},
rankCheck(input: {
keyword: string;
keywordId: string;
locationCode: number;
languageCode: string;
device: "desktop" | "mobile";
targetDomain: string;
depth: number;
}) {
return meterDataforseoCall(
customer,
() => fetchRankCheckSerpRaw(input),
"rank_tracking",
);
},
local(input: {
keyword: string;
locationCoordinate?: string;
languageCode: string;
searchType: "maps" | "local_finder";
device: "desktop" | "mobile";
depth: number;
searchPlaces?: boolean;
}) {
return meterDataforseoCall(
customer,
() => fetchLocalSerpItemsRaw(input),
"local_seo",
);
},
},
keywordData: {
searchVolume(input: {
keywords: string[];
locationCode?: number;
languageCode?: string;
}) {
return meterDataforseoCall(customer, () =>
fetchKeywordSearchVolumeRaw(input),
);
},
},
labs: {
keywordOverview(input: {
keywords: string[];
locationCode: number;
languageCode: string;
}) {
return meterDataforseoCall(
customer,
() =>
fetchKeywordOverviewRaw(
input.keywords,
input.locationCode,
input.languageCode,
),
"rank_tracking",
);
},
serpCompetitors(input: {
keywords: string[];
locationCode: number;
languageCode: string;
itemTypes?: DataforseoLabsItemType[];
includeSubdomains?: boolean;
limit: number;
offset?: number;
}) {
return meterDataforseoCall(customer, () =>
fetchSerpCompetitorsRaw(input),
);
},
},
lighthouse: {
live(input: { url: string; strategy: LighthouseStrategy }) {
return meterDataforseoCall<StoredLighthousePayload>(customer, () =>
fetchDataforseoLighthouseResultRaw(input),
);
},
},
aiSearch: {
mentionsSearch(input: LlmMentionsSearchInput) {
return meterDataforseoCall(customer, () =>
fetchLlmMentionsSearchRaw(input),
);
},
aggregatedMetrics(input: LlmAggregatedMetricsInput) {
return meterDataforseoCall(customer, () =>
fetchLlmAggregatedMetricsRaw(input),
);
},
topPages(input: LlmTopPagesInput) {
return meterDataforseoCall(customer, () => fetchLlmTopPagesRaw(input));
},
llmResponse(input: LlmResponsesInput) {
return meterDataforseoCall(customer, () => fetchLlmResponseRaw(input));
},
},
} as const;
}
async function meterDataforseoCall<T>(
customer: BillingCustomerContext,
execute: () => Promise<DataforseoApiResponse<T>>,
creditFeature?: CreditFeature,
): Promise<T> {
const isHostedMode = await isHostedServerAuthMode();
if (!isHostedMode) {
const result = await execute();
return result.data;
}
const billingCustomer = await getOrCreateOrganizationCustomer(customer);
const { monthlyRemaining } = await assertSeoDataBalanceAvailable(
billingCustomer.id,
);
let result: DataforseoApiResponse<T>;
try {
result = await execute();
} catch (error) {
if (error instanceof DataforseoChargedTaskError) {
await trackDataforseoCost({
customer,
customerId: billingCustomer.id,
billing: error.billing,
monthlyRemaining,
creditFeature,
});
}
throw error;
}
await trackDataforseoCost({
customer,
customerId: billingCustomer.id,
billing: result.billing,
monthlyRemaining,
creditFeature,
});
return result.data;
}
async function assertSeoDataBalanceAvailable(customerId: string) {
const [monthlyCheck, topupCheck] = await Promise.all([
autumn.check({
customerId,
featureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID,
}),
autumn.check({
customerId,
featureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID,
}),
]);
const monthlyRemaining = monthlyCheck.balance?.remaining ?? 0;
const topupRemaining = topupCheck.balance?.remaining ?? 0;
if (monthlyRemaining + topupRemaining <= 0) {
throw new AppError("INSUFFICIENT_CREDITS");
}
return { monthlyRemaining };
}
async function trackDataforseoCost(args: {
customer: BillingCustomerContext;
customerId: string;
billing: DataforseoApiCallCost;
monthlyRemaining: number;
creditFeature?: CreditFeature;
}) {
const totalCostUsd = roundUsdForBilling(
args.billing.costUsd * SEO_DATA_COST_MARKUP,
);
const totalCostCredits = Math.ceil(
totalCostUsd * AUTUMN_SEO_DATA_CREDITS_PER_USD,
);
const monthlyDeduct = Math.min(args.monthlyRemaining, totalCostCredits);
const topupDeduct = totalCostCredits - monthlyDeduct;
const creditFeature =
args.creditFeature ?? mapDataforseoPathToCreditFeature(args.billing.path);
const properties = {
provider: "dataforseo",
currency: "USD",
paths: [args.billing.path.join("/")],
creditFeature,
totalCostUsd,
totalCostCredits,
fromCache: false,
};
if (monthlyDeduct > 0) {
await autumn.track({
customerId: args.customerId,
featureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID,
value: monthlyDeduct,
properties: {
...properties,
balanceFeatureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID,
},
});
}
if (topupDeduct > 0) {
await autumn.track({
customerId: args.customerId,
featureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID,
value: topupDeduct,
properties: {
...properties,
balanceFeatureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID,
},
});
}
if (totalCostCredits > 0) {
await captureServerEvent({
distinctId: args.customer.userId,
event: "usage:credits_consume",
organizationId: args.customer.organizationId,
properties: {
project_id: args.customer.projectId,
credit_feature: creditFeature,
monthly_credits: monthlyDeduct,
topup_credits: topupDeduct,
total_credits: totalCostCredits,
cost_usd: totalCostUsd,
},
});
}
}
export type { LabsKeywordDataItem, SerpLiveItem };

22
src/server/lib/dataforseoCost.ts
View File

@ -1,22 +0,0 @@
import { AppError } from "@/server/lib/errors";
export type DataforseoApiCallCost = {
path: string[];
costUsd: number;
resultCount: number | null;
};
export type DataforseoApiResponse<T> = {
data: T;
billing: DataforseoApiCallCost;
};
export class DataforseoChargedTaskError extends AppError {
constructor(
message: string,
public readonly billing: DataforseoApiCallCost,
) {
super("INTERNAL_ERROR", message);
this.name = "DataforseoChargedTaskError";
}
}

60
src/server/lib/dataforseoLighthouse.ts
View File

@ -1,60 +0,0 @@
import { env } from "cloudflare:workers";
import {
parseDataforseoLighthousePayload,
requestCategories,
type LighthouseStrategy,
} from "@/server/lib/dataforseoLighthousePayload";
import type { DataforseoApiResponse } from "@/server/lib/dataforseoCost";
import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload";
const DATAFORSEO_LIGHTHOUSE_ENDPOINT =
"https://api.dataforseo.com/v3/on_page/lighthouse/live/json";
export async function fetchDataforseoLighthouseResultRaw(input: {
url: string;
strategy: LighthouseStrategy;
}): Promise<DataforseoApiResponse<StoredLighthousePayload>> {
const response = await fetch(DATAFORSEO_LIGHTHOUSE_ENDPOINT, {
method: "POST",
headers: {
Authorization: `Basic ${env.DATAFORSEO_API_KEY?.trim() ?? ""}`,
"Content-Type": "application/json",
},
body: JSON.stringify([
{
url: input.url,
for_mobile: input.strategy === "mobile",
categories: requestCategories,
},
]),
signal: AbortSignal.timeout(60_000),
});
const rawText = await response.text();
if (!response.ok) {
throw new Error(
`DataForSEO Lighthouse request failed (${response.status}): ${rawText}`,
);
}
let payload: unknown;
try {
payload = JSON.parse(rawText);
} catch {
throw new Error(
`DataForSEO Lighthouse returned non-JSON content (content-type: ${response.headers.get("content-type") ?? "unknown"}): ${rawText}`,
);
}
const data = parseDataforseoLighthousePayload(payload, input);
return {
data,
billing: {
path: ["v3", "on_page", "lighthouse", "live", "json"],
costUsd: data.metadata.cost ?? 0,
resultCount: 1,
},
};
}

382
src/server/lib/dataforseoLlm.ts
View File

@ -1,382 +0,0 @@
import { z } from "zod";
import {
llmAggregatedTotalSchema,
llmMentionItemSchema,
llmResponseEnvelopeSchema,
llmResponseResultSchema,
llmTopPagesItemSchema,
type LlmAggregatedTotal,
type LlmDataforseoTask,
type LlmMentionItem,
type LlmResponseResult,
type LlmTopPagesItem,
} from "@/server/lib/dataforseoLlmSchemas";
import type { DataforseoApiResponse } from "@/server/lib/dataforseoCost";
import { createDataforseoAccessClassifier } from "@/server/lib/dataforseoAccessClassification";
import { AppError } from "@/server/lib/errors";
import { getRequiredEnvValue } from "@/server/lib/runtime-env";
/**
* Raw HTTP wrappers for DataForSEO AI Optimization endpoints.
*
* The official `dataforseo-client` SDK doesn't ship typed bindings for these
* endpoints yet, so we POST raw JSON. Every wrapper returns billing metadata
* alongside parsed data so the calling client can meter usage with Autumn.
*/
const API_BASE = "https://api.dataforseo.com";
const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600;
// ChatGPT mention/response data is only available for US/en per DataForSEO docs.
export const CHATGPT_LOCATION_CODE = 2840;
export const CHATGPT_LANGUAGE_CODE = "en";
export type LlmPlatform = "chat_gpt" | "google";
// ---------------------------------------------------------------------------
// Shared HTTP / response handling
// ---------------------------------------------------------------------------
async function postLlm(path: string, payload: unknown): Promise<unknown> {
const apiKey = await getRequiredEnvValue("DATAFORSEO_API_KEY");
const response = await fetch(`${API_BASE}${path}`, {
method: "POST",
headers: {
Authorization: `Basic ${apiKey}`,
"Content-Type": "application/json",
},
body: JSON.stringify(payload),
});
const rawText = await response.text();
if (!response.ok) {
throw (
classifyAiSearchError(response.status, rawText, path) ??
new AppError(
"INTERNAL_ERROR",
`DataForSEO HTTP ${response.status} on ${path}. Response: ${truncate(rawText)}`,
)
);
}
try {
return JSON.parse(rawText);
} catch {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned non-JSON response: ${truncate(rawText)}`,
);
}
}
const classifyAiSearchError = createDataforseoAccessClassifier({
pathPrefix: "/ai_optimization/",
notEnabledCode: "AI_SEARCH_NOT_ENABLED",
notEnabledMessage:
"AI Optimization is not enabled for the connected DataForSEO account",
billingIssueCode: "AI_SEARCH_BILLING_ISSUE",
billingIssueMessage:
"The connected DataForSEO account has a billing or balance issue",
});
function truncate(text: string): string {
return text.length > MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH
? `${text.slice(0, MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH)}... [truncated]`
: text;
}
function parseEnvelope(path: string, raw: unknown): LlmDataforseoTask {
const envelope = llmResponseEnvelopeSchema.safeParse(raw);
if (!envelope.success) {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned an invalid envelope: ${envelope.error.issues
.slice(0, 3)
.map((i) => i.message)
.join("; ")}`,
);
}
const data = envelope.data;
if (data.status_code !== 20000) {
const message = data.status_message || `DataForSEO ${path} request failed`;
throw (
classifyAiSearchError(data.status_code, message, path) ??
new AppError("INTERNAL_ERROR", message)
);
}
const task = data.tasks?.[0];
if (!task) {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} response missing task`,
);
}
if (task.status_code !== 20000) {
const message = task.status_message || `DataForSEO ${path} task failed`;
throw (
classifyAiSearchError(task.status_code, message, path) ??
new AppError("INTERNAL_ERROR", message)
);
}
return task;
}
function buildBilling(task: LlmDataforseoTask) {
return {
path: task.path,
costUsd: task.cost,
resultCount: task.result_count ?? null,
};
}
// ---------------------------------------------------------------------------
// Target builders — DataForSEO's `target` array accepts domain OR keyword
// entries. We always pass exactly one target per call.
// ---------------------------------------------------------------------------
type LlmTarget =
| {
domain: string;
include_subdomains?: boolean;
search_filter?: "include" | "exclude";
search_scope?: string[];
}
| {
keyword: string;
search_filter?: "include" | "exclude";
search_scope?: string[];
match_type?: "word_match" | "partial_match";
};
export function buildLlmTarget(input: {
type: "domain" | "keyword";
value: string;
}): LlmTarget {
if (input.type === "domain") {
return {
domain: input.value,
include_subdomains: true,
search_filter: "include",
search_scope: ["any"],
};
}
return {
keyword: input.value,
search_filter: "include",
search_scope: ["any", "brand_entities"],
match_type: "word_match",
};
}
// ---------------------------------------------------------------------------
// LLM Mentions Search Live
// ---------------------------------------------------------------------------
export type LlmMentionsSearchInput = {
target: LlmTarget;
platform: LlmPlatform;
locationCode: number;
languageCode: string;
limit?: number;
};
export async function fetchLlmMentionsSearchRaw(
input: LlmMentionsSearchInput,
): Promise<DataforseoApiResponse<LlmMentionItem[]>> {
const path = "/v3/ai_optimization/llm_mentions/search/live";
const payload = [
{
target: [input.target],
platform: input.platform,
location_code: input.locationCode,
language_code: input.languageCode,
limit: clampLimit(input.limit ?? 100, 1, 1000),
},
];
const raw = await postLlm(path, payload);
const task = parseEnvelope(path, raw);
const items = z
.array(llmMentionItemSchema)
.safeParse(extractItems(task.result));
if (!items.success) {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned an invalid mention items shape`,
);
}
return { data: items.data, billing: buildBilling(task) };
}
// ---------------------------------------------------------------------------
// LLM Mentions Aggregated Metrics Live
// ---------------------------------------------------------------------------
export type LlmAggregatedMetricsInput = {
target: LlmTarget;
platform: LlmPlatform;
locationCode: number;
languageCode: string;
internalListLimit?: number;
};
export async function fetchLlmAggregatedMetricsRaw(
input: LlmAggregatedMetricsInput,
): Promise<DataforseoApiResponse<LlmAggregatedTotal>> {
const path = "/v3/ai_optimization/llm_mentions/aggregated_metrics/live";
const payload = [
{
target: [input.target],
platform: input.platform,
location_code: input.locationCode,
language_code: input.languageCode,
internal_list_limit: clampLimit(input.internalListLimit ?? 10, 1, 20),
},
];
const raw = await postLlm(path, payload);
const task = parseEnvelope(path, raw);
const totalRaw = extractFirstResult(task.result)?.total ?? {};
const total = llmAggregatedTotalSchema.safeParse(totalRaw);
if (!total.success) {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned an invalid aggregated metrics shape`,
);
}
return { data: total.data, billing: buildBilling(task) };
}
// ---------------------------------------------------------------------------
// LLM Mentions Top Pages Live
// ---------------------------------------------------------------------------
export type LlmTopPagesInput = {
target: LlmTarget;
platform: LlmPlatform;
locationCode: number;
languageCode: string;
itemsListLimit?: number;
};
export async function fetchLlmTopPagesRaw(
input: LlmTopPagesInput,
): Promise<DataforseoApiResponse<LlmTopPagesItem[]>> {
const path = "/v3/ai_optimization/llm_mentions/top_pages/live";
const payload = [
{
target: [input.target],
platform: input.platform,
location_code: input.locationCode,
language_code: input.languageCode,
links_scope: "sources",
items_list_limit: clampLimit(input.itemsListLimit ?? 10, 1, 10),
internal_list_limit: 5,
},
];
const raw = await postLlm(path, payload);
const task = parseEnvelope(path, raw);
const items = z
.array(llmTopPagesItemSchema)
.safeParse(extractFirstResult(task.result)?.items ?? []);
if (!items.success) {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned an invalid top pages shape`,
);
}
return { data: items.data, billing: buildBilling(task) };
}
// ---------------------------------------------------------------------------
// LLM Responses Live (per-model)
// ---------------------------------------------------------------------------
export type LlmResponseModelSlug =
| "chat_gpt"
| "claude"
| "gemini"
| "perplexity";
export type LlmResponsesInput = {
userPrompt: string;
modelSlug: LlmResponseModelSlug;
modelName: string;
webSearch?: boolean;
maxOutputTokens?: number;
/** Two-letter ISO country code used to geolocate the web-search component. */
webSearchCountryCode?: string;
};
export async function fetchLlmResponseRaw(
input: LlmResponsesInput,
): Promise<DataforseoApiResponse<LlmResponseResult>> {
const path = `/v3/ai_optimization/${input.modelSlug}/llm_responses/live`;
// DataForSEO's Gemini endpoint rejects `web_search_country_iso_code` with a
// 40501 "Invalid Field" error. The other three models accept it.
const supportsCountry = input.modelSlug !== "gemini";
const payload = [
{
user_prompt: input.userPrompt,
model_name: input.modelName,
web_search: input.webSearch ?? true,
max_output_tokens: clampLimit(input.maxOutputTokens ?? 1024, 256, 4096),
...(supportsCountry &&
input.webSearchCountryCode && {
web_search_country_iso_code: input.webSearchCountryCode,
}),
},
];
const raw = await postLlm(path, payload);
const task = parseEnvelope(path, raw);
const first = extractFirstResult(task.result) as unknown;
const result = llmResponseResultSchema.safeParse(first ?? {});
if (!result.success) {
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${path} returned an invalid response shape`,
);
}
return { data: result.data, billing: buildBilling(task) };
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function clampLimit(value: number, min: number, max: number): number {
return Math.min(max, Math.max(min, Math.floor(value)));
}
function extractFirstResult(
result: LlmDataforseoTask["result"],
): Record<string, unknown> | null {
const first = result?.[0];
return isRecord(first) ? first : null;
}
function extractItems(result: LlmDataforseoTask["result"]): unknown[] {
const first = extractFirstResult(result);
if (!first) return [];
const items = first.items;
return Array.isArray(items) ? items : [];
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}

28
src/server/lib/dataforseoLlmSchemas.ts
View File

@ -129,31 +129,3 @@ export const llmResponseResultSchema = z
.passthrough();
export type LlmResponseResult = z.infer<typeof llmResponseResultSchema>;
// ---------------------------------------------------------------------------
// Top-level envelope used by every AI Optimization endpoint.
// We re-declare here (instead of reusing dataforseoSchemas.ts) because the
// `result` shape differs from Labs/SERP — items are not always under
// `result[0].items` and totals/items can both be present.
// ---------------------------------------------------------------------------
const llmTaskSchema = z
.object({
status_code: z.number().optional(),
status_message: z.string().optional(),
path: z.array(z.string()),
cost: z.number(),
result_count: z.number().nullable().optional(),
result: z.array(z.unknown()).nullable().optional(),
})
.passthrough();
export type LlmDataforseoTask = z.infer<typeof llmTaskSchema>;
export const llmResponseEnvelopeSchema = z
.object({
status_code: z.number().optional(),
status_message: z.string().optional(),
tasks: z.array(llmTaskSchema).optional(),
})
.passthrough();

75
src/server/lib/dataforseoSchemas.test.ts
View File

@ -1,75 +0,0 @@
import { describe, expect, it } from "vitest";
import {
domainRankedKeywordItemSchema,
parseTaskItems,
relatedKeywordItemSchema,
successfulDataforseoTaskSchema,
} from "@/server/lib/dataforseoSchemas";
describe("dataforseoSchemas", () => {
it("accepts null items for empty successful tasks", () => {
const task = {
id: "04042314-1577-0387-0000-33dc4b485cfd",
status_code: 20000,
status_message: "Ok.",
path: ["v3", "dataforseo_labs", "google", "related_keywords", "live"],
cost: 0.02,
result_count: 1,
result: [
{
se_type: "google",
seed_keyword: "canva ai video alternative",
location_code: 2840,
language_code: "en",
total_count: null,
items_count: 0,
items: null,
},
],
};
const parsedTask = successfulDataforseoTaskSchema.parse(task);
expect(
parseTaskItems(
"google-related-keywords-live",
parsedTask,
relatedKeywordItemSchema,
),
).toEqual([]);
});
it("accepts empty ranked keyword tasks with null items", () => {
const task = {
id: "04070246-1577-0381-0000-2c56c059f67e",
status_code: 20000,
status_message: "Ok.",
path: ["v3", "dataforseo_labs", "google", "ranked_keywords", "live"],
cost: 0.01,
result_count: 1,
result: [
{
se_type: "google",
target: "openseo.so",
location_code: 2840,
language_code: "en",
total_count: null,
items_count: 0,
metrics: null,
metrics_absolute: null,
items: null,
},
],
};
const parsedTask = successfulDataforseoTaskSchema.parse(task);
expect(
parseTaskItems(
"google-ranked-keywords-live",
parsedTask,
domainRankedKeywordItemSchema,
),
).toEqual([]);
});
});

238
src/server/lib/dataforseoSchemas.ts
View File

@ -1,238 +0,0 @@
import { z } from "zod";
import { AppError } from "@/server/lib/errors";
const dataforseoTaskSchema = z
.object({
status_code: z.number().optional(),
status_message: z.string().optional(),
path: z.array(z.string()),
cost: z.number(),
result_count: z.number().nullable(),
result: z
.array(
z
.object({
items: z.array(z.unknown()).nullable().optional(),
})
.passthrough(),
)
.nullable()
.optional(),
})
.passthrough();
export type DataforseoTask = z.infer<typeof dataforseoTaskSchema>;
export const successfulDataforseoTaskSchema = dataforseoTaskSchema;
export const dataforseoResponseSchema = z
.object({
status_code: z.number().optional(),
status_message: z.string().optional(),
tasks: z.array(dataforseoTaskSchema).optional(),
})
.passthrough();
const monthlySearchSchema = z
.object({
year: z.number().int(),
month: z.number().int().min(1).max(12),
search_volume: z.number().nullable(),
})
.passthrough();
const keywordInfoSchema = z
.object({
search_volume: z.number().nullable().optional(),
cpc: z.number().nullable().optional(),
competition: z.number().nullable().optional(),
monthly_searches: z.array(monthlySearchSchema).nullable().optional(),
})
.passthrough();
const keywordInfoWithClickstreamSchema = z
.object({
search_volume: z.number().nullable().optional(),
monthly_searches: z.array(monthlySearchSchema).nullable().optional(),
})
.passthrough();
const searchIntentInfoSchema = z
.object({
main_intent: z.string().nullable().optional(),
})
.passthrough();
const keywordPropertiesSchema = z
.object({
keyword_difficulty: z.number().nullable().optional(),
})
.passthrough();
export const relatedKeywordItemSchema = z
.object({
keyword_data: z
.object({
keyword: z.string().optional(),
keyword_info: keywordInfoSchema.optional(),
keyword_info_normalized_with_clickstream:
keywordInfoWithClickstreamSchema.optional(),
search_intent_info: searchIntentInfoSchema.nullable().optional(),
keyword_properties: keywordPropertiesSchema.nullable().optional(),
})
.passthrough(),
})
.passthrough();
export const labsKeywordDataItemSchema = z
.object({
keyword: z.string(),
keyword_info: keywordInfoSchema.optional(),
keyword_info_normalized_with_clickstream:
keywordInfoWithClickstreamSchema.optional(),
search_intent_info: searchIntentInfoSchema.nullable().optional(),
keyword_properties: keywordPropertiesSchema.nullable().optional(),
})
.passthrough();
const domainMetricsValueSchema = z
.object({
etv: z.number().nullable().optional(),
count: z.number().nullable().optional(),
})
.passthrough();
export const domainMetricsItemSchema = z
.object({
metrics: z.record(
z.string(),
domainMetricsValueSchema.nullable().optional(),
),
})
.passthrough();
export const relevantPagesItemSchema = z
.object({
page_address: z.string().nullable().optional(),
metrics: z
.record(z.string(), domainMetricsValueSchema.nullable().optional())
.nullable()
.optional(),
})
.passthrough();
const rankedKeywordInfoSchema = z
.object({
search_volume: z.number().nullable().optional(),
cpc: z.number().nullable().optional(),
keyword_difficulty: z.number().nullable().optional(),
})
.passthrough();
const rankedKeywordDataSchema = z
.object({
keyword: z.string().nullable().optional(),
keyword_info: rankedKeywordInfoSchema.nullable().optional(),
keyword_properties: keywordPropertiesSchema.nullable().optional(),
})
.passthrough();
const rankedSerpItemSchema = z
.object({
url: z.string().nullable().optional(),
relative_url: z.string().nullable().optional(),
rank_absolute: z.number().nullable().optional(),
etv: z.number().nullable().optional(),
})
.passthrough();
const rankedSerpElementSchema = z
.object({
serp_item: rankedSerpItemSchema.nullable().optional(),
url: z.string().nullable().optional(),
relative_url: z.string().nullable().optional(),
rank_absolute: z.number().nullable().optional(),
etv: z.number().nullable().optional(),
})
.passthrough();
export const domainRankedKeywordItemSchema = z
.object({
keyword_data: rankedKeywordDataSchema.nullable().optional(),
ranked_serp_element: rankedSerpElementSchema.nullable().optional(),
keyword: z.string().nullable().optional(),
rank_absolute: z.number().nullable().optional(),
etv: z.number().nullable().optional(),
keyword_difficulty: z.number().nullable().optional(),
})
.passthrough();
export const serpSnapshotItemSchema = z
.object({
type: z.string(),
rank_group: z.number().nullable().optional(),
rank_absolute: z.number().nullable().optional(),
domain: z.string().nullable().optional(),
title: z.string().nullable().optional(),
url: z.string().nullable().optional(),
description: z.string().nullable().optional(),
breadcrumb: z.string().nullable().optional(),
etv: z.number().nullable().optional(),
estimated_paid_traffic_cost: z.number().nullable().optional(),
backlinks_info: z
.object({
referring_domains: z.number().nullable().optional(),
backlinks: z.number().nullable().optional(),
})
.passthrough()
.nullable()
.optional(),
rank_changes: z
.object({
previous_rank_absolute: z.number().nullable().optional(),
is_new: z.boolean().nullable().optional(),
is_up: z.boolean().nullable().optional(),
is_down: z.boolean().nullable().optional(),
})
.passthrough()
.nullable()
.optional(),
})
.passthrough();
export const keywordOverviewItemSchema = z
.object({
keyword: z.string(),
keyword_info: keywordInfoSchema.optional(),
keyword_properties: keywordPropertiesSchema.nullable().optional(),
search_intent_info: searchIntentInfoSchema.nullable().optional(),
})
.passthrough();
export type KeywordOverviewItem = z.infer<typeof keywordOverviewItemSchema>;
export type RelatedKeywordItem = z.infer<typeof relatedKeywordItemSchema>;
export type LabsKeywordDataItem = z.infer<typeof labsKeywordDataItemSchema>;
export type DomainMetricsItem = z.infer<typeof domainMetricsItemSchema>;
export type DomainRankedKeywordItem = z.infer<
typeof domainRankedKeywordItemSchema
>;
export type RelevantPagesItem = z.infer<typeof relevantPagesItemSchema>;
export type SerpLiveItem = z.infer<typeof serpSnapshotItemSchema>;
export function parseTaskItems<T extends z.ZodType>(
endpointName: string,
task: DataforseoTask,
itemSchema: T,
): z.infer<T>[] {
const parsed = z.array(itemSchema).safeParse(task.result?.[0]?.items ?? []);
if (!parsed.success) {
console.error(
`dataforseo.${endpointName}.invalid-payload`,
parsed.error.issues.slice(0, 5),
);
throw new AppError(
"INTERNAL_ERROR",
`DataForSEO ${endpointName} returned an invalid response shape`,
);
}
return parsed.data;
}

2
src/server/mcp/tools/dataforseo-research-tools.test.ts
View File

@ -13,7 +13,7 @@ vi.mock("cloudflare:workers", () => ({
env: {},
}));
vi.mock("@/server/lib/dataforseoClient", () => ({
vi.mock("@/server/lib/dataforseo", () => ({
createDataforseoClient: mocks.createDataforseoClient,
}));

2
src/server/mcp/tools/dataforseo-research-tools.ts
View File

@ -1,6 +1,6 @@
/* eslint-disable max-lines */
import { z } from "zod";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import { buildProjectMeta } from "@/server/mcp/context";
import { mcpResponse } from "@/server/mcp/formatters";
import {

2
src/server/mcp/tools/get-serp-results.ts
View File

@ -1,5 +1,5 @@
import { z } from "zod";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import { mcpResponse } from "@/server/mcp/formatters";
import { buildProjectMeta } from "@/server/mcp/context";
import { optionalMetaOutputSchema } from "@/server/mcp/output-schemas";

2
src/server/workflows/RankCheckWorkflow.ts
View File

@ -8,7 +8,7 @@ import type { BillingCustomerContext } from "@/server/billing/subscription";
import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository";
import { failRunIfActive } from "@/server/features/rank-tracking/services/rankCheckRunGuards";
import { runLiveCheck } from "@/server/workflows/rankCheckPaths";
import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import { createDataforseoClient } from "@/server/lib/dataforseo";
import { captureServerEvent } from "@/server/lib/posthog";
import { AppError } from "@/server/lib/errors";
import { autumn } from "@/server/billing/autumn";

2
src/server/workflows/rankCheckPaths.ts
View File

@ -1,6 +1,6 @@
import type { WorkflowStep } from "cloudflare:workers";
import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository";
import type { createDataforseoClient } from "@/server/lib/dataforseoClient";
import type { createDataforseoClient } from "@/server/lib/dataforseo";
import type { RankCheckResult } from "@/server/lib/dataforseo";
import type { RankTrackingConfig } from "@/types/schemas/rank-tracking";
import { KEYWORDS_PER_BATCH } from "@/shared/rank-tracking";