diff --git a/scripts/brand-lookup-cost-profile.ts b/scripts/brand-lookup-cost-profile.ts index 620626d..72959b6 100644 --- a/scripts/brand-lookup-cost-profile.ts +++ b/scripts/brand-lookup-cost-profile.ts @@ -3,11 +3,11 @@ import { buildLlmTarget, CHATGPT_LANGUAGE_CODE, CHATGPT_LOCATION_CODE, - fetchLlmAggregatedMetricsRaw, - fetchLlmMentionsSearchRaw, - fetchLlmTopPagesRaw, + fetchLlmAggregatedMetrics, + fetchLlmMentionsSearch, + fetchLlmTopPages, type LlmPlatform, -} from "@/server/lib/dataforseoLlm"; +} from "@/server/lib/dataforseo/ai"; import { applyBillingMarkupUsd } from "@/shared/billing"; import { loadLocalEnv, parseArgs } from "./cli-utils"; @@ -65,7 +65,7 @@ async function main() { const languageCode = platform === "chat_gpt" ? CHATGPT_LANGUAGE_CODE : userLanguageCode; - const aggregated = await fetchLlmAggregatedMetricsRaw({ + const aggregated = await fetchLlmAggregatedMetrics({ target: llmTarget, platform, locationCode, @@ -74,7 +74,7 @@ async function main() { }); calls.push(toRecord(platform, "aggregated_metrics", aggregated.billing)); - const topPages = await fetchLlmTopPagesRaw({ + const topPages = await fetchLlmTopPages({ target: llmTarget, platform, locationCode, @@ -83,7 +83,7 @@ async function main() { }); calls.push(toRecord(platform, "top_pages", topPages.billing)); - const mentions = await fetchLlmMentionsSearchRaw({ + const mentions = await fetchLlmMentionsSearch({ target: llmTarget, platform, locationCode, @@ -133,7 +133,6 @@ type CallRecord = { platform: LlmPlatform; endpoint: string; path: string; - resultCount: number | null; rawUsd: number; billedUsd: number; }; @@ -148,13 +147,12 @@ type RunSummary = { function toRecord( platform: LlmPlatform, endpoint: string, - billing: { costUsd: number; path: string[]; resultCount: number | null }, + billing: { costUsd: number; path: string[] }, ): CallRecord { return { platform, endpoint, path: billing.path.join("/"), - resultCount: billing.resultCount, rawUsd: round(billing.costUsd), billedUsd: applyBillingMarkupUsd(billing.costUsd), }; diff --git a/scripts/seed-rank-tracking.ts b/scripts/seed-rank-tracking.ts index 40b0bfd..f6ba69b 100644 --- a/scripts/seed-rank-tracking.ts +++ b/scripts/seed-rank-tracking.ts @@ -19,11 +19,7 @@ import { DataforseoLabsGoogleRankedKeywordsLiveRequestInfo, } from "dataforseo-client"; import * as schema from "../src/db/schema"; -import { - domainRankedKeywordItemSchema, - type DomainRankedKeywordItem, -} from "../src/server/lib/dataforseoSchemas"; -import { z } from "zod"; +import type { DomainRankedKeywordItem } from "../src/server/lib/dataforseo"; import { loadLocalEnv, parseArgs } from "./cli-utils"; loadLocalEnv(); @@ -207,14 +203,7 @@ async function fetchRankedKeywords( ); } - const rawItems = task.result?.[0]?.items ?? []; - const parsed = z.array(domainRankedKeywordItemSchema).safeParse(rawItems); - if (!parsed.success) { - console.error("Schema validation issues:", parsed.error.issues.slice(0, 3)); - throw new Error("DataForSEO response failed schema validation"); - } - - return parsed.data; + return task.result?.[0]?.items ?? []; } // --------------------------------------------------------------------------- diff --git a/src/server/features/ai-search/services/brandLookup.ts b/src/server/features/ai-search/services/brandLookup.ts index 06b65f7..1dc8e93 100644 --- a/src/server/features/ai-search/services/brandLookup.ts +++ b/src/server/features/ai-search/services/brandLookup.ts @@ -1,13 +1,13 @@ import { waitUntil } from "cloudflare:workers"; import { sortBy } from "remeda"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { buildLlmTarget, CHATGPT_LANGUAGE_CODE, CHATGPT_LOCATION_CODE, type LlmPlatform, -} from "@/server/lib/dataforseoLlm"; +} from "@/server/lib/dataforseo"; import type { LlmAggregatedTotal, LlmMentionItem, diff --git a/src/server/features/ai-search/services/promptExplorer.ts b/src/server/features/ai-search/services/promptExplorer.ts index 796262b..1aaf5dd 100644 --- a/src/server/features/ai-search/services/promptExplorer.ts +++ b/src/server/features/ai-search/services/promptExplorer.ts @@ -1,6 +1,6 @@ import { waitUntil } from "cloudflare:workers"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import type { LlmResponseResult } from "@/server/lib/dataforseoLlmSchemas"; import { AppError } from "@/server/lib/errors"; import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; diff --git a/src/server/features/backlinks/services/BacklinksService.billing.test.ts b/src/server/features/backlinks/services/BacklinksService.billing.test.ts index d4a77e8..0d87779 100644 --- a/src/server/features/backlinks/services/BacklinksService.billing.test.ts +++ b/src/server/features/backlinks/services/BacklinksService.billing.test.ts @@ -15,11 +15,8 @@ vi.mock("@/server/lib/r2-cache", () => ({ setCached: vi.fn(async () => undefined), })); -vi.mock("@/server/lib/dataforseoBacklinks", () => ({ +vi.mock("@/server/lib/dataforseo", () => ({ normalizeBacklinksTarget: vi.fn(), -})); - -vi.mock("@/server/lib/dataforseoClient", () => ({ createDataforseoClient: vi.fn(() => ({ backlinks: { summary: backlinksSummaryMock, @@ -31,7 +28,7 @@ vi.mock("@/server/lib/dataforseoClient", () => ({ })), })); -import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinks"; +import { normalizeBacklinksTarget } from "@/server/lib/dataforseo"; import { createBacklinksService } from "./BacklinksService"; const billingCustomer = { diff --git a/src/server/features/backlinks/services/BacklinksService.ts b/src/server/features/backlinks/services/BacklinksService.ts index 2c7fab6..37d40c9 100644 --- a/src/server/features/backlinks/services/BacklinksService.ts +++ b/src/server/features/backlinks/services/BacklinksService.ts @@ -1,5 +1,5 @@ import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; -import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinks"; +import { normalizeBacklinksTarget } from "@/server/lib/dataforseo"; import { normalizeBacklinksSpamFilterOptions, type BacklinksSpamFilterOptions, diff --git a/src/server/features/backlinks/services/backlinksServiceData.ts b/src/server/features/backlinks/services/backlinksServiceData.ts index f359903..6157c84 100644 --- a/src/server/features/backlinks/services/backlinksServiceData.ts +++ b/src/server/features/backlinks/services/backlinksServiceData.ts @@ -1,14 +1,14 @@ import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; import { - type fetchBacklinksHistoryRaw, - type fetchBacklinksRowsRaw, - type fetchBacklinksSummaryRaw, - type fetchDomainPagesSummaryRaw, - type fetchReferringDomainsRaw, + createDataforseoClient, normalizeBacklinksTarget, -} from "@/server/lib/dataforseoBacklinks"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; + type BacklinksHistoryItem, + type BacklinksItem, + type BacklinksSummaryItem, + type DomainPageSummaryItem, + type ReferringDomainItem, +} from "@/server/lib/dataforseo"; import { normalizeBacklinksSpamFilterOptions, type BacklinksSpamFilterOptions, @@ -204,9 +204,9 @@ function buildBacklinksDateRange(now: Date): BacklinksDateRange { function buildOverviewResult(args: { normalizedTarget: ReturnType; now: Date; - summary: Awaited>["data"]; - backlinks: Awaited>["data"]; - history: Awaited>["data"]; + summary: BacklinksSummaryItem; + backlinks: BacklinksItem[]; + history: BacklinksHistoryItem[]; }): BacklinksOverviewResult { const historyRows = args.history .map((item) => ({ @@ -277,9 +277,7 @@ function normalizeHistoryDate(value: string | null | undefined) { return value ? value.slice(0, 10) : null; } -function mapBacklinksRows( - rows: Awaited>["data"], -) { +function mapBacklinksRows(rows: BacklinksItem[]) { return rows.map((item) => ({ domainFrom: item.domain_from ?? null, urlFrom: item.url_from ?? null, @@ -300,9 +298,7 @@ function mapBacklinksRows( })); } -function mapReferringDomainsRows( - rows: Awaited>["data"], -) { +function mapReferringDomainsRows(rows: ReferringDomainItem[]) { return rows.map((item) => ({ domain: item.domain ?? null, backlinks: item.backlinks ?? null, @@ -315,9 +311,7 @@ function mapReferringDomainsRows( })); } -function mapTopPagesRows( - rows: Awaited>["data"], -) { +function mapTopPagesRows(rows: DomainPageSummaryItem[]) { return rows.map((item) => ({ page: item.page ?? item.url ?? null, backlinks: item.backlinks ?? null, diff --git a/src/server/features/domain/services/DomainService.ts b/src/server/features/domain/services/DomainService.ts index dc88ace..fe9e48b 100644 --- a/src/server/features/domain/services/DomainService.ts +++ b/src/server/features/domain/services/DomainService.ts @@ -1,7 +1,7 @@ import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { normalizeDomainInput } from "@/server/lib/domainUtils"; import { mapKeywordItem } from "@/server/features/domain/services/domainKeywordMapper"; import { getKeywordsPage } from "@/server/features/domain/services/domainKeywordsPage"; diff --git a/src/server/features/domain/services/domainKeywordsPage.ts b/src/server/features/domain/services/domainKeywordsPage.ts index 78e5397..d57bfeb 100644 --- a/src/server/features/domain/services/domainKeywordsPage.ts +++ b/src/server/features/domain/services/domainKeywordsPage.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; import { normalizeDomainInput } from "@/server/lib/domainUtils"; import { mapKeywordItem } from "@/server/features/domain/services/domainKeywordMapper"; diff --git a/src/server/features/domain/services/domainPagesPage.ts b/src/server/features/domain/services/domainPagesPage.ts index 3fb4015..4475ad8 100644 --- a/src/server/features/domain/services/domainPagesPage.ts +++ b/src/server/features/domain/services/domainPagesPage.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; import { normalizeDomainInput, toRelativePath } from "@/server/lib/domainUtils"; import type { RelevantPagesItem } from "@/server/lib/dataforseo"; diff --git a/src/server/features/keywords/services/research/research-data.ts b/src/server/features/keywords/services/research/research-data.ts index a0ae509..6af70ea 100644 --- a/src/server/features/keywords/services/research/research-data.ts +++ b/src/server/features/keywords/services/research/research-data.ts @@ -1,6 +1,6 @@ -import { type LabsKeywordDataItem } from "@/server/lib/dataforseoClient"; +import { type LabsKeywordDataItem } from "@/server/lib/dataforseo"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { normalizeIntent, normalizeKeyword, @@ -37,8 +37,8 @@ function mapKeywordDataItems(items: LabsKeywordDataItem[]): EnrichedKeyword[] { keyword: normalized, searchVolume: keywordInfo?.search_volume ?? null, trend: (keywordInfo?.monthly_searches ?? []).map((entry) => ({ - year: entry.year, - month: entry.month, + year: entry.year ?? 0, + month: entry.month ?? 0, searchVolume: entry.search_volume ?? 0, })), cpc: item.keyword_info?.cpc ?? null, @@ -63,40 +63,13 @@ async function fetchRelatedRows( depth: 3, }); - const rows: EnrichedKeyword[] = []; - const seen = new Set(); - - for (const item of items) { - const keywordData = item.keyword_data; - const keyword = keywordData.keyword; - if (!keyword) continue; - - const normalized = normalizeKeyword(keyword); - if (seen.has(normalized)) continue; - seen.add(normalized); - - const keywordInfo = keywordData.keyword_info_normalized_with_clickstream - ?.search_volume - ? keywordData.keyword_info_normalized_with_clickstream - : keywordData.keyword_info; - - rows.push({ - keyword: normalized, - searchVolume: keywordInfo?.search_volume ?? null, - trend: (keywordInfo?.monthly_searches ?? []).map((entry) => ({ - year: entry.year, - month: entry.month, - searchVolume: entry.search_volume ?? 0, - })), - cpc: keywordData.keyword_info?.cpc ?? null, - competition: keywordData.keyword_info?.competition ?? null, - keywordDifficulty: - keywordData.keyword_properties?.keyword_difficulty ?? null, - intent: normalizeIntent(keywordData.search_intent_info?.main_intent), - }); - } - - return rows; + // Related items wrap the keyword payload one level deeper; unwrap and reuse + // the same mapper as suggestions/ideas. + return mapKeywordDataItems( + items + .map((item) => item.keyword_data) + .filter((data): data is NonNullable => data != null), + ); } export async function fetchResearchRowsBySource( diff --git a/src/server/features/keywords/services/research/serp.ts b/src/server/features/keywords/services/research/serp.ts index d8b33eb..2339d72 100644 --- a/src/server/features/keywords/services/research/serp.ts +++ b/src/server/features/keywords/services/research/serp.ts @@ -1,9 +1,9 @@ -import { type SerpLiveItem } from "@/server/lib/dataforseoClient"; +import { type SerpLiveItem } from "@/server/lib/dataforseo"; import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; import type { SerpResultItem } from "@/types/keywords"; import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { normalizeKeyword } from "./helpers"; const SERP_CACHE_TTL_SECONDS = 12 * 60 * 60; diff --git a/src/server/features/rank-tracking/services/RankTrackingService.ts b/src/server/features/rank-tracking/services/RankTrackingService.ts index 937c95e..bffb76f 100644 --- a/src/server/features/rank-tracking/services/RankTrackingService.ts +++ b/src/server/features/rank-tracking/services/RankTrackingService.ts @@ -1,6 +1,6 @@ import { env } from "cloudflare:workers"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository"; import { AppError } from "@/server/lib/errors"; import type { @@ -270,6 +270,7 @@ async function refreshKeywordMetrics( } >(); for (const item of items) { + if (!item.keyword) continue; metricsMap.set(item.keyword.toLowerCase(), { searchVolume: item.keyword_info?.search_volume ?? null, keywordDifficulty: item.keyword_properties?.keyword_difficulty ?? null, diff --git a/src/server/lib/audit/lighthouse.ts b/src/server/lib/audit/lighthouse.ts index 1db8975..85a1b19 100644 --- a/src/server/lib/audit/lighthouse.ts +++ b/src/server/lib/audit/lighthouse.ts @@ -1,6 +1,6 @@ import { detectUrlTemplate } from "./url-utils"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import type { LighthouseResult, LighthouseStrategy } from "./types"; import { putTextToR2 } from "@/server/lib/r2"; diff --git a/src/server/lib/dataforseo.test.ts b/src/server/lib/dataforseo.test.ts deleted file mode 100644 index 97342a3..0000000 --- a/src/server/lib/dataforseo.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; - -vi.mock("cloudflare:workers", () => ({ - env: { - DATAFORSEO_API_KEY: "encoded-key", - }, -})); - -describe("DataForSEO raw wrappers", () => { - beforeEach(() => { - vi.resetModules(); - vi.restoreAllMocks(); - }); - - it("uses the live endpoint for Google Business Q&A", async () => { - const fetchMock = vi.fn().mockResolvedValue( - Response.json({ - status_code: 20000, - tasks: [ - { - status_code: 20000, - path: [ - "v3", - "business_data", - "google", - "questions_and_answers", - "live", - ], - cost: 0.0006, - result_count: 1, - result: [ - { - items: [ - { - question_text: "Do you offer indoor storage?", - answer_text: "Yes.", - }, - ], - }, - ], - }, - ], - }), - ); - vi.stubGlobal("fetch", fetchMock); - - const { fetchBusinessQuestionsAnswersRaw } = await import("./dataforseo"); - const result = await fetchBusinessQuestionsAnswersRaw({ - keyword: "Acme Storage", - locationCoordinate: "33.1234568,-84.9876543,5000", - languageCode: "en", - depth: 20, - }); - - expect( - fetchMock.mock.calls.map(([url]) => - typeof url === "string" || url instanceof URL - ? url.toString() - : url.url, - ), - ).toEqual([ - "https://api.dataforseo.com/v3/business_data/google/questions_and_answers/live", - ]); - expect(result.data).toEqual([ - { - question_text: "Do you offer indoor storage?", - answer_text: "Yes.", - }, - ]); - expect(result.billing).toEqual({ - path: ["v3", "business_data", "google", "questions_and_answers", "live"], - costUsd: 0.0006, - resultCount: 1, - }); - }); - - it("does not send location_name for keyword search volume", async () => { - const fetchMock = vi.fn().mockResolvedValue( - Response.json({ - status_code: 20000, - tasks: [ - { - status_code: 20000, - path: [ - "v3", - "keywords_data", - "google_ads", - "search_volume", - "live", - ], - cost: 0.0001, - result_count: 1, - result: [ - { - items: [ - { - keyword: "storage units", - location_code: 2840, - language_code: "en", - search_volume: 1000, - }, - ], - }, - ], - }, - ], - }), - ); - vi.stubGlobal("fetch", fetchMock); - - const { fetchKeywordSearchVolumeRaw } = await import("./dataforseo"); - await fetchKeywordSearchVolumeRaw({ - keywords: ["storage units"], - locationCode: 2840, - languageCode: "en", - }); - - const init = fetchMock.mock.calls[0]?.[1]; - expect(typeof init?.body).toBe("string"); - const body = init?.body; - if (typeof body !== "string") { - throw new Error("Expected DataForSEO request body to be a string"); - } - const payload = JSON.parse(body) as unknown; - expect(payload).toEqual([ - { - keywords: ["storage units"], - location_code: 2840, - language_code: "en", - }, - ]); - expect(JSON.stringify(payload)).not.toContain("location_name"); - }); -}); diff --git a/src/server/lib/dataforseo.ts b/src/server/lib/dataforseo.ts deleted file mode 100644 index 1c722ac..0000000 --- a/src/server/lib/dataforseo.ts +++ /dev/null @@ -1,798 +0,0 @@ -/* eslint-disable max-lines */ -import { - DataforseoLabsApi, - DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo, - DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo, - DataforseoLabsGoogleKeywordIdeasLiveRequestInfo, - DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo, - DataforseoLabsGoogleRankedKeywordsLiveRequestInfo, - DataforseoLabsGoogleRelevantPagesLiveRequestInfo, -} from "dataforseo-client"; -import { env } from "cloudflare:workers"; -import { z } from "zod"; -import { - DataforseoChargedTaskError, - type DataforseoApiCallCost, - type DataforseoApiResponse, -} from "@/server/lib/dataforseoCost"; -import { AppError } from "@/server/lib/errors"; -import { - dataforseoResponseSchema, - domainMetricsItemSchema, - domainRankedKeywordItemSchema, - keywordOverviewItemSchema, - labsKeywordDataItemSchema, - parseTaskItems, - relatedKeywordItemSchema, - relevantPagesItemSchema, - serpSnapshotItemSchema, - type DataforseoTask, - type DomainMetricsItem, - type DomainRankedKeywordItem, - type KeywordOverviewItem, - type LabsKeywordDataItem, - type RelatedKeywordItem, - type RelevantPagesItem, - type SerpLiveItem, - successfulDataforseoTaskSchema, -} from "@/server/lib/dataforseoSchemas"; -export type { - DomainRankedKeywordItem, - LabsKeywordDataItem, - RelevantPagesItem, - SerpLiveItem, -} from "@/server/lib/dataforseoSchemas"; - -// --------------------------------------------------------------------------- -// SDK client factories (lazily created per-request using the env secret) -// --------------------------------------------------------------------------- - -function createAuthenticatedFetch() { - return async (url: RequestInfo, init?: RequestInit): Promise => { - const headers = new Headers(init?.headers); - headers.set("Authorization", `Basic ${env.DATAFORSEO_API_KEY}`); - - const newInit: RequestInit = { - ...init, - headers, - }; - const response = await fetch(url, newInit); - - if (!response.ok) { - const rawText = await response.text(); - const path = formatDataforseoRequestPath(url); - const err = new AppError( - response.status === 429 ? "RATE_LIMITED" : "INTERNAL_ERROR", - `DataForSEO HTTP ${response.status} on ${path}`, - { - provider: "dataforseo", - providerStatus: String(response.status), - providerPath: path, - responseBody: formatDataforseoErrorPayload(rawText), - }, - ); - err.name = "DataForSEOHttpError"; - throw err; - } - - return response; - }; -} - -const API_BASE = "https://api.dataforseo.com"; -const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600; - -function getLabsApi() { - return new DataforseoLabsApi(API_BASE, { fetch: createAuthenticatedFetch() }); -} - -function formatDataforseoRequestPath(url: RequestInfo): string { - const rawUrl = typeof url === "string" ? url : url.url; - try { - return new URL(rawUrl).pathname; - } catch { - return rawUrl; - } -} - -async function postDataforseo( - path: string, - payload: unknown, -): Promise { - const authenticatedFetch = createAuthenticatedFetch(); - const response = await authenticatedFetch(`${API_BASE}${path}`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - }); - - const rawText = await response.text(); - - try { - return JSON.parse(rawText); - } catch { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned a non-JSON response. Response: ${formatDataforseoErrorPayload(rawText)}`, - ); - } -} - -// --------------------------------------------------------------------------- -// Response helpers -// --------------------------------------------------------------------------- - -/** - * Validate that the top-level response and first task both succeeded. - * Throws a descriptive error on failure. Returns the first task. - */ -type DataforseoTaskLike = { - id?: string; - status_code?: number; - status_message?: string; - path?: string[]; - cost?: number; - result_count?: number | null; - data?: unknown; - result?: DataforseoTask["result"]; -}; - -const dataforseoGenericItemSchema = z.record(z.string(), z.unknown()); - -const dataforseoGenericResultSchema = z - .object({ - total_count: z.number().nullable().optional(), - count: z.number().nullable().optional(), - offset: z.number().nullable().optional(), - items: z.array(dataforseoGenericItemSchema).nullable().optional(), - items_without_answers: z - .array(dataforseoGenericItemSchema) - .nullable() - .optional(), - }) - .passthrough(); - -type GenericDataforseoItem = z.infer; - -const dataforseoChargedTaskSchema = z.object({ - path: z.array(z.string()), - cost: z.number(), - result_count: z.number().nullable().optional(), -}); - -function formatDataforseoErrorPayload(value: unknown): string { - const text = - typeof value === "string" - ? value - : (() => { - try { - return JSON.stringify(value); - } catch { - return String(value); - } - })(); - - return text.length > MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH - ? `${text.slice(0, MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH)}... [truncated]` - : text; -} - -function compactObject( - value: Record, -): Record { - return Object.fromEntries( - Object.entries(value).filter(([, entry]) => entry !== undefined), - ); -} - -function extractGenericItems(task: DataforseoTask): GenericDataforseoItem[] { - const result = task.result ?? []; - const resultItems = z.array(dataforseoGenericResultSchema).parse(result); - const resultHasNestedItems = resultItems.some((item) => - Object.hasOwn(item, "items"), - ); - - return resultHasNestedItems - ? resultItems.flatMap((item) => [ - ...(item.items ?? []), - ...(item.items_without_answers ?? []), - ]) - : z.array(dataforseoGenericItemSchema).parse(result); -} - -async function postGenericItems( - path: string, - payload: Record, -): Promise> { - const responseRaw = await postDataforseo(path, [compactObject(payload)]); - const response = dataforseoResponseSchema.parse(responseRaw); - const task = assertOk(response); - - return { - data: extractGenericItems(task), - billing: buildTaskBilling(task), - }; -} - -function getTaskDebugPayload(task: DataforseoTaskLike) { - return { - id: task.id ?? null, - status_code: task.status_code ?? null, - status_message: task.status_message ?? null, - path: task.path ?? null, - cost: task.cost ?? null, - result_count: task.result_count ?? null, - data: task.data ?? null, - result_length: Array.isArray(task.result) ? task.result.length : null, - result_preview: Array.isArray(task.result) - ? (task.result[0] ?? null) - : null, - }; -} - -function assertOk( - response: { - status_code?: number; - status_message?: string; - tasks?: T[]; - } | null, -): DataforseoTask { - if (!response) { - throw new AppError( - "INTERNAL_ERROR", - "DataForSEO returned an empty response", - ); - } - if (response.status_code !== 20000) { - throw new AppError( - "INTERNAL_ERROR", - response.status_message || "DataForSEO request failed", - ); - } - const task = response.tasks?.[0]; - if (!task) { - throw new AppError("INTERNAL_ERROR", "DataForSEO response missing task"); - } - if (task.status_code !== 20000) { - const chargedTask = dataforseoChargedTaskSchema.safeParse(task); - if (chargedTask.success) { - throw new DataforseoChargedTaskError( - task.status_message || "DataForSEO task failed", - buildTaskBilling(chargedTask.data), - ); - } - throw new AppError( - "INTERNAL_ERROR", - task.status_message || "DataForSEO task failed", - ); - } - - const parsedTask = successfulDataforseoTaskSchema.safeParse(task); - if (!parsedTask.success) { - const issueSummary = parsedTask.error.issues - .slice(0, 5) - .map((issue) => { - const path = issue.path.length > 0 ? issue.path.join(".") : "task"; - return `${path}: ${issue.message}`; - }) - .join("; "); - const responseSummary = formatDataforseoErrorPayload({ - status_code: response.status_code ?? null, - status_message: response.status_message ?? null, - task: getTaskDebugPayload(task), - }); - - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO task missing billing metadata (${issueSummary}). Response: ${responseSummary}`, - ); - } - - return parsedTask.data; -} - -function buildTaskBilling(task: { - path: string[]; - cost: number; - result_count?: number | null; -}): DataforseoApiCallCost { - return { - path: task.path, - costUsd: task.cost, - resultCount: task.result_count ?? null, - }; -} - -// --------------------------------------------------------------------------- -// DataForSEO Labs API wrappers -// --------------------------------------------------------------------------- - -export async function fetchRelatedKeywordsRaw( - keyword: string, - locationCode: number, - languageCode: string, - limit: number, - depth: number = 3, -): Promise> { - const api = getLabsApi(); - const req = new DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo({ - keyword, - location_code: locationCode, - language_code: languageCode, - limit, - depth, - include_clickstream_data: true, - include_serp_info: false, - }); - - const endpoint = "google-related-keywords-live"; - const response = await api.googleRelatedKeywordsLive([req]); - const task = assertOk(response); - const data = parseTaskItems(endpoint, task, relatedKeywordItemSchema); - - return { - data, - billing: buildTaskBilling(task), - }; -} - -export async function fetchKeywordSuggestionsRaw( - keyword: string, - locationCode: number, - languageCode: string, - limit: number, -): Promise> { - const api = getLabsApi(); - const req = new DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo({ - keyword, - location_code: locationCode, - language_code: languageCode, - limit, - include_clickstream_data: true, - include_serp_info: false, - include_seed_keyword: true, - ignore_synonyms: false, - exact_match: false, - }); - - const endpoint = "google-keyword-suggestions-live"; - const response = await api.googleKeywordSuggestionsLive([req]); - const task = assertOk(response); - const data = parseTaskItems(endpoint, task, labsKeywordDataItemSchema); - - return { - data, - billing: buildTaskBilling(task), - }; -} - -export async function fetchKeywordIdeasRaw( - keyword: string, - locationCode: number, - languageCode: string, - limit: number, -): Promise> { - const api = getLabsApi(); - const req = new DataforseoLabsGoogleKeywordIdeasLiveRequestInfo({ - keywords: [keyword], - location_code: locationCode, - language_code: languageCode, - limit, - include_clickstream_data: true, - include_serp_info: false, - ignore_synonyms: false, - closely_variants: false, - }); - - const endpoint = "google-keyword-ideas-live"; - const response = await api.googleKeywordIdeasLive([req]); - const task = assertOk(response); - const data = parseTaskItems(endpoint, task, labsKeywordDataItemSchema); - - return { - data, - billing: buildTaskBilling(task), - }; -} - -// --------------------------------------------------------------------------- -// Domain API wrappers -// --------------------------------------------------------------------------- - -export async function fetchDomainRankOverviewRaw( - target: string, - locationCode: number, - languageCode: string, -): Promise> { - const api = getLabsApi(); - const req = new DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo({ - target, - location_code: locationCode, - language_code: languageCode, - limit: 1, - }); - - const endpoint = "google-domain-rank-overview-live"; - const response = await api.googleDomainRankOverviewLive([req]); - const task = assertOk(response); - const data = parseTaskItems(endpoint, task, domainMetricsItemSchema); - - return { - data, - billing: buildTaskBilling(task), - }; -} - -type RankedKeywordsPage = { - items: DomainRankedKeywordItem[]; - totalCount: number | null; -}; - -export async function fetchRankedKeywordsRaw(input: { - target: string; - locationCode: number; - languageCode: string; - limit: number; - offset?: number; - orderBy?: string[]; - filters?: unknown[]; - itemTypes?: DataforseoLabsItemType[]; - includeSubdomains?: boolean; -}): Promise> { - const api = getLabsApi(); - const req = new DataforseoLabsGoogleRankedKeywordsLiveRequestInfo({ - target: input.target, - location_code: input.locationCode, - language_code: input.languageCode, - limit: input.limit, - offset: input.offset, - order_by: input.orderBy, - filters: input.filters, - item_types: input.itemTypes, - include_subdomains: input.includeSubdomains, - }); - - const endpoint = "google-ranked-keywords-live"; - const response = await api.googleRankedKeywordsLive([req]); - const task = assertOk(response); - const items = parseTaskItems(endpoint, task, domainRankedKeywordItemSchema); - const rawTotal = (task.result?.[0] as Record | undefined) - ?.total_count; - const totalCount = typeof rawTotal === "number" ? rawTotal : null; - - return { - data: { items, totalCount }, - billing: buildTaskBilling(task), - }; -} - -type RelevantPagesPage = { - items: RelevantPagesItem[]; - totalCount: number | null; -}; - -export async function fetchRelevantPagesRaw(input: { - target: string; - locationCode: number; - languageCode: string; - limit: number; - offset?: number; - orderBy?: string[]; - filters?: unknown[]; -}): Promise> { - const api = getLabsApi(); - const req = new DataforseoLabsGoogleRelevantPagesLiveRequestInfo({ - target: input.target, - location_code: input.locationCode, - language_code: input.languageCode, - limit: input.limit, - offset: input.offset, - order_by: input.orderBy, - filters: input.filters, - }); - - const endpoint = "google-relevant-pages-live"; - const response = await api.googleRelevantPagesLive([req]); - const task = assertOk(response); - const items = parseTaskItems(endpoint, task, relevantPagesItemSchema); - const rawTotal = (task.result?.[0] as Record | undefined) - ?.total_count; - const totalCount = typeof rawTotal === "number" ? rawTotal : null; - - return { - data: { items, totalCount }, - billing: buildTaskBilling(task), - }; -} - -// --------------------------------------------------------------------------- -// SERP Analysis API wrapper (Google Organic Live) -// --------------------------------------------------------------------------- - -export async function fetchLiveSerpItemsRaw( - keyword: string, - locationCode: number, - languageCode: string, -): Promise> { - const responseRaw = await postDataforseo( - "/v3/serp/google/organic/live/advanced", - [ - { - keyword, - location_code: locationCode, - language_code: languageCode, - device: "desktop", - os: "windows", - depth: 100, - }, - ], - ); - const response = dataforseoResponseSchema.parse(responseRaw); - const endpoint = "google-organic-live-advanced"; - const task = assertOk(response); - const data = parseTaskItems(endpoint, task, serpSnapshotItemSchema); - - return { - data, - billing: buildTaskBilling(task), - }; -} - -export async function fetchLocalSerpItemsRaw(input: { - keyword: string; - locationCoordinate?: string; - languageCode: string; - searchType: "maps" | "local_finder"; - device: "desktop" | "mobile"; - depth: number; - searchPlaces?: boolean; -}): Promise> { - const path = - input.searchType === "maps" - ? "/v3/serp/google/maps/live/advanced" - : "/v3/serp/google/local_finder/live/advanced"; - return postGenericItems(path, { - keyword: input.keyword, - location_coordinate: input.locationCoordinate, - language_code: input.languageCode, - device: input.device, - os: input.device === "desktop" ? "windows" : "android", - depth: input.depth, - search_places: input.searchPlaces, - }); -} - -// --------------------------------------------------------------------------- -// SERP Rank Check API wrapper (Google Organic Live with target matching) -// --------------------------------------------------------------------------- - -export interface RankCheckResult { - keywordId: string; - keyword: string; - position: number | null; - url: string | null; - serpFeatures: string[]; -} - -export async function fetchRankCheckSerpRaw(input: { - keyword: string; - keywordId: string; - locationCode: number; - languageCode: string; - device: "desktop" | "mobile"; - targetDomain: string; - depth: number; -}): Promise> { - const depth = Math.min(100, Math.max(10, input.depth)); - const responseRaw = await postDataforseo( - "/v3/serp/google/organic/live/advanced", - [ - { - keyword: input.keyword, - location_code: input.locationCode, - language_code: input.languageCode, - device: input.device, - os: input.device === "desktop" ? "windows" : "android", - depth, - }, - ], - ); - - const response = dataforseoResponseSchema.parse(responseRaw); - - if (response.status_code !== 20000) { - throw new AppError( - "INTERNAL_ERROR", - response.status_message || "DataForSEO request failed", - ); - } - - const task = response.tasks?.[0]; - if (!task) { - throw new AppError("INTERNAL_ERROR", "DataForSEO response missing task"); - } - - // "No Search Results" (40501) is valid for obscure/new keywords — - // treat as empty result set rather than failing the entire run. - const isNoResults = - task.status_code === 40501 || - task.status_message?.toLowerCase().includes("no search results"); - if (task.status_code !== 20000 && !isNoResults) { - throw new AppError( - "INTERNAL_ERROR", - task.status_message || "DataForSEO task failed", - ); - } - - const parsedTask = successfulDataforseoTaskSchema.safeParse(task); - if (!parsedTask.success) { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO rank check task missing billing metadata`, - ); - } - - const items = z - .array(serpSnapshotItemSchema) - .parse(parsedTask.data.result?.[0]?.items ?? []); - - const target = input.targetDomain.toLowerCase(); - const organicMatch = items.find((item) => { - if (item.type !== "organic" || item.domain == null) return false; - const d = item.domain.toLowerCase(); - return d === target || d.endsWith(`.${target}`); - }); - - return { - data: { - keywordId: input.keywordId, - keyword: input.keyword, - position: organicMatch - ? (organicMatch.rank_absolute ?? organicMatch.rank_group ?? null) - : null, - url: organicMatch?.url ?? null, - serpFeatures: [ - ...new Set(items.map((item) => item.type).filter(Boolean)), - ], - }, - billing: buildTaskBilling(parsedTask.data), - }; -} - -// --------------------------------------------------------------------------- -// DataForSEO Labs — Keyword Overview (batch up to 700 keywords) -// --------------------------------------------------------------------------- - -export async function fetchKeywordOverviewRaw( - keywords: string[], - locationCode: number, - languageCode: string, -): Promise> { - const responseRaw = await postDataforseo( - "/v3/dataforseo_labs/google/keyword_overview/live", - [ - { - keywords, - location_code: locationCode, - language_code: languageCode, - }, - ], - ); - - const response = dataforseoResponseSchema.parse(responseRaw); - - if (response.status_code !== 20000) { - throw new AppError( - "INTERNAL_ERROR", - response.status_message || "DataForSEO keyword overview request failed", - ); - } - - const task = response.tasks?.[0]; - if (!task) { - throw new AppError( - "INTERNAL_ERROR", - "DataForSEO keyword overview response missing task", - ); - } - - if (task.status_code !== 20000) { - throw new AppError( - "INTERNAL_ERROR", - task.status_message || "DataForSEO keyword overview task failed", - ); - } - - const parsedTask = successfulDataforseoTaskSchema.safeParse(task); - if (!parsedTask.success) { - throw new AppError( - "INTERNAL_ERROR", - "DataForSEO keyword overview task missing billing metadata", - ); - } - - const data = parseTaskItems( - "google-keyword-overview-live", - parsedTask.data, - keywordOverviewItemSchema, - ); - - return { - data, - billing: buildTaskBilling(parsedTask.data), - }; -} - -export async function fetchBusinessListingsSearchRaw(input: { - categories?: string[]; - title?: string; - locationCoordinate: string; - orderBy?: string[]; - limit: number; -}): Promise> { - return postGenericItems("/v3/business_data/business_listings/search/live", { - categories: input.categories, - title: input.title, - location_coordinate: input.locationCoordinate, - order_by: input.orderBy, - limit: input.limit, - }); -} - -export async function fetchBusinessQuestionsAnswersRaw(input: { - keyword: string; - locationCoordinate: string; - languageCode: string; - depth: number; -}): Promise> { - return postGenericItems( - "/v3/business_data/google/questions_and_answers/live", - { - keyword: input.keyword, - location_coordinate: input.locationCoordinate, - language_code: input.languageCode, - depth: input.depth, - }, - ); -} - -export async function fetchKeywordSearchVolumeRaw(input: { - keywords: string[]; - locationCode?: number; - languageCode?: string; -}): Promise> { - return postGenericItems("/v3/keywords_data/google_ads/search_volume/live", { - keywords: input.keywords, - location_code: input.locationCode, - language_code: input.languageCode, - }); -} - -export type DataforseoLabsItemType = - | "organic" - | "paid" - | "featured_snippet" - | "local_pack" - | "ai_overview_reference"; - -export async function fetchSerpCompetitorsRaw(input: { - keywords: string[]; - locationCode: number; - languageCode: string; - itemTypes?: DataforseoLabsItemType[]; - includeSubdomains?: boolean; - limit: number; - offset?: number; -}): Promise> { - return postGenericItems("/v3/dataforseo_labs/google/serp_competitors/live", { - keywords: input.keywords, - location_code: input.locationCode, - language_code: input.languageCode, - item_types: input.itemTypes, - include_subdomains: input.includeSubdomains, - limit: input.limit, - offset: input.offset, - }); -} diff --git a/src/server/lib/dataforseo/ai.ts b/src/server/lib/dataforseo/ai.ts new file mode 100644 index 0000000..2792eae --- /dev/null +++ b/src/server/lib/dataforseo/ai.ts @@ -0,0 +1,333 @@ +import { z } from "zod"; +import { + AiOptimizationChatGptLlmResponsesLiveRequestInfo, + AiOptimizationClaudeLlmResponsesLiveRequestInfo, + AiOptimizationGeminiLlmResponsesLiveRequestInfo, + AiOptimizationLLmMentionsDomainElement, + AiOptimizationLLmMentionsKeywordElement, + AiOptimizationLlmMentionsAggregatedMetricsLiveRequestInfo, + AiOptimizationLlmMentionsSearchLiveRequestInfo, + AiOptimizationLlmMentionsTopPagesLiveRequestInfo, + type BaseAiOptimizationLLmMentionsTargetElement, + type AiOptimizationPerplexityLlmResponsesLiveRequestInfo, +} from "dataforseo-client"; +import { + llmAggregatedTotalSchema, + llmMentionItemSchema, + llmResponseResultSchema, + llmTopPagesItemSchema, + type LlmAggregatedTotal, + type LlmMentionItem, + type LlmResponseResult, + type LlmTopPagesItem, +} from "@/server/lib/dataforseoLlmSchemas"; +import { createDataforseoAccessClassifier } from "@/server/lib/dataforseoAccessClassification"; +import { AppError } from "@/server/lib/errors"; +import { aiOptimizationApi } from "@/server/lib/dataforseo/core"; +import { + assertOk, + buildTaskBilling, + isRecord, + type DataforseoApiResponse, + type DataforseoTaskLike, +} from "@/server/lib/dataforseo/envelope"; + +// ChatGPT mention/response data is only available for US/en per DataForSEO docs. +export const CHATGPT_LOCATION_CODE = 2840; +export const CHATGPT_LANGUAGE_CODE = "en"; + +export type LlmPlatform = "chat_gpt" | "google"; + +const classifyAiSearchError = createDataforseoAccessClassifier({ + pathPrefix: "/ai_optimization/", + notEnabledCode: "AI_SEARCH_NOT_ENABLED", + notEnabledMessage: + "AI Optimization is not enabled for the connected DataForSEO account", + billingIssueCode: "AI_SEARCH_BILLING_ISSUE", + billingIssueMessage: + "The connected DataForSEO account has a billing or balance issue", +}); + +const assertOptions = (path: string) => + ({ classify: classifyAiSearchError, classifyPath: path }) as const; + +// --------------------------------------------------------------------------- +// Target builders — DataForSEO's `target` array accepts domain OR keyword +// entries. We always pass exactly one target per call. +// --------------------------------------------------------------------------- + +type LlmTarget = + | { + domain: string; + include_subdomains?: boolean; + search_filter?: "include" | "exclude"; + search_scope?: string[]; + } + | { + keyword: string; + search_filter?: "include" | "exclude"; + search_scope?: string[]; + match_type?: "word_match" | "partial_match"; + }; + +export function buildLlmTarget(input: { + type: "domain" | "keyword"; + value: string; +}): LlmTarget { + if (input.type === "domain") { + return { + domain: input.value, + include_subdomains: true, + search_filter: "include", + search_scope: ["any"], + }; + } + return { + keyword: input.value, + search_filter: "include", + search_scope: ["any", "brand_entities"], + match_type: "word_match", + }; +} + +function clampLimit(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, Math.floor(value))); +} + +function targetList( + target: LlmTarget, +): BaseAiOptimizationLLmMentionsTargetElement[] { + return [ + "domain" in target + ? new AiOptimizationLLmMentionsDomainElement(target) + : new AiOptimizationLLmMentionsKeywordElement(target), + ]; +} + +function firstResult(task: DataforseoTaskLike): Record | null { + const first = task.result?.[0]; + return isRecord(first) ? first : null; +} + +// --------------------------------------------------------------------------- +// LLM Mentions Search +// --------------------------------------------------------------------------- + +type LlmMentionsSearchInput = { + target: LlmTarget; + platform: LlmPlatform; + locationCode: number; + languageCode: string; + limit?: number; +}; + +export async function fetchLlmMentionsSearch( + input: LlmMentionsSearchInput, +): Promise> { + const response = await aiOptimizationApi( + classifyAiSearchError, + ).llmMentionsSearchLive([ + new AiOptimizationLlmMentionsSearchLiveRequestInfo({ + target: targetList(input.target), + platform: input.platform, + location_code: input.locationCode, + language_code: input.languageCode, + limit: clampLimit(input.limit ?? 100, 1, 1000), + }), + ]); + const task = assertOk( + response, + assertOptions("/v3/ai_optimization/llm_mentions/search/live"), + ); + + const items = z + .array(llmMentionItemSchema) + .safeParse(firstResult(task)?.items ?? []); + if (!items.success) { + throw new AppError( + "INTERNAL_ERROR", + "DataForSEO llm_mentions/search returned an invalid mention items shape", + ); + } + return { data: items.data, billing: buildTaskBilling(task) }; +} + +// --------------------------------------------------------------------------- +// LLM Mentions Aggregated Metrics +// --------------------------------------------------------------------------- + +type LlmAggregatedMetricsInput = { + target: LlmTarget; + platform: LlmPlatform; + locationCode: number; + languageCode: string; + internalListLimit?: number; +}; + +export async function fetchLlmAggregatedMetrics( + input: LlmAggregatedMetricsInput, +): Promise> { + const response = await aiOptimizationApi( + classifyAiSearchError, + ).llmMentionsAggregatedMetricsLive([ + new AiOptimizationLlmMentionsAggregatedMetricsLiveRequestInfo({ + target: targetList(input.target), + platform: input.platform, + location_code: input.locationCode, + language_code: input.languageCode, + internal_list_limit: clampLimit(input.internalListLimit ?? 10, 1, 20), + }), + ]); + const task = assertOk( + response, + assertOptions("/v3/ai_optimization/llm_mentions/aggregated_metrics/live"), + ); + + const total = llmAggregatedTotalSchema.safeParse( + firstResult(task)?.total ?? {}, + ); + if (!total.success) { + throw new AppError( + "INTERNAL_ERROR", + "DataForSEO llm_mentions/aggregated_metrics returned an invalid shape", + ); + } + return { data: total.data, billing: buildTaskBilling(task) }; +} + +// --------------------------------------------------------------------------- +// LLM Mentions Top Pages +// --------------------------------------------------------------------------- + +type LlmTopPagesInput = { + target: LlmTarget; + platform: LlmPlatform; + locationCode: number; + languageCode: string; + itemsListLimit?: number; +}; + +export async function fetchLlmTopPages( + input: LlmTopPagesInput, +): Promise> { + const response = await aiOptimizationApi( + classifyAiSearchError, + ).llmMentionsTopPagesLive([ + new AiOptimizationLlmMentionsTopPagesLiveRequestInfo({ + target: targetList(input.target), + platform: input.platform, + location_code: input.locationCode, + language_code: input.languageCode, + links_scope: "sources", + items_list_limit: clampLimit(input.itemsListLimit ?? 10, 1, 10), + internal_list_limit: 5, + }), + ]); + const task = assertOk( + response, + assertOptions("/v3/ai_optimization/llm_mentions/top_pages/live"), + ); + + const items = z + .array(llmTopPagesItemSchema) + .safeParse(firstResult(task)?.items ?? []); + if (!items.success) { + throw new AppError( + "INTERNAL_ERROR", + "DataForSEO llm_mentions/top_pages returned an invalid shape", + ); + } + return { data: items.data, billing: buildTaskBilling(task) }; +} + +// --------------------------------------------------------------------------- +// LLM Responses (per-model) +// --------------------------------------------------------------------------- + +type LlmResponseModelSlug = "chat_gpt" | "claude" | "gemini" | "perplexity"; + +type LlmResponsesInput = { + userPrompt: string; + modelSlug: LlmResponseModelSlug; + modelName: string; + webSearch?: boolean; + maxOutputTokens?: number; + /** Two-letter ISO country code used to geolocate the web-search component. */ + webSearchCountryCode?: string; +}; + +type LlmResponseRequestFields = { + user_prompt: string; + model_name: string; + web_search: boolean; + max_output_tokens: number; + web_search_country_iso_code?: string; +}; + +function buildPerplexityLlmResponseRequest( + fields: LlmResponseRequestFields, +): AiOptimizationPerplexityLlmResponsesLiveRequestInfo { + return { + ...fields, + init(data?: unknown) { + if (isRecord(data)) Object.assign(this, data); + }, + toJSON(data?: unknown) { + return { + ...(isRecord(data) ? data : {}), + ...fields, + }; + }, + }; +} + +export async function fetchLlmResponse( + input: LlmResponsesInput, +): Promise> { + // DataForSEO's Gemini endpoint rejects `web_search_country_iso_code` with a + // 40501 "Invalid Field" error. The other three models accept it. + const supportsCountry = input.modelSlug !== "gemini"; + const fields: LlmResponseRequestFields = { + user_prompt: input.userPrompt, + model_name: input.modelName, + web_search: input.webSearch ?? true, + max_output_tokens: clampLimit(input.maxOutputTokens ?? 1024, 256, 4096), + ...(supportsCountry && input.webSearchCountryCode + ? { web_search_country_iso_code: input.webSearchCountryCode } + : {}), + }; + + const api = aiOptimizationApi(classifyAiSearchError); + const response = + input.modelSlug === "chat_gpt" + ? await api.chatGptLlmResponsesLive([ + new AiOptimizationChatGptLlmResponsesLiveRequestInfo(fields), + ]) + : input.modelSlug === "claude" + ? await api.claudeLlmResponsesLive([ + new AiOptimizationClaudeLlmResponsesLiveRequestInfo(fields), + ]) + : input.modelSlug === "gemini" + ? await api.geminiLlmResponsesLive([ + new AiOptimizationGeminiLlmResponsesLiveRequestInfo(fields), + ]) + : await api.perplexityLlmResponsesLive([ + // The generated Perplexity request class drops `web_search` in + // toJSON(), while the SDK method only JSON.stringify's this body. + buildPerplexityLlmResponseRequest(fields), + ]); + + const task = assertOk( + response, + assertOptions(`/v3/ai_optimization/${input.modelSlug}/llm_responses/live`), + ); + + const result = llmResponseResultSchema.safeParse(firstResult(task) ?? {}); + if (!result.success) { + throw new AppError( + "INTERNAL_ERROR", + "DataForSEO llm_responses returned an invalid response shape", + ); + } + return { data: result.data, billing: buildTaskBilling(task) }; +} diff --git a/src/server/lib/dataforseoBacklinks.test.ts b/src/server/lib/dataforseo/backlinks.test.ts similarity index 69% rename from src/server/lib/dataforseoBacklinks.test.ts rename to src/server/lib/dataforseo/backlinks.test.ts index 1ca970b..9b976af 100644 --- a/src/server/lib/dataforseoBacklinks.test.ts +++ b/src/server/lib/dataforseo/backlinks.test.ts @@ -1,6 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { AppError } from "@/server/lib/errors"; -import type * as DataforseoBacklinksSupport from "@/server/lib/dataforseoBacklinksSupport"; vi.mock("@/server/lib/runtime-env", () => ({ getRequiredEnvValue: vi.fn(async () => "test-api-key"), @@ -10,19 +9,25 @@ const { classifyBacklinksError } = vi.hoisted(() => ({ classifyBacklinksError: vi.fn(), })); -vi.mock("@/server/lib/dataforseoBacklinksSupport", async () => { - const actual = await vi.importActual( - "@/server/lib/dataforseoBacklinksSupport", - ); - return { ...actual, classifyBacklinksError }; -}); +// The classifier is built inside backlinks.ts via createDataforseoAccessClassifier; +// returning our hoisted mock lets the test drive classification. +vi.mock("@/server/lib/dataforseoAccessClassification", () => ({ + createDataforseoAccessClassifier: () => classifyBacklinksError, +})); import { - fetchBacklinksHistoryRaw, - fetchBacklinksRowsRaw, - fetchBacklinksSummaryRaw, + fetchBacklinksHistory, + fetchBacklinksRows, + fetchBacklinksSummary, normalizeBacklinksTarget, -} from "@/server/lib/dataforseoBacklinks"; +} from "@/server/lib/dataforseo/backlinks"; + +// A successful DataForSEO task always carries billing metadata (path + cost). +const billed = { + path: ["v3", "backlinks", "summary", "live"], + cost: 0.02, + result_count: 0, +}; describe("normalizeBacklinksTarget", () => { it("treats explicit homepage URLs as page lookups", () => { @@ -43,14 +48,6 @@ describe("normalizeBacklinksTarget", () => { }); }); - it("keeps trailing slashes for root page URLs", () => { - expect(normalizeBacklinksTarget("https://example.com/")).toEqual({ - apiTarget: "https://example.com/", - displayTarget: "https://example.com/", - scope: "page", - }); - }); - it("treats bare hostnames as domain lookups", () => { expect(normalizeBacklinksTarget("Example.com")).toEqual({ apiTarget: "example.com", @@ -71,21 +68,6 @@ describe("normalizeBacklinksTarget", () => { }); }); - it("normalizes domain scope for URLs with query strings or fragments", () => { - expect( - normalizeBacklinksTarget( - "https://Example.com/pricing?utm_source=newsletter#hero", - { - scope: "domain", - }, - ), - ).toEqual({ - apiTarget: "example.com", - displayTarget: "example.com", - scope: "domain", - }); - }); - it("lets callers force page scope for bare hostnames", () => { expect(normalizeBacklinksTarget("Example.com", { scope: "page" })).toEqual({ apiTarget: "https://example.com/", @@ -111,7 +93,7 @@ describe("normalizeBacklinksTarget", () => { }); }); -describe("fetchBacklinksSummaryRaw", () => { +describe("fetchBacklinksSummary", () => { beforeEach(() => { vi.stubGlobal("fetch", vi.fn()); }); @@ -143,9 +125,7 @@ describe("fetchBacklinksSummaryRaw", () => { }); await expect( - fetchBacklinksSummaryRaw({ - target: "example.com", - }), + fetchBacklinksSummary({ target: "example.com" }), ).rejects.toMatchObject({ code: "BACKLINKS_NOT_ENABLED" }); expect(classifyBacklinksError).toHaveBeenCalledWith( @@ -165,6 +145,7 @@ describe("fetchBacklinksSummaryRaw", () => { { status_code: 20000, status_message: "Ok.", + ...billed, result: [null], }, ], @@ -175,9 +156,7 @@ describe("fetchBacklinksSummaryRaw", () => { classifyBacklinksError.mockReturnValue(null); await expect( - fetchBacklinksSummaryRaw({ - target: "not-a-real-input.example", - }), + fetchBacklinksSummary({ target: "not-a-real-input.example" }), ).resolves.toMatchObject({ data: {} }); }); @@ -191,6 +170,7 @@ describe("fetchBacklinksSummaryRaw", () => { { status_code: 20000, status_message: "Ok.", + ...billed, result: [], }, ], @@ -201,55 +181,37 @@ describe("fetchBacklinksSummaryRaw", () => { classifyBacklinksError.mockReturnValue(null); await expect( - fetchBacklinksSummaryRaw({ - target: "example.com", - }), + fetchBacklinksSummary({ target: "example.com" }), ).resolves.toMatchObject({ data: {} }); }); it("treats empty backlinks rows and history results as valid empty arrays", async () => { - vi.mocked(fetch) - .mockResolvedValueOnce( - new Response( - JSON.stringify({ - status_code: 20000, - status_message: "Ok.", - tasks: [ - { - status_code: 20000, - status_message: "Ok.", - result: [], - }, - ], - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ), - ) - .mockResolvedValueOnce( - new Response( - JSON.stringify({ - status_code: 20000, - status_message: "Ok.", - tasks: [ - { - status_code: 20000, - status_message: "Ok.", - result: [], - }, - ], - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ), + const emptyOk = () => + new Response( + JSON.stringify({ + status_code: 20000, + status_message: "Ok.", + tasks: [ + { + status_code: 20000, + status_message: "Ok.", + ...billed, + result: [], + }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, ); + vi.mocked(fetch) + .mockResolvedValueOnce(emptyOk()) + .mockResolvedValueOnce(emptyOk()); classifyBacklinksError.mockReturnValue(null); await expect( - fetchBacklinksRowsRaw({ - target: "example.com", - }), + fetchBacklinksRows({ target: "example.com" }), ).resolves.toMatchObject({ data: [] }); await expect( - fetchBacklinksHistoryRaw({ + fetchBacklinksHistory({ target: "example.com", dateFrom: "2025-01-01", dateTo: "2025-12-31", diff --git a/src/server/lib/dataforseo/backlinks.ts b/src/server/lib/dataforseo/backlinks.ts new file mode 100644 index 0000000..cd8d402 --- /dev/null +++ b/src/server/lib/dataforseo/backlinks.ts @@ -0,0 +1,280 @@ +import { z } from "zod"; +import { + BacklinksBacklinksLiveRequestInfo, + BacklinksDomainPagesSummaryLiveRequestInfo, + BacklinksHistoryLiveRequestInfo, + BacklinksReferringDomainsLiveRequestInfo, + BacklinksSummaryLiveRequestInfo, +} from "dataforseo-client"; +import { + normalizeBacklinksSpamFilterOptions, + type BacklinksSpamFilterOptions, +} from "@/types/schemas/backlinks"; +import { createDataforseoAccessClassifier } from "@/server/lib/dataforseoAccessClassification"; +import { AppError } from "@/server/lib/errors"; +import { backlinksApi } from "@/server/lib/dataforseo/core"; +import { + assertOk, + buildTaskBilling, + parseTaskItems, + type DataforseoApiResponse, +} from "@/server/lib/dataforseo/envelope"; + +export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget"; + +type BacklinksRequest = { target: string }; +type BacklinksListRequest = BacklinksRequest & + BacklinksSpamFilterOptions & { limit?: number }; +type BacklinksTimeseriesRequest = { + target: string; + dateFrom: string; + dateTo: string; +}; + +const classifyBacklinksError = createDataforseoAccessClassifier({ + pathPrefix: "/backlinks/", + notEnabledCode: "BACKLINKS_NOT_ENABLED", + notEnabledMessage: + "Backlinks is not enabled for the connected DataForSEO account", + billingIssueCode: "BACKLINKS_BILLING_ISSUE", + billingIssueMessage: + "The connected DataForSEO account has a billing or balance issue", +}); + +// DataForSEO ships both the misspelled (`*_reffering_*`) and corrected keys; we +// accept both via passthrough so callers can read whichever is present. +export const backlinksSummaryItemSchema = z + .object({ + target: z.string().optional(), + rank: z.number().nullable().optional(), + backlinks: z.number().nullable().optional(), + referring_pages: z.number().nullable().optional(), + referring_domains: z.number().nullable().optional(), + broken_backlinks: z.number().nullable().optional(), + broken_pages: z.number().nullable().optional(), + new_backlinks: z.number().nullable().optional(), + lost_backlinks: z.number().nullable().optional(), + new_reffering_domains: z.number().nullable().optional(), + lost_reffering_domains: z.number().nullable().optional(), + new_referring_domains: z.number().nullable().optional(), + lost_referring_domains: z.number().nullable().optional(), + backlinks_spam_score: z.number().nullable().optional(), + info: z + .object({ target_spam_score: z.number().nullable().optional() }) + .passthrough() + .nullable() + .optional(), + }) + .passthrough(); + +export const backlinksItemSchema = z + .object({ + domain_from: z.string().nullable().optional(), + url_from: z.string().nullable().optional(), + url_to: z.string().nullable().optional(), + anchor: z.string().nullable().optional(), + item_type: z.string().nullable().optional(), + dofollow: z.boolean().nullable().optional(), + rank: z.number().nullable().optional(), + domain_from_rank: z.number().nullable().optional(), + page_from_rank: z.number().nullable().optional(), + backlinks_spam_score: z.number().nullable().optional(), + backlink_spam_score: z.number().nullable().optional(), + first_seen: z.string().nullable().optional(), + last_visited: z.string().nullable().optional(), + lost_date: z.string().nullable().optional(), + is_new: z.boolean().nullable().optional(), + is_lost: z.boolean().nullable().optional(), + is_broken: z.boolean().nullable().optional(), + links_count: z.number().nullable().optional(), + rel_attributes: z.array(z.string()).nullable().optional(), + attributes: z.array(z.string()).nullable().optional(), + }) + .passthrough(); + +export const referringDomainItemSchema = z + .object({ + domain: z.string().nullable().optional(), + backlinks: z.number().nullable().optional(), + referring_pages: z.number().nullable().optional(), + rank: z.number().nullable().optional(), + first_seen: z.string().nullable().optional(), + broken_backlinks: z.number().nullable().optional(), + broken_pages: z.number().nullable().optional(), + backlinks_spam_score: z.number().nullable().optional(), + target_spam_score: z.number().nullable().optional(), + }) + .passthrough(); + +export const domainPageSummaryItemSchema = z + .object({ + page: z.string().nullable().optional(), + url: z.string().nullable().optional(), + backlinks: z.number().nullable().optional(), + referring_domains: z.number().nullable().optional(), + rank: z.number().nullable().optional(), + broken_backlinks: z.number().nullable().optional(), + }) + .passthrough(); + +export const backlinksHistoryItemSchema = z + .object({ + date: z.string().nullable().optional(), + rank: z.number().nullable().optional(), + backlinks: z.number().nullable().optional(), + referring_domains: z.number().nullable().optional(), + new_backlinks: z.number().nullable().optional(), + lost_backlinks: z.number().nullable().optional(), + new_reffering_domains: z.number().nullable().optional(), + lost_reffering_domains: z.number().nullable().optional(), + new_referring_domains: z.number().nullable().optional(), + lost_referring_domains: z.number().nullable().optional(), + }) + .passthrough(); + +function buildCommonPayload(input: BacklinksRequest) { + return { + target: input.target, + include_subdomains: true, + include_indirect_links: true, + exclude_internal_backlinks: true, + backlinks_status_type: "live", + rank_scale: "one_hundred", + }; +} + +const assertOptions = (path: string) => + ({ classify: classifyBacklinksError, classifyPath: path }) as const; + +export async function fetchBacklinksSummary(input: BacklinksRequest) { + const response = await backlinksApi(classifyBacklinksError).summaryLive([ + new BacklinksSummaryLiveRequestInfo(buildCommonPayload(input)), + ]); + const task = assertOk(response, assertOptions("/v3/backlinks/summary/live")); + + const firstResult = task.result?.[0]; + if (firstResult) { + const parsed = backlinksSummaryItemSchema.safeParse(firstResult); + if (!parsed.success) { + console.error( + "dataforseo.backlinks-summary-live.invalid-result", + parsed.error.issues.slice(0, 5), + ); + throw new AppError( + "INTERNAL_ERROR", + "DataForSEO backlinks-summary-live returned an invalid response shape", + ); + } + return { + data: parsed.data, + billing: buildTaskBilling(task), + } satisfies DataforseoApiResponse; + } + + return { + data: {} as z.infer, + billing: buildTaskBilling(task), + }; +} + +export async function fetchBacklinksRows(input: BacklinksListRequest) { + const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input); + const filters = spamFilterOptions.hideSpam + ? [["backlink_spam_score", "<=", spamFilterOptions.spamThreshold]] + : undefined; + const response = await backlinksApi(classifyBacklinksError).backlinksLive([ + new BacklinksBacklinksLiveRequestInfo({ + ...buildCommonPayload(input), + limit: input.limit ?? 100, + order_by: ["rank,desc"], + ...(filters ? { filters } : {}), + }), + ]); + const task = assertOk( + response, + assertOptions("/v3/backlinks/backlinks/live"), + ); + return { + data: parseTaskItems("backlinks-live", task, backlinksItemSchema), + billing: buildTaskBilling(task), + }; +} + +export async function fetchReferringDomains(input: BacklinksListRequest) { + const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input); + const filters = spamFilterOptions.hideSpam + ? [["backlinks_spam_score", "<=", spamFilterOptions.spamThreshold]] + : undefined; + const response = await backlinksApi( + classifyBacklinksError, + ).referringDomainsLive([ + new BacklinksReferringDomainsLiveRequestInfo({ + ...buildCommonPayload(input), + limit: input.limit ?? 100, + order_by: ["backlinks,desc"], + ...(filters ? { filters } : {}), + }), + ]); + const task = assertOk( + response, + assertOptions("/v3/backlinks/referring_domains/live"), + ); + return { + data: parseTaskItems( + "referring-domains-live", + task, + referringDomainItemSchema, + ), + billing: buildTaskBilling(task), + }; +} + +export async function fetchDomainPagesSummary(input: BacklinksListRequest) { + const response = await backlinksApi( + classifyBacklinksError, + ).domainPagesSummaryLive([ + new BacklinksDomainPagesSummaryLiveRequestInfo({ + ...buildCommonPayload(input), + limit: input.limit ?? 100, + order_by: ["backlinks,desc"], + }), + ]); + const task = assertOk( + response, + assertOptions("/v3/backlinks/domain_pages_summary/live"), + ); + return { + data: parseTaskItems( + "domain-pages-summary-live", + task, + domainPageSummaryItemSchema, + ), + billing: buildTaskBilling(task), + }; +} + +export async function fetchBacklinksHistory(input: BacklinksTimeseriesRequest) { + const response = await backlinksApi(classifyBacklinksError).historyLive([ + new BacklinksHistoryLiveRequestInfo({ + target: input.target, + date_from: input.dateFrom, + date_to: input.dateTo, + rank_scale: "one_hundred", + }), + ]); + const task = assertOk(response, assertOptions("/v3/backlinks/history/live")); + return { + data: parseTaskItems( + "backlinks-history-live", + task, + backlinksHistoryItemSchema, + ), + billing: buildTaskBilling(task), + }; +} + +export type BacklinksSummaryItem = z.infer; +export type BacklinksItem = z.infer; +export type ReferringDomainItem = z.infer; +export type DomainPageSummaryItem = z.infer; +export type BacklinksHistoryItem = z.infer; diff --git a/src/server/lib/dataforseo/business.ts b/src/server/lib/dataforseo/business.ts new file mode 100644 index 0000000..15074f1 --- /dev/null +++ b/src/server/lib/dataforseo/business.ts @@ -0,0 +1,83 @@ +import { z } from "zod"; +import { + BusinessDataBusinessListingsSearchLiveRequestInfo, + BusinessDataGoogleQuestionsAndAnswersLiveRequestInfo, + type BusinessDataBusinessListingsSearchLiveItem, +} from "dataforseo-client"; +import { businessDataApi } from "@/server/lib/dataforseo/core"; +import { + assertOk, + buildTaskBilling, + type DataforseoApiResponse, +} from "@/server/lib/dataforseo/envelope"; + +type BusinessListingItem = BusinessDataBusinessListingsSearchLiveItem; + +export async function fetchBusinessListingsSearch(input: { + categories?: string[]; + title?: string; + locationCoordinate: string; + orderBy?: string[]; + limit: number; +}): Promise> { + const response = await businessDataApi().businessListingsSearchLive([ + new BusinessDataBusinessListingsSearchLiveRequestInfo({ + categories: input.categories, + title: input.title, + location_coordinate: input.locationCoordinate, + order_by: input.orderBy, + limit: input.limit, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; +} + +// Q&A results carry both answered (`items`) and unanswered +// (`items_without_answers`) rows; the SDK types this result as `any`, so we +// validate a generic record shape and flatten both. +const questionsResultSchema = z + .object({ + items: z.array(z.record(z.string(), z.unknown())).nullable().optional(), + items_without_answers: z + .array(z.record(z.string(), z.unknown())) + .nullable() + .optional(), + }) + .passthrough(); + +function combinedQuestionItems(results: unknown): Record[] { + const list = Array.isArray(results) ? results : []; + return list.flatMap((result) => { + const parsed = questionsResultSchema.safeParse(result ?? {}); + if (!parsed.success) return []; + return [ + ...(parsed.data.items ?? []), + ...(parsed.data.items_without_answers ?? []), + ]; + }); +} + +export async function fetchQuestionsAnswers(input: { + keyword: string; + locationCoordinate: string; + languageCode: string; + depth: number; +}): Promise[]>> { + const response = await businessDataApi().googleQuestionsAndAnswersLive([ + new BusinessDataGoogleQuestionsAndAnswersLiveRequestInfo({ + keyword: input.keyword, + location_coordinate: input.locationCoordinate, + language_code: input.languageCode, + depth: input.depth, + }), + ]); + const task = assertOk(response); + return { + data: combinedQuestionItems(task.result), + billing: buildTaskBilling(task), + }; +} diff --git a/src/server/lib/dataforseoClient.test.ts b/src/server/lib/dataforseo/client.test.ts similarity index 87% rename from src/server/lib/dataforseoClient.test.ts rename to src/server/lib/dataforseo/client.test.ts index 5062168..3d527ce 100644 --- a/src/server/lib/dataforseoClient.test.ts +++ b/src/server/lib/dataforseo/client.test.ts @@ -45,45 +45,53 @@ vi.mock("@/server/lib/posthog", () => ({ captureServerEvent: vi.fn(), })); -vi.mock("@/server/lib/dataforseo", () => ({ - fetchKeywordIdeasRaw: vi.fn(), - fetchKeywordSuggestionsRaw: vi.fn(), - fetchRelatedKeywordsRaw: vi.fn(), - fetchBusinessListingsSearchRaw: vi.fn(), - fetchBusinessQuestionsAnswersRaw: vi.fn(), - fetchDomainRankOverviewRaw: vi.fn(), - fetchKeywordSearchVolumeRaw: vi.fn(), - fetchLocalSerpItemsRaw: vi.fn(), - fetchSerpCompetitorsRaw: vi.fn(), - fetchRankedKeywordsRaw: vi.fn(), - fetchLiveSerpItemsRaw: vi.fn(), +// Mock every section module the client wraps so meterDataforseoCall's +// `execute()` resolves to a controllable fixture. +vi.mock("@/server/lib/dataforseo/labs", () => ({ + fetchRelatedKeywords: vi.fn(), + fetchKeywordSuggestions: vi.fn(), + fetchKeywordIdeas: vi.fn(), + fetchDomainRankOverview: vi.fn(), + fetchRankedKeywords: vi.fn(), + fetchRelevantPages: vi.fn(), + fetchKeywordOverview: vi.fn(), + fetchSerpCompetitors: vi.fn(), })); - -vi.mock("@/server/lib/dataforseoLighthouse", () => ({ - fetchDataforseoLighthouseResultRaw: vi.fn(), +vi.mock("@/server/lib/dataforseo/serp", () => ({ + fetchLiveSerp: vi.fn(), + fetchRankCheckSerp: vi.fn(), + fetchLocalSerp: vi.fn(), })); - -vi.mock("@/server/lib/dataforseoBacklinks", () => ({ - fetchBacklinksHistoryRaw: vi.fn(), - fetchBacklinksRowsRaw: vi.fn(), - fetchBacklinksSummaryRaw: vi.fn(), - fetchDomainPagesSummaryRaw: vi.fn(), - fetchReferringDomainsRaw: vi.fn(), +vi.mock("@/server/lib/dataforseo/keywordsData", () => ({ + fetchKeywordSearchVolume: vi.fn(), })); - -vi.mock("@/server/lib/dataforseoLlm", () => ({ - fetchLlmResponseRaw: vi.fn(), - fetchLlmAggregatedMetricsRaw: vi.fn(), - fetchLlmMentionsSearchRaw: vi.fn(), - fetchLlmTopPagesRaw: vi.fn(), +vi.mock("@/server/lib/dataforseo/business", () => ({ + fetchBusinessListingsSearch: vi.fn(), + fetchQuestionsAnswers: vi.fn(), +})); +vi.mock("@/server/lib/dataforseo/backlinks", () => ({ + fetchBacklinksSummary: vi.fn(), + fetchBacklinksRows: vi.fn(), + fetchReferringDomains: vi.fn(), + fetchDomainPagesSummary: vi.fn(), + fetchBacklinksHistory: vi.fn(), +})); +vi.mock("@/server/lib/dataforseo/lighthouse", () => ({ + fetchLighthouseResult: vi.fn(), +})); +vi.mock("@/server/lib/dataforseo/ai", () => ({ + fetchLlmMentionsSearch: vi.fn(), + fetchLlmAggregatedMetrics: vi.fn(), + fetchLlmTopPages: vi.fn(), + fetchLlmResponse: vi.fn(), })); import { createDataforseoClient, mapDataforseoPathToCreditFeature, -} from "./dataforseoClient"; -import { DataforseoChargedTaskError } from "./dataforseoCost"; -import { fetchBacklinksSummaryRaw } from "./dataforseoBacklinks"; +} from "@/server/lib/dataforseo/client"; +import { DataforseoChargedTaskError } from "@/server/lib/dataforseo/envelope"; +import { fetchBacklinksSummary } from "@/server/lib/dataforseo/backlinks"; const billingCustomer = { organizationId: "org_123", @@ -113,9 +121,9 @@ function mockBalances(monthly: number, topup: number) { } function mockDataforseoResult(costUsd: number) { - vi.mocked(fetchBacklinksSummaryRaw).mockResolvedValue({ + vi.mocked(fetchBacklinksSummary).mockResolvedValue({ data: { rank: 42 }, - billing: { costUsd, path: ["backlinks", "summary"], resultCount: 1 }, + billing: { costUsd, path: ["backlinks", "summary"] }, }); } @@ -238,11 +246,10 @@ describe("meterDataforseoCall with split balances", () => { it("meters charged DataForSEO task errors before rethrowing", async () => { setupHostedMode(); mockBalances(5000, 3000); - vi.mocked(fetchBacklinksSummaryRaw).mockRejectedValue( + vi.mocked(fetchBacklinksSummary).mockRejectedValue( new DataforseoChargedTaskError("DataForSEO task failed", { costUsd: RAW_COST, path: ["v3", "backlinks", "summary", "live"], - resultCount: 0, }), ); diff --git a/src/server/lib/dataforseo/client.ts b/src/server/lib/dataforseo/client.ts new file mode 100644 index 0000000..6eccc0c --- /dev/null +++ b/src/server/lib/dataforseo/client.ts @@ -0,0 +1,261 @@ +import { + AUTUMN_SEO_DATA_BALANCE_FEATURE_ID, + AUTUMN_SEO_DATA_CREDITS_PER_USD, + AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID, + SEO_DATA_COST_MARKUP, + roundUsdForBilling, +} from "@/shared/billing"; +import { + type CreditFeature, + mapDataforseoPathToCreditFeature, +} from "@/shared/billing-credit-features"; +import { autumn } from "@/server/billing/autumn"; +import { getOrCreateOrganizationCustomer } from "@/server/billing/subscription"; +import type { BillingCustomerContext } from "@/server/billing/subscription"; +import { + fetchBusinessListingsSearch, + fetchQuestionsAnswers, +} from "@/server/lib/dataforseo/business"; +import { + fetchBacklinksHistory, + fetchBacklinksRows, + fetchBacklinksSummary, + fetchDomainPagesSummary, + fetchReferringDomains, +} from "@/server/lib/dataforseo/backlinks"; +import { + fetchDomainRankOverview, + fetchKeywordIdeas, + fetchKeywordOverview, + fetchKeywordSuggestions, + fetchRankedKeywords, + fetchRelatedKeywords, + fetchRelevantPages, + fetchSerpCompetitors, +} from "@/server/lib/dataforseo/labs"; +import { + fetchLiveSerp, + fetchLocalSerp, + fetchRankCheckSerp, +} from "@/server/lib/dataforseo/serp"; +import { fetchKeywordSearchVolume } from "@/server/lib/dataforseo/keywordsData"; +import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse"; +import { + fetchLlmAggregatedMetrics, + fetchLlmMentionsSearch, + fetchLlmResponse, + fetchLlmTopPages, +} from "@/server/lib/dataforseo/ai"; +import { + DataforseoChargedTaskError, + type DataforseoApiCallCost, + type DataforseoApiResponse, +} from "@/server/lib/dataforseo/envelope"; +import { AppError } from "@/server/lib/errors"; +import { captureServerEvent } from "@/server/lib/posthog"; +import { isHostedServerAuthMode } from "@/server/lib/runtime-env"; + +export { mapDataforseoPathToCreditFeature }; + +/** + * Wraps a section fetcher with billing metering. Each entry on the client is + * `meter(customer, fetcher, creditFeature?)`, which returns a function with the + * fetcher's own input type and resolves to its unwrapped `.data`. + */ +function meter( + customer: BillingCustomerContext, + fetcher: (input: I) => Promise>, + creditFeature?: CreditFeature, +): (input: I) => Promise { + return (input: I) => + meterDataforseoCall(customer, () => fetcher(input), creditFeature); +} + +export function createDataforseoClient(customer: BillingCustomerContext) { + return { + business: { + businessListings: meter( + customer, + fetchBusinessListingsSearch, + "local_seo", + ), + questionsAnswers: meter(customer, fetchQuestionsAnswers, "local_seo"), + }, + backlinks: { + summary: meter(customer, fetchBacklinksSummary), + rows: meter(customer, fetchBacklinksRows), + referringDomains: meter(customer, fetchReferringDomains), + domainPages: meter(customer, fetchDomainPagesSummary), + history: meter(customer, fetchBacklinksHistory), + }, + keywords: { + related: meter(customer, fetchRelatedKeywords), + suggestions: meter(customer, fetchKeywordSuggestions), + ideas: meter(customer, fetchKeywordIdeas), + }, + domain: { + rankOverview: meter(customer, fetchDomainRankOverview), + rankedKeywords: meter(customer, fetchRankedKeywords), + relevantPages: meter(customer, fetchRelevantPages), + }, + serp: { + live: meter(customer, fetchLiveSerp), + rankCheck: meter(customer, fetchRankCheckSerp, "rank_tracking"), + local: meter(customer, fetchLocalSerp, "local_seo"), + }, + keywordData: { + searchVolume: meter(customer, fetchKeywordSearchVolume), + }, + labs: { + keywordOverview: meter(customer, fetchKeywordOverview, "rank_tracking"), + serpCompetitors: meter(customer, fetchSerpCompetitors), + }, + lighthouse: { + live: meter(customer, fetchLighthouseResult), + }, + aiSearch: { + mentionsSearch: meter(customer, fetchLlmMentionsSearch), + aggregatedMetrics: meter(customer, fetchLlmAggregatedMetrics), + topPages: meter(customer, fetchLlmTopPages), + llmResponse: meter(customer, fetchLlmResponse), + }, + } as const; +} + +async function meterDataforseoCall( + customer: BillingCustomerContext, + execute: () => Promise>, + creditFeature?: CreditFeature, +): Promise { + const isHostedMode = await isHostedServerAuthMode(); + + if (!isHostedMode) { + const result = await execute(); + return result.data; + } + + const billingCustomer = await getOrCreateOrganizationCustomer(customer); + + const { monthlyRemaining } = await assertSeoDataBalanceAvailable( + billingCustomer.id, + ); + + let result: DataforseoApiResponse; + try { + result = await execute(); + } catch (error) { + if (error instanceof DataforseoChargedTaskError) { + await trackDataforseoCost({ + customer, + customerId: billingCustomer.id, + billing: error.billing, + monthlyRemaining, + creditFeature, + }); + } + throw error; + } + + await trackDataforseoCost({ + customer, + customerId: billingCustomer.id, + billing: result.billing, + monthlyRemaining, + creditFeature, + }); + + return result.data; +} + +async function assertSeoDataBalanceAvailable(customerId: string) { + const [monthlyCheck, topupCheck] = await Promise.all([ + autumn.check({ + customerId, + featureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID, + }), + autumn.check({ + customerId, + featureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID, + }), + ]); + + const monthlyRemaining = monthlyCheck.balance?.remaining ?? 0; + const topupRemaining = topupCheck.balance?.remaining ?? 0; + + if (monthlyRemaining + topupRemaining <= 0) { + throw new AppError("INSUFFICIENT_CREDITS"); + } + + return { monthlyRemaining }; +} + +async function trackDataforseoCost(args: { + customer: BillingCustomerContext; + customerId: string; + billing: DataforseoApiCallCost; + monthlyRemaining: number; + creditFeature?: CreditFeature; +}) { + const totalCostUsd = roundUsdForBilling( + args.billing.costUsd * SEO_DATA_COST_MARKUP, + ); + const totalCostCredits = Math.ceil( + totalCostUsd * AUTUMN_SEO_DATA_CREDITS_PER_USD, + ); + + const monthlyDeduct = Math.min(args.monthlyRemaining, totalCostCredits); + const topupDeduct = totalCostCredits - monthlyDeduct; + + const creditFeature = + args.creditFeature ?? mapDataforseoPathToCreditFeature(args.billing.path); + + const properties = { + provider: "dataforseo", + currency: "USD", + paths: [args.billing.path.join("/")], + creditFeature, + totalCostUsd, + totalCostCredits, + fromCache: false, + }; + + if (monthlyDeduct > 0) { + await autumn.track({ + customerId: args.customerId, + featureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID, + value: monthlyDeduct, + properties: { + ...properties, + balanceFeatureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID, + }, + }); + } + + if (topupDeduct > 0) { + await autumn.track({ + customerId: args.customerId, + featureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID, + value: topupDeduct, + properties: { + ...properties, + balanceFeatureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID, + }, + }); + } + + if (totalCostCredits > 0) { + await captureServerEvent({ + distinctId: args.customer.userId, + event: "usage:credits_consume", + organizationId: args.customer.organizationId, + properties: { + project_id: args.customer.projectId, + credit_feature: creditFeature, + monthly_credits: monthlyDeduct, + topup_credits: topupDeduct, + total_credits: totalCostCredits, + cost_usd: totalCostUsd, + }, + }); + } +} diff --git a/src/server/lib/dataforseo/core.ts b/src/server/lib/dataforseo/core.ts new file mode 100644 index 0000000..627c7f7 --- /dev/null +++ b/src/server/lib/dataforseo/core.ts @@ -0,0 +1,110 @@ +import { + AiOptimizationApi, + BacklinksApi, + BusinessDataApi, + DataforseoLabsApi, + KeywordsDataApi, + OnPageApi, + SerpApi, +} from "dataforseo-client"; +import { AppError } from "@/server/lib/errors"; +import { getRequiredEnvValue } from "@/server/lib/runtime-env"; + +const API_BASE = "https://api.dataforseo.com"; +const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600; +// Safety ceiling on any live call (Lighthouse is the slowest, ~tens of seconds). +const DATAFORSEO_REQUEST_TIMEOUT_MS = 60_000; + +/** + * Translates a DataForSEO HTTP/task failure into a product-specific AppError + * (e.g. "backlinks not enabled", "billing issue"). Returns null when the + * failure isn't one this classifier recognises, so the caller can fall back to + * a generic error. See {@link createDataforseoAccessClassifier}. + */ +export type DataforseoErrorClassifier = ( + status: number | undefined, + details: string, + path: string, +) => AppError | null; + +function formatDataforseoErrorPayload(value: unknown): string { + const text = + typeof value === "string" + ? value + : (() => { + try { + return JSON.stringify(value); + } catch { + return String(value); + } + })(); + + return text.length > MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH + ? `${text.slice(0, MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH)}... [truncated]` + : text; +} + +function formatDataforseoRequestPath(url: RequestInfo): string { + const rawUrl = typeof url === "string" ? url : url.url; + try { + return new URL(rawUrl).pathname; + } catch { + return rawUrl; + } +} + +/** + * The single authenticated `fetch` used by every DataForSEO SDK call. Throws on + * non-2xx so the SDK's own `ApiException` path never fires; task-level failures + * (which return HTTP 200) are handled downstream by {@link assertOk}. An + * optional classifier maps recognised HTTP failures to product errors. + */ +function createAuthenticatedFetch(classify?: DataforseoErrorClassifier) { + return async (url: RequestInfo, init?: RequestInit): Promise => { + const apiKey = await getRequiredEnvValue("DATAFORSEO_API_KEY"); + const headers = new Headers(init?.headers); + headers.set("Authorization", `Basic ${apiKey}`); + + const response = await fetch(url, { + ...init, + headers, + signal: + init?.signal ?? AbortSignal.timeout(DATAFORSEO_REQUEST_TIMEOUT_MS), + }); + if (response.ok) return response; + + const rawText = await response.text(); + const path = formatDataforseoRequestPath(url); + const classified = classify?.(response.status, rawText, path); + if (classified) throw classified; + + const error = new AppError( + response.status === 429 ? "RATE_LIMITED" : "INTERNAL_ERROR", + `DataForSEO HTTP ${response.status} on ${path}`, + { + provider: "dataforseo", + providerStatus: String(response.status), + providerPath: path, + responseBody: formatDataforseoErrorPayload(rawText), + }, + ); + error.name = "DataForSEOHttpError"; + throw error; + }; +} + +function http(classify?: DataforseoErrorClassifier) { + return { fetch: createAuthenticatedFetch(classify) }; +} + +// Per-section API factories. Each is created per-request so the auth secret is +// read lazily (it lives in the Worker env, not in module scope). +export const labsApi = () => new DataforseoLabsApi(API_BASE, http()); +export const serpApi = () => new SerpApi(API_BASE, http()); +export const keywordsDataApi = () => new KeywordsDataApi(API_BASE, http()); +export const businessDataApi = () => new BusinessDataApi(API_BASE, http()); +export const onPageApi = () => new OnPageApi(API_BASE, http()); +export const backlinksApi = (classify?: DataforseoErrorClassifier) => + new BacklinksApi(API_BASE, http(classify)); +export const aiOptimizationApi = (classify?: DataforseoErrorClassifier) => + new AiOptimizationApi(API_BASE, http(classify)); diff --git a/src/server/lib/dataforseo/endpoints.test.ts b/src/server/lib/dataforseo/endpoints.test.ts new file mode 100644 index 0000000..cd2a7c3 --- /dev/null +++ b/src/server/lib/dataforseo/endpoints.test.ts @@ -0,0 +1,332 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/server/lib/runtime-env", () => ({ + getRequiredEnvValue: vi.fn(async () => "test-api-key"), +})); + +import { fetchQuestionsAnswers } from "@/server/lib/dataforseo/business"; +import { + buildLlmTarget, + fetchLlmAggregatedMetrics, + fetchLlmMentionsSearch, + fetchLlmResponse, + fetchLlmTopPages, +} from "@/server/lib/dataforseo/ai"; +import { fetchKeywordSearchVolume } from "@/server/lib/dataforseo/keywordsData"; + +function parseDataforseoRequestBody(init: RequestInit | undefined): unknown { + const body = init?.body; + if (typeof body !== "string") { + throw new Error("Expected DataForSEO request body to be a string"); + } + return JSON.parse(body) as unknown; +} + +describe("DataForSEO SDK-backed endpoints", () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it("uses the live endpoint for Google Business Q&A and returns items + billing", async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + status_code: 20000, + tasks: [ + { + status_code: 20000, + path: [ + "v3", + "business_data", + "google", + "questions_and_answers", + "live", + ], + cost: 0.0006, + result_count: 1, + result: [ + { + items: [ + { + question_text: "Do you offer indoor storage?", + answer_text: "Yes.", + }, + ], + }, + ], + }, + ], + }), + ); + vi.stubGlobal("fetch", fetchMock); + + const result = await fetchQuestionsAnswers({ + keyword: "Acme Storage", + locationCoordinate: "33.1234568,-84.9876543,5000", + languageCode: "en", + depth: 20, + }); + + expect( + fetchMock.mock.calls.map(([url]) => + typeof url === "string" || url instanceof URL + ? url.toString() + : url.url, + ), + ).toEqual([ + "https://api.dataforseo.com/v3/business_data/google/questions_and_answers/live", + ]); + expect(result.data).toEqual([ + { question_text: "Do you offer indoor storage?", answer_text: "Yes." }, + ]); + expect(result.billing).toEqual({ + path: ["v3", "business_data", "google", "questions_and_answers", "live"], + costUsd: 0.0006, + }); + }); + + it("does not send location_name for keyword search volume", async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + status_code: 20000, + tasks: [ + { + status_code: 20000, + path: [ + "v3", + "keywords_data", + "google_ads", + "search_volume", + "live", + ], + cost: 0.0001, + result_count: 1, + result: [ + { + keyword: "storage units", + location_code: 2840, + language_code: "en", + search_volume: 1000, + }, + ], + }, + ], + }), + ); + vi.stubGlobal("fetch", fetchMock); + + await fetchKeywordSearchVolume({ + keywords: ["storage units"], + locationCode: 2840, + languageCode: "en", + }); + + const payload = parseDataforseoRequestBody(fetchMock.mock.calls[0]?.[1]); + expect(payload).toEqual([ + { + keywords: ["storage units"], + location_code: 2840, + language_code: "en", + }, + ]); + expect(JSON.stringify(payload)).not.toContain("location_name"); + }); + + it("serializes LLM mentions domain targets for all live endpoints", async () => { + const fetchMock = vi.fn().mockImplementation((url) => { + const path = + typeof url === "string" || url instanceof URL + ? url.toString() + : url.url; + const result = path.includes("/aggregated_metrics/") + ? { total: { platform: [] } } + : { items: [] }; + + return Promise.resolve( + Response.json({ + status_code: 20000, + tasks: [ + { + status_code: 20000, + path: new URL(path).pathname.split("/").filter(Boolean), + cost: 0.0001, + result_count: 1, + result: [result], + }, + ], + }), + ); + }); + vi.stubGlobal("fetch", fetchMock); + + const target = buildLlmTarget({ + type: "domain", + value: "example.com", + }); + + await fetchLlmMentionsSearch({ + target, + platform: "google", + locationCode: 2840, + languageCode: "en", + }); + await fetchLlmAggregatedMetrics({ + target, + platform: "google", + locationCode: 2840, + languageCode: "en", + }); + await fetchLlmTopPages({ + target, + platform: "google", + locationCode: 2840, + languageCode: "en", + }); + + const expectedTarget = [ + { + search_scope: ["any"], + search_filter: "include", + domain: "example.com", + include_subdomains: true, + }, + ]; + const payloads = fetchMock.mock.calls.map(([, init]) => + parseDataforseoRequestBody(init), + ); + + expect(payloads).toEqual([ + [ + { + target: expectedTarget, + location_code: 2840, + language_code: "en", + platform: "google", + limit: 100, + }, + ], + [ + { + target: expectedTarget, + location_code: 2840, + language_code: "en", + platform: "google", + internal_list_limit: 10, + }, + ], + [ + { + target: expectedTarget, + location_code: 2840, + language_code: "en", + platform: "google", + links_scope: "sources", + items_list_limit: 10, + internal_list_limit: 5, + }, + ], + ]); + }); + + it("serializes LLM mentions keyword targets", async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + status_code: 20000, + tasks: [ + { + status_code: 20000, + path: ["v3", "ai_optimization", "llm_mentions", "search", "live"], + cost: 0.0001, + result_count: 1, + result: [{ items: [] }], + }, + ], + }), + ); + vi.stubGlobal("fetch", fetchMock); + + await fetchLlmMentionsSearch({ + target: buildLlmTarget({ + type: "keyword", + value: "Acme Storage", + }), + platform: "chat_gpt", + locationCode: 2840, + languageCode: "en", + }); + + expect(parseDataforseoRequestBody(fetchMock.mock.calls[0]?.[1])).toEqual([ + { + target: [ + { + search_scope: ["any", "brand_entities"], + search_filter: "include", + keyword: "Acme Storage", + match_type: "word_match", + }, + ], + location_code: 2840, + language_code: "en", + platform: "chat_gpt", + limit: 100, + }, + ]); + }); + + it("preserves web_search for Perplexity LLM responses", async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + status_code: 20000, + tasks: [ + { + status_code: 20000, + path: [ + "v3", + "ai_optimization", + "perplexity", + "llm_responses", + "live", + ], + cost: 0.0001, + result_count: 1, + result: [ + { + model_name: "sonar", + output_tokens: 12, + web_search: false, + items: [], + }, + ], + }, + ], + }), + ); + vi.stubGlobal("fetch", fetchMock); + + await fetchLlmResponse({ + userPrompt: "What is OpenSEO?", + modelSlug: "perplexity", + modelName: "sonar", + webSearch: false, + webSearchCountryCode: "US", + }); + + expect( + fetchMock.mock.calls.map(([url]) => + typeof url === "string" || url instanceof URL + ? url.toString() + : url.url, + ), + ).toEqual([ + "https://api.dataforseo.com/v3/ai_optimization/perplexity/llm_responses/live", + ]); + expect(parseDataforseoRequestBody(fetchMock.mock.calls[0]?.[1])).toEqual([ + { + user_prompt: "What is OpenSEO?", + model_name: "sonar", + web_search: false, + max_output_tokens: 1024, + web_search_country_iso_code: "US", + }, + ]); + }); +}); diff --git a/src/server/lib/dataforseo/envelope.test.ts b/src/server/lib/dataforseo/envelope.test.ts new file mode 100644 index 0000000..b834a2f --- /dev/null +++ b/src/server/lib/dataforseo/envelope.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; +import { + assertOk, + DataforseoChargedTaskError, + parseTaskItems, +} from "@/server/lib/dataforseo/envelope"; +import { AppError } from "@/server/lib/errors"; + +const itemSchema = z.object({ keyword: z.string().optional() }).passthrough(); + +describe("parseTaskItems", () => { + it("returns [] when the result items are null", () => { + const task = { status_code: 20000, result: [{ items: null }] }; + expect(parseTaskItems("x", task, itemSchema)).toEqual([]); + }); + + it("returns [] when there is no result", () => { + expect(parseTaskItems("x", { result: undefined }, itemSchema)).toEqual([]); + }); + + it("parses present items", () => { + const task = { result: [{ items: [{ keyword: "seo" }] }] }; + expect(parseTaskItems("x", task, itemSchema)).toEqual([{ keyword: "seo" }]); + }); +}); + +describe("assertOk", () => { + const okTask = { + status_code: 20000, + path: ["v3", "backlinks", "summary", "live"], + cost: 0.1, + result_count: 1, + result: [], + }; + + it("returns the first task on success", () => { + expect(assertOk({ status_code: 20000, tasks: [okTask] })).toBe(okTask); + }); + + it("throws DataforseoChargedTaskError when a charged task fails", () => { + const task = { + status_code: 40000, + status_message: "fail", + path: ["v3", "backlinks", "summary", "live"], + cost: 0.05, + result_count: 0, + }; + try { + assertOk({ status_code: 20000, tasks: [task] }); + throw new Error("expected assertOk to throw"); + } catch (error) { + expect(error).toBeInstanceOf(DataforseoChargedTaskError); + if (error instanceof DataforseoChargedTaskError) { + expect(error.billing).toEqual({ + path: task.path, + costUsd: 0.05, + }); + } + } + }); + + it("uses the classifier for non-charged (no-cost) failures", () => { + const classify = vi.fn(() => new AppError("BACKLINKS_NOT_ENABLED", "nope")); + const task = { + status_code: 40204, + status_message: "subscription required", + }; + expect(() => + assertOk( + { status_code: 20000, tasks: [task] }, + { classify, classifyPath: "/v3/backlinks/summary/live" }, + ), + ).toThrow("nope"); + expect(classify).toHaveBeenCalledWith( + 40204, + "subscription required", + "/v3/backlinks/summary/live", + ); + }); + + it.each([ + [40204, "BACKLINKS_NOT_ENABLED"], + [403, "BACKLINKS_NOT_ENABLED"], + [40200, "BACKLINKS_BILLING_ISSUE"], + ] as const)( + "uses the classifier for account failure %s before charging billed task metadata", + (status, code) => { + const classify = vi.fn( + () => new AppError(code, "Classified DataForSEO account failure"), + ); + const task = { + status_code: status, + status_message: "Backlinks subscription required", + path: ["v3", "backlinks", "summary", "live"], + cost: 0.05, + result_count: 0, + }; + + try { + assertOk({ status_code: 20000, tasks: [task] }, { classify }); + throw new Error("expected assertOk to throw"); + } catch (error) { + expect(error).not.toBeInstanceOf(DataforseoChargedTaskError); + expect(error).toMatchObject({ code }); + } + expect(classify).toHaveBeenCalledWith( + status, + "Backlinks subscription required", + "/v3/backlinks/summary/live", + ); + }, + ); + + it("treats 40501 as an empty success when asked", () => { + const task = { + status_code: 40501, + status_message: "No Search Results", + path: ["v3", "serp", "google", "organic", "live", "advanced"], + cost: 0.0, + }; + expect( + assertOk( + { status_code: 20000, tasks: [task] }, + { treatNoResultsAsEmpty: true }, + ), + ).toBe(task); + }); +}); diff --git a/src/server/lib/dataforseo/envelope.ts b/src/server/lib/dataforseo/envelope.ts new file mode 100644 index 0000000..bcb90c8 --- /dev/null +++ b/src/server/lib/dataforseo/envelope.ts @@ -0,0 +1,171 @@ +import { z } from "zod"; +import { AppError } from "@/server/lib/errors"; +import type { DataforseoErrorClassifier } from "@/server/lib/dataforseo/core"; + +// --------------------------------------------------------------------------- +// Billing envelope — the load-bearing seam that carries each call's USD cost +// out to the single metering point in client.ts. Every section fetcher returns +// DataforseoApiResponse; nothing else constructs a billing object. +// --------------------------------------------------------------------------- + +export type DataforseoApiCallCost = { + path: string[]; + costUsd: number; +}; + +export type DataforseoApiResponse = { + data: T; + billing: DataforseoApiCallCost; +}; + +/** + * Thrown when a DataForSEO task fails *after* it was billed (cost + path are + * present). meterDataforseoCall catches this to charge the customer for the + * failed-but-charged call before rethrowing. Do not throw this for access / + * balance failures; classify those first even when DataForSEO includes billing + * metadata on the failed task. + */ +export class DataforseoChargedTaskError extends AppError { + constructor( + message: string, + public readonly billing: DataforseoApiCallCost, + ) { + super("INTERNAL_ERROR", message); + this.name = "DataforseoChargedTaskError"; + } +} + +// The SDK types cost / path / result_count as optional with no runtime +// validation, so this is the one guard that guarantees we can bill a call. +const billingMetadataSchema = z.object({ + path: z.array(z.string()), + cost: z.number(), + result_count: z.number().nullable().optional(), +}); + +export interface DataforseoTaskLike { + status_code?: number; + status_message?: string; + path?: string[]; + cost?: number; + result_count?: number; + result?: unknown[]; + [key: string]: unknown; +} + +interface DataforseoResponseLike { + status_code?: number; + status_message?: string; + tasks?: T[]; + [key: string]: unknown; +} + +function tryBuildTaskBilling(task: unknown): DataforseoApiCallCost | null { + const parsed = billingMetadataSchema.safeParse(task); + if (!parsed.success) return null; + return { + path: parsed.data.path, + costUsd: parsed.data.cost, + }; +} + +export function buildTaskBilling( + task: DataforseoTaskLike, +): DataforseoApiCallCost { + const billing = tryBuildTaskBilling(task); + if (!billing) { + throw new AppError( + "INTERNAL_ERROR", + "DataForSEO task is missing billing metadata (path/cost)", + ); + } + return billing; +} + +type AssertOkOptions = { + /** Maps a recognised access / billing failure to a product error. */ + classify?: DataforseoErrorClassifier; + /** Request path string handed to the classifier (e.g. "/v3/backlinks/summary/live"). */ + classifyPath?: string; + /** Treat DataForSEO's "no search results" (40501) as an empty success. */ + treatNoResultsAsEmpty?: boolean; +}; + +/** + * Validates that the top-level response and its first task both succeeded, and + * returns that (SDK-typed) task. The single status / billing ladder shared by + * every endpoint: + * - access / balance failure -> classified AppError + * - charged-but-failed task (cost present) -> DataforseoChargedTaskError + */ +export function assertOk( + response: DataforseoResponseLike | null, + options: AssertOkOptions = {}, +): T { + if (!response) { + throw new AppError( + "INTERNAL_ERROR", + "DataForSEO returned an empty response", + ); + } + const { classify, classifyPath, treatNoResultsAsEmpty } = options; + + if (response.status_code !== 20000) { + const message = response.status_message || "DataForSEO request failed"; + throw ( + classify?.(response.status_code, message, classifyPath ?? "") ?? + new AppError("INTERNAL_ERROR", message) + ); + } + + const task = response.tasks?.[0]; + if (!task) { + throw new AppError("INTERNAL_ERROR", "DataForSEO response missing task"); + } + + if (task.status_code !== 20000) { + const isNoResults = + task.status_code === 40501 || + (task.status_message?.toLowerCase().includes("no search results") ?? + false); + if (treatNoResultsAsEmpty && isNoResults) return task; + + const message = task.status_message || "DataForSEO task failed"; + const path = classifyPath ?? (task.path ? `/${task.path.join("/")}` : ""); + const classified = classify?.(task.status_code, message, path); + if (classified) throw classified; + + const billing = tryBuildTaskBilling(task); + if (billing) throw new DataforseoChargedTaskError(message, billing); + + throw new AppError("INTERNAL_ERROR", message); + } + + return task; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +/** Reads `task.result[0].items`, validating against a Zod schema for loosely-typed endpoints. */ +export function parseTaskItems( + endpoint: string, + task: DataforseoTaskLike, + itemSchema: T, +): Array> { + const first = task.result?.[0]; + const items = isRecord(first) ? first.items : []; + const parsed = z.array(itemSchema).safeParse(items ?? []); + if (!parsed.success) { + console.error( + `dataforseo.${endpoint}.invalid-payload`, + parsed.error.issues.slice(0, 5), + ); + throw new AppError( + "INTERNAL_ERROR", + `DataForSEO ${endpoint} returned an invalid response shape`, + ); + } + return parsed.data; +} diff --git a/src/server/lib/dataforseo/index.ts b/src/server/lib/dataforseo/index.ts new file mode 100644 index 0000000..27aa81a --- /dev/null +++ b/src/server/lib/dataforseo/index.ts @@ -0,0 +1,33 @@ +// Public surface of the DataForSEO integration. Internals live in the +// per-section files (labs / serp / keywordsData / business / backlinks / ai / +// lighthouse); everything funnels through envelope.ts (status + billing) and is +// metered in client.ts. + +export { createDataforseoClient } from "@/server/lib/dataforseo/client"; + +export { + type LabsKeywordDataItem, + type DomainRankedKeywordItem, + type RelevantPagesItem, +} from "@/server/lib/dataforseo/labs"; + +export { + type SerpLiveItem, + type RankCheckResult, +} from "@/server/lib/dataforseo/serp"; + +export { + normalizeBacklinksTarget, + type BacklinksSummaryItem, + type BacklinksItem, + type ReferringDomainItem, + type DomainPageSummaryItem, + type BacklinksHistoryItem, +} from "@/server/lib/dataforseo/backlinks"; + +export { + buildLlmTarget, + CHATGPT_LANGUAGE_CODE, + CHATGPT_LOCATION_CODE, + type LlmPlatform, +} from "@/server/lib/dataforseo/ai"; diff --git a/src/server/lib/dataforseo/keywordsData.ts b/src/server/lib/dataforseo/keywordsData.ts new file mode 100644 index 0000000..ad814c6 --- /dev/null +++ b/src/server/lib/dataforseo/keywordsData.ts @@ -0,0 +1,34 @@ +import { + KeywordsDataGoogleAdsSearchVolumeLiveRequestInfo, + type KeywordsDataGoogleAdsSearchVolumeLiveResultInfo, +} from "dataforseo-client"; +import { keywordsDataApi } from "@/server/lib/dataforseo/core"; +import { + assertOk, + buildTaskBilling, + type DataforseoApiResponse, +} from "@/server/lib/dataforseo/envelope"; + +// The search_volume live response returns keyword rows directly as the task +// result array (no nested `items`). `competition` is a string enum here +// ("LOW"/"MEDIUM"/"HIGH"), distinct from the numeric Labs competition. +type KeywordSearchVolumeRow = KeywordsDataGoogleAdsSearchVolumeLiveResultInfo; + +export async function fetchKeywordSearchVolume(input: { + keywords: string[]; + locationCode?: number; + languageCode?: string; +}): Promise> { + const response = await keywordsDataApi().googleAdsSearchVolumeLive([ + new KeywordsDataGoogleAdsSearchVolumeLiveRequestInfo({ + keywords: input.keywords, + location_code: input.locationCode, + language_code: input.languageCode, + }), + ]); + const task = assertOk(response); + return { + data: task.result ?? [], + billing: buildTaskBilling(task), + }; +} diff --git a/src/server/lib/dataforseo/labs.ts b/src/server/lib/dataforseo/labs.ts new file mode 100644 index 0000000..2e522d3 --- /dev/null +++ b/src/server/lib/dataforseo/labs.ts @@ -0,0 +1,317 @@ +import { z } from "zod"; +import { + DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo, + DataforseoLabsGoogleKeywordIdeasLiveRequestInfo, + DataforseoLabsGoogleKeywordOverviewLiveRequestInfo, + DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo, + DataforseoLabsGoogleRankedKeywordsLiveRequestInfo, + DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo, + DataforseoLabsGoogleRelevantPagesLiveRequestInfo, + DataforseoLabsGoogleSerpCompetitorsLiveRequestInfo, + type DataforseoLabsDomainRankOverviewLiveItem, + type DataforseoLabsGoogleKeywordOverviewLiveItem, + type DataforseoLabsRelatedKeywordsLiveItem, + type DataforseoLabsRelevantPagesLiveItem, + type DataforseoLabsSerpCompetitorsLiveItem, + type KeywordDataInfo, +} from "dataforseo-client"; +import { labsApi } from "@/server/lib/dataforseo/core"; +import { + assertOk, + buildTaskBilling, + parseTaskItems, + type DataforseoApiResponse, +} from "@/server/lib/dataforseo/envelope"; + +// SDK item models are 1:1 supersets of what we need, so we expose them directly +// under the names the rest of the app already uses (no hand-written Zod). +export type LabsKeywordDataItem = KeywordDataInfo; +type RelatedKeywordItem = DataforseoLabsRelatedKeywordsLiveItem; +type DomainMetricsItem = DataforseoLabsDomainRankOverviewLiveItem; +export type RelevantPagesItem = DataforseoLabsRelevantPagesLiveItem; +type KeywordOverviewItem = DataforseoLabsGoogleKeywordOverviewLiveItem; +type SerpCompetitorItem = DataforseoLabsSerpCompetitorsLiveItem; + +// Ranked keywords is the one Labs endpoint the SDK types loosely: its +// `ranked_serp_element.serp_item` is the base element item, so the url / etv / +// rank fields we read are untyped (`any`). Keep a focused schema so the +// domain-keyword mapper stays type-safe. +const rankedSerpItemSchema = z + .object({ + url: z.string().nullable().optional(), + relative_url: z.string().nullable().optional(), + rank_absolute: z.number().nullable().optional(), + etv: z.number().nullable().optional(), + }) + .passthrough(); + +const domainRankedKeywordItemSchema = z + .object({ + keyword_data: z + .object({ + keyword: z.string().nullable().optional(), + keyword_info: z + .object({ + search_volume: z.number().nullable().optional(), + cpc: z.number().nullable().optional(), + keyword_difficulty: z.number().nullable().optional(), + }) + .passthrough() + .nullable() + .optional(), + keyword_properties: z + .object({ + keyword_difficulty: z.number().nullable().optional(), + }) + .passthrough() + .nullable() + .optional(), + }) + .passthrough() + .nullable() + .optional(), + ranked_serp_element: z + .object({ + serp_item: rankedSerpItemSchema.nullable().optional(), + url: z.string().nullable().optional(), + relative_url: z.string().nullable().optional(), + rank_absolute: z.number().nullable().optional(), + etv: z.number().nullable().optional(), + }) + .passthrough() + .nullable() + .optional(), + keyword: z.string().nullable().optional(), + }) + .passthrough(); + +export type DomainRankedKeywordItem = z.infer< + typeof domainRankedKeywordItemSchema +>; + +type DataforseoLabsItemType = + | "organic" + | "paid" + | "featured_snippet" + | "local_pack" + | "ai_overview_reference"; + +export async function fetchRelatedKeywords(input: { + keyword: string; + locationCode: number; + languageCode: string; + limit: number; + depth?: number; +}): Promise> { + const response = await labsApi().googleRelatedKeywordsLive([ + new DataforseoLabsGoogleRelatedKeywordsLiveRequestInfo({ + keyword: input.keyword, + location_code: input.locationCode, + language_code: input.languageCode, + limit: input.limit, + depth: input.depth ?? 3, + include_clickstream_data: true, + include_serp_info: false, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; +} + +export async function fetchKeywordSuggestions(input: { + keyword: string; + locationCode: number; + languageCode: string; + limit: number; +}): Promise> { + const response = await labsApi().googleKeywordSuggestionsLive([ + new DataforseoLabsGoogleKeywordSuggestionsLiveRequestInfo({ + keyword: input.keyword, + location_code: input.locationCode, + language_code: input.languageCode, + limit: input.limit, + include_clickstream_data: true, + include_serp_info: false, + include_seed_keyword: true, + ignore_synonyms: false, + exact_match: false, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; +} + +export async function fetchKeywordIdeas(input: { + keyword: string; + locationCode: number; + languageCode: string; + limit: number; +}): Promise> { + const response = await labsApi().googleKeywordIdeasLive([ + new DataforseoLabsGoogleKeywordIdeasLiveRequestInfo({ + keywords: [input.keyword], + location_code: input.locationCode, + language_code: input.languageCode, + limit: input.limit, + include_clickstream_data: true, + include_serp_info: false, + ignore_synonyms: false, + closely_variants: false, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; +} + +export async function fetchDomainRankOverview(input: { + target: string; + locationCode: number; + languageCode: string; +}): Promise> { + const response = await labsApi().googleDomainRankOverviewLive([ + new DataforseoLabsGoogleDomainRankOverviewLiveRequestInfo({ + target: input.target, + location_code: input.locationCode, + language_code: input.languageCode, + limit: 1, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; +} + +type RankedKeywordsPage = { + items: DomainRankedKeywordItem[]; + totalCount: number | null; +}; + +export async function fetchRankedKeywords(input: { + target: string; + locationCode: number; + languageCode: string; + limit: number; + offset?: number; + orderBy?: string[]; + filters?: unknown[]; + itemTypes?: DataforseoLabsItemType[]; + includeSubdomains?: boolean; +}): Promise> { + const response = await labsApi().googleRankedKeywordsLive([ + new DataforseoLabsGoogleRankedKeywordsLiveRequestInfo({ + target: input.target, + location_code: input.locationCode, + language_code: input.languageCode, + limit: input.limit, + offset: input.offset, + order_by: input.orderBy, + filters: input.filters, + item_types: input.itemTypes, + include_subdomains: input.includeSubdomains, + }), + ]); + const task = assertOk(response); + return { + data: { + items: parseTaskItems( + "google-ranked-keywords-live", + task, + domainRankedKeywordItemSchema, + ), + totalCount: task.result?.[0]?.total_count ?? null, + }, + billing: buildTaskBilling(task), + }; +} + +type RelevantPagesPage = { + items: RelevantPagesItem[]; + totalCount: number | null; +}; + +export async function fetchRelevantPages(input: { + target: string; + locationCode: number; + languageCode: string; + limit: number; + offset?: number; + orderBy?: string[]; + filters?: unknown[]; +}): Promise> { + const response = await labsApi().googleRelevantPagesLive([ + new DataforseoLabsGoogleRelevantPagesLiveRequestInfo({ + target: input.target, + location_code: input.locationCode, + language_code: input.languageCode, + limit: input.limit, + offset: input.offset, + order_by: input.orderBy, + filters: input.filters, + }), + ]); + const task = assertOk(response); + return { + data: { + items: task.result?.[0]?.items ?? [], + totalCount: task.result?.[0]?.total_count ?? null, + }, + billing: buildTaskBilling(task), + }; +} + +export async function fetchKeywordOverview(input: { + keywords: string[]; + locationCode: number; + languageCode: string; +}): Promise> { + const response = await labsApi().googleKeywordOverviewLive([ + new DataforseoLabsGoogleKeywordOverviewLiveRequestInfo({ + keywords: input.keywords, + location_code: input.locationCode, + language_code: input.languageCode, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; +} + +export async function fetchSerpCompetitors(input: { + keywords: string[]; + locationCode: number; + languageCode: string; + itemTypes?: DataforseoLabsItemType[]; + includeSubdomains?: boolean; + limit: number; + offset?: number; +}): Promise> { + const response = await labsApi().googleSerpCompetitorsLive([ + new DataforseoLabsGoogleSerpCompetitorsLiveRequestInfo({ + keywords: input.keywords, + location_code: input.locationCode, + language_code: input.languageCode, + item_types: input.itemTypes, + include_subdomains: input.includeSubdomains, + limit: input.limit, + offset: input.offset, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; +} diff --git a/src/server/lib/dataforseo/lighthouse.ts b/src/server/lib/dataforseo/lighthouse.ts new file mode 100644 index 0000000..f36d43e --- /dev/null +++ b/src/server/lib/dataforseo/lighthouse.ts @@ -0,0 +1,32 @@ +import { OnPageLighthouseLiveJsonRequestInfo } from "dataforseo-client"; +import { + parseDataforseoLighthousePayload, + requestCategories, + type LighthouseStrategy, +} from "@/server/lib/dataforseoLighthousePayload"; +import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload"; +import { onPageApi } from "@/server/lib/dataforseo/core"; +import { + assertOk, + buildTaskBilling, + type DataforseoApiResponse, +} from "@/server/lib/dataforseo/envelope"; + +export async function fetchLighthouseResult(input: { + url: string; + strategy: LighthouseStrategy; +}): Promise> { + const response = await onPageApi().lighthouseLiveJson([ + new OnPageLighthouseLiveJsonRequestInfo({ + url: input.url, + for_mobile: input.strategy === "mobile", + categories: [...requestCategories], + }), + ]); + + // assertOk handles status / charged-task billing; parse extracts the scores. + const task = assertOk(response); + const data = parseDataforseoLighthousePayload(response, input); + + return { data, billing: buildTaskBilling(task) }; +} diff --git a/src/server/lib/dataforseo/serp.ts b/src/server/lib/dataforseo/serp.ts new file mode 100644 index 0000000..3f18c0b --- /dev/null +++ b/src/server/lib/dataforseo/serp.ts @@ -0,0 +1,188 @@ +import { z } from "zod"; +import { + SerpGoogleLocalFinderLiveAdvancedRequestInfo, + SerpGoogleMapsLiveAdvancedRequestInfo, + SerpGoogleOrganicLiveAdvancedRequestInfo, +} from "dataforseo-client"; +import { serpApi } from "@/server/lib/dataforseo/core"; +import { + assertOk, + buildTaskBilling, + parseTaskItems, + type DataforseoApiResponse, +} from "@/server/lib/dataforseo/envelope"; + +// Kept as a hand-written schema: the SDK's BaseSerpApiElementItem type omits +// etv / estimated_paid_traffic_cost / backlinks_info / rank_changes, which we +// rely on. The fields survive deserialization (the SDK copies unknown keys), so +// validating here is both our type-safety guard and how we read those fields. +const serpSnapshotItemSchema = z + .object({ + type: z.string(), + rank_group: z.number().nullable().optional(), + rank_absolute: z.number().nullable().optional(), + domain: z.string().nullable().optional(), + title: z.string().nullable().optional(), + url: z.string().nullable().optional(), + description: z.string().nullable().optional(), + breadcrumb: z.string().nullable().optional(), + etv: z.number().nullable().optional(), + estimated_paid_traffic_cost: z.number().nullable().optional(), + backlinks_info: z + .object({ + referring_domains: z.number().nullable().optional(), + backlinks: z.number().nullable().optional(), + }) + .passthrough() + .nullable() + .optional(), + rank_changes: z + .object({ + previous_rank_absolute: z.number().nullable().optional(), + is_new: z.boolean().nullable().optional(), + is_up: z.boolean().nullable().optional(), + is_down: z.boolean().nullable().optional(), + }) + .passthrough() + .nullable() + .optional(), + }) + .passthrough(); + +export type SerpLiveItem = z.infer; + +export async function fetchLiveSerp(input: { + keyword: string; + locationCode: number; + languageCode: string; +}): Promise> { + const response = await serpApi().googleOrganicLiveAdvanced([ + new SerpGoogleOrganicLiveAdvancedRequestInfo({ + keyword: input.keyword, + location_code: input.locationCode, + language_code: input.languageCode, + device: "desktop", + os: "windows", + depth: 100, + }), + ]); + const task = assertOk(response); + return { + data: parseTaskItems( + "google-organic-live-advanced", + task, + serpSnapshotItemSchema, + ), + billing: buildTaskBilling(task), + }; +} + +export interface RankCheckResult { + keywordId: string; + keyword: string; + position: number | null; + url: string | null; + serpFeatures: string[]; +} + +export async function fetchRankCheckSerp(input: { + keyword: string; + keywordId: string; + locationCode: number; + languageCode: string; + device: "desktop" | "mobile"; + targetDomain: string; + depth: number; +}): Promise> { + const depth = Math.min(100, Math.max(10, input.depth)); + const response = await serpApi().googleOrganicLiveAdvanced([ + new SerpGoogleOrganicLiveAdvancedRequestInfo({ + keyword: input.keyword, + location_code: input.locationCode, + language_code: input.languageCode, + device: input.device, + os: input.device === "desktop" ? "windows" : "android", + depth, + }), + ]); + + // "No Search Results" (40501) is valid for obscure/new keywords — treat as an + // empty result set rather than failing the whole rank-tracking run. + const task = assertOk(response, { treatNoResultsAsEmpty: true }); + const items = parseTaskItems( + "google-organic-live-advanced", + task, + serpSnapshotItemSchema, + ); + + const target = input.targetDomain.toLowerCase(); + const organicMatch = items.find((item) => { + if (item.type !== "organic" || item.domain == null) return false; + const domain = item.domain.toLowerCase(); + return domain === target || domain.endsWith(`.${target}`); + }); + + return { + data: { + keywordId: input.keywordId, + keyword: input.keyword, + position: organicMatch + ? (organicMatch.rank_absolute ?? organicMatch.rank_group ?? null) + : null, + url: organicMatch?.url ?? null, + serpFeatures: [ + ...new Set(items.map((item) => item.type).filter(Boolean)), + ], + }, + billing: buildTaskBilling(task), + }; +} + +export async function fetchLocalSerp(input: { + keyword: string; + locationCoordinate?: string; + languageCode: string; + searchType: "maps" | "local_finder"; + device: "desktop" | "mobile"; + depth: number; + searchPlaces?: boolean; +}): Promise[]>> { + const os = input.device === "desktop" ? "windows" : "android"; + + // Maps and Local Finder return different SDK item models; both carry an index + // signature, so the typed items assign cleanly to the generic row shape. + if (input.searchType === "maps") { + const response = await serpApi().googleMapsLiveAdvanced([ + new SerpGoogleMapsLiveAdvancedRequestInfo({ + keyword: input.keyword, + location_coordinate: input.locationCoordinate, + language_code: input.languageCode, + device: input.device, + os, + depth: input.depth, + search_places: input.searchPlaces, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; + } + + const response = await serpApi().googleLocalFinderLiveAdvanced([ + new SerpGoogleLocalFinderLiveAdvancedRequestInfo({ + keyword: input.keyword, + location_coordinate: input.locationCoordinate, + language_code: input.languageCode, + device: input.device, + os, + depth: input.depth, + }), + ]); + const task = assertOk(response); + return { + data: task.result?.[0]?.items ?? [], + billing: buildTaskBilling(task), + }; +} diff --git a/src/server/lib/dataforseoBacklinks.ts b/src/server/lib/dataforseoBacklinks.ts deleted file mode 100644 index 7b5775b..0000000 --- a/src/server/lib/dataforseoBacklinks.ts +++ /dev/null @@ -1,292 +0,0 @@ -import { AppError } from "@/server/lib/errors"; -import { - normalizeBacklinksSpamFilterOptions, - type BacklinksSpamFilterOptions, -} from "@/types/schemas/backlinks"; -import type { - DataforseoApiCallCost, - DataforseoApiResponse, -} from "@/server/lib/dataforseoCost"; -import { getRequiredEnvValue } from "@/server/lib/runtime-env"; -import { - classifyBacklinksError, - type BacklinksTaskResult, - backlinksHistoryItemSchema, - backlinksItemSchema, - backlinksSummaryItemSchema, - domainPageSummaryItemSchema, - parseItems, - referringDomainItemSchema, - responseSchema, -} from "@/server/lib/dataforseoBacklinksSupport"; -export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget"; - -const API_BASE = "https://api.dataforseo.com"; - -export type BacklinksRequest = { - target: string; -}; - -export type BacklinksListRequest = BacklinksRequest & - BacklinksSpamFilterOptions & { - limit?: number; - }; - -export type BacklinksTimeseriesRequest = { - target: string; - dateFrom: string; - dateTo: string; -}; - -type DataforseoTaskResponse = { - results: BacklinksTaskResult[]; - billing: DataforseoApiCallCost; -}; - -async function createAuthenticatedFetch() { - const apiKey = await getRequiredEnvValue("DATAFORSEO_API_KEY"); - - return (url: RequestInfo, init?: RequestInit): Promise => { - const headers = new Headers(init?.headers); - headers.set("Authorization", `Basic ${apiKey}`); - - return fetch(url, { - ...init, - headers, - }); - }; -} - -async function postBacklinks(path: string, payload: unknown) { - const authenticatedFetch = await createAuthenticatedFetch(); - const response = await authenticatedFetch(`${API_BASE}${path}`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - }); - - const rawText = await response.text(); - if (!response.ok) { - const classifiedError = classifyBacklinksError( - response.status, - rawText, - path, - ); - if (classifiedError) throw classifiedError; - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO HTTP ${response.status} on ${path}`, - ); - } - - let raw: unknown; - try { - raw = JSON.parse(rawText); - } catch { - const classifiedError = classifyBacklinksError( - response.status, - rawText, - path, - ); - if (classifiedError) throw classifiedError; - console.error( - `dataforseo.${path}.non-json-response`, - rawText.slice(0, 800), - ); - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned a non-JSON response`, - ); - } - - const parsed = responseSchema.safeParse(raw); - if (!parsed.success) { - const classifiedError = classifyBacklinksError( - response.status, - rawText, - path, - ); - if (classifiedError) throw classifiedError; - console.error( - `dataforseo.${path}.invalid-top-level-shape`, - rawText.slice(0, 800), - ); - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned an invalid response shape`, - ); - } - - const responseData = parsed.data; - if (responseData.status_code !== 20000) { - const classifiedError = classifyBacklinksError( - responseData.status_code, - `${responseData.status_message ?? ""} ${rawText}`, - path, - ); - if (classifiedError) throw classifiedError; - throw new AppError( - "INTERNAL_ERROR", - responseData.status_message || "DataForSEO request failed", - ); - } - - const task = responseData.tasks?.[0]; - if (!task) { - throw new AppError("INTERNAL_ERROR", "DataForSEO response missing task"); - } - - if (task.status_code !== 20000) { - const classifiedError = classifyBacklinksError( - task.status_code, - `${task.status_message ?? ""} ${rawText}`, - path, - ); - if (classifiedError) throw classifiedError; - throw new AppError( - "INTERNAL_ERROR", - task.status_message || "DataForSEO task failed", - ); - } - - return { - results: (task.result ?? []).filter( - (r): r is BacklinksTaskResult => r != null, - ), - billing: { - path: task.path ?? [], - costUsd: task.cost ?? responseData.cost ?? 0, - resultCount: task.result_count ?? null, - }, - } satisfies DataforseoTaskResponse; -} - -function buildCommonPayload(input: BacklinksRequest) { - return { - target: input.target, - include_subdomains: true, - include_indirect_links: true, - exclude_internal_backlinks: true, - backlinks_status_type: "live", - rank_scale: "one_hundred", - }; -} - -export async function fetchBacklinksSummaryRaw(input: BacklinksRequest) { - const response = await postBacklinks("/v3/backlinks/summary/live", [ - buildCommonPayload(input), - ]); - const firstResult = response.results[0]; - const parsed = firstResult - ? backlinksSummaryItemSchema.safeParse(firstResult) - : null; - if (parsed && !parsed.success) { - console.error( - "dataforseo.backlinks-summary-live.invalid-result", - parsed.error.issues.slice(0, 5), - ); - throw new AppError( - "INTERNAL_ERROR", - "DataForSEO backlinks-summary-live returned an invalid response shape", - ); - } - const data = parsed?.data ?? {}; - return { - data, - billing: response.billing, - } satisfies DataforseoApiResponse; -} - -export async function fetchBacklinksRowsRaw(input: BacklinksListRequest) { - const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input); - const filters = spamFilterOptions.hideSpam - ? [["backlink_spam_score", "<=", spamFilterOptions.spamThreshold]] - : undefined; - const response = await postBacklinks("/v3/backlinks/backlinks/live", [ - { - ...buildCommonPayload(input), - limit: input.limit ?? 100, - order_by: ["rank,desc"], - ...(filters ? { filters } : {}), - }, - ]); - const data = parseItems( - "backlinks-live", - response.results, - backlinksItemSchema, - ); - return { - data, - billing: response.billing, - } satisfies DataforseoApiResponse; -} - -export async function fetchReferringDomainsRaw(input: BacklinksListRequest) { - const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input); - const filters = spamFilterOptions.hideSpam - ? [["backlinks_spam_score", "<=", spamFilterOptions.spamThreshold]] - : undefined; - const response = await postBacklinks("/v3/backlinks/referring_domains/live", [ - { - ...buildCommonPayload(input), - limit: input.limit ?? 100, - order_by: ["backlinks,desc"], - ...(filters ? { filters } : {}), - }, - ]); - const data = parseItems( - "referring-domains-live", - response.results, - referringDomainItemSchema, - ); - return { - data, - billing: response.billing, - } satisfies DataforseoApiResponse; -} - -export async function fetchDomainPagesSummaryRaw(input: BacklinksListRequest) { - const response = await postBacklinks( - "/v3/backlinks/domain_pages_summary/live", - [ - { - ...buildCommonPayload(input), - limit: input.limit ?? 100, - order_by: ["backlinks,desc"], - }, - ], - ); - const data = parseItems( - "domain-pages-summary-live", - response.results, - domainPageSummaryItemSchema, - ); - return { - data, - billing: response.billing, - } satisfies DataforseoApiResponse; -} - -export async function fetchBacklinksHistoryRaw( - input: BacklinksTimeseriesRequest, -) { - const response = await postBacklinks("/v3/backlinks/history/live", [ - { - target: input.target, - date_from: input.dateFrom, - date_to: input.dateTo, - rank_scale: "one_hundred", - }, - ]); - const data = parseItems( - "backlinks-history-live", - response.results, - backlinksHistoryItemSchema, - ); - return { - data, - billing: response.billing, - } satisfies DataforseoApiResponse; -} diff --git a/src/server/lib/dataforseoBacklinksSupport.ts b/src/server/lib/dataforseoBacklinksSupport.ts deleted file mode 100644 index 8fff3f8..0000000 --- a/src/server/lib/dataforseoBacklinksSupport.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { z } from "zod"; -import { createDataforseoAccessClassifier } from "@/server/lib/dataforseoAccessClassification"; -import { AppError } from "@/server/lib/errors"; - -const taskResultSchema = z - .object({ - items: z.array(z.unknown()).nullable().optional(), - }) - .passthrough(); - -export type BacklinksTaskResult = z.infer; - -const taskSchema = z - .object({ - status_code: z.number().optional(), - status_message: z.string().optional(), - cost: z.number().nullable().optional(), - result_count: z.number().nullable().optional(), - path: z.array(z.string()).optional(), - result: z.array(taskResultSchema.nullable()).nullable().optional(), - }) - .passthrough(); - -export const responseSchema = z - .object({ - status_code: z.number().optional(), - status_message: z.string().optional(), - cost: z.number().nullable().optional(), - tasks: z.array(taskSchema).optional(), - }) - .passthrough(); - -export const backlinksSummaryItemSchema = z - .object({ - target: z.string().optional(), - rank: z.number().nullable().optional(), - backlinks: z.number().nullable().optional(), - referring_pages: z.number().nullable().optional(), - referring_domains: z.number().nullable().optional(), - broken_backlinks: z.number().nullable().optional(), - broken_pages: z.number().nullable().optional(), - new_backlinks: z.number().nullable().optional(), - lost_backlinks: z.number().nullable().optional(), - new_reffering_domains: z.number().nullable().optional(), - lost_reffering_domains: z.number().nullable().optional(), - new_referring_domains: z.number().nullable().optional(), - lost_referring_domains: z.number().nullable().optional(), - backlinks_spam_score: z.number().nullable().optional(), - info: z - .object({ - target_spam_score: z.number().nullable().optional(), - }) - .passthrough() - .nullable() - .optional(), - }) - .passthrough(); - -export const backlinksItemSchema = z - .object({ - domain_from: z.string().nullable().optional(), - url_from: z.string().nullable().optional(), - url_to: z.string().nullable().optional(), - anchor: z.string().nullable().optional(), - item_type: z.string().nullable().optional(), - dofollow: z.boolean().nullable().optional(), - rank: z.number().nullable().optional(), - domain_from_rank: z.number().nullable().optional(), - page_from_rank: z.number().nullable().optional(), - backlinks_spam_score: z.number().nullable().optional(), - backlink_spam_score: z.number().nullable().optional(), - first_seen: z.string().nullable().optional(), - last_visited: z.string().nullable().optional(), - lost_date: z.string().nullable().optional(), - is_new: z.boolean().nullable().optional(), - is_lost: z.boolean().nullable().optional(), - is_broken: z.boolean().nullable().optional(), - links_count: z.number().nullable().optional(), - rel_attributes: z.array(z.string()).nullable().optional(), - attributes: z.array(z.string()).nullable().optional(), - }) - .passthrough(); - -export const referringDomainItemSchema = z - .object({ - domain: z.string().nullable().optional(), - backlinks: z.number().nullable().optional(), - referring_pages: z.number().nullable().optional(), - rank: z.number().nullable().optional(), - first_seen: z.string().nullable().optional(), - broken_backlinks: z.number().nullable().optional(), - broken_pages: z.number().nullable().optional(), - backlinks_spam_score: z.number().nullable().optional(), - target_spam_score: z.number().nullable().optional(), - }) - .passthrough(); - -export const domainPageSummaryItemSchema = z - .object({ - page: z.string().nullable().optional(), - url: z.string().nullable().optional(), - backlinks: z.number().nullable().optional(), - referring_domains: z.number().nullable().optional(), - rank: z.number().nullable().optional(), - broken_backlinks: z.number().nullable().optional(), - }) - .passthrough(); - -export const backlinksHistoryItemSchema = z - .object({ - date: z.string().nullable().optional(), - rank: z.number().nullable().optional(), - backlinks: z.number().nullable().optional(), - referring_domains: z.number().nullable().optional(), - new_backlinks: z.number().nullable().optional(), - lost_backlinks: z.number().nullable().optional(), - new_reffering_domains: z.number().nullable().optional(), - lost_reffering_domains: z.number().nullable().optional(), - new_referring_domains: z.number().nullable().optional(), - lost_referring_domains: z.number().nullable().optional(), - }) - .passthrough(); - -export const classifyBacklinksError = createDataforseoAccessClassifier({ - pathPrefix: "/backlinks/", - notEnabledCode: "BACKLINKS_NOT_ENABLED", - notEnabledMessage: - "Backlinks is not enabled for the connected DataForSEO account", - billingIssueCode: "BACKLINKS_BILLING_ISSUE", - billingIssueMessage: - "The connected DataForSEO account has a billing or balance issue", -}); - -export function parseItems( - endpointName: string, - results: BacklinksTaskResult[], - itemSchema: T, -): Array> { - const firstResult = results[0] ?? null; - if (firstResult == null) { - return []; - } - - const parsed = z.array(itemSchema).safeParse(firstResult.items ?? []); - if (!parsed.success) { - console.error( - `dataforseo.${endpointName}.invalid-items`, - parsed.error.issues.slice(0, 5), - ); - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${endpointName} returned an invalid response shape`, - ); - } - - return parsed.data; -} diff --git a/src/server/lib/dataforseoClient.ts b/src/server/lib/dataforseoClient.ts deleted file mode 100644 index a1ac929..0000000 --- a/src/server/lib/dataforseoClient.ts +++ /dev/null @@ -1,470 +0,0 @@ -/* eslint-disable max-lines, max-lines-per-function */ -import { - AUTUMN_SEO_DATA_BALANCE_FEATURE_ID, - AUTUMN_SEO_DATA_CREDITS_PER_USD, - AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID, - SEO_DATA_COST_MARKUP, - roundUsdForBilling, -} from "@/shared/billing"; -import { - type CreditFeature, - mapDataforseoPathToCreditFeature, -} from "@/shared/billing-credit-features"; -import { autumn } from "@/server/billing/autumn"; -import { getOrCreateOrganizationCustomer } from "@/server/billing/subscription"; -import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { - fetchKeywordIdeasRaw, - fetchKeywordOverviewRaw, - fetchKeywordSuggestionsRaw, - fetchRelatedKeywordsRaw, - fetchBusinessListingsSearchRaw, - fetchBusinessQuestionsAnswersRaw, - fetchDomainRankOverviewRaw, - fetchKeywordSearchVolumeRaw, - fetchLocalSerpItemsRaw, - fetchRankedKeywordsRaw, - fetchRelevantPagesRaw, - fetchSerpCompetitorsRaw, - type DataforseoLabsItemType, - fetchLiveSerpItemsRaw, - fetchRankCheckSerpRaw, - type LabsKeywordDataItem, - type SerpLiveItem, -} from "@/server/lib/dataforseo"; -import { - fetchLlmAggregatedMetricsRaw, - fetchLlmMentionsSearchRaw, - fetchLlmResponseRaw, - fetchLlmTopPagesRaw, - type LlmAggregatedMetricsInput, - type LlmMentionsSearchInput, - type LlmResponsesInput, - type LlmTopPagesInput, -} from "@/server/lib/dataforseoLlm"; -import { fetchDataforseoLighthouseResultRaw } from "@/server/lib/dataforseoLighthouse"; -import type { LighthouseStrategy } from "@/server/lib/dataforseoLighthousePayload"; -import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload"; -import { - fetchBacklinksHistoryRaw, - fetchBacklinksRowsRaw, - fetchBacklinksSummaryRaw, - fetchDomainPagesSummaryRaw, - fetchReferringDomainsRaw, - type BacklinksListRequest, - type BacklinksRequest, - type BacklinksTimeseriesRequest, -} from "@/server/lib/dataforseoBacklinks"; -import { - type DataforseoApiResponse, - type DataforseoApiCallCost, - DataforseoChargedTaskError, -} from "@/server/lib/dataforseoCost"; -import { AppError } from "@/server/lib/errors"; -import { captureServerEvent } from "@/server/lib/posthog"; -import { isHostedServerAuthMode } from "@/server/lib/runtime-env"; - -export { mapDataforseoPathToCreditFeature }; - -export function createDataforseoClient(customer: BillingCustomerContext) { - return { - business: { - businessListings(input: { - categories?: string[]; - title?: string; - locationCoordinate: string; - orderBy?: string[]; - limit: number; - }) { - return meterDataforseoCall( - customer, - () => fetchBusinessListingsSearchRaw(input), - "local_seo", - ); - }, - questionsAnswers(input: { - keyword: string; - locationCoordinate: string; - languageCode: string; - depth: number; - }) { - return meterDataforseoCall( - customer, - () => fetchBusinessQuestionsAnswersRaw(input), - "local_seo", - ); - }, - }, - backlinks: { - summary(input: BacklinksRequest) { - return meterDataforseoCall(customer, () => - fetchBacklinksSummaryRaw(input), - ); - }, - rows(input: BacklinksListRequest) { - return meterDataforseoCall(customer, () => - fetchBacklinksRowsRaw(input), - ); - }, - referringDomains(input: BacklinksListRequest) { - return meterDataforseoCall(customer, () => - fetchReferringDomainsRaw(input), - ); - }, - domainPages(input: BacklinksListRequest) { - return meterDataforseoCall(customer, () => - fetchDomainPagesSummaryRaw(input), - ); - }, - history(input: BacklinksTimeseriesRequest) { - return meterDataforseoCall(customer, () => - fetchBacklinksHistoryRaw(input), - ); - }, - }, - keywords: { - related(input: { - keyword: string; - locationCode: number; - languageCode: string; - limit: number; - depth?: number; - }) { - return meterDataforseoCall(customer, () => - fetchRelatedKeywordsRaw( - input.keyword, - input.locationCode, - input.languageCode, - input.limit, - input.depth, - ), - ); - }, - suggestions(input: { - keyword: string; - locationCode: number; - languageCode: string; - limit: number; - }) { - return meterDataforseoCall(customer, () => - fetchKeywordSuggestionsRaw( - input.keyword, - input.locationCode, - input.languageCode, - input.limit, - ), - ); - }, - ideas(input: { - keyword: string; - locationCode: number; - languageCode: string; - limit: number; - }) { - return meterDataforseoCall(customer, () => - fetchKeywordIdeasRaw( - input.keyword, - input.locationCode, - input.languageCode, - input.limit, - ), - ); - }, - }, - domain: { - rankOverview(input: { - target: string; - locationCode: number; - languageCode: string; - }) { - return meterDataforseoCall(customer, () => - fetchDomainRankOverviewRaw( - input.target, - input.locationCode, - input.languageCode, - ), - ); - }, - rankedKeywords(input: { - target: string; - locationCode: number; - languageCode: string; - limit: number; - offset?: number; - orderBy?: string[]; - filters?: unknown[]; - itemTypes?: DataforseoLabsItemType[]; - includeSubdomains?: boolean; - }) { - return meterDataforseoCall(customer, () => - fetchRankedKeywordsRaw(input), - ); - }, - relevantPages(input: { - target: string; - locationCode: number; - languageCode: string; - limit: number; - offset?: number; - orderBy?: string[]; - filters?: unknown[]; - }) { - return meterDataforseoCall(customer, () => - fetchRelevantPagesRaw(input), - ); - }, - }, - serp: { - live(input: { - keyword: string; - locationCode: number; - languageCode: string; - }) { - return meterDataforseoCall(customer, () => - fetchLiveSerpItemsRaw( - input.keyword, - input.locationCode, - input.languageCode, - ), - ); - }, - rankCheck(input: { - keyword: string; - keywordId: string; - locationCode: number; - languageCode: string; - device: "desktop" | "mobile"; - targetDomain: string; - depth: number; - }) { - return meterDataforseoCall( - customer, - () => fetchRankCheckSerpRaw(input), - "rank_tracking", - ); - }, - local(input: { - keyword: string; - locationCoordinate?: string; - languageCode: string; - searchType: "maps" | "local_finder"; - device: "desktop" | "mobile"; - depth: number; - searchPlaces?: boolean; - }) { - return meterDataforseoCall( - customer, - () => fetchLocalSerpItemsRaw(input), - "local_seo", - ); - }, - }, - keywordData: { - searchVolume(input: { - keywords: string[]; - locationCode?: number; - languageCode?: string; - }) { - return meterDataforseoCall(customer, () => - fetchKeywordSearchVolumeRaw(input), - ); - }, - }, - labs: { - keywordOverview(input: { - keywords: string[]; - locationCode: number; - languageCode: string; - }) { - return meterDataforseoCall( - customer, - () => - fetchKeywordOverviewRaw( - input.keywords, - input.locationCode, - input.languageCode, - ), - "rank_tracking", - ); - }, - serpCompetitors(input: { - keywords: string[]; - locationCode: number; - languageCode: string; - itemTypes?: DataforseoLabsItemType[]; - includeSubdomains?: boolean; - limit: number; - offset?: number; - }) { - return meterDataforseoCall(customer, () => - fetchSerpCompetitorsRaw(input), - ); - }, - }, - lighthouse: { - live(input: { url: string; strategy: LighthouseStrategy }) { - return meterDataforseoCall(customer, () => - fetchDataforseoLighthouseResultRaw(input), - ); - }, - }, - aiSearch: { - mentionsSearch(input: LlmMentionsSearchInput) { - return meterDataforseoCall(customer, () => - fetchLlmMentionsSearchRaw(input), - ); - }, - aggregatedMetrics(input: LlmAggregatedMetricsInput) { - return meterDataforseoCall(customer, () => - fetchLlmAggregatedMetricsRaw(input), - ); - }, - topPages(input: LlmTopPagesInput) { - return meterDataforseoCall(customer, () => fetchLlmTopPagesRaw(input)); - }, - llmResponse(input: LlmResponsesInput) { - return meterDataforseoCall(customer, () => fetchLlmResponseRaw(input)); - }, - }, - } as const; -} - -async function meterDataforseoCall( - customer: BillingCustomerContext, - execute: () => Promise>, - creditFeature?: CreditFeature, -): Promise { - const isHostedMode = await isHostedServerAuthMode(); - - if (!isHostedMode) { - const result = await execute(); - return result.data; - } - - const billingCustomer = await getOrCreateOrganizationCustomer(customer); - - const { monthlyRemaining } = await assertSeoDataBalanceAvailable( - billingCustomer.id, - ); - - let result: DataforseoApiResponse; - try { - result = await execute(); - } catch (error) { - if (error instanceof DataforseoChargedTaskError) { - await trackDataforseoCost({ - customer, - customerId: billingCustomer.id, - billing: error.billing, - monthlyRemaining, - creditFeature, - }); - } - throw error; - } - - await trackDataforseoCost({ - customer, - customerId: billingCustomer.id, - billing: result.billing, - monthlyRemaining, - creditFeature, - }); - - return result.data; -} - -async function assertSeoDataBalanceAvailable(customerId: string) { - const [monthlyCheck, topupCheck] = await Promise.all([ - autumn.check({ - customerId, - featureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID, - }), - autumn.check({ - customerId, - featureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID, - }), - ]); - - const monthlyRemaining = monthlyCheck.balance?.remaining ?? 0; - const topupRemaining = topupCheck.balance?.remaining ?? 0; - - if (monthlyRemaining + topupRemaining <= 0) { - throw new AppError("INSUFFICIENT_CREDITS"); - } - - return { monthlyRemaining }; -} - -async function trackDataforseoCost(args: { - customer: BillingCustomerContext; - customerId: string; - billing: DataforseoApiCallCost; - monthlyRemaining: number; - creditFeature?: CreditFeature; -}) { - const totalCostUsd = roundUsdForBilling( - args.billing.costUsd * SEO_DATA_COST_MARKUP, - ); - const totalCostCredits = Math.ceil( - totalCostUsd * AUTUMN_SEO_DATA_CREDITS_PER_USD, - ); - - const monthlyDeduct = Math.min(args.monthlyRemaining, totalCostCredits); - const topupDeduct = totalCostCredits - monthlyDeduct; - - const creditFeature = - args.creditFeature ?? mapDataforseoPathToCreditFeature(args.billing.path); - - const properties = { - provider: "dataforseo", - currency: "USD", - paths: [args.billing.path.join("/")], - creditFeature, - totalCostUsd, - totalCostCredits, - fromCache: false, - }; - - if (monthlyDeduct > 0) { - await autumn.track({ - customerId: args.customerId, - featureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID, - value: monthlyDeduct, - properties: { - ...properties, - balanceFeatureId: AUTUMN_SEO_DATA_BALANCE_FEATURE_ID, - }, - }); - } - - if (topupDeduct > 0) { - await autumn.track({ - customerId: args.customerId, - featureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID, - value: topupDeduct, - properties: { - ...properties, - balanceFeatureId: AUTUMN_SEO_DATA_TOPUP_BALANCE_FEATURE_ID, - }, - }); - } - - if (totalCostCredits > 0) { - await captureServerEvent({ - distinctId: args.customer.userId, - event: "usage:credits_consume", - organizationId: args.customer.organizationId, - properties: { - project_id: args.customer.projectId, - credit_feature: creditFeature, - monthly_credits: monthlyDeduct, - topup_credits: topupDeduct, - total_credits: totalCostCredits, - cost_usd: totalCostUsd, - }, - }); - } -} - -export type { LabsKeywordDataItem, SerpLiveItem }; diff --git a/src/server/lib/dataforseoCost.ts b/src/server/lib/dataforseoCost.ts deleted file mode 100644 index 7701f89..0000000 --- a/src/server/lib/dataforseoCost.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { AppError } from "@/server/lib/errors"; - -export type DataforseoApiCallCost = { - path: string[]; - costUsd: number; - resultCount: number | null; -}; - -export type DataforseoApiResponse = { - data: T; - billing: DataforseoApiCallCost; -}; - -export class DataforseoChargedTaskError extends AppError { - constructor( - message: string, - public readonly billing: DataforseoApiCallCost, - ) { - super("INTERNAL_ERROR", message); - this.name = "DataforseoChargedTaskError"; - } -} diff --git a/src/server/lib/dataforseoLighthouse.ts b/src/server/lib/dataforseoLighthouse.ts deleted file mode 100644 index 185c203..0000000 --- a/src/server/lib/dataforseoLighthouse.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { env } from "cloudflare:workers"; -import { - parseDataforseoLighthousePayload, - requestCategories, - type LighthouseStrategy, -} from "@/server/lib/dataforseoLighthousePayload"; -import type { DataforseoApiResponse } from "@/server/lib/dataforseoCost"; -import type { StoredLighthousePayload } from "@/server/lib/lighthouseStoredPayload"; - -const DATAFORSEO_LIGHTHOUSE_ENDPOINT = - "https://api.dataforseo.com/v3/on_page/lighthouse/live/json"; - -export async function fetchDataforseoLighthouseResultRaw(input: { - url: string; - strategy: LighthouseStrategy; -}): Promise> { - const response = await fetch(DATAFORSEO_LIGHTHOUSE_ENDPOINT, { - method: "POST", - headers: { - Authorization: `Basic ${env.DATAFORSEO_API_KEY?.trim() ?? ""}`, - "Content-Type": "application/json", - }, - body: JSON.stringify([ - { - url: input.url, - for_mobile: input.strategy === "mobile", - categories: requestCategories, - }, - ]), - signal: AbortSignal.timeout(60_000), - }); - - const rawText = await response.text(); - - if (!response.ok) { - throw new Error( - `DataForSEO Lighthouse request failed (${response.status}): ${rawText}`, - ); - } - - let payload: unknown; - try { - payload = JSON.parse(rawText); - } catch { - throw new Error( - `DataForSEO Lighthouse returned non-JSON content (content-type: ${response.headers.get("content-type") ?? "unknown"}): ${rawText}`, - ); - } - - const data = parseDataforseoLighthousePayload(payload, input); - - return { - data, - billing: { - path: ["v3", "on_page", "lighthouse", "live", "json"], - costUsd: data.metadata.cost ?? 0, - resultCount: 1, - }, - }; -} diff --git a/src/server/lib/dataforseoLlm.ts b/src/server/lib/dataforseoLlm.ts deleted file mode 100644 index 39ca7ac..0000000 --- a/src/server/lib/dataforseoLlm.ts +++ /dev/null @@ -1,382 +0,0 @@ -import { z } from "zod"; -import { - llmAggregatedTotalSchema, - llmMentionItemSchema, - llmResponseEnvelopeSchema, - llmResponseResultSchema, - llmTopPagesItemSchema, - type LlmAggregatedTotal, - type LlmDataforseoTask, - type LlmMentionItem, - type LlmResponseResult, - type LlmTopPagesItem, -} from "@/server/lib/dataforseoLlmSchemas"; -import type { DataforseoApiResponse } from "@/server/lib/dataforseoCost"; -import { createDataforseoAccessClassifier } from "@/server/lib/dataforseoAccessClassification"; -import { AppError } from "@/server/lib/errors"; -import { getRequiredEnvValue } from "@/server/lib/runtime-env"; - -/** - * Raw HTTP wrappers for DataForSEO AI Optimization endpoints. - * - * The official `dataforseo-client` SDK doesn't ship typed bindings for these - * endpoints yet, so we POST raw JSON. Every wrapper returns billing metadata - * alongside parsed data so the calling client can meter usage with Autumn. - */ - -const API_BASE = "https://api.dataforseo.com"; -const MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH = 1600; - -// ChatGPT mention/response data is only available for US/en per DataForSEO docs. -export const CHATGPT_LOCATION_CODE = 2840; -export const CHATGPT_LANGUAGE_CODE = "en"; - -export type LlmPlatform = "chat_gpt" | "google"; - -// --------------------------------------------------------------------------- -// Shared HTTP / response handling -// --------------------------------------------------------------------------- - -async function postLlm(path: string, payload: unknown): Promise { - const apiKey = await getRequiredEnvValue("DATAFORSEO_API_KEY"); - const response = await fetch(`${API_BASE}${path}`, { - method: "POST", - headers: { - Authorization: `Basic ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - }); - - const rawText = await response.text(); - - if (!response.ok) { - throw ( - classifyAiSearchError(response.status, rawText, path) ?? - new AppError( - "INTERNAL_ERROR", - `DataForSEO HTTP ${response.status} on ${path}. Response: ${truncate(rawText)}`, - ) - ); - } - - try { - return JSON.parse(rawText); - } catch { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned non-JSON response: ${truncate(rawText)}`, - ); - } -} - -const classifyAiSearchError = createDataforseoAccessClassifier({ - pathPrefix: "/ai_optimization/", - notEnabledCode: "AI_SEARCH_NOT_ENABLED", - notEnabledMessage: - "AI Optimization is not enabled for the connected DataForSEO account", - billingIssueCode: "AI_SEARCH_BILLING_ISSUE", - billingIssueMessage: - "The connected DataForSEO account has a billing or balance issue", -}); - -function truncate(text: string): string { - return text.length > MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH - ? `${text.slice(0, MAX_DATAFORSEO_ERROR_PAYLOAD_LENGTH)}... [truncated]` - : text; -} - -function parseEnvelope(path: string, raw: unknown): LlmDataforseoTask { - const envelope = llmResponseEnvelopeSchema.safeParse(raw); - if (!envelope.success) { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned an invalid envelope: ${envelope.error.issues - .slice(0, 3) - .map((i) => i.message) - .join("; ")}`, - ); - } - - const data = envelope.data; - if (data.status_code !== 20000) { - const message = data.status_message || `DataForSEO ${path} request failed`; - throw ( - classifyAiSearchError(data.status_code, message, path) ?? - new AppError("INTERNAL_ERROR", message) - ); - } - - const task = data.tasks?.[0]; - if (!task) { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} response missing task`, - ); - } - - if (task.status_code !== 20000) { - const message = task.status_message || `DataForSEO ${path} task failed`; - throw ( - classifyAiSearchError(task.status_code, message, path) ?? - new AppError("INTERNAL_ERROR", message) - ); - } - - return task; -} - -function buildBilling(task: LlmDataforseoTask) { - return { - path: task.path, - costUsd: task.cost, - resultCount: task.result_count ?? null, - }; -} - -// --------------------------------------------------------------------------- -// Target builders — DataForSEO's `target` array accepts domain OR keyword -// entries. We always pass exactly one target per call. -// --------------------------------------------------------------------------- - -type LlmTarget = - | { - domain: string; - include_subdomains?: boolean; - search_filter?: "include" | "exclude"; - search_scope?: string[]; - } - | { - keyword: string; - search_filter?: "include" | "exclude"; - search_scope?: string[]; - match_type?: "word_match" | "partial_match"; - }; - -export function buildLlmTarget(input: { - type: "domain" | "keyword"; - value: string; -}): LlmTarget { - if (input.type === "domain") { - return { - domain: input.value, - include_subdomains: true, - search_filter: "include", - search_scope: ["any"], - }; - } - return { - keyword: input.value, - search_filter: "include", - search_scope: ["any", "brand_entities"], - match_type: "word_match", - }; -} - -// --------------------------------------------------------------------------- -// LLM Mentions Search Live -// --------------------------------------------------------------------------- - -export type LlmMentionsSearchInput = { - target: LlmTarget; - platform: LlmPlatform; - locationCode: number; - languageCode: string; - limit?: number; -}; - -export async function fetchLlmMentionsSearchRaw( - input: LlmMentionsSearchInput, -): Promise> { - const path = "/v3/ai_optimization/llm_mentions/search/live"; - const payload = [ - { - target: [input.target], - platform: input.platform, - location_code: input.locationCode, - language_code: input.languageCode, - limit: clampLimit(input.limit ?? 100, 1, 1000), - }, - ]; - - const raw = await postLlm(path, payload); - const task = parseEnvelope(path, raw); - - const items = z - .array(llmMentionItemSchema) - .safeParse(extractItems(task.result)); - if (!items.success) { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned an invalid mention items shape`, - ); - } - - return { data: items.data, billing: buildBilling(task) }; -} - -// --------------------------------------------------------------------------- -// LLM Mentions Aggregated Metrics Live -// --------------------------------------------------------------------------- - -export type LlmAggregatedMetricsInput = { - target: LlmTarget; - platform: LlmPlatform; - locationCode: number; - languageCode: string; - internalListLimit?: number; -}; - -export async function fetchLlmAggregatedMetricsRaw( - input: LlmAggregatedMetricsInput, -): Promise> { - const path = "/v3/ai_optimization/llm_mentions/aggregated_metrics/live"; - const payload = [ - { - target: [input.target], - platform: input.platform, - location_code: input.locationCode, - language_code: input.languageCode, - internal_list_limit: clampLimit(input.internalListLimit ?? 10, 1, 20), - }, - ]; - - const raw = await postLlm(path, payload); - const task = parseEnvelope(path, raw); - - const totalRaw = extractFirstResult(task.result)?.total ?? {}; - const total = llmAggregatedTotalSchema.safeParse(totalRaw); - if (!total.success) { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned an invalid aggregated metrics shape`, - ); - } - - return { data: total.data, billing: buildBilling(task) }; -} - -// --------------------------------------------------------------------------- -// LLM Mentions Top Pages Live -// --------------------------------------------------------------------------- - -export type LlmTopPagesInput = { - target: LlmTarget; - platform: LlmPlatform; - locationCode: number; - languageCode: string; - itemsListLimit?: number; -}; - -export async function fetchLlmTopPagesRaw( - input: LlmTopPagesInput, -): Promise> { - const path = "/v3/ai_optimization/llm_mentions/top_pages/live"; - const payload = [ - { - target: [input.target], - platform: input.platform, - location_code: input.locationCode, - language_code: input.languageCode, - links_scope: "sources", - items_list_limit: clampLimit(input.itemsListLimit ?? 10, 1, 10), - internal_list_limit: 5, - }, - ]; - - const raw = await postLlm(path, payload); - const task = parseEnvelope(path, raw); - - const items = z - .array(llmTopPagesItemSchema) - .safeParse(extractFirstResult(task.result)?.items ?? []); - if (!items.success) { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned an invalid top pages shape`, - ); - } - - return { data: items.data, billing: buildBilling(task) }; -} - -// --------------------------------------------------------------------------- -// LLM Responses Live (per-model) -// --------------------------------------------------------------------------- - -export type LlmResponseModelSlug = - | "chat_gpt" - | "claude" - | "gemini" - | "perplexity"; - -export type LlmResponsesInput = { - userPrompt: string; - modelSlug: LlmResponseModelSlug; - modelName: string; - webSearch?: boolean; - maxOutputTokens?: number; - /** Two-letter ISO country code used to geolocate the web-search component. */ - webSearchCountryCode?: string; -}; - -export async function fetchLlmResponseRaw( - input: LlmResponsesInput, -): Promise> { - const path = `/v3/ai_optimization/${input.modelSlug}/llm_responses/live`; - // DataForSEO's Gemini endpoint rejects `web_search_country_iso_code` with a - // 40501 "Invalid Field" error. The other three models accept it. - const supportsCountry = input.modelSlug !== "gemini"; - const payload = [ - { - user_prompt: input.userPrompt, - model_name: input.modelName, - web_search: input.webSearch ?? true, - max_output_tokens: clampLimit(input.maxOutputTokens ?? 1024, 256, 4096), - ...(supportsCountry && - input.webSearchCountryCode && { - web_search_country_iso_code: input.webSearchCountryCode, - }), - }, - ]; - - const raw = await postLlm(path, payload); - const task = parseEnvelope(path, raw); - - const first = extractFirstResult(task.result) as unknown; - const result = llmResponseResultSchema.safeParse(first ?? {}); - if (!result.success) { - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${path} returned an invalid response shape`, - ); - } - - return { data: result.data, billing: buildBilling(task) }; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function clampLimit(value: number, min: number, max: number): number { - return Math.min(max, Math.max(min, Math.floor(value))); -} - -function extractFirstResult( - result: LlmDataforseoTask["result"], -): Record | null { - const first = result?.[0]; - return isRecord(first) ? first : null; -} - -function extractItems(result: LlmDataforseoTask["result"]): unknown[] { - const first = extractFirstResult(result); - if (!first) return []; - const items = first.items; - return Array.isArray(items) ? items : []; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; -} diff --git a/src/server/lib/dataforseoLlmSchemas.ts b/src/server/lib/dataforseoLlmSchemas.ts index 6d9e0fb..486fe9c 100644 --- a/src/server/lib/dataforseoLlmSchemas.ts +++ b/src/server/lib/dataforseoLlmSchemas.ts @@ -129,31 +129,3 @@ export const llmResponseResultSchema = z .passthrough(); export type LlmResponseResult = z.infer; - -// --------------------------------------------------------------------------- -// Top-level envelope used by every AI Optimization endpoint. -// We re-declare here (instead of reusing dataforseoSchemas.ts) because the -// `result` shape differs from Labs/SERP — items are not always under -// `result[0].items` and totals/items can both be present. -// --------------------------------------------------------------------------- - -const llmTaskSchema = z - .object({ - status_code: z.number().optional(), - status_message: z.string().optional(), - path: z.array(z.string()), - cost: z.number(), - result_count: z.number().nullable().optional(), - result: z.array(z.unknown()).nullable().optional(), - }) - .passthrough(); - -export type LlmDataforseoTask = z.infer; - -export const llmResponseEnvelopeSchema = z - .object({ - status_code: z.number().optional(), - status_message: z.string().optional(), - tasks: z.array(llmTaskSchema).optional(), - }) - .passthrough(); diff --git a/src/server/lib/dataforseoSchemas.test.ts b/src/server/lib/dataforseoSchemas.test.ts deleted file mode 100644 index 079ae91..0000000 --- a/src/server/lib/dataforseoSchemas.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - domainRankedKeywordItemSchema, - parseTaskItems, - relatedKeywordItemSchema, - successfulDataforseoTaskSchema, -} from "@/server/lib/dataforseoSchemas"; - -describe("dataforseoSchemas", () => { - it("accepts null items for empty successful tasks", () => { - const task = { - id: "04042314-1577-0387-0000-33dc4b485cfd", - status_code: 20000, - status_message: "Ok.", - path: ["v3", "dataforseo_labs", "google", "related_keywords", "live"], - cost: 0.02, - result_count: 1, - result: [ - { - se_type: "google", - seed_keyword: "canva ai video alternative", - location_code: 2840, - language_code: "en", - total_count: null, - items_count: 0, - items: null, - }, - ], - }; - - const parsedTask = successfulDataforseoTaskSchema.parse(task); - - expect( - parseTaskItems( - "google-related-keywords-live", - parsedTask, - relatedKeywordItemSchema, - ), - ).toEqual([]); - }); - - it("accepts empty ranked keyword tasks with null items", () => { - const task = { - id: "04070246-1577-0381-0000-2c56c059f67e", - status_code: 20000, - status_message: "Ok.", - path: ["v3", "dataforseo_labs", "google", "ranked_keywords", "live"], - cost: 0.01, - result_count: 1, - result: [ - { - se_type: "google", - target: "openseo.so", - location_code: 2840, - language_code: "en", - total_count: null, - items_count: 0, - metrics: null, - metrics_absolute: null, - items: null, - }, - ], - }; - - const parsedTask = successfulDataforseoTaskSchema.parse(task); - - expect( - parseTaskItems( - "google-ranked-keywords-live", - parsedTask, - domainRankedKeywordItemSchema, - ), - ).toEqual([]); - }); -}); diff --git a/src/server/lib/dataforseoSchemas.ts b/src/server/lib/dataforseoSchemas.ts deleted file mode 100644 index 5182429..0000000 --- a/src/server/lib/dataforseoSchemas.ts +++ /dev/null @@ -1,238 +0,0 @@ -import { z } from "zod"; -import { AppError } from "@/server/lib/errors"; - -const dataforseoTaskSchema = z - .object({ - status_code: z.number().optional(), - status_message: z.string().optional(), - path: z.array(z.string()), - cost: z.number(), - result_count: z.number().nullable(), - result: z - .array( - z - .object({ - items: z.array(z.unknown()).nullable().optional(), - }) - .passthrough(), - ) - .nullable() - .optional(), - }) - .passthrough(); - -export type DataforseoTask = z.infer; -export const successfulDataforseoTaskSchema = dataforseoTaskSchema; - -export const dataforseoResponseSchema = z - .object({ - status_code: z.number().optional(), - status_message: z.string().optional(), - tasks: z.array(dataforseoTaskSchema).optional(), - }) - .passthrough(); - -const monthlySearchSchema = z - .object({ - year: z.number().int(), - month: z.number().int().min(1).max(12), - search_volume: z.number().nullable(), - }) - .passthrough(); - -const keywordInfoSchema = z - .object({ - search_volume: z.number().nullable().optional(), - cpc: z.number().nullable().optional(), - competition: z.number().nullable().optional(), - monthly_searches: z.array(monthlySearchSchema).nullable().optional(), - }) - .passthrough(); - -const keywordInfoWithClickstreamSchema = z - .object({ - search_volume: z.number().nullable().optional(), - monthly_searches: z.array(monthlySearchSchema).nullable().optional(), - }) - .passthrough(); - -const searchIntentInfoSchema = z - .object({ - main_intent: z.string().nullable().optional(), - }) - .passthrough(); - -const keywordPropertiesSchema = z - .object({ - keyword_difficulty: z.number().nullable().optional(), - }) - .passthrough(); - -export const relatedKeywordItemSchema = z - .object({ - keyword_data: z - .object({ - keyword: z.string().optional(), - keyword_info: keywordInfoSchema.optional(), - keyword_info_normalized_with_clickstream: - keywordInfoWithClickstreamSchema.optional(), - search_intent_info: searchIntentInfoSchema.nullable().optional(), - keyword_properties: keywordPropertiesSchema.nullable().optional(), - }) - .passthrough(), - }) - .passthrough(); - -export const labsKeywordDataItemSchema = z - .object({ - keyword: z.string(), - keyword_info: keywordInfoSchema.optional(), - keyword_info_normalized_with_clickstream: - keywordInfoWithClickstreamSchema.optional(), - search_intent_info: searchIntentInfoSchema.nullable().optional(), - keyword_properties: keywordPropertiesSchema.nullable().optional(), - }) - .passthrough(); - -const domainMetricsValueSchema = z - .object({ - etv: z.number().nullable().optional(), - count: z.number().nullable().optional(), - }) - .passthrough(); - -export const domainMetricsItemSchema = z - .object({ - metrics: z.record( - z.string(), - domainMetricsValueSchema.nullable().optional(), - ), - }) - .passthrough(); - -export const relevantPagesItemSchema = z - .object({ - page_address: z.string().nullable().optional(), - metrics: z - .record(z.string(), domainMetricsValueSchema.nullable().optional()) - .nullable() - .optional(), - }) - .passthrough(); - -const rankedKeywordInfoSchema = z - .object({ - search_volume: z.number().nullable().optional(), - cpc: z.number().nullable().optional(), - keyword_difficulty: z.number().nullable().optional(), - }) - .passthrough(); - -const rankedKeywordDataSchema = z - .object({ - keyword: z.string().nullable().optional(), - keyword_info: rankedKeywordInfoSchema.nullable().optional(), - keyword_properties: keywordPropertiesSchema.nullable().optional(), - }) - .passthrough(); - -const rankedSerpItemSchema = z - .object({ - url: z.string().nullable().optional(), - relative_url: z.string().nullable().optional(), - rank_absolute: z.number().nullable().optional(), - etv: z.number().nullable().optional(), - }) - .passthrough(); - -const rankedSerpElementSchema = z - .object({ - serp_item: rankedSerpItemSchema.nullable().optional(), - url: z.string().nullable().optional(), - relative_url: z.string().nullable().optional(), - rank_absolute: z.number().nullable().optional(), - etv: z.number().nullable().optional(), - }) - .passthrough(); - -export const domainRankedKeywordItemSchema = z - .object({ - keyword_data: rankedKeywordDataSchema.nullable().optional(), - ranked_serp_element: rankedSerpElementSchema.nullable().optional(), - keyword: z.string().nullable().optional(), - rank_absolute: z.number().nullable().optional(), - etv: z.number().nullable().optional(), - keyword_difficulty: z.number().nullable().optional(), - }) - .passthrough(); - -export const serpSnapshotItemSchema = z - .object({ - type: z.string(), - rank_group: z.number().nullable().optional(), - rank_absolute: z.number().nullable().optional(), - domain: z.string().nullable().optional(), - title: z.string().nullable().optional(), - url: z.string().nullable().optional(), - description: z.string().nullable().optional(), - breadcrumb: z.string().nullable().optional(), - etv: z.number().nullable().optional(), - estimated_paid_traffic_cost: z.number().nullable().optional(), - backlinks_info: z - .object({ - referring_domains: z.number().nullable().optional(), - backlinks: z.number().nullable().optional(), - }) - .passthrough() - .nullable() - .optional(), - rank_changes: z - .object({ - previous_rank_absolute: z.number().nullable().optional(), - is_new: z.boolean().nullable().optional(), - is_up: z.boolean().nullable().optional(), - is_down: z.boolean().nullable().optional(), - }) - .passthrough() - .nullable() - .optional(), - }) - .passthrough(); - -export const keywordOverviewItemSchema = z - .object({ - keyword: z.string(), - keyword_info: keywordInfoSchema.optional(), - keyword_properties: keywordPropertiesSchema.nullable().optional(), - search_intent_info: searchIntentInfoSchema.nullable().optional(), - }) - .passthrough(); - -export type KeywordOverviewItem = z.infer; -export type RelatedKeywordItem = z.infer; -export type LabsKeywordDataItem = z.infer; -export type DomainMetricsItem = z.infer; -export type DomainRankedKeywordItem = z.infer< - typeof domainRankedKeywordItemSchema ->; -export type RelevantPagesItem = z.infer; -export type SerpLiveItem = z.infer; - -export function parseTaskItems( - endpointName: string, - task: DataforseoTask, - itemSchema: T, -): z.infer[] { - const parsed = z.array(itemSchema).safeParse(task.result?.[0]?.items ?? []); - if (!parsed.success) { - console.error( - `dataforseo.${endpointName}.invalid-payload`, - parsed.error.issues.slice(0, 5), - ); - throw new AppError( - "INTERNAL_ERROR", - `DataForSEO ${endpointName} returned an invalid response shape`, - ); - } - return parsed.data; -} diff --git a/src/server/mcp/tools/dataforseo-research-tools.test.ts b/src/server/mcp/tools/dataforseo-research-tools.test.ts index abbd909..5dea9d8 100644 --- a/src/server/mcp/tools/dataforseo-research-tools.test.ts +++ b/src/server/mcp/tools/dataforseo-research-tools.test.ts @@ -13,7 +13,7 @@ vi.mock("cloudflare:workers", () => ({ env: {}, })); -vi.mock("@/server/lib/dataforseoClient", () => ({ +vi.mock("@/server/lib/dataforseo", () => ({ createDataforseoClient: mocks.createDataforseoClient, })); diff --git a/src/server/mcp/tools/dataforseo-research-tools.ts b/src/server/mcp/tools/dataforseo-research-tools.ts index b2edafb..5cf93f5 100644 --- a/src/server/mcp/tools/dataforseo-research-tools.ts +++ b/src/server/mcp/tools/dataforseo-research-tools.ts @@ -1,6 +1,6 @@ /* eslint-disable max-lines */ import { z } from "zod"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { buildProjectMeta } from "@/server/mcp/context"; import { mcpResponse } from "@/server/mcp/formatters"; import { diff --git a/src/server/mcp/tools/get-serp-results.ts b/src/server/mcp/tools/get-serp-results.ts index 218d57e..767dc39 100644 --- a/src/server/mcp/tools/get-serp-results.ts +++ b/src/server/mcp/tools/get-serp-results.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { mcpResponse } from "@/server/mcp/formatters"; import { buildProjectMeta } from "@/server/mcp/context"; import { optionalMetaOutputSchema } from "@/server/mcp/output-schemas"; diff --git a/src/server/workflows/RankCheckWorkflow.ts b/src/server/workflows/RankCheckWorkflow.ts index 7520e8b..50ebeb3 100644 --- a/src/server/workflows/RankCheckWorkflow.ts +++ b/src/server/workflows/RankCheckWorkflow.ts @@ -8,7 +8,7 @@ import type { BillingCustomerContext } from "@/server/billing/subscription"; import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository"; import { failRunIfActive } from "@/server/features/rank-tracking/services/rankCheckRunGuards"; import { runLiveCheck } from "@/server/workflows/rankCheckPaths"; -import { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import { createDataforseoClient } from "@/server/lib/dataforseo"; import { captureServerEvent } from "@/server/lib/posthog"; import { AppError } from "@/server/lib/errors"; import { autumn } from "@/server/billing/autumn"; diff --git a/src/server/workflows/rankCheckPaths.ts b/src/server/workflows/rankCheckPaths.ts index ab5a632..a12053d 100644 --- a/src/server/workflows/rankCheckPaths.ts +++ b/src/server/workflows/rankCheckPaths.ts @@ -1,6 +1,6 @@ import type { WorkflowStep } from "cloudflare:workers"; import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository"; -import type { createDataforseoClient } from "@/server/lib/dataforseoClient"; +import type { createDataforseoClient } from "@/server/lib/dataforseo"; import type { RankCheckResult } from "@/server/lib/dataforseo"; import type { RankTrackingConfig } from "@/types/schemas/rank-tracking"; import { KEYWORDS_PER_BATCH } from "@/shared/rank-tracking";