From 868924e1fe25bb674b3568e8bf331499f1ec9e28 Mon Sep 17 00:00:00 2001 From: Ben Senescu <44480372+bensenescu@users.noreply.github.com> Date: Tue, 30 Jun 2026 17:23:27 -0400 Subject: [PATCH] refactor(keywords): unify keyword-metric fetching behind one helper (#320) --- .../services/research/refresh-metrics.ts | 144 ++++-------------- .../services/RankTrackingService.ts | 103 ++++--------- src/server/lib/dataforseo/index.ts | 11 +- .../lib/dataforseo/keyword-metrics.test.ts | 132 ++++++++++++++++ src/server/lib/dataforseo/keyword-metrics.ts | 141 +++++++++++++++++ ...taforseo-research-tools.google-ads.test.ts | 15 +- .../tools/dataforseo-research-tools.test.ts | 15 +- .../mcp/tools/dataforseo-research-tools.ts | 85 ++++------- 8 files changed, 396 insertions(+), 250 deletions(-) create mode 100644 src/server/lib/dataforseo/keyword-metrics.test.ts create mode 100644 src/server/lib/dataforseo/keyword-metrics.ts diff --git a/src/server/features/keywords/services/research/refresh-metrics.ts b/src/server/features/keywords/services/research/refresh-metrics.ts index 337f059..7214f6c 100644 --- a/src/server/features/keywords/services/research/refresh-metrics.ts +++ b/src/server/features/keywords/services/research/refresh-metrics.ts @@ -1,89 +1,11 @@ import { KeywordResearchRepository } from "@/server/features/keywords/repositories/KeywordResearchRepository"; import { normalizeIntent } from "@/server/features/keywords/services/research/helpers"; -import { createDataforseoClient } from "@/server/lib/dataforseo"; +import { + createDataforseoClient, + fetchKeywordMetricsForList, +} from "@/server/lib/dataforseo"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import type { KeywordIntent, MonthlySearch } from "@/types/keywords"; import type { RefreshSavedKeywordMetricsInput } from "@/types/schemas/keywords"; -import { getKeywordDataProvider } from "@/shared/keyword-locations"; - -const REFRESH_BATCH_SIZE = 700; - -// Match the shape the research/save flow persists so a refresh never degrades -// stored metrics (see research-data.ts mapKeywordDataItems / mapAdsKeywordItems). -function toMonthlySearchesJson( - entries: - | { - year?: number | null; - month?: number | null; - search_volume?: number | null; - }[] - | null - | undefined, -): string { - const trend: MonthlySearch[] = (entries ?? []).map((entry) => ({ - year: entry.year ?? 0, - month: entry.month ?? 0, - searchVolume: entry.search_volume ?? 0, - })); - return JSON.stringify(trend); -} - -type RefreshedMetric = { - searchVolume: number | null; - cpc: number | null; - competition: number | null; - keywordDifficulty: number | null; - intent: KeywordIntent; - monthlySearchesJson: string; -}; - -// Fetch fresh metrics for one homogeneous batch and key them by lowercase -// keyword. Labs covers most countries; the rest fall back to Google Ads, which -// carries volume/CPC/competition but no difficulty or intent. -async function fetchBatchMetrics( - client: ReturnType, - request: { keywords: string[]; locationCode: number; languageCode: string }, - useGoogleAds: boolean, -): Promise> { - const metricsMap = new Map(); - - if (useGoogleAds) { - const items = await client.keywords.adsSearchVolume({ - ...request, - creditFeature: "keyword_research", - }); - for (const item of items) { - if (!item.keyword) continue; - metricsMap.set(item.keyword.toLowerCase(), { - searchVolume: item.search_volume ?? null, - cpc: item.cpc ?? null, - // competition_index is 0-100; the rest of the app stores a 0-1 ratio. - competition: - item.competition_index != null ? item.competition_index / 100 : null, - keywordDifficulty: null, - intent: "unknown", - monthlySearchesJson: toMonthlySearchesJson(item.monthly_searches), - }); - } - return metricsMap; - } - - const items = await client.labs.keywordOverview(request); - for (const item of items) { - if (!item.keyword) continue; - metricsMap.set(item.keyword.toLowerCase(), { - searchVolume: item.keyword_info?.search_volume ?? null, - cpc: item.keyword_info?.cpc ?? null, - competition: item.keyword_info?.competition ?? null, - keywordDifficulty: item.keyword_properties?.keyword_difficulty ?? null, - intent: normalizeIntent(item.search_intent_info?.main_intent), - monthlySearchesJson: toMonthlySearchesJson( - item.keyword_info?.monthly_searches, - ), - }); - } - return metricsMap; -} export async function refreshSavedKeywordMetrics( input: RefreshSavedKeywordMetricsInput, @@ -98,7 +20,7 @@ export async function refreshSavedKeywordMetrics( const client = createDataforseoClient(billingCustomer); let updated = 0; - // Group by (locationCode, languageCode) so each DataForSEO call is homogeneous. + // Group by (locationCode, languageCode) so each provider call is homogeneous. const groups = new Map(); for (const row of rows) { const key = `${row.row.locationCode}:${row.row.languageCode}`; @@ -109,36 +31,36 @@ export async function refreshSavedKeywordMetrics( for (const groupRows of groups.values()) { const { locationCode, languageCode } = groupRows[0].row; - const useGoogleAds = getKeywordDataProvider(locationCode) === "google_ads"; + const metrics = await fetchKeywordMetricsForList(client, { + keywords: groupRows.map((r) => r.row.keyword), + locationCode, + languageCode, + creditFeature: "keyword_research", + }); + const byKeyword = new Map( + metrics.map((metric) => [metric.keyword.toLowerCase(), metric]), + ); - for (let i = 0; i < groupRows.length; i += REFRESH_BATCH_SIZE) { - const batch = groupRows.slice(i, i + REFRESH_BATCH_SIZE); - const keywords = batch.map((r) => r.row.keyword); - const request = { keywords, locationCode, languageCode }; + await Promise.all( + groupRows.map((r) => { + const metric = byKeyword.get(r.row.keyword.toLowerCase()); + if (!metric) return Promise.resolve(); + return KeywordResearchRepository.upsertKeywordMetric({ + projectId: input.projectId, + keyword: r.row.keyword, + locationCode, + languageCode, + searchVolume: metric.searchVolume, + cpc: metric.cpc, + competition: metric.competition, + keywordDifficulty: metric.keywordDifficulty, + intent: normalizeIntent(metric.intent), + monthlySearchesJson: JSON.stringify(metric.monthlySearches), + }); + }), + ); - const metricsMap = await fetchBatchMetrics(client, request, useGoogleAds); - - await Promise.all( - batch.map((r) => { - const metrics = metricsMap.get(r.row.keyword.toLowerCase()); - if (!metrics) return Promise.resolve(); - return KeywordResearchRepository.upsertKeywordMetric({ - projectId: input.projectId, - keyword: r.row.keyword, - locationCode, - languageCode, - searchVolume: metrics.searchVolume, - cpc: metrics.cpc, - competition: metrics.competition, - keywordDifficulty: metrics.keywordDifficulty, - intent: metrics.intent, - monthlySearchesJson: metrics.monthlySearchesJson, - }); - }), - ); - - updated += metricsMap.size; - } + updated += byKeyword.size; } return { updated }; diff --git a/src/server/features/rank-tracking/services/RankTrackingService.ts b/src/server/features/rank-tracking/services/RankTrackingService.ts index b165d3d..3eb34f0 100644 --- a/src/server/features/rank-tracking/services/RankTrackingService.ts +++ b/src/server/features/rank-tracking/services/RankTrackingService.ts @@ -1,7 +1,9 @@ import { env } from "cloudflare:workers"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { createDataforseoClient } from "@/server/lib/dataforseo"; -import { getKeywordDataProvider } from "@/shared/keyword-locations"; +import { + createDataforseoClient, + fetchKeywordMetricsForList, +} from "@/server/lib/dataforseo"; import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository"; import { AppError } from "@/server/lib/errors"; import type { @@ -260,8 +262,6 @@ async function getLatestRun(configId: string, projectId: string) { // Keyword metrics (volume, difficulty, CPC) // --------------------------------------------------------------------------- -const KEYWORD_OVERVIEW_BATCH_SIZE = 700; - async function refreshKeywordMetrics( configId: string, projectId: string, @@ -272,76 +272,35 @@ async function refreshKeywordMetrics( if (keywords.length === 0) return { updated: 0 }; const client = createDataforseoClient(billingCustomer); + const metrics = await fetchKeywordMetricsForList(client, { + keywords: keywords.map((kw) => kw.keyword), + locationCode: config.locationCode, + languageCode: config.languageCode, + creditFeature: "rank_tracking", + }); + const byKeyword = new Map( + metrics.map((metric) => [metric.keyword.toLowerCase(), metric]), + ); + const now = new Date().toISOString(); - let updated = 0; + const updates = keywords + .map((kw) => { + const metric = byKeyword.get(kw.keyword.toLowerCase()); + if (!metric) return null; + // Rank tracking only tracks volume / difficulty / CPC. + return { + id: kw.id, + searchVolume: metric.searchVolume, + keywordDifficulty: metric.keywordDifficulty, + cpc: metric.cpc, + metricsFetchedAt: now, + }; + }) + .filter((u): u is NonNullable => u !== null); - // Countries Labs doesn't cover get volume/CPC from Google Ads (no KD). - const useGoogleAds = - getKeywordDataProvider(config.locationCode) === "google_ads"; - - for (let i = 0; i < keywords.length; i += KEYWORD_OVERVIEW_BATCH_SIZE) { - const batch = keywords.slice(i, i + KEYWORD_OVERVIEW_BATCH_SIZE); - const request = { - keywords: batch.map((kw) => kw.keyword), - locationCode: config.locationCode, - languageCode: config.languageCode, - }; - - // Build a lookup by lowercase keyword - const metricsMap = new Map< - string, - { - searchVolume: number | null; - keywordDifficulty: number | null; - cpc: number | null; - } - >(); - if (useGoogleAds) { - const adsItems = await client.keywords.adsSearchVolume({ - ...request, - creditFeature: "rank_tracking", - }); - for (const item of adsItems) { - if (!item.keyword) continue; - metricsMap.set(item.keyword.toLowerCase(), { - searchVolume: item.search_volume ?? null, - keywordDifficulty: null, - cpc: item.cpc ?? null, - }); - } - } else { - for (const item of await client.labs.keywordOverview(request)) { - if (!item.keyword) continue; - metricsMap.set(item.keyword.toLowerCase(), { - searchVolume: item.keyword_info?.search_volume ?? null, - keywordDifficulty: - item.keyword_properties?.keyword_difficulty ?? null, - cpc: item.keyword_info?.cpc ?? null, - }); - } - } - - const updates = batch - .map((kw) => { - const metrics = metricsMap.get(kw.keyword.toLowerCase()); - if (!metrics) return null; - return { - id: kw.id, - searchVolume: metrics.searchVolume, - keywordDifficulty: metrics.keywordDifficulty, - cpc: metrics.cpc, - metricsFetchedAt: now, - }; - }) - .filter((u): u is NonNullable => u !== null); - - if (updates.length > 0) { - await RankTrackingRepository.updateKeywordMetrics(updates); - updated += updates.length; - } - } - - return { updated }; + if (updates.length === 0) return { updated: 0 }; + await RankTrackingRepository.updateKeywordMetrics(updates); + return { updated: updates.length }; } // --------------------------------------------------------------------------- diff --git a/src/server/lib/dataforseo/index.ts b/src/server/lib/dataforseo/index.ts index 29e3ace..2fe98ac 100644 --- a/src/server/lib/dataforseo/index.ts +++ b/src/server/lib/dataforseo/index.ts @@ -5,17 +5,18 @@ export { createDataforseoClient } from "@/server/lib/dataforseo/client"; +export { + fetchKeywordMetricsForList, + type KeywordMetricRow, +} from "@/server/lib/dataforseo/keyword-metrics"; + export { type LabsKeywordDataItem, type DomainRankedKeywordItem, type RelevantPagesItem, - type KeywordOverviewItem, } from "@/server/lib/dataforseo/labs"; -export { - type AdsKeywordItem, - type AdsKeywordIdeaItem, -} from "@/server/lib/dataforseo/google-ads"; +export { type AdsKeywordIdeaItem } from "@/server/lib/dataforseo/google-ads"; export { fetchRankCheckTaskResult, diff --git a/src/server/lib/dataforseo/keyword-metrics.test.ts b/src/server/lib/dataforseo/keyword-metrics.test.ts new file mode 100644 index 0000000..795ce3b --- /dev/null +++ b/src/server/lib/dataforseo/keyword-metrics.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it, vi } from "vitest"; + +import { fetchKeywordMetricsForList } from "./keyword-metrics"; + +type Client = Parameters[0]; + +// Minimal fake client exposing only the two endpoints the helper touches. +function fakeClient(overrides: { + keywordOverview?: Client["labs"]["keywordOverview"]; + adsSearchVolume?: Client["keywords"]["adsSearchVolume"]; +}): Client { + return { + labs: { keywordOverview: overrides.keywordOverview ?? vi.fn() }, + keywords: { adsSearchVolume: overrides.adsSearchVolume ?? vi.fn() }, + }; +} + +describe("fetchKeywordMetricsForList", () => { + it("normalizes Labs items and prefers clickstream-refined volume/trend", async () => { + const keywordOverview = vi.fn().mockResolvedValue([ + { + keyword: "best crm", + keyword_info: { + search_volume: 1000, + cpc: 3.2, + competition: 0.5, + competition_level: "MEDIUM", + monthly_searches: [{ year: 2026, month: 1, search_volume: 1000 }], + }, + keyword_info_normalized_with_clickstream: { + search_volume: 880, + monthly_searches: [{ year: 2026, month: 1, search_volume: 880 }], + }, + keyword_properties: { keyword_difficulty: 42 }, + search_intent_info: { main_intent: "commercial" }, + }, + ]); + const client = fakeClient({ keywordOverview }); + + const rows = await fetchKeywordMetricsForList(client, { + keywords: ["best crm"], + locationCode: 2840, + languageCode: "en", + includeClickstreamData: true, + creditFeature: "keyword_research", + }); + + expect(keywordOverview).toHaveBeenCalledWith({ + keywords: ["best crm"], + locationCode: 2840, + languageCode: "en", + includeClickstreamData: true, + creditFeature: "keyword_research", + }); + expect(rows).toEqual([ + { + keyword: "best crm", + searchVolume: 880, + cpc: 3.2, + competition: 0.5, + competitionLevel: "MEDIUM", + keywordDifficulty: 42, + intent: "commercial", + monthlySearches: [{ year: 2026, month: 1, searchVolume: 880 }], + }, + ]); + }); + + it("routes Google Ads locations and scales competition_index to a 0-1 ratio", async () => { + const adsSearchVolume = vi.fn().mockResolvedValue([ + { + keyword: "hotel reykjavik", + search_volume: 1300, + cpc: 2.54, + competition: "HIGH", + competition_index: 42, + monthly_searches: [{ year: 2026, month: 5, search_volume: 1300 }], + }, + { keyword: undefined }, + ]); + const client = fakeClient({ adsSearchVolume }); + + const rows = await fetchKeywordMetricsForList(client, { + keywords: ["hotel reykjavik"], + locationCode: 2352, + languageCode: "is", + creditFeature: "rank_tracking", + }); + + expect(adsSearchVolume).toHaveBeenCalledWith({ + keywords: ["hotel reykjavik"], + locationCode: 2352, + languageCode: "is", + creditFeature: "rank_tracking", + }); + // Item without a keyword is dropped; Ads carries no KD/intent. + expect(rows).toEqual([ + { + keyword: "hotel reykjavik", + searchVolume: 1300, + cpc: 2.54, + competition: 0.42, + competitionLevel: "HIGH", + keywordDifficulty: null, + intent: null, + monthlySearches: [{ year: 2026, month: 5, searchVolume: 1300 }], + }, + ]); + }); + + it("batches keyword lists above the per-call cap", async () => { + const keywordOverview = vi + .fn() + .mockImplementation(({ keywords }: { keywords: string[] }) => + Promise.resolve( + keywords.map((keyword) => ({ keyword, keyword_info: {} })), + ), + ); + const client = fakeClient({ keywordOverview }); + const keywords = Array.from({ length: 1500 }, (_, i) => `kw-${i}`); + + const rows = await fetchKeywordMetricsForList(client, { + keywords, + locationCode: 2840, + languageCode: "en", + creditFeature: "keyword_research", + }); + + expect(keywordOverview).toHaveBeenCalledTimes(3); // 700 + 700 + 100 + expect(rows).toHaveLength(1500); + }); +}); diff --git a/src/server/lib/dataforseo/keyword-metrics.ts b/src/server/lib/dataforseo/keyword-metrics.ts new file mode 100644 index 0000000..a5a119e --- /dev/null +++ b/src/server/lib/dataforseo/keyword-metrics.ts @@ -0,0 +1,141 @@ +import type { createDataforseoClient } from "@/server/lib/dataforseo/client"; +import type { AdsKeywordItem } from "@/server/lib/dataforseo/google-ads"; +import type { KeywordOverviewItem } from "@/server/lib/dataforseo/labs"; +import type { CreditFeature } from "@/shared/billing-credit-features"; +import { getKeywordDataProvider } from "@/shared/keyword-locations"; +import type { MonthlySearch } from "@/types/keywords"; + +type DataforseoClient = ReturnType; + +// Narrowed to the two endpoints the helper uses, so tests can fake it cheaply. +type KeywordMetricsClient = { + labs: Pick; + keywords: Pick; +}; + +// DataForSEO's batch metric endpoints accept up to ~700 keywords per request. +const KEYWORD_METRICS_BATCH_SIZE = 700; + +// `intent` is the raw `main_intent` (null for Google Ads); run it through +// `normalizeIntent` for the app enum. `competition` is a 0-1 ratio. +export type KeywordMetricRow = { + keyword: string; + searchVolume: number | null; + cpc: number | null; + competition: number | null; + competitionLevel: string | null; + keywordDifficulty: number | null; + intent: string | null; + monthlySearches: MonthlySearch[]; +}; + +function toMonthlySearches( + entries: + | { + year?: number | null; + month?: number | null; + search_volume?: number | null; + }[] + | null + | undefined, +): MonthlySearch[] { + return (entries ?? []).map((entry) => ({ + year: entry.year ?? 0, + month: entry.month ?? 0, + searchVolume: entry.search_volume ?? 0, + })); +} + +function normalizeKeywordOverview( + item: KeywordOverviewItem, + keyword: string, +): KeywordMetricRow { + const info = item.keyword_info; + // The clickstream-normalized block only exists when the caller opted into + // clickstream data (it doubles request cost); prefer it when present. + const clickstreamInfo = item.keyword_info_normalized_with_clickstream; + const usesClickstream = clickstreamInfo?.search_volume != null; + return { + keyword, + searchVolume: clickstreamInfo?.search_volume ?? info?.search_volume ?? null, + cpc: info?.cpc ?? null, + competition: info?.competition ?? null, + competitionLevel: info?.competition_level ?? null, + keywordDifficulty: item.keyword_properties?.keyword_difficulty ?? null, + intent: item.search_intent_info?.main_intent ?? null, + monthlySearches: toMonthlySearches( + usesClickstream + ? clickstreamInfo?.monthly_searches + : info?.monthly_searches, + ), + }; +} + +// Google Ads items (countries Labs doesn't cover) carry volume/CPC/competition +// but no keyword difficulty or search intent. +function normalizeAdsKeyword( + item: AdsKeywordItem, + keyword: string, +): KeywordMetricRow { + return { + keyword, + searchVolume: item.search_volume ?? null, + cpc: item.cpc ?? null, + // competition_index is a 0-100 scale; the app stores a 0-1 ratio. + competition: + item.competition_index != null ? item.competition_index / 100 : null, + competitionLevel: item.competition ?? null, + keywordDifficulty: null, + intent: null, + monthlySearches: toMonthlySearches(item.monthly_searches), + }; +} + +// Hydrate a keyword list with fresh metrics: route by location (Labs vs Google +// Ads), batch under the per-call cap, and drop items DataForSEO returns without +// a keyword. `creditFeature` is required so spend is always attributed. +export async function fetchKeywordMetricsForList( + client: KeywordMetricsClient, + params: { + keywords: string[]; + locationCode: number; + languageCode: string; + creditFeature: CreditFeature; + includeClickstreamData?: boolean; + }, +): Promise { + const useGoogleAds = + getKeywordDataProvider(params.locationCode) === "google_ads"; + const rows: KeywordMetricRow[] = []; + + for (let i = 0; i < params.keywords.length; i += KEYWORD_METRICS_BATCH_SIZE) { + const keywords = params.keywords.slice(i, i + KEYWORD_METRICS_BATCH_SIZE); + + if (useGoogleAds) { + const items = await client.keywords.adsSearchVolume({ + keywords, + locationCode: params.locationCode, + languageCode: params.languageCode, + creditFeature: params.creditFeature, + }); + for (const item of items) { + if (!item.keyword) continue; + rows.push(normalizeAdsKeyword(item, item.keyword)); + } + } else { + const items = await client.labs.keywordOverview({ + keywords, + locationCode: params.locationCode, + languageCode: params.languageCode, + includeClickstreamData: params.includeClickstreamData ?? false, + creditFeature: params.creditFeature, + }); + for (const item of items) { + if (!item.keyword) continue; + rows.push(normalizeKeywordOverview(item, item.keyword)); + } + } + } + + return rows; +} diff --git a/src/server/mcp/tools/dataforseo-research-tools.google-ads.test.ts b/src/server/mcp/tools/dataforseo-research-tools.google-ads.test.ts index f12a8d1..db1c471 100644 --- a/src/server/mcp/tools/dataforseo-research-tools.google-ads.test.ts +++ b/src/server/mcp/tools/dataforseo-research-tools.google-ads.test.ts @@ -3,6 +3,7 @@ import type { ToolExtra } from "@/server/mcp/context"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { z } from "zod"; import { MCP_AUTH_CONTEXT_PROP } from "@/server/mcp/context"; +import type { fetchKeywordMetricsForList as FetchKeywordMetricsForList } from "@/server/lib/dataforseo/keyword-metrics"; const mocks = vi.hoisted(() => ({ createDataforseoClient: vi.fn(), @@ -13,9 +14,17 @@ vi.mock("cloudflare:workers", () => ({ env: {}, })); -vi.mock("@/server/lib/dataforseo", () => ({ - createDataforseoClient: mocks.createDataforseoClient, -})); +// Keep the real fetchKeywordMetricsForList (it only routes provider calls onto +// the supplied client) so the handler's normalization is exercised end-to-end. +vi.mock("@/server/lib/dataforseo", async () => { + const keywordMetrics = await vi.importActual<{ + fetchKeywordMetricsForList: typeof FetchKeywordMetricsForList; + }>("@/server/lib/dataforseo/keyword-metrics"); + return { + createDataforseoClient: mocks.createDataforseoClient, + fetchKeywordMetricsForList: keywordMetrics.fetchKeywordMetricsForList, + }; +}); vi.mock("@/server/features/projects/services/ProjectService", () => ({ ProjectService: { diff --git a/src/server/mcp/tools/dataforseo-research-tools.test.ts b/src/server/mcp/tools/dataforseo-research-tools.test.ts index a65e877..16228ea 100644 --- a/src/server/mcp/tools/dataforseo-research-tools.test.ts +++ b/src/server/mcp/tools/dataforseo-research-tools.test.ts @@ -3,6 +3,7 @@ import type { ToolExtra } from "@/server/mcp/context"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { z } from "zod"; import { MCP_AUTH_CONTEXT_PROP } from "@/server/mcp/context"; +import type { fetchKeywordMetricsForList as FetchKeywordMetricsForList } from "@/server/lib/dataforseo/keyword-metrics"; const mocks = vi.hoisted(() => ({ createDataforseoClient: vi.fn(), @@ -13,9 +14,17 @@ vi.mock("cloudflare:workers", () => ({ env: {}, })); -vi.mock("@/server/lib/dataforseo", () => ({ - createDataforseoClient: mocks.createDataforseoClient, -})); +// Keep the real fetchKeywordMetricsForList (it only routes provider calls onto +// the supplied client) so the handler's normalization is exercised end-to-end. +vi.mock("@/server/lib/dataforseo", async () => { + const keywordMetrics = await vi.importActual<{ + fetchKeywordMetricsForList: typeof FetchKeywordMetricsForList; + }>("@/server/lib/dataforseo/keyword-metrics"); + return { + createDataforseoClient: mocks.createDataforseoClient, + fetchKeywordMetricsForList: keywordMetrics.fetchKeywordMetricsForList, + }; +}); vi.mock("@/server/features/projects/services/ProjectService", () => ({ ProjectService: { diff --git a/src/server/mcp/tools/dataforseo-research-tools.ts b/src/server/mcp/tools/dataforseo-research-tools.ts index 9c5a016..532bfca 100644 --- a/src/server/mcp/tools/dataforseo-research-tools.ts +++ b/src/server/mcp/tools/dataforseo-research-tools.ts @@ -2,10 +2,9 @@ import { z } from "zod"; import { createDataforseoClient, - type AdsKeywordItem, - type KeywordOverviewItem, + fetchKeywordMetricsForList, + type KeywordMetricRow, } from "@/server/lib/dataforseo"; -import { getKeywordDataProvider } from "@/shared/keyword-locations"; import { buildProjectMeta } from "@/server/mcp/context"; import { mcpResponse } from "@/server/mcp/formatters"; import { @@ -439,45 +438,31 @@ function sortCompetitors( }); } -function normalizeKeywordOverview(item: KeywordOverviewItem) { - const info = item.keyword_info; - // Only present when the caller opted into clickstream-refined volumes. - const clickstreamInfo = item.keyword_info_normalized_with_clickstream; +// Project the shared canonical metric row onto this tool's snake_case API +// shape (kept stable for MCP clients); an absent trend stays null as before. +function toMcpKeywordMetricRow(row: KeywordMetricRow) { return { - keyword: item.keyword, - search_volume: - clickstreamInfo?.search_volume ?? info?.search_volume ?? null, - keyword_difficulty: item.keyword_properties?.keyword_difficulty ?? null, - main_intent: item.search_intent_info?.main_intent ?? null, - cpc: info?.cpc ?? null, - competition: info?.competition ?? null, - competition_level: info?.competition_level ?? null, - monthly_searches: - (clickstreamInfo?.search_volume - ? clickstreamInfo.monthly_searches - : info?.monthly_searches) ?? null, + keyword: row.keyword, + search_volume: row.searchVolume, + keyword_difficulty: row.keywordDifficulty, + main_intent: row.intent, + cpc: row.cpc, + competition: row.competition, + competition_level: row.competitionLevel, + monthly_searches: row.monthlySearches.length + ? row.monthlySearches.map((entry) => ({ + year: entry.year, + month: entry.month, + search_volume: entry.searchVolume, + })) + : null, }; } -type KeywordMetricRow = ReturnType; - -// Google Ads items (countries Labs doesn't cover) have no difficulty/intent. -function normalizeAdsKeyword(item: AdsKeywordItem): KeywordMetricRow { - return { - keyword: item.keyword, - search_volume: item.search_volume ?? null, - keyword_difficulty: null, - main_intent: null, - cpc: item.cpc ?? null, - competition: - item.competition_index != null ? item.competition_index / 100 : null, - competition_level: item.competition ?? null, - monthly_searches: item.monthly_searches ?? null, - }; -} +type McpKeywordMetricRow = ReturnType; function sortKeywordMetricRows( - rows: KeywordMetricRow[], + rows: McpKeywordMetricRow[], sortBy: NonNullable = "search_volume", ) { return rows.toSorted((a, b) => { @@ -846,27 +831,15 @@ export const getKeywordMetricsTool = { const client = createDataforseoClient(context.billing); const locationCode = args.locationCode ?? DEFAULT_LOCATION_CODE; const languageCode = args.languageCode ?? DEFAULT_LANGUAGE_CODE; - const normalized = - getKeywordDataProvider(locationCode) === "google_ads" - ? ( - await client.keywords.adsSearchVolume({ - keywords: args.keywords, - locationCode, - languageCode, - creditFeature: "keyword_research", - }) - ).map(normalizeAdsKeyword) - : ( - await client.labs.keywordOverview({ - keywords: args.keywords, - locationCode, - languageCode, - includeClickstreamData: args.includeClickstreamData ?? false, - creditFeature: "keyword_research", - }) - ).map(normalizeKeywordOverview); + const metrics = await fetchKeywordMetricsForList(client, { + keywords: args.keywords, + locationCode, + languageCode, + includeClickstreamData: args.includeClickstreamData ?? false, + creditFeature: "keyword_research", + }); const rows = sortKeywordMetricRows( - normalized, + metrics.map(toMcpKeywordMetricRow), args.sortBy ?? "search_volume", ).map((row) => args.includeMonthlyTrends === false