refactor(keywords): unify keyword-metric fetching behind one helper (#320)
This commit is contained in:
Ben Senescu
Ben Senescu
parent
116719a019
commit
868924e1fe
@ -1,89 +1,11 @@
|
||||
import { KeywordResearchRepository } from "@/server/features/keywords/repositories/KeywordResearchRepository";
|
||||
import { normalizeIntent } from "@/server/features/keywords/services/research/helpers";
|
||||
import { createDataforseoClient } from "@/server/lib/dataforseo";
|
||||
import {
|
||||
createDataforseoClient,
|
||||
fetchKeywordMetricsForList,
|
||||
} from "@/server/lib/dataforseo";
|
||||
import type { BillingCustomerContext } from "@/server/billing/subscription";
|
||||
import type { KeywordIntent, MonthlySearch } from "@/types/keywords";
|
||||
import type { RefreshSavedKeywordMetricsInput } from "@/types/schemas/keywords";
|
||||
import { getKeywordDataProvider } from "@/shared/keyword-locations";
|
||||
|
||||
const REFRESH_BATCH_SIZE = 700;
|
||||
|
||||
// Match the shape the research/save flow persists so a refresh never degrades
|
||||
// stored metrics (see research-data.ts mapKeywordDataItems / mapAdsKeywordItems).
|
||||
function toMonthlySearchesJson(
|
||||
entries:
|
||||
| {
|
||||
year?: number | null;
|
||||
month?: number | null;
|
||||
search_volume?: number | null;
|
||||
}[]
|
||||
| null
|
||||
| undefined,
|
||||
): string {
|
||||
const trend: MonthlySearch[] = (entries ?? []).map((entry) => ({
|
||||
year: entry.year ?? 0,
|
||||
month: entry.month ?? 0,
|
||||
searchVolume: entry.search_volume ?? 0,
|
||||
}));
|
||||
return JSON.stringify(trend);
|
||||
}
|
||||
|
||||
type RefreshedMetric = {
|
||||
searchVolume: number | null;
|
||||
cpc: number | null;
|
||||
competition: number | null;
|
||||
keywordDifficulty: number | null;
|
||||
intent: KeywordIntent;
|
||||
monthlySearchesJson: string;
|
||||
};
|
||||
|
||||
// Fetch fresh metrics for one homogeneous batch and key them by lowercase
|
||||
// keyword. Labs covers most countries; the rest fall back to Google Ads, which
|
||||
// carries volume/CPC/competition but no difficulty or intent.
|
||||
async function fetchBatchMetrics(
|
||||
client: ReturnType<typeof createDataforseoClient>,
|
||||
request: { keywords: string[]; locationCode: number; languageCode: string },
|
||||
useGoogleAds: boolean,
|
||||
): Promise<Map<string, RefreshedMetric>> {
|
||||
const metricsMap = new Map<string, RefreshedMetric>();
|
||||
|
||||
if (useGoogleAds) {
|
||||
const items = await client.keywords.adsSearchVolume({
|
||||
...request,
|
||||
creditFeature: "keyword_research",
|
||||
});
|
||||
for (const item of items) {
|
||||
if (!item.keyword) continue;
|
||||
metricsMap.set(item.keyword.toLowerCase(), {
|
||||
searchVolume: item.search_volume ?? null,
|
||||
cpc: item.cpc ?? null,
|
||||
// competition_index is 0-100; the rest of the app stores a 0-1 ratio.
|
||||
competition:
|
||||
item.competition_index != null ? item.competition_index / 100 : null,
|
||||
keywordDifficulty: null,
|
||||
intent: "unknown",
|
||||
monthlySearchesJson: toMonthlySearchesJson(item.monthly_searches),
|
||||
});
|
||||
}
|
||||
return metricsMap;
|
||||
}
|
||||
|
||||
const items = await client.labs.keywordOverview(request);
|
||||
for (const item of items) {
|
||||
if (!item.keyword) continue;
|
||||
metricsMap.set(item.keyword.toLowerCase(), {
|
||||
searchVolume: item.keyword_info?.search_volume ?? null,
|
||||
cpc: item.keyword_info?.cpc ?? null,
|
||||
competition: item.keyword_info?.competition ?? null,
|
||||
keywordDifficulty: item.keyword_properties?.keyword_difficulty ?? null,
|
||||
intent: normalizeIntent(item.search_intent_info?.main_intent),
|
||||
monthlySearchesJson: toMonthlySearchesJson(
|
||||
item.keyword_info?.monthly_searches,
|
||||
),
|
||||
});
|
||||
}
|
||||
return metricsMap;
|
||||
}
|
||||
|
||||
export async function refreshSavedKeywordMetrics(
|
||||
input: RefreshSavedKeywordMetricsInput,
|
||||
@ -98,7 +20,7 @@ export async function refreshSavedKeywordMetrics(
|
||||
const client = createDataforseoClient(billingCustomer);
|
||||
let updated = 0;
|
||||
|
||||
// Group by (locationCode, languageCode) so each DataForSEO call is homogeneous.
|
||||
// Group by (locationCode, languageCode) so each provider call is homogeneous.
|
||||
const groups = new Map<string, typeof rows>();
|
||||
for (const row of rows) {
|
||||
const key = `${row.row.locationCode}:${row.row.languageCode}`;
|
||||
@ -109,36 +31,36 @@ export async function refreshSavedKeywordMetrics(
|
||||
|
||||
for (const groupRows of groups.values()) {
|
||||
const { locationCode, languageCode } = groupRows[0].row;
|
||||
const useGoogleAds = getKeywordDataProvider(locationCode) === "google_ads";
|
||||
const metrics = await fetchKeywordMetricsForList(client, {
|
||||
keywords: groupRows.map((r) => r.row.keyword),
|
||||
locationCode,
|
||||
languageCode,
|
||||
creditFeature: "keyword_research",
|
||||
});
|
||||
const byKeyword = new Map(
|
||||
metrics.map((metric) => [metric.keyword.toLowerCase(), metric]),
|
||||
);
|
||||
|
||||
for (let i = 0; i < groupRows.length; i += REFRESH_BATCH_SIZE) {
|
||||
const batch = groupRows.slice(i, i + REFRESH_BATCH_SIZE);
|
||||
const keywords = batch.map((r) => r.row.keyword);
|
||||
const request = { keywords, locationCode, languageCode };
|
||||
await Promise.all(
|
||||
groupRows.map((r) => {
|
||||
const metric = byKeyword.get(r.row.keyword.toLowerCase());
|
||||
if (!metric) return Promise.resolve();
|
||||
return KeywordResearchRepository.upsertKeywordMetric({
|
||||
projectId: input.projectId,
|
||||
keyword: r.row.keyword,
|
||||
locationCode,
|
||||
languageCode,
|
||||
searchVolume: metric.searchVolume,
|
||||
cpc: metric.cpc,
|
||||
competition: metric.competition,
|
||||
keywordDifficulty: metric.keywordDifficulty,
|
||||
intent: normalizeIntent(metric.intent),
|
||||
monthlySearchesJson: JSON.stringify(metric.monthlySearches),
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
const metricsMap = await fetchBatchMetrics(client, request, useGoogleAds);
|
||||
|
||||
await Promise.all(
|
||||
batch.map((r) => {
|
||||
const metrics = metricsMap.get(r.row.keyword.toLowerCase());
|
||||
if (!metrics) return Promise.resolve();
|
||||
return KeywordResearchRepository.upsertKeywordMetric({
|
||||
projectId: input.projectId,
|
||||
keyword: r.row.keyword,
|
||||
locationCode,
|
||||
languageCode,
|
||||
searchVolume: metrics.searchVolume,
|
||||
cpc: metrics.cpc,
|
||||
competition: metrics.competition,
|
||||
keywordDifficulty: metrics.keywordDifficulty,
|
||||
intent: metrics.intent,
|
||||
monthlySearchesJson: metrics.monthlySearchesJson,
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
updated += metricsMap.size;
|
||||
}
|
||||
updated += byKeyword.size;
|
||||
}
|
||||
|
||||
return { updated };
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
import { env } from "cloudflare:workers";
|
||||
import type { BillingCustomerContext } from "@/server/billing/subscription";
|
||||
import { createDataforseoClient } from "@/server/lib/dataforseo";
|
||||
import { getKeywordDataProvider } from "@/shared/keyword-locations";
|
||||
import {
|
||||
createDataforseoClient,
|
||||
fetchKeywordMetricsForList,
|
||||
} from "@/server/lib/dataforseo";
|
||||
import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository";
|
||||
import { AppError } from "@/server/lib/errors";
|
||||
import type {
|
||||
@ -260,8 +262,6 @@ async function getLatestRun(configId: string, projectId: string) {
|
||||
// Keyword metrics (volume, difficulty, CPC)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const KEYWORD_OVERVIEW_BATCH_SIZE = 700;
|
||||
|
||||
async function refreshKeywordMetrics(
|
||||
configId: string,
|
||||
projectId: string,
|
||||
@ -272,76 +272,35 @@ async function refreshKeywordMetrics(
|
||||
if (keywords.length === 0) return { updated: 0 };
|
||||
|
||||
const client = createDataforseoClient(billingCustomer);
|
||||
const metrics = await fetchKeywordMetricsForList(client, {
|
||||
keywords: keywords.map((kw) => kw.keyword),
|
||||
locationCode: config.locationCode,
|
||||
languageCode: config.languageCode,
|
||||
creditFeature: "rank_tracking",
|
||||
});
|
||||
const byKeyword = new Map(
|
||||
metrics.map((metric) => [metric.keyword.toLowerCase(), metric]),
|
||||
);
|
||||
|
||||
const now = new Date().toISOString();
|
||||
let updated = 0;
|
||||
const updates = keywords
|
||||
.map((kw) => {
|
||||
const metric = byKeyword.get(kw.keyword.toLowerCase());
|
||||
if (!metric) return null;
|
||||
// Rank tracking only tracks volume / difficulty / CPC.
|
||||
return {
|
||||
id: kw.id,
|
||||
searchVolume: metric.searchVolume,
|
||||
keywordDifficulty: metric.keywordDifficulty,
|
||||
cpc: metric.cpc,
|
||||
metricsFetchedAt: now,
|
||||
};
|
||||
})
|
||||
.filter((u): u is NonNullable<typeof u> => u !== null);
|
||||
|
||||
// Countries Labs doesn't cover get volume/CPC from Google Ads (no KD).
|
||||
const useGoogleAds =
|
||||
getKeywordDataProvider(config.locationCode) === "google_ads";
|
||||
|
||||
for (let i = 0; i < keywords.length; i += KEYWORD_OVERVIEW_BATCH_SIZE) {
|
||||
const batch = keywords.slice(i, i + KEYWORD_OVERVIEW_BATCH_SIZE);
|
||||
const request = {
|
||||
keywords: batch.map((kw) => kw.keyword),
|
||||
locationCode: config.locationCode,
|
||||
languageCode: config.languageCode,
|
||||
};
|
||||
|
||||
// Build a lookup by lowercase keyword
|
||||
const metricsMap = new Map<
|
||||
string,
|
||||
{
|
||||
searchVolume: number | null;
|
||||
keywordDifficulty: number | null;
|
||||
cpc: number | null;
|
||||
}
|
||||
>();
|
||||
if (useGoogleAds) {
|
||||
const adsItems = await client.keywords.adsSearchVolume({
|
||||
...request,
|
||||
creditFeature: "rank_tracking",
|
||||
});
|
||||
for (const item of adsItems) {
|
||||
if (!item.keyword) continue;
|
||||
metricsMap.set(item.keyword.toLowerCase(), {
|
||||
searchVolume: item.search_volume ?? null,
|
||||
keywordDifficulty: null,
|
||||
cpc: item.cpc ?? null,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
for (const item of await client.labs.keywordOverview(request)) {
|
||||
if (!item.keyword) continue;
|
||||
metricsMap.set(item.keyword.toLowerCase(), {
|
||||
searchVolume: item.keyword_info?.search_volume ?? null,
|
||||
keywordDifficulty:
|
||||
item.keyword_properties?.keyword_difficulty ?? null,
|
||||
cpc: item.keyword_info?.cpc ?? null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const updates = batch
|
||||
.map((kw) => {
|
||||
const metrics = metricsMap.get(kw.keyword.toLowerCase());
|
||||
if (!metrics) return null;
|
||||
return {
|
||||
id: kw.id,
|
||||
searchVolume: metrics.searchVolume,
|
||||
keywordDifficulty: metrics.keywordDifficulty,
|
||||
cpc: metrics.cpc,
|
||||
metricsFetchedAt: now,
|
||||
};
|
||||
})
|
||||
.filter((u): u is NonNullable<typeof u> => u !== null);
|
||||
|
||||
if (updates.length > 0) {
|
||||
await RankTrackingRepository.updateKeywordMetrics(updates);
|
||||
updated += updates.length;
|
||||
}
|
||||
}
|
||||
|
||||
return { updated };
|
||||
if (updates.length === 0) return { updated: 0 };
|
||||
await RankTrackingRepository.updateKeywordMetrics(updates);
|
||||
return { updated: updates.length };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@ -5,17 +5,18 @@
|
||||
|
||||
export { createDataforseoClient } from "@/server/lib/dataforseo/client";
|
||||
|
||||
export {
|
||||
fetchKeywordMetricsForList,
|
||||
type KeywordMetricRow,
|
||||
} from "@/server/lib/dataforseo/keyword-metrics";
|
||||
|
||||
export {
|
||||
type LabsKeywordDataItem,
|
||||
type DomainRankedKeywordItem,
|
||||
type RelevantPagesItem,
|
||||
type KeywordOverviewItem,
|
||||
} from "@/server/lib/dataforseo/labs";
|
||||
|
||||
export {
|
||||
type AdsKeywordItem,
|
||||
type AdsKeywordIdeaItem,
|
||||
} from "@/server/lib/dataforseo/google-ads";
|
||||
export { type AdsKeywordIdeaItem } from "@/server/lib/dataforseo/google-ads";
|
||||
|
||||
export {
|
||||
fetchRankCheckTaskResult,
|
||||
|
||||
132
src/server/lib/dataforseo/keyword-metrics.test.ts
Normal file
132
src/server/lib/dataforseo/keyword-metrics.test.ts
Normal file
@ -0,0 +1,132 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { fetchKeywordMetricsForList } from "./keyword-metrics";
|
||||
|
||||
type Client = Parameters<typeof fetchKeywordMetricsForList>[0];
|
||||
|
||||
// Minimal fake client exposing only the two endpoints the helper touches.
|
||||
function fakeClient(overrides: {
|
||||
keywordOverview?: Client["labs"]["keywordOverview"];
|
||||
adsSearchVolume?: Client["keywords"]["adsSearchVolume"];
|
||||
}): Client {
|
||||
return {
|
||||
labs: { keywordOverview: overrides.keywordOverview ?? vi.fn() },
|
||||
keywords: { adsSearchVolume: overrides.adsSearchVolume ?? vi.fn() },
|
||||
};
|
||||
}
|
||||
|
||||
describe("fetchKeywordMetricsForList", () => {
|
||||
it("normalizes Labs items and prefers clickstream-refined volume/trend", async () => {
|
||||
const keywordOverview = vi.fn().mockResolvedValue([
|
||||
{
|
||||
keyword: "best crm",
|
||||
keyword_info: {
|
||||
search_volume: 1000,
|
||||
cpc: 3.2,
|
||||
competition: 0.5,
|
||||
competition_level: "MEDIUM",
|
||||
monthly_searches: [{ year: 2026, month: 1, search_volume: 1000 }],
|
||||
},
|
||||
keyword_info_normalized_with_clickstream: {
|
||||
search_volume: 880,
|
||||
monthly_searches: [{ year: 2026, month: 1, search_volume: 880 }],
|
||||
},
|
||||
keyword_properties: { keyword_difficulty: 42 },
|
||||
search_intent_info: { main_intent: "commercial" },
|
||||
},
|
||||
]);
|
||||
const client = fakeClient({ keywordOverview });
|
||||
|
||||
const rows = await fetchKeywordMetricsForList(client, {
|
||||
keywords: ["best crm"],
|
||||
locationCode: 2840,
|
||||
languageCode: "en",
|
||||
includeClickstreamData: true,
|
||||
creditFeature: "keyword_research",
|
||||
});
|
||||
|
||||
expect(keywordOverview).toHaveBeenCalledWith({
|
||||
keywords: ["best crm"],
|
||||
locationCode: 2840,
|
||||
languageCode: "en",
|
||||
includeClickstreamData: true,
|
||||
creditFeature: "keyword_research",
|
||||
});
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
keyword: "best crm",
|
||||
searchVolume: 880,
|
||||
cpc: 3.2,
|
||||
competition: 0.5,
|
||||
competitionLevel: "MEDIUM",
|
||||
keywordDifficulty: 42,
|
||||
intent: "commercial",
|
||||
monthlySearches: [{ year: 2026, month: 1, searchVolume: 880 }],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("routes Google Ads locations and scales competition_index to a 0-1 ratio", async () => {
|
||||
const adsSearchVolume = vi.fn().mockResolvedValue([
|
||||
{
|
||||
keyword: "hotel reykjavik",
|
||||
search_volume: 1300,
|
||||
cpc: 2.54,
|
||||
competition: "HIGH",
|
||||
competition_index: 42,
|
||||
monthly_searches: [{ year: 2026, month: 5, search_volume: 1300 }],
|
||||
},
|
||||
{ keyword: undefined },
|
||||
]);
|
||||
const client = fakeClient({ adsSearchVolume });
|
||||
|
||||
const rows = await fetchKeywordMetricsForList(client, {
|
||||
keywords: ["hotel reykjavik"],
|
||||
locationCode: 2352,
|
||||
languageCode: "is",
|
||||
creditFeature: "rank_tracking",
|
||||
});
|
||||
|
||||
expect(adsSearchVolume).toHaveBeenCalledWith({
|
||||
keywords: ["hotel reykjavik"],
|
||||
locationCode: 2352,
|
||||
languageCode: "is",
|
||||
creditFeature: "rank_tracking",
|
||||
});
|
||||
// Item without a keyword is dropped; Ads carries no KD/intent.
|
||||
expect(rows).toEqual([
|
||||
{
|
||||
keyword: "hotel reykjavik",
|
||||
searchVolume: 1300,
|
||||
cpc: 2.54,
|
||||
competition: 0.42,
|
||||
competitionLevel: "HIGH",
|
||||
keywordDifficulty: null,
|
||||
intent: null,
|
||||
monthlySearches: [{ year: 2026, month: 5, searchVolume: 1300 }],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("batches keyword lists above the per-call cap", async () => {
|
||||
const keywordOverview = vi
|
||||
.fn()
|
||||
.mockImplementation(({ keywords }: { keywords: string[] }) =>
|
||||
Promise.resolve(
|
||||
keywords.map((keyword) => ({ keyword, keyword_info: {} })),
|
||||
),
|
||||
);
|
||||
const client = fakeClient({ keywordOverview });
|
||||
const keywords = Array.from({ length: 1500 }, (_, i) => `kw-${i}`);
|
||||
|
||||
const rows = await fetchKeywordMetricsForList(client, {
|
||||
keywords,
|
||||
locationCode: 2840,
|
||||
languageCode: "en",
|
||||
creditFeature: "keyword_research",
|
||||
});
|
||||
|
||||
expect(keywordOverview).toHaveBeenCalledTimes(3); // 700 + 700 + 100
|
||||
expect(rows).toHaveLength(1500);
|
||||
});
|
||||
});
|
||||
141
src/server/lib/dataforseo/keyword-metrics.ts
Normal file
141
src/server/lib/dataforseo/keyword-metrics.ts
Normal file
@ -0,0 +1,141 @@
|
||||
import type { createDataforseoClient } from "@/server/lib/dataforseo/client";
|
||||
import type { AdsKeywordItem } from "@/server/lib/dataforseo/google-ads";
|
||||
import type { KeywordOverviewItem } from "@/server/lib/dataforseo/labs";
|
||||
import type { CreditFeature } from "@/shared/billing-credit-features";
|
||||
import { getKeywordDataProvider } from "@/shared/keyword-locations";
|
||||
import type { MonthlySearch } from "@/types/keywords";
|
||||
|
||||
type DataforseoClient = ReturnType<typeof createDataforseoClient>;
|
||||
|
||||
// Narrowed to the two endpoints the helper uses, so tests can fake it cheaply.
|
||||
type KeywordMetricsClient = {
|
||||
labs: Pick<DataforseoClient["labs"], "keywordOverview">;
|
||||
keywords: Pick<DataforseoClient["keywords"], "adsSearchVolume">;
|
||||
};
|
||||
|
||||
// DataForSEO's batch metric endpoints accept up to ~700 keywords per request.
|
||||
const KEYWORD_METRICS_BATCH_SIZE = 700;
|
||||
|
||||
// `intent` is the raw `main_intent` (null for Google Ads); run it through
|
||||
// `normalizeIntent` for the app enum. `competition` is a 0-1 ratio.
|
||||
export type KeywordMetricRow = {
|
||||
keyword: string;
|
||||
searchVolume: number | null;
|
||||
cpc: number | null;
|
||||
competition: number | null;
|
||||
competitionLevel: string | null;
|
||||
keywordDifficulty: number | null;
|
||||
intent: string | null;
|
||||
monthlySearches: MonthlySearch[];
|
||||
};
|
||||
|
||||
function toMonthlySearches(
|
||||
entries:
|
||||
| {
|
||||
year?: number | null;
|
||||
month?: number | null;
|
||||
search_volume?: number | null;
|
||||
}[]
|
||||
| null
|
||||
| undefined,
|
||||
): MonthlySearch[] {
|
||||
return (entries ?? []).map((entry) => ({
|
||||
year: entry.year ?? 0,
|
||||
month: entry.month ?? 0,
|
||||
searchVolume: entry.search_volume ?? 0,
|
||||
}));
|
||||
}
|
||||
|
||||
function normalizeKeywordOverview(
|
||||
item: KeywordOverviewItem,
|
||||
keyword: string,
|
||||
): KeywordMetricRow {
|
||||
const info = item.keyword_info;
|
||||
// The clickstream-normalized block only exists when the caller opted into
|
||||
// clickstream data (it doubles request cost); prefer it when present.
|
||||
const clickstreamInfo = item.keyword_info_normalized_with_clickstream;
|
||||
const usesClickstream = clickstreamInfo?.search_volume != null;
|
||||
return {
|
||||
keyword,
|
||||
searchVolume: clickstreamInfo?.search_volume ?? info?.search_volume ?? null,
|
||||
cpc: info?.cpc ?? null,
|
||||
competition: info?.competition ?? null,
|
||||
competitionLevel: info?.competition_level ?? null,
|
||||
keywordDifficulty: item.keyword_properties?.keyword_difficulty ?? null,
|
||||
intent: item.search_intent_info?.main_intent ?? null,
|
||||
monthlySearches: toMonthlySearches(
|
||||
usesClickstream
|
||||
? clickstreamInfo?.monthly_searches
|
||||
: info?.monthly_searches,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
// Google Ads items (countries Labs doesn't cover) carry volume/CPC/competition
|
||||
// but no keyword difficulty or search intent.
|
||||
function normalizeAdsKeyword(
|
||||
item: AdsKeywordItem,
|
||||
keyword: string,
|
||||
): KeywordMetricRow {
|
||||
return {
|
||||
keyword,
|
||||
searchVolume: item.search_volume ?? null,
|
||||
cpc: item.cpc ?? null,
|
||||
// competition_index is a 0-100 scale; the app stores a 0-1 ratio.
|
||||
competition:
|
||||
item.competition_index != null ? item.competition_index / 100 : null,
|
||||
competitionLevel: item.competition ?? null,
|
||||
keywordDifficulty: null,
|
||||
intent: null,
|
||||
monthlySearches: toMonthlySearches(item.monthly_searches),
|
||||
};
|
||||
}
|
||||
|
||||
// Hydrate a keyword list with fresh metrics: route by location (Labs vs Google
|
||||
// Ads), batch under the per-call cap, and drop items DataForSEO returns without
|
||||
// a keyword. `creditFeature` is required so spend is always attributed.
|
||||
export async function fetchKeywordMetricsForList(
|
||||
client: KeywordMetricsClient,
|
||||
params: {
|
||||
keywords: string[];
|
||||
locationCode: number;
|
||||
languageCode: string;
|
||||
creditFeature: CreditFeature;
|
||||
includeClickstreamData?: boolean;
|
||||
},
|
||||
): Promise<KeywordMetricRow[]> {
|
||||
const useGoogleAds =
|
||||
getKeywordDataProvider(params.locationCode) === "google_ads";
|
||||
const rows: KeywordMetricRow[] = [];
|
||||
|
||||
for (let i = 0; i < params.keywords.length; i += KEYWORD_METRICS_BATCH_SIZE) {
|
||||
const keywords = params.keywords.slice(i, i + KEYWORD_METRICS_BATCH_SIZE);
|
||||
|
||||
if (useGoogleAds) {
|
||||
const items = await client.keywords.adsSearchVolume({
|
||||
keywords,
|
||||
locationCode: params.locationCode,
|
||||
languageCode: params.languageCode,
|
||||
creditFeature: params.creditFeature,
|
||||
});
|
||||
for (const item of items) {
|
||||
if (!item.keyword) continue;
|
||||
rows.push(normalizeAdsKeyword(item, item.keyword));
|
||||
}
|
||||
} else {
|
||||
const items = await client.labs.keywordOverview({
|
||||
keywords,
|
||||
locationCode: params.locationCode,
|
||||
languageCode: params.languageCode,
|
||||
includeClickstreamData: params.includeClickstreamData ?? false,
|
||||
creditFeature: params.creditFeature,
|
||||
});
|
||||
for (const item of items) {
|
||||
if (!item.keyword) continue;
|
||||
rows.push(normalizeKeywordOverview(item, item.keyword));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
@ -3,6 +3,7 @@ import type { ToolExtra } from "@/server/mcp/context";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { MCP_AUTH_CONTEXT_PROP } from "@/server/mcp/context";
|
||||
import type { fetchKeywordMetricsForList as FetchKeywordMetricsForList } from "@/server/lib/dataforseo/keyword-metrics";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
createDataforseoClient: vi.fn(),
|
||||
@ -13,9 +14,17 @@ vi.mock("cloudflare:workers", () => ({
|
||||
env: {},
|
||||
}));
|
||||
|
||||
vi.mock("@/server/lib/dataforseo", () => ({
|
||||
createDataforseoClient: mocks.createDataforseoClient,
|
||||
}));
|
||||
// Keep the real fetchKeywordMetricsForList (it only routes provider calls onto
|
||||
// the supplied client) so the handler's normalization is exercised end-to-end.
|
||||
vi.mock("@/server/lib/dataforseo", async () => {
|
||||
const keywordMetrics = await vi.importActual<{
|
||||
fetchKeywordMetricsForList: typeof FetchKeywordMetricsForList;
|
||||
}>("@/server/lib/dataforseo/keyword-metrics");
|
||||
return {
|
||||
createDataforseoClient: mocks.createDataforseoClient,
|
||||
fetchKeywordMetricsForList: keywordMetrics.fetchKeywordMetricsForList,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/server/features/projects/services/ProjectService", () => ({
|
||||
ProjectService: {
|
||||
|
||||
@ -3,6 +3,7 @@ import type { ToolExtra } from "@/server/mcp/context";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { MCP_AUTH_CONTEXT_PROP } from "@/server/mcp/context";
|
||||
import type { fetchKeywordMetricsForList as FetchKeywordMetricsForList } from "@/server/lib/dataforseo/keyword-metrics";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
createDataforseoClient: vi.fn(),
|
||||
@ -13,9 +14,17 @@ vi.mock("cloudflare:workers", () => ({
|
||||
env: {},
|
||||
}));
|
||||
|
||||
vi.mock("@/server/lib/dataforseo", () => ({
|
||||
createDataforseoClient: mocks.createDataforseoClient,
|
||||
}));
|
||||
// Keep the real fetchKeywordMetricsForList (it only routes provider calls onto
|
||||
// the supplied client) so the handler's normalization is exercised end-to-end.
|
||||
vi.mock("@/server/lib/dataforseo", async () => {
|
||||
const keywordMetrics = await vi.importActual<{
|
||||
fetchKeywordMetricsForList: typeof FetchKeywordMetricsForList;
|
||||
}>("@/server/lib/dataforseo/keyword-metrics");
|
||||
return {
|
||||
createDataforseoClient: mocks.createDataforseoClient,
|
||||
fetchKeywordMetricsForList: keywordMetrics.fetchKeywordMetricsForList,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/server/features/projects/services/ProjectService", () => ({
|
||||
ProjectService: {
|
||||
|
||||
@ -2,10 +2,9 @@
|
||||
import { z } from "zod";
|
||||
import {
|
||||
createDataforseoClient,
|
||||
type AdsKeywordItem,
|
||||
type KeywordOverviewItem,
|
||||
fetchKeywordMetricsForList,
|
||||
type KeywordMetricRow,
|
||||
} from "@/server/lib/dataforseo";
|
||||
import { getKeywordDataProvider } from "@/shared/keyword-locations";
|
||||
import { buildProjectMeta } from "@/server/mcp/context";
|
||||
import { mcpResponse } from "@/server/mcp/formatters";
|
||||
import {
|
||||
@ -439,45 +438,31 @@ function sortCompetitors(
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeKeywordOverview(item: KeywordOverviewItem) {
|
||||
const info = item.keyword_info;
|
||||
// Only present when the caller opted into clickstream-refined volumes.
|
||||
const clickstreamInfo = item.keyword_info_normalized_with_clickstream;
|
||||
// Project the shared canonical metric row onto this tool's snake_case API
|
||||
// shape (kept stable for MCP clients); an absent trend stays null as before.
|
||||
function toMcpKeywordMetricRow(row: KeywordMetricRow) {
|
||||
return {
|
||||
keyword: item.keyword,
|
||||
search_volume:
|
||||
clickstreamInfo?.search_volume ?? info?.search_volume ?? null,
|
||||
keyword_difficulty: item.keyword_properties?.keyword_difficulty ?? null,
|
||||
main_intent: item.search_intent_info?.main_intent ?? null,
|
||||
cpc: info?.cpc ?? null,
|
||||
competition: info?.competition ?? null,
|
||||
competition_level: info?.competition_level ?? null,
|
||||
monthly_searches:
|
||||
(clickstreamInfo?.search_volume
|
||||
? clickstreamInfo.monthly_searches
|
||||
: info?.monthly_searches) ?? null,
|
||||
keyword: row.keyword,
|
||||
search_volume: row.searchVolume,
|
||||
keyword_difficulty: row.keywordDifficulty,
|
||||
main_intent: row.intent,
|
||||
cpc: row.cpc,
|
||||
competition: row.competition,
|
||||
competition_level: row.competitionLevel,
|
||||
monthly_searches: row.monthlySearches.length
|
||||
? row.monthlySearches.map((entry) => ({
|
||||
year: entry.year,
|
||||
month: entry.month,
|
||||
search_volume: entry.searchVolume,
|
||||
}))
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
type KeywordMetricRow = ReturnType<typeof normalizeKeywordOverview>;
|
||||
|
||||
// Google Ads items (countries Labs doesn't cover) have no difficulty/intent.
|
||||
function normalizeAdsKeyword(item: AdsKeywordItem): KeywordMetricRow {
|
||||
return {
|
||||
keyword: item.keyword,
|
||||
search_volume: item.search_volume ?? null,
|
||||
keyword_difficulty: null,
|
||||
main_intent: null,
|
||||
cpc: item.cpc ?? null,
|
||||
competition:
|
||||
item.competition_index != null ? item.competition_index / 100 : null,
|
||||
competition_level: item.competition ?? null,
|
||||
monthly_searches: item.monthly_searches ?? null,
|
||||
};
|
||||
}
|
||||
type McpKeywordMetricRow = ReturnType<typeof toMcpKeywordMetricRow>;
|
||||
|
||||
function sortKeywordMetricRows(
|
||||
rows: KeywordMetricRow[],
|
||||
rows: McpKeywordMetricRow[],
|
||||
sortBy: NonNullable<GetKeywordMetricsArgs["sortBy"]> = "search_volume",
|
||||
) {
|
||||
return rows.toSorted((a, b) => {
|
||||
@ -846,27 +831,15 @@ export const getKeywordMetricsTool = {
|
||||
const client = createDataforseoClient(context.billing);
|
||||
const locationCode = args.locationCode ?? DEFAULT_LOCATION_CODE;
|
||||
const languageCode = args.languageCode ?? DEFAULT_LANGUAGE_CODE;
|
||||
const normalized =
|
||||
getKeywordDataProvider(locationCode) === "google_ads"
|
||||
? (
|
||||
await client.keywords.adsSearchVolume({
|
||||
keywords: args.keywords,
|
||||
locationCode,
|
||||
languageCode,
|
||||
creditFeature: "keyword_research",
|
||||
})
|
||||
).map(normalizeAdsKeyword)
|
||||
: (
|
||||
await client.labs.keywordOverview({
|
||||
keywords: args.keywords,
|
||||
locationCode,
|
||||
languageCode,
|
||||
includeClickstreamData: args.includeClickstreamData ?? false,
|
||||
creditFeature: "keyword_research",
|
||||
})
|
||||
).map(normalizeKeywordOverview);
|
||||
const metrics = await fetchKeywordMetricsForList(client, {
|
||||
keywords: args.keywords,
|
||||
locationCode,
|
||||
languageCode,
|
||||
includeClickstreamData: args.includeClickstreamData ?? false,
|
||||
creditFeature: "keyword_research",
|
||||
});
|
||||
const rows = sortKeywordMetricRows(
|
||||
normalized,
|
||||
metrics.map(toMcpKeywordMetricRow),
|
||||
args.sortBy ?? "search_volume",
|
||||
).map((row) =>
|
||||
args.includeMonthlyTrends === false
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user