feat: add keyword metrics (volume, difficulty, CPC) to rank tracking (#118)

drizzle/0009_smart_kitty_pryde.sql

@@ -0,0 +1,4 @@
+ALTER TABLE `rank_tracking_keywords` ADD `search_volume` integer;--> statement-breakpoint
+ALTER TABLE `rank_tracking_keywords` ADD `keyword_difficulty` integer;--> statement-breakpoint
+ALTER TABLE `rank_tracking_keywords` ADD `cpc` real;--> statement-breakpoint
+ALTER TABLE `rank_tracking_keywords` ADD `metrics_fetched_at` text;

drizzle/meta/_journal.json

@@ -64,6 +64,13 @@
       "when": 1776288697036,
       "tag": "0008_luxuriant_colossus",
       "breakpoints": true
+    },
+    {
+      "idx": 9,
+      "version": "6",
+      "when": 1776347182067,
+      "tag": "0009_smart_kitty_pryde",
+      "breakpoints": true
     }
   ]
 }

src/client/features/rank-tracking/ActionsMenu.tsx

@@ -1,18 +1,22 @@
 import { useState } from "react";
-import { MoreHorizontal, Play, Download, Copy } from "lucide-react";
+import { MoreHorizontal, Play, Download, Copy, RefreshCw } from "lucide-react";
 
 export function ActionsMenu({
   onCheckNow,
   onExport,
   onCopyKeywords,
+  onRefreshMetrics,
   isRunning,
+  metricsRefreshing,
   hasData,
   checkDisabled,
 }: {
   onCheckNow: () => void;
   onExport: () => void;
   onCopyKeywords: () => void;
+  onRefreshMetrics: () => void;
   isRunning: boolean;
+  metricsRefreshing: boolean;
   hasData: boolean;
   checkDisabled?: boolean;
 }) {
@@ -42,6 +46,19 @@ export function ActionsMenu({
                 {isRunning ? "Running..." : "Check Now"}
               </button>
             )}
+            <button
+              className="flex w-full items-center gap-2 px-3 py-2 text-sm hover:bg-base-200"
+              onClick={() => {
+                onRefreshMetrics();
+                setOpen(false);
+              }}
+              disabled={metricsRefreshing || !hasData}
+            >
+              <RefreshCw
+                className={`size-3.5 ${metricsRefreshing ? "animate-spin" : ""}`}
+              />
+              {metricsRefreshing ? "Refreshing..." : "Refresh Metrics"}
+            </button>
             <button
               className="flex w-full items-center gap-2 px-3 py-2 text-sm hover:bg-base-200"
               onClick={() => {

src/client/features/rank-tracking/RankTrackingColumns.tsx

@@ -4,13 +4,19 @@ import type { ColumnDef, SortingFn } from "@tanstack/react-table";
 import type { RankTrackingRow } from "@/types/schemas/rank-tracking";
 import {
   comparePositions,
+  CpcCell,
   DeviceRankCell,
   DeviceUrlCell,
+  DifficultyCell,
   SerpFeatureTags,
+  VolumeCell,
 } from "./RankTrackingTableParts";
 
 const HEADER_TOOLTIPS: Record<string, string> = {
   keyword: "The search term being tracked in Google",
+  volume: "Estimated monthly search volume from Google",
+  kd: "Keyword difficulty score (0-100) — higher means harder to rank",
+  cpc: "Average cost per click in Google Ads (USD)",
   desktopPosition:
     "Current Google ranking position, showing change from the comparison period",
   mobilePosition:
@@ -53,6 +59,46 @@ export function SortableHeader({
   );
 }
 
+const nullsLastNumeric: SortingFn<RankTrackingRow> = (rowA, rowB, columnId) => {
+  const a = rowA.getValue<number | null>(columnId);
+  const b = rowB.getValue<number | null>(columnId);
+  if (a == null && b == null) return 0;
+  if (a == null) return 1;
+  if (b == null) return -1;
+  return a - b;
+};
+
+const volumeColumn: ColumnDef<RankTrackingRow> = {
+  id: "volume",
+  accessorKey: "searchVolume",
+  header: ({ column }) => (
+    <SortableHeader column={column} label="Volume" id="volume" />
+  ),
+  size: 90,
+  cell: ({ getValue }) => <VolumeCell value={getValue<number | null>()} />,
+  sortingFn: nullsLastNumeric,
+};
+
+const kdColumn: ColumnDef<RankTrackingRow> = {
+  id: "kd",
+  accessorKey: "keywordDifficulty",
+  header: ({ column }) => <SortableHeader column={column} label="KD" id="kd" />,
+  size: 70,
+  cell: ({ getValue }) => <DifficultyCell value={getValue<number | null>()} />,
+  sortingFn: nullsLastNumeric,
+};
+
+const cpcColumn: ColumnDef<RankTrackingRow> = {
+  id: "cpc",
+  accessorKey: "cpc",
+  header: ({ column }) => (
+    <SortableHeader column={column} label="CPC" id="cpc" />
+  ),
+  size: 80,
+  cell: ({ getValue }) => <CpcCell value={getValue<number | null>()} />,
+  sortingFn: nullsLastNumeric,
+};
+
 const positionSort: SortingFn<RankTrackingRow> = (rowA, rowB, columnId) => {
   const device = columnId === "desktopPosition" ? "desktop" : "mobile";
   return comparePositions(
@@ -166,11 +212,16 @@ export function useRankTrackingColumns(
     if (showDesktop) {
       cols.push(makeDeviceColumn("desktop"));
       cols.push(makeUrlColumn("desktop", domain));
-      cols.push(makeSerpColumn("desktop"));
     }
     if (showMobile) {
       cols.push(makeDeviceColumn("mobile"));
       cols.push(makeUrlColumn("mobile", domain));
+    }
+    cols.push(volumeColumn, kdColumn, cpcColumn);
+    if (showDesktop) {
+      cols.push(makeSerpColumn("desktop"));
+    }
+    if (showMobile) {
       cols.push(makeSerpColumn("mobile"));
     }
     return cols;

src/client/features/rank-tracking/RankTrackingDomainDetail.tsx

@@ -40,6 +40,7 @@ import {
 } from "./RankTrackingFilters";
 import { CheckConfirmModal } from "./CheckConfirmModal";
 import { SegmentedToggle } from "@/client/components/SegmentedToggle";
+import { useMetricsRefresh } from "./useMetricsRefresh";
 import { useRankCheckTrigger } from "./useRankCheckTrigger";
 import { useRankRunPolling } from "./useRankRunPolling";
 
@@ -172,6 +173,9 @@ function RankTrackingDomainDetailInner({
     onSuccess: () => setPendingCheck(null),
   });
 
+  const { refresh: refreshMetrics, isRefreshing: metricsRefreshing } =
+    useMetricsRefresh(projectId, config.id);
+
   const requestCheck = (count: number, keywordIds?: string[]) => {
     if (count < 50) {
       startCheck({ keywordIds });
@@ -356,6 +360,8 @@ function RankTrackingDomainDetailInner({
               const count = costEstimate?.keywordCount ?? rows?.length ?? 0;
               if (count > 0) requestCheck(count);
             }}
+            onRefreshMetrics={() => refreshMetrics()}
+            metricsRefreshing={metricsRefreshing}
             onExport={() =>
               exportRankTrackingCsv(
                 filtered,

src/client/features/rank-tracking/RankTrackingTableParts.tsx

@@ -127,6 +127,37 @@ export function DeviceUrlCell({
   );
 }
 
+const compactFormatter = new Intl.NumberFormat("en-US", {
+  notation: "compact",
+  maximumFractionDigits: 1,
+});
+
+export function VolumeCell({ value }: { value: number | null }) {
+  if (value == null) return <span className="text-base-content/40">-</span>;
+  return (
+    <span className="font-mono text-sm">{compactFormatter.format(value)}</span>
+  );
+}
+
+export function DifficultyCell({ value }: { value: number | null }) {
+  if (value == null) return <span className="text-base-content/40">-</span>;
+  let badgeClass = "bg-success/20 text-success";
+  if (value > 60) badgeClass = "bg-error/20 text-error";
+  else if (value > 30) badgeClass = "bg-warning/20 text-warning";
+  return (
+    <span
+      className={`font-mono rounded px-1.5 py-0.5 text-xs font-semibold ${badgeClass}`}
+    >
+      {value}
+    </span>
+  );
+}
+
+export function CpcCell({ value }: { value: number | null }) {
+  if (value == null) return <span className="text-base-content/40">-</span>;
+  return <span className="font-mono text-sm">${value.toFixed(2)}</span>;
+}
+
 export function comparePositions(a: number | null, b: number | null): number {
   if (a === null && b === null) return 0;
   if (a === null) return 1; // nulls sort last
@@ -156,6 +187,9 @@ export function exportRankTrackingCsv(
   }
   const headers = [
     "Keyword",
+    "Volume",
+    "KD",
+    "CPC",
     ...(showDesktop
       ? [
           "Desktop Position",
@@ -175,6 +209,9 @@ export function exportRankTrackingCsv(
   ];
   const csvRows = sorted.map((row) => [
     row.keyword,
+    row.searchVolume ?? "",
+    row.keywordDifficulty ?? "",
+    row.cpc != null ? row.cpc.toFixed(2) : "",
     ...(showDesktop
       ? [
           row.desktop.position ?? "Not ranking",

src/client/features/rank-tracking/useMetricsRefresh.ts

@@ -0,0 +1,23 @@
+import { toast } from "sonner";
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+import { refreshTrackingKeywordMetrics } from "@/serverFunctions/rank-tracking";
+
+export function useMetricsRefresh(projectId: string, configId: string) {
+  const queryClient = useQueryClient();
+  const mutation = useMutation({
+    mutationFn: () =>
+      refreshTrackingKeywordMetrics({
+        data: { projectId, configId },
+      }),
+    onSuccess: (result) => {
+      void queryClient.invalidateQueries({
+        queryKey: ["rankTrackingResults", projectId, configId],
+      });
+      toast.success(`Metrics updated for ${result.updated} keywords`);
+    },
+    onError: () => {
+      toast.error("Failed to refresh keyword metrics");
+    },
+  });
+  return { refresh: mutation.mutate, isRefreshing: mutation.isPending };
+}

src/db/app.schema.ts

@@ -152,6 +152,10 @@ export const rankTrackingKeywords = sqliteTable(
       .notNull()
       .references(() => rankTrackingConfigs.id, { onDelete: "cascade" }),
     keyword: text("keyword").notNull(),
+    searchVolume: integer("search_volume"),
+    keywordDifficulty: integer("keyword_difficulty"),
+    cpc: real("cpc"),
+    metricsFetchedAt: text("metrics_fetched_at"),
     createdAt: text("created_at")
       .notNull()
       .default(sql`(current_timestamp)`),

src/server/features/rank-tracking/repositories/RankTrackingRepository.ts

@@ -332,6 +332,28 @@ async function getConfigSummaries(projectId: string) {
   }));
 }
 
+async function updateKeywordMetrics(
+  updates: Array<{
+    id: string;
+    searchVolume: number | null;
+    keywordDifficulty: number | null;
+    cpc: number | null;
+    metricsFetchedAt: string;
+  }>,
+) {
+  await executeInBatches(updates, (u) =>
+    db
+      .update(rankTrackingKeywords)
+      .set({
+        searchVolume: u.searchVolume,
+        keywordDifficulty: u.keywordDifficulty,
+        cpc: u.cpc,
+        metricsFetchedAt: u.metricsFetchedAt,
+      })
+      .where(eq(rankTrackingKeywords.id, u.id)),
+  );
+}
+
 async function getKeywordCountForConfig(configId: string) {
   const rows = await db
     .select({ value: count() })
@@ -359,6 +381,7 @@ export const RankTrackingRepository = {
   getKeywordsForConfig,
   addKeywordsToConfig,
   removeKeywordsFromConfig,
+  updateKeywordMetrics,
   getKeywordCountForConfig,
   getConfigSummaries,
   getLatestSnapshotsForKeywords,

src/server/features/rank-tracking/services/RankTrackingService.ts

@@ -1,5 +1,6 @@
 import { env } from "cloudflare:workers";
 import type { BillingCustomerContext } from "@/server/billing/subscription";
+import { createDataforseoClient } from "@/server/lib/dataforseoClient";
 import { RankTrackingRepository } from "@/server/features/rank-tracking/repositories/RankTrackingRepository";
 import { AppError } from "@/server/lib/errors";
 import type {
@@ -232,6 +233,73 @@ async function getLatestRun(configId: string, projectId: string) {
   return formatRun(run);
 }
 
+// ---------------------------------------------------------------------------
+// Keyword metrics (volume, difficulty, CPC)
+// ---------------------------------------------------------------------------
+
+const KEYWORD_OVERVIEW_BATCH_SIZE = 700;
+
+async function refreshKeywordMetrics(
+  configId: string,
+  projectId: string,
+  billingCustomer: BillingCustomerContext,
+): Promise<{ updated: number }> {
+  const config = await getValidatedConfig(configId, projectId);
+  const keywords = await RankTrackingRepository.getKeywordsForConfig(configId);
+  if (keywords.length === 0) return { updated: 0 };
+
+  const client = createDataforseoClient(billingCustomer);
+  const now = new Date().toISOString();
+  let updated = 0;
+
+  for (let i = 0; i < keywords.length; i += KEYWORD_OVERVIEW_BATCH_SIZE) {
+    const batch = keywords.slice(i, i + KEYWORD_OVERVIEW_BATCH_SIZE);
+    const items = await client.labs.keywordOverview({
+      keywords: batch.map((kw) => kw.keyword),
+      locationCode: config.locationCode,
+      languageCode: config.languageCode,
+    });
+
+    // Build a lookup by lowercase keyword
+    const metricsMap = new Map<
+      string,
+      {
+        searchVolume: number | null;
+        keywordDifficulty: number | null;
+        cpc: number | null;
+      }
+    >();
+    for (const item of items) {
+      metricsMap.set(item.keyword.toLowerCase(), {
+        searchVolume: item.keyword_info?.search_volume ?? null,
+        keywordDifficulty: item.keyword_properties?.keyword_difficulty ?? null,
+        cpc: item.keyword_info?.cpc ?? null,
+      });
+    }
+
+    const updates = batch
+      .map((kw) => {
+        const metrics = metricsMap.get(kw.keyword.toLowerCase());
+        if (!metrics) return null;
+        return {
+          id: kw.id,
+          searchVolume: metrics.searchVolume,
+          keywordDifficulty: metrics.keywordDifficulty,
+          cpc: metrics.cpc,
+          metricsFetchedAt: now,
+        };
+      })
+      .filter((u): u is NonNullable<typeof u> => u !== null);
+
+    if (updates.length > 0) {
+      await RankTrackingRepository.updateKeywordMetrics(updates);
+      updated += updates.length;
+    }
+  }
+
+  return { updated };
+}
+
 // ---------------------------------------------------------------------------
 // Cost estimation
 // ---------------------------------------------------------------------------
@@ -314,4 +382,5 @@ export const RankTrackingService = {
   triggerCheck,
   getLatestRun,
   estimateCost,
+  refreshKeywordMetrics,
 };

src/server/features/rank-tracking/services/rankTrackingResults.ts

@@ -89,6 +89,9 @@ export async function getLatestResults(
       {
         trackingKeywordId: keyword.id,
         keyword: keyword.keyword,
+        searchVolume: keyword.searchVolume,
+        keywordDifficulty: keyword.keywordDifficulty,
+        cpc: keyword.cpc,
         desktop: createEmptyDeviceResult(
           previousPositions.get(`${keyword.id}:desktop`) ?? null,
         ),

src/server/lib/dataforseo.ts

@@ -15,6 +15,7 @@ import {
   dataforseoResponseSchema,
   domainMetricsItemSchema,
   domainRankedKeywordItemSchema,
+  keywordOverviewItemSchema,
   labsKeywordDataItemSchema,
   parseTaskItems,
   relatedKeywordItemSchema,
@@ -22,6 +23,7 @@ import {
   type DataforseoTask,
   type DomainMetricsItem,
   type DomainRankedKeywordItem,
+  type KeywordOverviewItem,
   type LabsKeywordDataItem,
   type RelatedKeywordItem,
   type SerpLiveItem,
@@ -479,3 +481,67 @@ export async function fetchRankCheckSerpRaw(input: {
     billing: buildTaskBilling(parsedTask.data),
   };
 }
+
+// ---------------------------------------------------------------------------
+// DataForSEO Labs — Keyword Overview (batch up to 700 keywords)
+// ---------------------------------------------------------------------------
+
+export async function fetchKeywordOverviewRaw(
+  keywords: string[],
+  locationCode: number,
+  languageCode: string,
+): Promise<DataforseoApiResponse<KeywordOverviewItem[]>> {
+  const responseRaw = await postDataforseo(
+    "/v3/dataforseo_labs/google/keyword_overview/live",
+    [
+      {
+        keywords,
+        location_code: locationCode,
+        language_code: languageCode,
+      },
+    ],
+  );
+
+  const response = dataforseoResponseSchema.parse(responseRaw);
+
+  if (response.status_code !== 20000) {
+    throw new AppError(
+      "INTERNAL_ERROR",
+      response.status_message || "DataForSEO keyword overview request failed",
+    );
+  }
+
+  const task = response.tasks?.[0];
+  if (!task) {
+    throw new AppError(
+      "INTERNAL_ERROR",
+      "DataForSEO keyword overview response missing task",
+    );
+  }
+
+  if (task.status_code !== 20000) {
+    throw new AppError(
+      "INTERNAL_ERROR",
+      task.status_message || "DataForSEO keyword overview task failed",
+    );
+  }
+
+  const parsedTask = successfulDataforseoTaskSchema.safeParse(task);
+  if (!parsedTask.success) {
+    throw new AppError(
+      "INTERNAL_ERROR",
+      "DataForSEO keyword overview task missing billing metadata",
+    );
+  }
+
+  const data = parseTaskItems(
+    "google-keyword-overview-live",
+    parsedTask.data,
+    keywordOverviewItemSchema,
+  );
+
+  return {
+    data,
+    billing: buildTaskBilling(parsedTask.data),
+  };
+}

src/server/lib/dataforseoClient.ts

@@ -10,6 +10,7 @@ import { getOrCreateOrganizationCustomer } from "@/server/billing/subscription";
 import type { BillingCustomerContext } from "@/server/billing/subscription";
 import {
   fetchKeywordIdeasRaw,
+  fetchKeywordOverviewRaw,
   fetchKeywordSuggestionsRaw,
   fetchRelatedKeywordsRaw,
   fetchDomainRankOverviewRaw,
@@ -186,6 +187,24 @@ export function createDataforseoClient(customer: BillingCustomerContext) {
         );
       },
     },
+    labs: {
+      keywordOverview(input: {
+        keywords: string[];
+        locationCode: number;
+        languageCode: string;
+      }) {
+        return meterDataforseoCall(
+          customer,
+          () =>
+            fetchKeywordOverviewRaw(
+              input.keywords,
+              input.locationCode,
+              input.languageCode,
+            ),
+          "rank_tracking",
+        );
+      },
+    },
     serp: {
       live(input: {
         keyword: string;

src/server/lib/dataforseoSchemas.ts

@@ -189,6 +189,16 @@ export const serpSnapshotItemSchema = z
   })
   .passthrough();
 
+export const keywordOverviewItemSchema = z
+  .object({
+    keyword: z.string(),
+    keyword_info: keywordInfoSchema.optional(),
+    keyword_properties: keywordPropertiesSchema.nullable().optional(),
+    search_intent_info: searchIntentInfoSchema.nullable().optional(),
+  })
+  .passthrough();
+
+export type KeywordOverviewItem = z.infer<typeof keywordOverviewItemSchema>;
 export type RelatedKeywordItem = z.infer<typeof relatedKeywordItemSchema>;
 export type LabsKeywordDataItem = z.infer<typeof labsKeywordDataItemSchema>;
 export type DomainMetricsItem = z.infer<typeof domainMetricsItemSchema>;

src/serverFunctions/rank-tracking.ts

@@ -18,6 +18,7 @@ import {
   estimateCostSchema,
   addKeywordsSchema,
   removeKeywordsSchema,
+  refreshMetricsSchema,
 } from "@/types/schemas/rank-tracking";
 
 export const getRankTrackingConfigs = createServerFn({ method: "POST" })
@@ -190,6 +191,26 @@ export const addTrackingKeywords = createServerFn({ method: "POST" })
       }
     }
 
+    // Fetch keyword metrics (awaited so they're in the DB before client re-fetches)
+    if (result.added > 0) {
+      try {
+        await RankTrackingService.refreshKeywordMetrics(
+          data.configId,
+          context.projectId,
+          context,
+        );
+      } catch (err) {
+        const appErr = asAppError(err);
+        if (appErr?.code === "INSUFFICIENT_CREDITS") {
+          console.info(
+            "[rank-tracking] auto-metrics-refresh skipped: insufficient credits",
+          );
+        } else {
+          console.error("[rank-tracking] auto-metrics-refresh failed:", err);
+        }
+      }
+    }
+
     return { ...result, checkTriggered };
   });
 
@@ -204,3 +225,29 @@ export const removeTrackingKeywords = createServerFn({ method: "POST" })
     );
     return { removed: data.keywordIds.length };
   });
+
+export const refreshTrackingKeywordMetrics = createServerFn({ method: "POST" })
+  .middleware(requireProjectContext)
+  .inputValidator((data: unknown) => refreshMetricsSchema.parse(data))
+  .handler(async ({ data, context }) => {
+    const result = await RankTrackingService.refreshKeywordMetrics(
+      data.configId,
+      context.projectId,
+      context,
+    );
+
+    waitUntil(
+      captureServerEvent({
+        distinctId: context.userId,
+        event: "rank_tracking:metrics_refresh",
+        organizationId: context.organizationId,
+        properties: {
+          project_id: context.projectId,
+          config_id: data.configId,
+          updated: result.updated,
+        },
+      }),
+    );
+
+    return result;
+  });

src/types/schemas/rank-tracking.ts

@@ -34,6 +34,9 @@ export interface RankTrackingDeviceResult {
 export interface RankTrackingRow {
   trackingKeywordId: string;
   keyword: string;
+  searchVolume: number | null;
+  keywordDifficulty: number | null;
+  cpc: number | null;
   desktop: RankTrackingDeviceResult;
   mobile: RankTrackingDeviceResult;
 }
@@ -106,3 +109,8 @@ export const removeKeywordsSchema = z.object({
   configId: z.string().uuid(),
   keywordIds: z.array(z.string().uuid()).min(1).max(2000),
 });
+
+export const refreshMetricsSchema = z.object({
+  projectId: z.string().uuid(),
+  configId: z.string().uuid(),
+});