SERP analysis: depth 20 by default, load top 100 on demand; classify DataForSEO timeouts (#523)

src/client/features/keywords/components/SerpAnalysisCard.tsx

@@ -6,8 +6,11 @@ export function SerpAnalysisCard({
   items,
   keyword,
   loading,
+  loadingMore,
+  canLoadMore,
   error,
   onRetry,
+  deepFetchFailed,
   page,
   pageSize,
   onPageChange,
@@ -15,8 +18,14 @@ export function SerpAnalysisCard({
   items: SerpResultItem[];
   keyword?: string | null;
   loading: boolean;
+  /** A deeper snapshot is being fetched; `items` is still the shallow one. */
+  loadingMore: boolean;
+  /** Paging past the loaded results can buy a deeper snapshot. */
+  canLoadMore: boolean;
   error?: string | null;
   onRetry?: () => void;
+  /** The failure was the deeper crawl, so retrying restores the shallow one. */
+  deepFetchFailed: boolean;
   page: number;
   pageSize: number;
   onPageChange: (p: number) => void;
@@ -31,7 +40,7 @@ export function SerpAnalysisCard({
         <p>{error}</p>
         {onRetry ? (
           <button className="btn btn-xs" onClick={onRetry}>
-            Retry
+            {deepFetchFailed ? "Show top 20" : "Retry"}
           </button>
         ) : null}
       </div>
@@ -56,10 +65,16 @@ export function SerpAnalysisCard({
           feature="serp_analysis"
         />
       </div>
-      <SerpAnalysisTable items={pageItems} />
+      {pageItems.length === 0 && loadingMore ? (
+        <SerpAnalysisLoadingState />
+      ) : (
+        <SerpAnalysisTable items={pageItems} />
+      )}
       <SerpAnalysisPagination
         page={page}
         totalPages={totalPages}
+        loadingMore={loadingMore}
+        canLoadMore={canLoadMore}
         onPageChange={onPageChange}
       />
     </div>
@@ -113,23 +128,38 @@ function SerpAnalysisTable({ items }: { items: SerpResultItem[] }) {
 function SerpAnalysisPagination({
   page,
   totalPages,
+  loadingMore,
+  canLoadMore,
   onPageChange,
 }: {
   page: number;
   totalPages: number;
+  loadingMore: boolean;
+  canLoadMore: boolean;
   onPageChange: (p: number) => void;
 }) {
-  if (totalPages <= 1) return null;
+  if (totalPages <= 1 && !canLoadMore) return null;
+
+  // Past the loaded results, "Next" stops being free paging and buys a deeper
+  // crawl — say so on the button rather than spending silently.
+  const nextBuysDeeperSnapshot =
+    canLoadMore && !loadingMore && page >= totalPages - 1;
 
   return (
     <div className="flex items-center justify-between mt-3 pt-3 border-t border-base-200">
       <span className="text-xs text-base-content/50">
-        Page {page + 1} of {totalPages}
+        {loadingMore ? (
+          "Loading more results…"
+        ) : (
+          <>
+            Page {page + 1} of {totalPages}
+          </>
+        )}
       </span>
       <div className="flex gap-1">
         <button
           className="btn btn-ghost btn-xs"
-          disabled={page === 0}
+          disabled={page === 0 || loadingMore}
           onClick={() => onPageChange(page - 1)}
         >
           <ChevronLeft className="size-3.5" />
@@ -137,10 +167,10 @@ function SerpAnalysisPagination({
         </button>
         <button
           className="btn btn-ghost btn-xs"
-          disabled={page >= totalPages - 1}
+          disabled={loadingMore || (page >= totalPages - 1 && !canLoadMore)}
           onClick={() => onPageChange(page + 1)}
         >
-          Next
+          {nextBuysDeeperSnapshot ? "Load top 100" : "Next"}
           <ChevronRight className="size-3.5" />
         </button>
       </div>

src/client/features/keywords/hooks/useKeywordSerpAnalysis.ts

@@ -1,32 +1,67 @@
 import { useQuery } from "@tanstack/react-query";
-import { useState } from "react";
+import { useCallback, useState } from "react";
 import { getStandardErrorMessage } from "@/client/lib/error-messages";
 import { getSerpAnalysis } from "@/serverFunctions/keywords";
 
+const SERP_PAGE_SIZE = 10;
+/** Depth every SERP panel opens at — two pages of results, ~5 credits. */
+const SERP_INITIAL_DEPTH = 20;
+/**
+ * Depth we buy when a user pages past what's loaded. Google has no offset, so
+ * this re-crawls the top of the SERP: the deeper snapshot replaces the shallow
+ * one instead of extending it, and costs ~20 more credits.
+ */
+const SERP_DEEP_DEPTH = 100;
+
 export function useKeywordSerpAnalysis(
   projectId: string,
   locationCode: number | undefined,
 ) {
-  const [serpKeyword, setSerpKeyword] = useState<string | null>(null);
-  const [serpPage, setSerpPage] = useState(0);
-  const SERP_PAGE_SIZE = 10;
+  const [serpKeyword, setSerpKeywordState] = useState<string | null>(null);
+  const [serpPage, setSerpPageState] = useState(0);
+  const [requestedDepth, setRequestedDepth] = useState<20 | 100>(
+    SERP_INITIAL_DEPTH,
+  );
+
+  // Everything but the depth identifies the snapshot; the depth decides how
+  // deep it goes.
+  const snapshotKey = ["serpAnalysis", projectId, serpKeyword, locationCode];
 
   const serpQuery = useQuery({
-    queryKey: ["serpAnalysis", projectId, serpKeyword, locationCode],
+    queryKey: [...snapshotKey, requestedDepth],
     queryFn: () =>
       getSerpAnalysis({
         data: {
           projectId,
           keyword: serpKeyword!,
           locationCode,
+          depth: requestedDepth,
         },
       }),
+    // Keep the shallow snapshot on screen while the deeper refetch runs — but
+    // only when nothing except the depth changed, so a new keyword or market
+    // shows the loading state instead of the previous SERP.
+    placeholderData: (previous, previousQuery) => {
+      const previousKey = previousQuery?.queryKey;
+      const sameSnapshot = snapshotKey.every(
+        (part, index) => previousKey?.[index] === part,
+      );
+      return sameSnapshot ? previous : undefined;
+    },
+    // Every fetch here is billed, and a deep crawl can outrun DataForSEO's
+    // request deadline — a timed-out call may already be billed upstream while
+    // metering nothing. Never replay or re-issue one without the user asking;
+    // the card offers an explicit retry.
+    retry: false,
+    refetchOnReconnect: false,
+    refetchOnWindowFocus: false,
     enabled: !!serpKeyword,
     // Every attempt is a fresh billed DataForSEO task, so a failure must not be
     // retried automatically — the user re-runs the search instead.
     retry: false,
   });
 
+  const { refetch: refetchSerp } = serpQuery;
   const serpResults = serpQuery.data?.items ?? [];
   const activeSerpKeyword =
     serpKeyword ?? serpQuery.data?.requestedKeyword ?? null;
@@ -35,16 +70,72 @@ export function useKeywordSerpAnalysis(
     ? getStandardErrorMessage(serpQuery.error, "Failed to load SERP data.")
     : null;
 
+  // A deeper fetch is available until the loaded snapshot is the deep one.
+  const canLoadMoreSerp =
+    (serpQuery.data?.depth ?? SERP_DEEP_DEPTH) < SERP_DEEP_DEPTH;
+  const serpLoadingMore = serpQuery.isPlaceholderData;
+
+  // A failed deep fetch leaves nothing on screen, and retrying it re-buys the
+  // expensive call. The card offers the shallow snapshot back instead.
+  const deepFetchFailed =
+    serpQuery.isError && requestedDepth === SERP_DEEP_DEPTH;
+
+  // Retrying a failed deep crawl re-buys the expensive call that just timed
+  // out. Drop back to the shallow snapshot instead: it's still cached, so the
+  // user gets their results back for free and can ask for 100 again.
+  const retrySerp = useCallback(() => {
+    if (requestedDepth === SERP_DEEP_DEPTH) {
+      setRequestedDepth(SERP_INITIAL_DEPTH);
+      return;
+    }
+    void refetchSerp();
+  }, [refetchSerp, requestedDepth]);
+
+  const setSerpKeyword = useCallback((keyword: string | null) => {
+    setSerpKeywordState(keyword);
+    setRequestedDepth(SERP_INITIAL_DEPTH);
+  }, []);
+
+  const loadedPages = Math.max(
+    1,
+    Math.ceil(serpResults.length / SERP_PAGE_SIZE),
+  );
+  // While the deeper snapshot loads the user stays on the page they asked for;
+  // once it lands, a SERP too small to fill that page falls back to the last.
+  const visiblePage = serpLoadingMore
+    ? serpPage
+    : Math.min(serpPage, loadedPages - 1);
+
+  const setSerpPage = useCallback(
+    (nextPage: number) => {
+      // Paging forward past the loaded results buys the full 100-deep snapshot;
+      // the user lands on the requested page once it arrives.
+      if (
+        nextPage > visiblePage &&
+        nextPage >= loadedPages &&
+        canLoadMoreSerp
+      ) {
+        setRequestedDepth(SERP_DEEP_DEPTH);
+      }
+      setSerpPageState(nextPage);
+    },
+    [canLoadMoreSerp, loadedPages, visiblePage],
+  );
+
   return {
     serpKeyword,
     setSerpKeyword,
-    serpPage,
+    retrySerp,
+    serpPage: visiblePage,
     setSerpPage,
     SERP_PAGE_SIZE,
     serpQuery,
     serpResults,
     activeSerpKeyword,
     serpLoading,
+    serpLoadingMore,
+    canLoadMoreSerp,
+    deepFetchFailed,
     serpError,
   };
 }

src/client/features/keywords/page/KeywordResearchDesktopResults.tsx

@@ -373,8 +373,11 @@ function DesktopSerpPanel({ controller }: Props) {
             items={controller.serpResults}
             keyword={controller.activeSerpKeyword}
             loading={controller.serpLoading}
+            loadingMore={controller.serpLoadingMore}
+            canLoadMore={controller.canLoadMoreSerp}
             error={controller.serpError}
-            onRetry={() => void controller.serpQuery.refetch()}
+            onRetry={controller.retrySerp}
+            deepFetchFailed={controller.deepFetchFailed}
             page={controller.serpPage}
             pageSize={controller.SERP_PAGE_SIZE}
             onPageChange={controller.setSerpPage}

src/client/features/keywords/page/KeywordResearchMobileResults.tsx

@@ -68,8 +68,11 @@ export function KeywordResearchMobileResults({ controller }: Props) {
             items={controller.serpResults}
             keyword={controller.activeSerpKeyword}
             loading={controller.serpLoading}
+            loadingMore={controller.serpLoadingMore}
+            canLoadMore={controller.canLoadMoreSerp}
             error={controller.serpError}
-            onRetry={() => void controller.serpQuery.refetch()}
+            onRetry={controller.retrySerp}
+            deepFetchFailed={controller.deepFetchFailed}
             page={controller.serpPage}
             pageSize={controller.SERP_PAGE_SIZE}
             onPageChange={controller.setSerpPage}

src/client/features/keywords/state/useKeywordResearchController.ts

@@ -76,6 +76,10 @@ export function useKeywordResearchController(
     serpResults,
     activeSerpKeyword,
     serpLoading,
+    serpLoadingMore,
+    canLoadMoreSerp,
+    deepFetchFailed,
+    retrySerp,
     serpError,
   } = useKeywordSerpAnalysis(input.projectId, locationCode);
 
@@ -260,11 +264,15 @@ export function useKeywordResearchController(
     researchMutationError,
     retrySearch,
     resetFilters,
+    retrySerp,
     rows,
     searchedKeyword,
     selectedRows,
+    canLoadMoreSerp,
+    deepFetchFailed,
     serpError,
     serpLoading,
+    serpLoadingMore,
     serpPage,
     serpQuery,
     serpResults,

src/server/features/keywords/services/research/serp.test.ts

@@ -0,0 +1,124 @@
+import { describe, expect, it, vi } from "vitest";
+import type { BillingCustomerContext } from "@/server/billing/subscription";
+
+const mocks = vi.hoisted(() => ({
+  createDataforseoClient: vi.fn(),
+  getCached: vi.fn(),
+  setCached: vi.fn(async () => {}),
+}));
+
+vi.mock("cloudflare:workers", () => ({ waitUntil: vi.fn() }));
+
+vi.mock("@/server/lib/r2-cache", () => ({
+  buildCacheKey: vi.fn(async () => "serp:analysis:key"),
+  getCached: mocks.getCached,
+  setCached: mocks.setCached,
+}));
+
+vi.mock("@/server/lib/dataforseo", () => ({
+  SERP_ANALYSIS_DEPTH: 20,
+  createDataforseoClient: mocks.createDataforseoClient,
+}));
+
+import { getSerpAnalysis } from "./serp";
+
+const billingCustomer: BillingCustomerContext = {
+  organizationId: "org_1",
+  userId: "user_1",
+  userEmail: "alice@example.com",
+};
+
+const input = {
+  projectId: "proj_1",
+  keyword: "seo tools",
+  locationCode: 2840,
+  languageCode: "en",
+};
+
+function cachedSnapshot(depth: number) {
+  return {
+    requestedKeyword: "seo tools",
+    depth,
+    items: [
+      {
+        rank: 1,
+        title: "Cached",
+        url: "https://cached.example",
+        domain: "cached.example",
+        description: "",
+        etv: null,
+        estimatedPaidTrafficCost: null,
+        referringDomains: null,
+        backlinks: null,
+        isNew: false,
+        rankChange: null,
+      },
+    ],
+  };
+}
+
+function mockLiveSerp() {
+  const live = vi.fn().mockResolvedValue([
+    {
+      type: "organic",
+      rank_group: 1,
+      title: "Live",
+      url: "https://live.example",
+      domain: "live.example",
+    },
+  ]);
+  mocks.createDataforseoClient.mockReturnValue({ serp: { live } });
+  return live;
+}
+
+describe("getSerpAnalysis cache depth", () => {
+  it("keeps a good snapshot when the deeper crawl comes back empty", async () => {
+    mocks.getCached.mockResolvedValue(cachedSnapshot(20));
+    mocks.createDataforseoClient.mockReturnValue({
+      serp: { live: vi.fn().mockResolvedValue([]) },
+    });
+
+    const result = await getSerpAnalysis(
+      { ...input, depth: 100 },
+      billingCustomer,
+    );
+
+    expect(result.items).toEqual([]);
+    // Caching the empty deep result would answer every shallower request with
+    // nothing until the entry expires.
+    expect(mocks.setCached).not.toHaveBeenCalled();
+  });
+
+  it("serves a depth-100 snapshot for a depth-20 request", async () => {
+    mocks.getCached.mockResolvedValue(cachedSnapshot(100));
+    const live = mockLiveSerp();
+
+    const result = await getSerpAnalysis(
+      { ...input, depth: 20 },
+      billingCustomer,
+    );
+
+    expect(live).not.toHaveBeenCalled();
+    expect(result.depth).toBe(100);
+    expect(result.items[0]?.title).toBe("Cached");
+  });
+
+  it("refetches live at depth 100 when only a depth-20 snapshot is cached", async () => {
+    mocks.getCached.mockResolvedValue(cachedSnapshot(20));
+    const live = mockLiveSerp();
+
+    const result = await getSerpAnalysis(
+      { ...input, depth: 100 },
+      billingCustomer,
+    );
+
+    expect(live).toHaveBeenCalledWith(expect.objectContaining({ depth: 100 }));
+    expect(result.items[0]?.title).toBe("Live");
+    // The deeper snapshot replaces the shallow entry under the same key.
+    expect(mocks.setCached).toHaveBeenCalledWith(
+      "serp:analysis:key",
+      expect.objectContaining({ depth: 100 }),
+      expect.any(Number),
+    );
+  });
+});

src/server/features/keywords/services/research/serp.ts

@@ -1,5 +1,8 @@
 import { waitUntil } from "cloudflare:workers";
-import { type SerpLiveItem } from "@/server/lib/dataforseo";
+import {
+  SERP_ANALYSIS_DEPTH,
+  type SerpLiveItem,
+} from "@/server/lib/dataforseo";
 import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
 import type { SerpResultItem } from "@/types/keywords";
 import { z } from "zod";
@@ -14,6 +17,8 @@ type SerpAnalysisReason = "no_organic_results";
 type SerpAnalysisResult = {
   requestedKeyword: string;
   items: SerpResultItem[];
+  /** SERP depth this snapshot was crawled at — how deep the results go. */
+  depth: number;
   reason?: SerpAnalysisReason;
 };
 
@@ -34,6 +39,10 @@ const serpResultItemSchema = z.object({
 const serpCacheSchema = z.object({
   requestedKeyword: z.string(),
   items: z.array(serpResultItemSchema),
+  // Entries written before depth was tracked don't say how deep they went, so
+  // they're treated as the shallow floor: worst case a user re-buys a deeper
+  // crawl once, rather than being shown 20 results as if they were 100.
+  depth: z.number().int().default(SERP_ANALYSIS_DEPTH),
   reason: z.enum(["no_organic_results"]).optional(),
 });
 
@@ -61,10 +70,12 @@ async function getSerpLiveAnalysis(
     keyword: string;
     locationCode: number;
     languageCode: string;
+    depth: number;
   },
   billingCustomer: BillingCustomerContext,
 ): Promise<SerpAnalysisResult> {
   const keyword = normalizeKeyword(input.keyword);
+  const { depth } = input;
 
   const cacheKey = await buildCacheKey("serp:analysis", {
     organizationId: billingCustomer.organizationId,
@@ -74,9 +85,13 @@ async function getSerpLiveAnalysis(
     languageCode: input.languageCode,
   });
 
+  // Depth lives in the cached value, not the key: DataForSEO has no offset, so
+  // a deeper crawl re-fetches the head too. A depth-100 snapshot therefore also
+  // answers depth-20 requests, and a deeper request replaces the entry rather
+  // than appending to it.
   const cachedRaw = await getCached(cacheKey);
   const cached = serpCacheSchema.safeParse(cachedRaw);
-  if (cached.success) {
+  if (cached.success && cached.data.depth >= depth) {
     return cached.data;
   }
 
@@ -84,14 +99,27 @@ async function getSerpLiveAnalysis(
     keyword,
     locationCode: input.locationCode,
     languageCode: input.languageCode,
+    depth,
   });
 
   const items = mapOrganicSerpItems(liveItems);
-  const result: SerpAnalysisResult = { requestedKeyword: keyword, items };
+  const result: SerpAnalysisResult = {
+    requestedKeyword: keyword,
+    items,
+    depth,
+  };
   if (items.length === 0) {
     result.reason = "no_organic_results";
   }
 
+  // DataForSEO reports "no results" transiently, and a deeper crawl overwrites
+  // the same key. Without this guard one empty re-crawl would evict a good
+  // snapshot and, being the deeper entry, answer every shallower request with
+  // nothing for the rest of the TTL.
+  if (items.length === 0 && cached.success && cached.data.items.length > 0) {
+    return result;
+  }
+
   // waitUntil, not void: workerd cancels unregistered pending I/O once the
   // response is sent, so a fire-and-forget put never persists the cache.
   waitUntil(

src/server/lib/dataforseo/core.test.ts

@@ -29,4 +29,26 @@ describe("DataForSEO transport", () => {
       "Basic encoded-credentials",
     );
   });
+
+  // The request-deadline abort arrives as a bare DOMException. Left unclassified
+  // it escapes as an anonymous INTERNAL_ERROR; retrying it would replay a call
+  // DataForSEO may already have billed. Both names are reachable: the shared
+  // budget aborts with TimeoutError, Lighthouse's own controller with AbortError.
+  it.each(["TimeoutError", "AbortError"])(
+    "maps a %s abort to UPSTREAM_UNAVAILABLE without retrying",
+    async (name) => {
+      const fetchMock = vi
+        .fn<typeof fetch>()
+        .mockRejectedValue(new DOMException("aborted", name));
+      vi.stubGlobal("fetch", fetchMock);
+
+      await expect(
+        dataforseoPost("/v3/serp/google/organic/live/advanced", []),
+      ).rejects.toMatchObject({
+        code: "UPSTREAM_UNAVAILABLE",
+        name: "DataForSEOTimeoutError",
+      });
+      expect(fetchMock).toHaveBeenCalledOnce();
+    },
+  );
 });

src/server/lib/dataforseo/core.ts

@@ -75,7 +75,34 @@ function createAuthenticatedFetch(
       init?.signal ?? AbortSignal.timeout(DATAFORSEO_REQUEST_TIMEOUT_MS);
 
     for (let attempt = 0; ; attempt++) {
-      const response = await fetch(url, { ...init, headers, signal });
+      let response: Response;
+      try {
+        response = await fetch(url, { ...init, headers, signal });
+      } catch (error) {
+        // Both abort flavours mean "we ran out of time", and neither carries a
+        // useful name of its own: the shared budget above rejects with
+        // TimeoutError, while Lighthouse passes its own AbortController
+        // deadline via init.signal and rejects with AbortError. Classify them
+        // so callers get a provider-degradation error instead of an anonymous
+        // internal one. Deliberately not retried: a call that ran past the
+        // deadline may already be billed by DataForSEO, and because this is not
+        // a DataforseoChargedTaskError the customer is metered nothing for it,
+        // so replaying would be spend we eat twice.
+        if (
+          error instanceof Error &&
+          (error.name === "TimeoutError" || error.name === "AbortError")
+        ) {
+          const path = formatDataforseoRequestPath(url);
+          const timeoutError = new AppError(
+            "UPSTREAM_UNAVAILABLE",
+            `DataForSEO request timed out on ${path}`,
+            { provider: "dataforseo", providerPath: path },
+          );
+          timeoutError.name = "DataForSEOTimeoutError";
+          throw timeoutError;
+        }
+        throw error;
+      }
       if (response.ok) return response;
 
       // Transient upstream 5xx on an idempotent read -> back off and retry.

src/server/lib/dataforseo/index.ts

@@ -18,6 +18,8 @@ export {
   type LlmPlatform,
 } from "@/server/lib/dataforseo/shared";
 
+export { SERP_ANALYSIS_DEPTH } from "@/server/lib/dataforseo/serp";
+
 export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
 
 // Section fetchers called outside the metered client. Task collection is free

src/server/lib/dataforseo/serp.ts

@@ -13,6 +13,15 @@ import {
 } from "@/server/lib/dataforseo/envelope";
 import { AppError } from "@/server/lib/errors";
 
+// Default depth for keyword SERP analysis. DataForSEO crawls (and bills) one
+// Google page of 10 results at a time, and the crawls are sequential, so depth
+// is the single lever on both latency and cost here: every 10 results is
+// another page fetch against the shared 60s request budget. Keep this low —
+// callers that need to see deeper ranks pass an explicit depth. There is no
+// offset/cursor: a deeper request re-crawls pages 1..N/10 from the top, so it
+// replaces the shallow snapshot rather than extending it.
+export const SERP_ANALYSIS_DEPTH = 20;
+
 /** DataForSEO bills SERPs in pages of 10; depth outside 10-100 is rejected. */
 function clampSerpDepth(depth: number): number {
   return Math.min(100, Math.max(10, depth));
@@ -78,6 +87,7 @@ export async function fetchLiveSerp(input: {
   keyword: string;
   locationCode: number;
   languageCode: string;
+  depth?: number;
 }): Promise<DataforseoApiResponse<SerpLiveItem[]>> {
   const response = await dataforseoPost(
     "/v3/serp/google/organic/live/advanced",
@@ -88,7 +98,7 @@ export async function fetchLiveSerp(input: {
         language_code: input.languageCode,
         device: "desktop",
         os: "windows",
-        depth: 100,
+        depth: clampSerpDepth(input.depth ?? SERP_ANALYSIS_DEPTH),
       },
     ],
   );

src/server/mcp/tools/get-serp-results.ts

@@ -1,5 +1,8 @@
 import { z } from "zod";
-import { createDataforseoClient } from "@/server/lib/dataforseo";
+import {
+  createDataforseoClient,
+  SERP_ANALYSIS_DEPTH,
+} from "@/server/lib/dataforseo";
 import { mcpResponse } from "@/server/mcp/formatters";
 import { buildProjectMeta } from "@/server/mcp/context";
 import { optionalMetaOutputSchema } from "@/server/mcp/output-schemas";
@@ -43,6 +46,16 @@ const inputSchema = {
     .describe(
       "1-10 queries. Bulk-friendly — prefer this over multiple single-query calls.",
     ),
+  depth: z
+    .number()
+    .int()
+    .min(10)
+    .max(100)
+    .multipleOf(10)
+    .optional()
+    .describe(
+      "How many SERP rows to crawl per keyword — a multiple of 10 from 10 to 100, default 20. Google has no offset, so a deeper crawl re-fetches the top too: each additional 10 adds ~2.5 credits per keyword. Only raise it when you need ranks past the top 20.",
+    ),
 } as const;
 
 type Args = z.infer<z.ZodObject<typeof inputSchema>>;
@@ -52,7 +65,7 @@ export const getSerpResultsTool = {
   config: {
     title: "Get Google SERP results",
     description:
-      "Fetch live Google organic search results for 1-10 keywords. Use this to inspect who ranks for a query, verify competitors, compare SERPs across keywords, or gather source URLs before content planning. Charges credits per keyword (~30-60 each). Does not save results to OpenSEO. Per-keyword errors don't fail the batch.",
+      "Fetch live Google organic search results for 1-10 keywords. Use this to inspect who ranks for a query, verify competitors, compare SERPs across keywords, or gather source URLs before content planning. Returns the top `depth` result rows per keyword (default 20). Charges credits per keyword: ~5 each at the default depth 20, and each additional 10 of depth adds ~2.5. Does not save results to OpenSEO. Per-keyword errors don't fail the batch.",
     inputSchema,
     outputSchema: {
       results: z.array(
@@ -94,15 +107,17 @@ export const getSerpResultsTool = {
   },
   handler: withMcpProjectAuth(async (args: Args, context) => {
     const client = createDataforseoClient(context.billing);
+    const depth = args.depth ?? SERP_ANALYSIS_DEPTH;
     const results = await Promise.all(
       args.queries.map(async (q) => {
         try {
           const items = await client.serp.live({
             keyword: q.keyword,
             ...resolveMarket(q, context.project),
+            depth,
           });
           // Trim noise — return only essentials per item.
-          const trimmed = items.slice(0, 20).map((item) => ({
+          const trimmed = items.slice(0, depth).map((item) => ({
             type: item.type,
             rank: item.rank_absolute ?? item.rank_group ?? null,
             title: item.title ?? null,

src/server/mcp/tools/tool-text-output.test.ts

@@ -42,6 +42,7 @@ vi.mock("@/server/lib/dataforseo", async () => {
     createDataforseoClient: mocks.createDataforseoClient,
     fetchBusinessDataTaskResult: mocks.fetchBusinessDataTaskResult,
     normalizeBacklinksTarget: targets.normalizeBacklinksTarget,
+    SERP_ANALYSIS_DEPTH: 20,
   };
 });
 vi.mock("@/server/features/projects/services/ProjectService", () => ({
@@ -387,4 +388,31 @@ describe("MCP tool text output (service-backed tools)", () => {
       "1 | example.com | Best SEO Tools | https://example.com/best",
     );
   });
+
+  it("get_serp_results crawls and returns rows to the requested depth", async () => {
+    const live = vi.fn().mockResolvedValue(
+      Array.from({ length: 40 }, (_, index) => ({
+        type: "organic",
+        rank_absolute: index + 1,
+        title: `Result ${index + 1}`,
+        url: `https://example.com/${index + 1}`,
+        domain: "example.com",
+        description: "desc",
+      })),
+    );
+    mocks.createDataforseoClient.mockReturnValue({ serp: { live } });
+
+    const result = await getSerpResultsTool.handler(
+      {
+        projectId: "project_1",
+        queries: [{ keyword: "seo tools" }],
+        depth: 30,
+      },
+      toolContext,
+    );
+
+    expect(live).toHaveBeenCalledWith(expect.objectContaining({ depth: 30 }));
+    // Rows are trimmed to the depth that was crawled, not the fixed top 20.
+    expect(textContent(result)).toContain('"seo tools" (30 results)');
+  });
 });

src/types/schemas/keywords.ts

@@ -176,6 +176,10 @@ export const serpAnalysisSchema = z.object({
   keyword: z.string().min(1),
   locationCode: z.number().int().positive().optional(),
   languageCode: z.string().min(2).max(8).optional(),
+  // Only the two depths the app offers: the default top-20 snapshot, and the
+  // full 100 the SERP panel buys when a user pages past the loaded results.
+  // Each 10 of depth is another crawled Google page (~2.5 credits).
+  depth: z.union([z.literal(20), z.literal(100)]).default(20),
 });
 
 /* ------------------------------------------------------------------ */