diff --git a/e2e/domain-overview-test-utils.ts b/e2e/domain-overview-test-utils.ts index 89774b0..0be9242 100644 --- a/e2e/domain-overview-test-utils.ts +++ b/e2e/domain-overview-test-utils.ts @@ -190,7 +190,7 @@ export async function openDomainOverview(page: Page, tab: DomainTab) { const params = new URLSearchParams({ domain: PRIMARY_TEST_DOMAIN, - subdomains: "true", + scope: "subdomains", sort: "traffic", order: "desc", }); diff --git a/e2e/fixtures/domain-overview-fixtures.ts b/e2e/fixtures/domain-overview-fixtures.ts index 1e5896f..555677c 100644 --- a/e2e/fixtures/domain-overview-fixtures.ts +++ b/e2e/fixtures/domain-overview-fixtures.ts @@ -1,6 +1,12 @@ +import { parseResearchTarget } from "@/shared/researchScope"; + export function getFixtureOverview(domain: string) { + const parsed = parseResearchTarget(domain); + const target = parsed.ok ? parsed.target : null; return { - domain, + domain: target?.hostname ?? domain, + scope: target?.scope ?? "domain", + displayTarget: target?.display ?? domain, organicTraffic: 373, organicKeywords: 307, backlinks: null, diff --git a/scripts/backlinks-cost-profile.ts b/scripts/backlinks-cost-profile.ts index 8f5d415..7b9667f 100644 --- a/scripts/backlinks-cost-profile.ts +++ b/scripts/backlinks-cost-profile.ts @@ -5,6 +5,7 @@ import type { BacklinksLookupInput, BacklinksTargetScope, } from "@/types/schemas/backlinks"; +import { backlinksScopeParamSchema } from "@/types/schemas/backlinks"; import { loadLocalEnv, parseArgs } from "./cli-utils"; loadLocalEnv(); @@ -143,8 +144,11 @@ function parseScope( value: string | undefined, ): BacklinksTargetScope | undefined { if (!value) return undefined; - if (value === "domain" || value === "page") return value; - printUsageAndExit(`Invalid scope: ${value}. Expected domain or page.`); + const parsed = backlinksScopeParamSchema.safeParse(value); + if (parsed.success) return parsed.data; + printUsageAndExit( + `Invalid scope: ${value}. Expected domain, subdomains, or exact_url.`, + ); } function parsePositiveInteger(value: string | undefined, fallback: number) { @@ -156,7 +160,7 @@ function parsePositiveInteger(value: string | undefined, fallback: number) { function printUsageAndExit(message: string): never { console.error(message); console.error( - "Usage: pnpm billing:backlinks --target=example.com --confirmLive=true [--scope=domain|page] [--repeat=1] [--includeTabs=true|false] [--allowCi=true]", + "Usage: pnpm billing:backlinks --target=example.com --confirmLive=true [--scope=domain|subdomains|exact_url] [--repeat=1] [--includeTabs=true|false] [--allowCi=true]", ); process.exit(1); } diff --git a/src/client/components/ResearchScopeSelect.tsx b/src/client/components/ResearchScopeSelect.tsx new file mode 100644 index 0000000..262a7d8 --- /dev/null +++ b/src/client/components/ResearchScopeSelect.tsx @@ -0,0 +1,147 @@ +import { useEffect, useRef, useState } from "react"; +import { Check, ChevronDown } from "lucide-react"; +import { + RESEARCH_SCOPES, + RESEARCH_SCOPE_DESCRIPTIONS, + RESEARCH_SCOPE_EXAMPLES, + RESEARCH_SCOPE_LABELS, + type ResearchScope, +} from "@/shared/researchScope"; + +type Props = { + value: ResearchScope; + onChange: (scope: ResearchScope) => void; + /** Greys the whole control (e.g. a brand-keyword lookup) with this reason. */ + disabledReason?: string; + className?: string; + "aria-label"?: string; +}; + +/** + * The shared research-scope selector: Exact URL / Subfolder / Domain / + * Subdomains. A custom dropdown (not a native select) so each option can + * explain what it covers. Every research input that accepts a URL or domain + * renders this next to the input so scope is explicit instead of inferred. + */ +export function ResearchScopeSelect({ + value, + onChange, + disabledReason, + className = "", + "aria-label": ariaLabel = "Research scope", +}: Props) { + const [open, setOpen] = useState(false); + const [activeScope, setActiveScope] = useState(value); + const containerRef = useRef(null); + + useEffect(() => { + if (open) setActiveScope(value); + }, [open, value]); + + // Close on outside click so it behaves like the surrounding native selects. + useEffect(() => { + if (!open) return; + const handlePointerDown = (event: PointerEvent) => { + const target = event.target; + if (target instanceof Node && !containerRef.current?.contains(target)) { + setOpen(false); + } + }; + document.addEventListener("pointerdown", handlePointerDown); + return () => document.removeEventListener("pointerdown", handlePointerDown); + }, [open]); + + const select = (scope: ResearchScope) => { + onChange(scope); + setOpen(false); + }; + + const moveActive = (step: 1 | -1) => { + const currentIndex = RESEARCH_SCOPES.indexOf(activeScope); + const nextIndex = Math.min( + Math.max(currentIndex + step, 0), + RESEARCH_SCOPES.length - 1, + ); + const next = RESEARCH_SCOPES[nextIndex]; + if (next) setActiveScope(next); + }; + + const handleKeyDown = (event: React.KeyboardEvent) => { + if (!open) return; + switch (event.key) { + case "ArrowDown": + event.preventDefault(); + moveActive(1); + break; + case "ArrowUp": + event.preventDefault(); + moveActive(-1); + break; + case "Enter": + case " ": + event.preventDefault(); + select(activeScope); + break; + case "Escape": + event.preventDefault(); + setOpen(false); + break; + } + }; + + return ( +
+ + + {open ? ( +
    + {RESEARCH_SCOPES.map((scope) => { + const isSelected = scope === value; + return ( +
  • + +
  • + ); + })} +
+ ) : null} +
+ ); +} diff --git a/src/client/features/ai-search/BrandLookupPage.tsx b/src/client/features/ai-search/BrandLookupPage.tsx index 9540ba9..3540dfe 100644 --- a/src/client/features/ai-search/BrandLookupPage.tsx +++ b/src/client/features/ai-search/BrandLookupPage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState, type FormEvent } from "react"; +import { useEffect, useMemo, useRef, useState, type FormEvent } from "react"; import { useQuery } from "@tanstack/react-query"; import { Link } from "@tanstack/react-router"; import { @@ -25,14 +25,26 @@ import { parseCompetitorList, } from "@/types/schemas/ai-search"; import { detectTarget } from "@/shared/targetDetection"; +import { + parseResearchTarget, + toScopeSearchParam, + type ResearchScope, +} from "@/shared/researchScope"; type Props = { projectId: string; initialQuery: string; initialCompetitors: string[]; - onSearchChange: (nextQuery: string, nextCompetitors: string[]) => void; + initialScope: ResearchScope | undefined; + onSearchChange: ( + nextQuery: string, + nextCompetitors: string[], + nextScope: ResearchScope | undefined, + ) => void; }; +const KEYWORD_SCOPE_REASON = "Scopes apply to domain lookups"; + const BRAND_LOOKUP_BULLETS = [ { icon: TrendingUp, @@ -63,10 +75,15 @@ function BrandLookupPageInner({ projectId, initialQuery, initialCompetitors, + initialScope, onSearchChange, planGate, }: Props & { planGate: HostedPlanGateState }) { const [query, setQuery] = useState(initialQuery); + // The user's explicit scope pick, or undefined to follow the input's default. + const [scopeChoice, setScopeChoice] = useState( + initialScope, + ); // Raw comma-separated competitor text; parsed into a deduped array on submit. const [competitorsInput, setCompetitorsInput] = useState( initialCompetitors.join(", "), @@ -84,14 +101,36 @@ function BrandLookupPageInner({ // stable string key, since `initialCompetitors` is a fresh array each render. const competitorKey = initialCompetitors.join(","); + // Scope only applies to domain/URL inputs. The pick always stays selectable + // — an invalid one (Subfolder without a path) errors on submit instead of + // the select greying out or changing under the user. + const scopeTarget = useMemo(() => { + if (detectTarget(query).type !== "domain") return null; + const parsed = parseResearchTarget(query); + return parsed.ok ? parsed.target : null; + }, [query]); + + const selectedScope = scopeChoice ?? scopeTarget?.scope ?? "domain"; + // Only grey the control once the input is clearly a brand keyword — an + // empty box shouldn't look disabled before the user has typed anything. + const scopeDisabledReason = + query.trim() !== "" && !scopeTarget ? KEYWORD_SCOPE_REASON : undefined; + const lookupQuery = useQuery({ - queryKey: ["brand-lookup", projectId, trimmedInitialQuery, competitorKey], + queryKey: [ + "brand-lookup", + projectId, + trimmedInitialQuery, + competitorKey, + initialScope ?? "", + ], queryFn: () => lookupBrand({ data: { projectId, query: trimmedInitialQuery, competitors: initialCompetitors, + scope: initialScope, locationCode: 2840, languageCode: "en", }, @@ -117,18 +156,20 @@ function BrandLookupPageInner({ const lastAddedKeyRef = useRef(null); useEffect(() => { if (!hasActiveQuery || !lookupQuery.isSuccess) return; - const addedKey = `${trimmedInitialQuery}::${competitorKey}`; + const addedKey = `${trimmedInitialQuery}::${competitorKey}::${initialScope ?? ""}`; if (lastAddedKeyRef.current === addedKey) return; lastAddedKeyRef.current = addedKey; addSearch({ query: trimmedInitialQuery, competitors: competitorKey ? competitorKey.split(",") : [], + scope: initialScope, }); }, [ hasActiveQuery, lookupQuery.isSuccess, trimmedInitialQuery, competitorKey, + initialScope, addSearch, ]); @@ -176,8 +217,24 @@ function BrandLookupPageInner({ }); return; } + if ( + scopeTarget && + selectedScope === "subfolder" && + scopeTarget.path === "" + ) { + setValidationError({ + field: "query", + message: "Add a path to use Subfolder (e.g. example.com/blog)", + }); + return; + } setValidationError(null); - onSearchChange(trimmed, competitors); + // Keyword lookups never carry a scope; domain lookups omit it when it + // matches the query's implied default. + const explicitScope = scopeTarget + ? toScopeSearchParam(trimmed, selectedScope) + : undefined; + onSearchChange(trimmed, competitors, explicitScope); }; // The form inputs are reset whenever the URL `q`/`c` changes — including the @@ -188,8 +245,9 @@ function BrandLookupPageInner({ useEffect(() => { setQuery(initialQuery); setCompetitorsInput(competitorKey.split(",").join(", ")); + setScopeChoice(initialScope); setValidationError(null); - }, [initialQuery, competitorKey]); + }, [initialQuery, competitorKey, initialScope]); const isLoading = hasActiveQuery && lookupQuery.isPending; const errorMessage = @@ -222,6 +280,9 @@ function BrandLookupPageInner({ setQuery(next); if (validationError) setValidationError(null); }} + scope={selectedScope} + onScopeChange={setScopeChoice} + scopeDisabledReason={scopeDisabledReason} competitors={competitorsInput} onCompetitorsChange={(next) => { setCompetitorsInput(next); @@ -251,7 +312,7 @@ function BrandLookupPageInner({ from="/p/$projectId/brand-lookup" to="/p/$projectId/brand-lookup" params={{ projectId }} - search={{ q: undefined, c: undefined }} + search={{ q: undefined, c: undefined, scope: undefined }} replace className="btn btn-ghost btn-sm gap-2 px-0 text-base-content/70 hover:bg-transparent" > diff --git a/src/client/features/ai-search/components/BrandLookupCitationTables.tsx b/src/client/features/ai-search/components/BrandLookupCitationTables.tsx index 1f4d92f..e60c593 100644 --- a/src/client/features/ai-search/components/BrandLookupCitationTables.tsx +++ b/src/client/features/ai-search/components/BrandLookupCitationTables.tsx @@ -334,11 +334,17 @@ export function buildTopQueriesColumns({ ]; } -export function TopPagesTable({ table }: { table: Table }) { +export function TopPagesTable({ + table, + emptyMessage = "No cited sources to show.", +}: { + table: Table; + emptyMessage?: string; +}) { if (table.getRowModel().rows.length === 0) { return (

- No cited sources to show. + {emptyMessage}

); } @@ -346,11 +352,17 @@ export function TopPagesTable({ table }: { table: Table }) { return ; } -export function TopQueriesTable({ table }: { table: Table }) { +export function TopQueriesTable({ + table, + emptyMessage = "No matching queries found.", +}: { + table: Table; + emptyMessage?: string; +}) { if (table.getRowModel().rows.length === 0) { return (

- No matching queries found. + {emptyMessage}

); } diff --git a/src/client/features/ai-search/components/BrandLookupCitationsCard.tsx b/src/client/features/ai-search/components/BrandLookupCitationsCard.tsx index 568985a..3cb2ac8 100644 --- a/src/client/features/ai-search/components/BrandLookupCitationsCard.tsx +++ b/src/client/features/ai-search/components/BrandLookupCitationsCard.tsx @@ -60,8 +60,16 @@ export function CitationTabsCard({ ]; const showQueryPlatform = queryPlatforms.length > 1; const showPagePlatform = pagePlatforms.length > 1; + // Under a URL scope `resolvedTarget` carries the path; the "You" badge and + // the Prompt Explorer brand highlight both want the bare hostname. const targetDomain = - result.detectedTargetType === "domain" ? result.resolvedTarget : null; + result.detectedTargetType === "domain" + ? result.resolvedTarget.split("/")[0] + : null; + const brand = targetDomain ?? result.resolvedTarget; + // Page rows are already narrowed server-side; say so instead of implying the + // pre-scope "everything cited alongside the brand" set. + const isUrlScoped = result.aggregatesAreDomainLevel; const filteredPages = useMemo( () => filterTopPages(result.topPages, filters.pages.values), @@ -78,18 +86,18 @@ export function CitationTabsCard({ showPlatform: showPagePlatform, targetDomain, projectId, - brand: result.resolvedTarget, + brand, }), - [showPagePlatform, targetDomain, projectId, result.resolvedTarget], + [showPagePlatform, targetDomain, projectId, brand], ); const queriesColumns = useMemo( () => buildTopQueriesColumns({ showPlatform: showQueryPlatform, projectId, - brand: result.resolvedTarget, + brand, }), - [showQueryPlatform, projectId, result.resolvedTarget], + [showQueryPlatform, projectId, brand], ); const pagesTable = useAppTable({ @@ -229,19 +237,21 @@ export function CitationTabsCard({ {activeTab === "pages" ? ( <> - Pages cited alongside{" "} + {isUrlScoped ? "Cited pages within " : "Pages cited alongside "} {result.resolvedTarget} - {" "} - in AI answers. Prompt examples come from the fetched sample. + + {isUrlScoped ? "." : " in AI answers."} Prompt examples come from + the fetched sample. ) : ( <> Fetched sample of prompts whose AI answer cited{" "} + {isUrlScoped ? "a page within " : null} {result.resolvedTarget} - {" "} - in its text or sources. + + {isUrlScoped ? "." : " in its text or sources."} )} @@ -260,9 +270,26 @@ export function CitationTabsCard({ ) : null} {activeTab === "pages" ? ( - + ) : ( - + )} ); diff --git a/src/client/features/ai-search/components/BrandLookupHistorySection.tsx b/src/client/features/ai-search/components/BrandLookupHistorySection.tsx index 292c00d..aead719 100644 --- a/src/client/features/ai-search/components/BrandLookupHistorySection.tsx +++ b/src/client/features/ai-search/components/BrandLookupHistorySection.tsx @@ -5,6 +5,7 @@ import { SearchHistorySection, } from "@/client/features/ai-search/components/SearchHistorySection"; import type { BrandLookupSearchHistoryItem } from "@/client/hooks/useBrandLookupSearchHistory"; +import { RESEARCH_SCOPE_LABELS } from "@/shared/researchScope"; type Props = { projectId: string; @@ -31,6 +32,7 @@ export function BrandLookupHistorySection({ projectId, ...props }: Props) { item.competitors.length > 0 ? item.competitors.join(",") : undefined, + scope: item.scope, }} replace className={HISTORY_ITEM_LINK_CLASS} @@ -40,7 +42,16 @@ export function BrandLookupHistorySection({ projectId, ...props }: Props) { )} renderItem={(item) => (
-

{item.query}

+

+ {item.query} + {/* Only non-default scopes are stored, so this badge always adds + information the query string doesn't already carry. */} + {item.scope ? ( + + {RESEARCH_SCOPE_LABELS[item.scope]} + + ) : null} +

{item.competitors.length > 0 ? (

vs {item.competitors.join(", ")} diff --git a/src/client/features/ai-search/components/BrandLookupResults.tsx b/src/client/features/ai-search/components/BrandLookupResults.tsx index a910e22..207bfb2 100644 --- a/src/client/features/ai-search/components/BrandLookupResults.tsx +++ b/src/client/features/ai-search/components/BrandLookupResults.tsx @@ -8,6 +8,7 @@ import { PLATFORM_DOT_CLASS, } from "@/client/features/ai-search/platformLabels"; import type { BrandLookupResult } from "@/types/schemas/ai-search"; +import { RESEARCH_SCOPE_LABELS } from "@/shared/researchScope"; type Props = { result: BrandLookupResult; @@ -17,6 +18,24 @@ type Props = { type PlatformRow = BrandLookupResult["perPlatform"][number]; type MetricKey = "mentions" | "aiSearchVolume"; +const DOMAIN_LEVEL_TIP = + "AI search providers report mentions per domain, not per page. This number covers the whole domain — the cited pages below are limited to your scope."; + +/** + * Marks a metric that could not be narrowed to a URL scope, so a page-scoped + * lookup never reads as if the number belonged to that page. + */ +function DomainLevelBadge() { + return ( + + Domain-level + + ); +} + export function BrandLookupResults({ result, projectId }: Props) { if (!result.hasData) { const erroredPlatforms = result.perPlatform.filter( @@ -71,7 +90,12 @@ export function BrandLookupResults({ result, projectId }: Props) { > {hasTrendData ? : null} - {sov ? : null} + {sov ? ( + + ) : null}

@@ -89,6 +113,11 @@ function BrandHeader({ result }: { result: BrandLookupResult }) { {result.detectedTargetType} + {result.scope ? ( + + {RESEARCH_SCOPE_LABELS[result.scope]} + + ) : null}

Updated {formatRelative(result.fetchedAt)} @@ -107,6 +136,7 @@ function StatsCard({ result }: { result: BrandLookupResult }) { value={result.totalMentions} perPlatform={result.perPlatform} metric="mentions" + isDomainLevel={result.aggregatesAreDomainLevel} /> @@ -126,12 +157,14 @@ function StatBlock({ value, perPlatform, metric, + isDomainLevel, }: { label: string; tooltip: string; value: number | null; perPlatform: PlatformRow[]; metric: MetricKey; + isDomainLevel: boolean; }) { return (

@@ -140,6 +173,7 @@ function StatBlock({ + {isDomainLevel ? : null}

{formatCount(value)} @@ -191,10 +225,11 @@ function PlatformStatRow({ function MentionTrendCard({ result }: { result: BrandLookupResult }) { return (

-
+

Mention trend (last 12 months)

+ {result.aggregatesAreDomainLevel ? : null}
diff --git a/src/client/features/ai-search/components/BrandLookupSearchCard.tsx b/src/client/features/ai-search/components/BrandLookupSearchCard.tsx index 71052ea..f15b75c 100644 --- a/src/client/features/ai-search/components/BrandLookupSearchCard.tsx +++ b/src/client/features/ai-search/components/BrandLookupSearchCard.tsx @@ -2,11 +2,16 @@ import type { FormEvent } from "react"; import { Search } from "lucide-react"; import { isHostedClientAuthMode } from "@/lib/auth-mode"; import { applyBillingMarkupUsd } from "@/shared/billing"; +import { ResearchScopeSelect } from "@/client/components/ResearchScopeSelect"; +import type { ResearchScope } from "@/shared/researchScope"; import { BRAND_LOOKUP_MAX_INPUT_LENGTH } from "@/types/schemas/ai-search"; type Props = { query: string; onQueryChange: (next: string) => void; + scope: ResearchScope; + onScopeChange: (next: ResearchScope) => void; + scopeDisabledReason: string | undefined; competitors: string; onCompetitorsChange: (next: string) => void; onSubmit: (event: FormEvent) => void; @@ -42,6 +47,9 @@ const BRAND_LOOKUP_COMPETITOR_DISPLAYED_COST_USD = markup( export function BrandLookupSearchCard({ query, onQueryChange, + scope, + onScopeChange, + scopeDisabledReason, competitors, onCompetitorsChange, onSubmit, @@ -79,6 +87,12 @@ export function BrandLookupSearchCard({ /> + +
- {data.scope} + + {RESEARCH_SCOPE_LABELS[data.scope]} + Target: {data.displayTarget} - Updated {formatRelativeTimestamp(data.fetchedAt)} + {/* history/live can't exclude subdomains, so say so rather than imply + the charts match the domain-scoped totals. */} + {data.scope === "domain" ? ( + - Trends include subdomains + ) : null}
- {data.scope === "page" ? ( + {data.scope === "exact_url" ? (
- Showing backlinks for this exact page. Enter a bare domain for - site-wide results. Trend charts are only shown for domain-level - lookups. + Showing backlinks for this exact page. Switch the scope to Domain or + Subdomains for site-wide results — trend charts need one of those. + +
+ ) : null} + {data.scope === "subfolder" ? ( +
+ + Showing backlinks pointing into this subfolder. Counts come from + filtered backlink totals; rank, trends, and the referring-domains + breakdown need Domain or Subdomains scope.
) : null} @@ -68,7 +84,8 @@ function OverviewGrid({ data: BacklinksOverviewData; summaryStats: SummaryStat[]; }) { - const domainScope = data.scope === "domain"; + // Trend charts need history/live, which only takes a whole hostname. + const domainScope = data.scope === "domain" || data.scope === "subdomains"; return (
diff --git a/src/client/features/backlinks/BacklinksPageContent.tsx b/src/client/features/backlinks/BacklinksPageContent.tsx index b6840c8..c451a81 100644 --- a/src/client/features/backlinks/BacklinksPageContent.tsx +++ b/src/client/features/backlinks/BacklinksPageContent.tsx @@ -155,6 +155,7 @@ export function BacklinksBody({
- {BACKLINKS_RESULTS_TABS.map(({ label, tab }) => ( + {BACKLINKS_RESULTS_TABS.filter( + // Referring domains can't be filtered to a path prefix. + ({ tab }) => !(scope === "subfolder" && tab === "domains"), + ).map(({ label, tab }) => ( onPageChange(1)} + // The subfolder url-prefix group (and, on the backlinks tab, the + // server-appended spam condition) shares the 8-condition budget. + maxConditions={ + scope === "subfolder" + ? MAX_DATAFORSEO_FILTER_CONDITIONS - + BACKLINKS_SUBFOLDER_FILTER_CONDITIONS - + (activeTab === "pages" ? 0 : 1) + : undefined + } /> ) : null} diff --git a/src/client/features/backlinks/BacklinksSearchCard.tsx b/src/client/features/backlinks/BacklinksSearchCard.tsx index 8c6b301..815a7da 100644 --- a/src/client/features/backlinks/BacklinksSearchCard.tsx +++ b/src/client/features/backlinks/BacklinksSearchCard.tsx @@ -7,17 +7,19 @@ import { getFormError, shouldValidateFieldOnChange, } from "@/client/lib/forms"; -import type { BacklinksSearchState } from "./backlinksPageTypes"; +import { ResearchScopeSelect } from "@/client/components/ResearchScopeSelect"; import { - inferBacklinksSearchScopeFromTarget, - resolveBacklinksSearchScope, -} from "./backlinksSearchScope"; + defaultScopeForInput, + parseResearchTarget, +} from "@/shared/researchScope"; +import type { BacklinksSearchState } from "./backlinksPageTypes"; type SearchDraft = Pick; function getBacklinksValidationErrors( value: SearchDraft, shouldValidateUntouchedField: boolean, + validateFormat = false, ) { if (!value.target.trim()) { if (!shouldValidateUntouchedField) { @@ -31,6 +33,15 @@ function getBacklinksValidationErrors( }); } + if (validateFormat) { + const parsed = parseResearchTarget(value.target, value.scope); + if (!parsed.ok) { + return createFormValidationErrors({ + fields: { target: parsed.message }, + }); + } + } + return null; } @@ -52,21 +63,10 @@ export function BacklinksSearchCard({ value, shouldValidateFieldOnChange(formApi, "target"), ), - onSubmit: ({ value }) => getBacklinksValidationErrors(value, true), + onSubmit: ({ value }) => getBacklinksValidationErrors(value, true, true), }, onSubmit: ({ value }) => { - const target = value.target.trim(); - const scope = resolveBacklinksSearchScope({ - target, - selectedScope: value.scope, - userSelectedScope, - }); - - onSubmit({ - ...value, - target, - scope, - }); + onSubmit({ ...value, target: value.target.trim() }); }, }); @@ -105,7 +105,7 @@ export function BacklinksSearchCard({ if (!userSelectedScope) { form.setFieldValue( "scope", - inferBacklinksSearchScopeFromTarget(nextTarget), + defaultScopeForInput(nextTarget), ); } }} @@ -115,6 +115,18 @@ export function BacklinksSearchCard({ }} + + {(field) => ( + { + setUserSelectedScope(true); + field.handleChange(scope); + }} + /> + )} + + state.isSubmitting}> {(isSubmitting) => ( - - - )} - -
diff --git a/src/client/features/backlinks/backlinksSearchScope.test.ts b/src/client/features/backlinks/backlinksSearchScope.test.ts deleted file mode 100644 index 3a024c1..0000000 --- a/src/client/features/backlinks/backlinksSearchScope.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - getPersistedBacklinksSearchScope, - inferBacklinksSearchScopeFromTarget, - resolveBacklinksSearchScope, -} from "./backlinksSearchScope"; - -describe("inferBacklinksSearchScopeFromTarget", () => { - it("treats bare hostnames as domain lookups", () => { - expect(inferBacklinksSearchScopeFromTarget("example.com")).toBe("domain"); - }); - - it("treats path-based targets without a protocol as page lookups", () => { - expect(inferBacklinksSearchScopeFromTarget("example.com/pricing")).toBe( - "page", - ); - }); - - it("treats root urls with explicit protocol as domain lookups", () => { - expect(inferBacklinksSearchScopeFromTarget("https://example.com/")).toBe( - "domain", - ); - }); - - it("treats explicit urls with a path as page lookups", () => { - expect( - inferBacklinksSearchScopeFromTarget("https://example.com/pricing"), - ).toBe("page"); - }); - - it("uses inferred scope until the user overrides it", () => { - expect( - resolveBacklinksSearchScope({ - target: "example.com/pricing", - selectedScope: "domain", - userSelectedScope: false, - }), - ).toBe("page"); - }); - - it("preserves a manual scope override", () => { - expect( - resolveBacklinksSearchScope({ - target: "https://example.com/pricing?utm_source=newsletter", - selectedScope: "domain", - userSelectedScope: true, - }), - ).toBe("domain"); - }); - - it("omits persisted scope when it matches the inferred target scope", () => { - expect( - getPersistedBacklinksSearchScope("example.com/pricing", "page"), - ).toBe(undefined); - }); - - it("persists explicit scope overrides", () => { - expect( - getPersistedBacklinksSearchScope( - "https://example.com/pricing?utm_source=newsletter", - "domain", - ), - ).toBe("domain"); - }); -}); diff --git a/src/client/features/backlinks/backlinksSearchScope.ts b/src/client/features/backlinks/backlinksSearchScope.ts deleted file mode 100644 index dad8e5b..0000000 --- a/src/client/features/backlinks/backlinksSearchScope.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { BacklinksTargetScope } from "@/types/schemas/backlinks"; - -export function inferBacklinksSearchScopeFromTarget( - target: string, -): BacklinksTargetScope { - const trimmed = target.trim(); - if (!trimmed) { - return "domain"; - } - - const hasExplicitProtocol = /^[a-zA-Z][a-zA-Z\d+.-]*:\/\//.test(trimmed); - - try { - const parsed = new URL( - hasExplicitProtocol ? trimmed : `https://${trimmed}`, - ); - return parsed.pathname !== "/" ? "page" : "domain"; - } catch { - return "domain"; - } -} - -export function resolveBacklinksSearchScope({ - target, - selectedScope, - userSelectedScope, -}: { - target: string; - selectedScope: BacklinksTargetScope; - userSelectedScope: boolean; -}): BacklinksTargetScope { - if (userSelectedScope) { - return selectedScope; - } - - return inferBacklinksSearchScopeFromTarget(target); -} - -export function getPersistedBacklinksSearchScope( - target: string, - scope: BacklinksTargetScope, -): BacklinksTargetScope | undefined { - return scope === inferBacklinksSearchScopeFromTarget(target) - ? undefined - : scope; -} diff --git a/src/client/features/backlinks/useBacklinksPageData.ts b/src/client/features/backlinks/useBacklinksPageData.ts index 9548033..f0aa6ab 100644 --- a/src/client/features/backlinks/useBacklinksPageData.ts +++ b/src/client/features/backlinks/useBacklinksPageData.ts @@ -27,7 +27,7 @@ import { toTopPagesFiltersPayload, } from "./backlinksFilterTypes"; import type { BacklinksFiltersState } from "./useBacklinksFilters"; -import { getPersistedBacklinksSearchScope } from "./backlinksSearchScope"; +import { toScopeSearchParam } from "@/shared/researchScope"; type UseBacklinksPageDataArgs = { projectId: string; @@ -231,7 +231,7 @@ export function navigateToBacklinksSearch( search: (prev) => ({ ...prev, target: values.target, - scope: getPersistedBacklinksSearchScope(values.target, values.scope), + scope: toScopeSearchParam(values.target, values.scope), tab: undefined, page: undefined, sort: undefined, diff --git a/src/client/features/domain/DomainOverviewPage.tsx b/src/client/features/domain/DomainOverviewPage.tsx index fb51f8f..2998fae 100644 --- a/src/client/features/domain/DomainOverviewPage.tsx +++ b/src/client/features/domain/DomainOverviewPage.tsx @@ -30,10 +30,17 @@ import { useSearchTabNavigation } from "@/client/features/search-tabs/useSearchT import { formatMetric, getDefaultSortOrder, - normalizeDomainTarget, + getResearchInputPath, toSortOrderSearchParam, toSortSearchParam, } from "@/client/features/domain/utils"; +import { + RESEARCH_SCOPE_LABELS, + defaultScopeForPath, + parseResearchTarget, + toScopeSearchParam, + type ResearchScope, +} from "@/shared/researchScope"; import { createFormValidationErrors, shouldValidateFieldOnChange, @@ -133,7 +140,8 @@ function getHistorySearchUpdate( return { ...buildDomainFiltersClearSearchUpdate(), domain: item.domain, - subdomains: item.subdomains ? undefined : false, + scope: toScopeSearchParam(item.domain, item.scope), + subdomains: undefined, sort: toSortSearchParam(item.sort), order: undefined, tab: item.tab === "keywords" ? undefined : item.tab, @@ -144,7 +152,7 @@ function getHistorySearchUpdate( function getSearchSubmitUpdate({ domain, - subdomains, + scope, sort, locationCode, currentOrder, @@ -152,7 +160,7 @@ function getSearchSubmitUpdate({ defaultLocationCode, }: { domain: string; - subdomains: boolean; + scope: ResearchScope; sort: DomainSortMode; locationCode: number; currentOrder: SortOrder; @@ -162,7 +170,8 @@ function getSearchSubmitUpdate({ return { ...buildDomainFiltersClearSearchUpdate(), domain, - subdomains: subdomains ? undefined : false, + scope: toScopeSearchParam(domain, scope), + subdomains: undefined, sort: toSortSearchParam(sort), order: toSortOrderSearchParam(sort, currentOrder), tab: activeTab === "keywords" ? undefined : activeTab, @@ -181,6 +190,9 @@ function useDomainOverviewState({ projectId: string; }) { const lastTrackedKey = useRef(""); + // While editing the domain input, the scope tracks the input's default until + // the user picks one; a pick survives further edits unless it turns invalid. + const userPickedScope = useRef(false); const { history, @@ -264,7 +276,7 @@ function useDomainOverviewState({ const overviewQuery = useDomainOverviewQuery({ projectId, domain: routeState.domain, - includeSubdomains: routeState.subdomains, + scope: routeState.scope, locationCode: routeState.sentLocationCode, }); const overview = overviewQuery.data ?? null; @@ -273,7 +285,7 @@ function useDomainOverviewState({ const controlsForm = useForm({ defaultValues: { domain: routeState.domain, - subdomains: routeState.subdomains, + scope: routeState.scope, sort: routeState.sort, locationCode: routeState.locationCode, }, @@ -287,13 +299,15 @@ function useDomainOverviewState({ onSubmit: ({ value }) => getDomainSearchValidationErrors(value), }, onSubmit: ({ formApi, value }) => { - const target = normalizeDomainTarget(value.domain); - if (!target) return; - formApi.setFieldValue("domain", target); + const parsed = parseResearchTarget(value.domain, value.scope); + if (!parsed.ok) return; + const target = parsed.target; + formApi.setFieldValue("domain", target.display); + formApi.setFieldValue("scope", target.scope); setSearchParams( getSearchSubmitUpdate({ - domain: target, - subdomains: value.subdomains, + domain: target.display, + scope: target.scope, sort: value.sort, locationCode: value.locationCode, currentOrder: routeState.order, @@ -305,9 +319,10 @@ function useDomainOverviewState({ }); useEffect(() => { + userPickedScope.current = false; controlsForm.reset({ domain: routeState.domain, - subdomains: routeState.subdomains, + scope: routeState.scope, sort: routeState.sort, locationCode: routeState.locationCode, }); @@ -315,10 +330,29 @@ function useDomainOverviewState({ controlsForm, routeState.domain, routeState.locationCode, + routeState.scope, routeState.sort, - routeState.subdomains, ]); + const handleDomainChange = useCallback( + (nextDomain: string) => { + // An explicit pick sticks even when it stops fitting the input (e.g. + // Subfolder after the path is deleted) — submit validation explains + // instead of the select silently changing under the user. + if (userPickedScope.current) return; + const path = getResearchInputPath(nextDomain); + const nextScope = defaultScopeForPath(path); + if (nextScope !== controlsForm.getFieldValue("scope")) { + controlsForm.setFieldValue("scope", nextScope); + } + }, + [controlsForm], + ); + + const handleScopeChange = useCallback(() => { + userPickedScope.current = true; + }, []); + useEffect(() => { controlsForm.setErrorMap({ onSubmit: overviewQuery.error @@ -334,19 +368,19 @@ function useDomainOverviewState({ useEffect(() => { if (!overviewQuery.isSuccess || !overview) return; - const key = `${routeState.domain}|${routeState.subdomains}|${routeState.locationCode}`; + const key = `${routeState.domain}|${routeState.scope}|${routeState.locationCode}`; if (lastTrackedKey.current === key) return; lastTrackedKey.current = key; captureClientEvent("domain_overview:search_complete", { sort_mode: routeState.sort, - include_subdomains: routeState.subdomains, + scope: routeState.scope, result_count: overview.organicKeywords ?? 0, location_code: routeState.locationCode, }); addSearch({ domain: routeState.domain, - subdomains: routeState.subdomains, + scope: routeState.scope, sort: routeState.sort, tab: routeState.tab, locationCode: routeState.locationCode, @@ -360,8 +394,8 @@ function useDomainOverviewState({ overviewQuery.isSuccess, routeState.domain, routeState.locationCode, + routeState.scope, routeState.sort, - routeState.subdomains, routeState.tab, ]); @@ -401,6 +435,8 @@ function useDomainOverviewState({ setSearchParams, applySort, applyLocationChange, + handleDomainChange, + handleScopeChange, handleTabChange, handleSortColumnClick, handleHistorySelect, @@ -430,10 +466,10 @@ export function DomainOverviewPage({ return { type: "domain", domain: routeState.domain, - subdomains: routeState.subdomains, + scope: routeState.scope, locationCode: routeState.sentLocationCode, }; - }, [routeState.domain, routeState.sentLocationCode, routeState.subdomains]); + }, [routeState.domain, routeState.scope, routeState.sentLocationCode]); const navigateToSearchTab = useCallback( (input: SearchTabInput | null) => { @@ -450,7 +486,8 @@ export function DomainOverviewPage({ ...prev, ...buildDomainFiltersClearSearchUpdate(), domain: input.domain, - subdomains: input.subdomains ? undefined : false, + scope: toScopeSearchParam(input.domain, input.scope), + subdomains: undefined, sort: undefined, order: undefined, tab: undefined, @@ -482,6 +519,13 @@ export function DomainOverviewPage({ navigateToInput: navigateToSearchTab, }); + // domain_rank_overview can't be narrowed: its metrics always cover the + // hostname plus subdomains, so anything narrower needs a label. + const overviewMetricsHint = + state.overview && state.overview.scope !== "subdomains" + ? "Whole domain incl. subdomains" + : undefined; + const tabControls = routeState.domain ? (
@@ -523,6 +567,8 @@ export function DomainOverviewPage({ controlsForm={state.controlsForm} isLoading={state.isLoading} onSubmit={state.handleSearchSubmit} + onDomainChange={state.handleDomainChange} + onScopeChange={state.handleScopeChange} onSortChange={(sort) => state.applySort(sort, getDefaultSortOrder(sort)) } @@ -548,6 +594,14 @@ export function DomainOverviewPage({ ) : ( <> {tabControls} +
+ + {state.overview.displayTarget} + + + {RESEARCH_SCOPE_LABELS[state.overview.scope]} + +
{!state.overview.hasData ? (
- Not enough data for this domain yet. Try another domain or - include subdomains. + Not enough data for this scope yet. Try another domain or a + broader scope.
) : null} @@ -602,7 +658,9 @@ export function DomainOverviewPage({ = { textFields: ReadonlyArray>; rangeFields: ReadonlyArray>; countConditions: (values: TValues) => number; + /** Conditions left for user filters once scope filters take their share. */ + maxConditions?: number; onApply: (values: TValues) => void; onClear: () => void; /** Extra feature-specific controls (toggles etc.) bound to the draft. */ @@ -57,6 +59,7 @@ export function DomainFilterPanel({ textFields, rangeFields, countConditions, + maxConditions = MAX_DATAFORSEO_FILTER_CONDITIONS, onApply, onClear, renderExtra, @@ -79,8 +82,9 @@ export function DomainFilterPanel({ appliedFilters, fields, countConditions, + maxConditions, }), - [appliedFilters, countConditions, draftFilters, fields], + [appliedFilters, countConditions, draftFilters, fields, maxConditions], ); useDomainRenderDebug(debugName, { activeFilterCount, @@ -204,15 +208,14 @@ export function DomainFilterPanel({
- Too many filter conditions ({meta.conditionCount} of{" "} - {MAX_DATAFORSEO_FILTER_CONDITIONS} max). Remove some terms or ranges - before applying. + Too many filter conditions ({meta.conditionCount} of {maxConditions}{" "} + max). Remove some terms or ranges before applying.
) : null}
- {meta.conditionCount} / {MAX_DATAFORSEO_FILTER_CONDITIONS} conditions + {meta.conditionCount} / {maxConditions} conditions
diff --git a/src/client/features/domain/components/DomainSearchCard.tsx b/src/client/features/domain/components/DomainSearchCard.tsx index 2a77c64..ee9394f 100644 --- a/src/client/features/domain/components/DomainSearchCard.tsx +++ b/src/client/features/domain/components/DomainSearchCard.tsx @@ -6,11 +6,15 @@ import { toSortMode } from "@/client/features/domain/utils"; import type { DomainSortMode } from "@/client/features/domain/types"; import { LABS_LOCATION_OPTIONS } from "@/client/features/keywords/locations"; import { LocationSelect } from "@/client/components/LocationSelect"; +import { ResearchScopeSelect } from "@/client/components/ResearchScopeSelect"; +import type { ResearchScope } from "@/shared/researchScope"; type Props = { controlsForm: DomainOverviewControlsForm; isLoading: boolean; onSubmit: (event: FormEvent) => void; + onDomainChange: (domain: string) => void; + onScopeChange: (scope: ResearchScope) => void; onSortChange: (sort: DomainSortMode) => void; onLocationChange: (locationCode: number) => void; }; @@ -19,6 +23,8 @@ export function DomainSearchCard({ controlsForm, isLoading, onSubmit, + onDomainChange, + onScopeChange, onSortChange, onLocationChange, }: Props) { @@ -40,9 +46,12 @@ export function DomainSearchCard({ field.handleChange(event.target.value)} + onChange={(event) => { + field.handleChange(event.target.value); + onDomainChange(event.target.value); + }} aria-invalid={domainError ? true : undefined} aria-describedby={ domainError ? "domain-input-error" : undefined @@ -53,6 +62,19 @@ export function DomainSearchCard({ }} + + {(field) => ( + { + field.handleChange(scope); + onScopeChange(scope); + }} + /> + )} + + {(field) => ( - -
- -
); diff --git a/src/client/features/domain/components/KeywordsTab.tsx b/src/client/features/domain/components/KeywordsTab.tsx index 2b76bcf..8278c0b 100644 --- a/src/client/features/domain/components/KeywordsTab.tsx +++ b/src/client/features/domain/components/KeywordsTab.tsx @@ -25,6 +25,7 @@ import { useDomainKeywordsQuery } from "@/client/features/domain/hooks/useDomain import { useSaveKeywordsMutation } from "@/client/features/domain/mutations"; import { useDomainKeywordFilterPreferences } from "@/client/features/domain/useDomainFilterPreferences"; import { + EMPTY_DOMAIN_FILTERS, type DomainSortMode, type KeywordRow, type KeywordsFilterValues, @@ -38,6 +39,10 @@ import { MAX_DATAFORSEO_FILTER_CONDITIONS, type DomainSearchParams, } from "@/types/schemas/domain"; +import { + RESEARCH_SCOPE_FILTER_SLOTS, + type ResearchScope, +} from "@/shared/researchScope"; type SearchUpdate = Partial; @@ -64,7 +69,11 @@ const KEYWORD_RANGE_FILTERS = [ type Props = { projectId: string; - domain: string; + /** Research target as displayed: hostname, plus the path for URL scopes. */ + target: string; + /** Hostname only, for resolving relative result URLs. */ + hostname: string; + scope: ResearchScope; routeState: DomainOverviewRouteState; canSaveKeywords: boolean; setSearchParams: (updates: SearchUpdate) => void; @@ -75,7 +84,9 @@ type Props = { export function KeywordsTab({ projectId, - domain, + target, + hostname, + scope, routeState, canSaveKeywords, setSearchParams, @@ -88,29 +99,41 @@ export function KeywordsTab({ new Set(), ); const [showFilters, setShowFilters] = useState(false); + // Scope filters consume part of DataForSEO's fixed filter budget. + const maxConditions = + MAX_DATAFORSEO_FILTER_CONDITIONS - + RESEARCH_SCOPE_FILTER_SLOTS.keywords[scope]; const filterPreferences = useDomainKeywordFilterPreferences( - `${projectId}:${domain}`, + `${projectId}:${target}`, ); const { filters: preferredFilters, save: savePreferredFilters, clear: clearPreferredFilters, } = filterPreferences; - const appliedFilters = routeState.hasAppliedKeywordFilters + const restoredFilters = routeState.hasAppliedKeywordFilters ? routeState.appliedFilters : preferredFilters; + // Filters restored from the URL or saved preferences can exceed this + // scope's tighter budget; sending them would make the server reject the + // whole query, so hold them back and let the panel explain. + const filtersOverBudget = + countKeywordFilterConditions(restoredFilters) > maxConditions; + const appliedFilters = filtersOverBudget + ? EMPTY_DOMAIN_FILTERS + : restoredFilters; const query = useDomainKeywordsQuery({ projectId, - domain, - includeSubdomains: routeState.subdomains, + domain: target, + scope, locationCode: routeState.sentLocationCode, page: routeState.page, pageSize: routeState.pageSize, sortMode: routeState.sort, sortOrder: routeState.order, appliedFilters, - enabled: Boolean(domain), + enabled: Boolean(target), }); const rows = query.data?.keywords ?? EMPTY_KEYWORDS; @@ -168,16 +191,13 @@ export function KeywordsTab({ const applyFilters = useCallback( (values: KeywordsFilterValues) => { - if ( - countKeywordFilterConditions(values) > MAX_DATAFORSEO_FILTER_CONDITIONS - ) - return; + if (countKeywordFilterConditions(values) > maxConditions) return; const update = buildKeywordsSearchUpdate(values); debugDomain("KeywordsTab:apply-filters", { values, update }); savePreferredFilters(values); setSearchParams(update); }, - [savePreferredFilters, setSearchParams], + [maxConditions, savePreferredFilters, setSearchParams], ); const resetFilters = useCallback(() => { @@ -190,12 +210,13 @@ export function KeywordsTab({ const activeFilterCount = useMemo( () => - KEYWORD_FILTER_FIELDS.filter((k) => appliedFilters[k].trim() !== "") + KEYWORD_FILTER_FIELDS.filter((k) => restoredFilters[k].trim() !== "") .length, - [appliedFilters], + [restoredFilters], ); const exportTable = useMemo(() => keywordsToTable(rows), [rows]); + const fileNamePrefix = target.replaceAll("/", "-"); const selectedExportTable = useMemo( () => keywordsToTable(rows.filter((r) => selectedKeywords.has(r.keyword))), [rows, selectedKeywords], @@ -214,7 +235,7 @@ export function KeywordsTab({ }; const handleDownload = (extension: "csv" | "xls") => { downloadCsv( - `${domain}-keywords.${extension}`, + `${fileNamePrefix}-keywords.${extension}`, buildCsv(exportTable.headers, exportTable.rows), ); if (extension === "csv") { @@ -233,7 +254,7 @@ export function KeywordsTab({ }; const handleDownloadSelectionCsv = () => { downloadCsv( - `${domain}-selected-keywords.csv`, + `${fileNamePrefix}-selected-keywords.csv`, buildCsv(selectedExportTable.headers, selectedExportTable.rows), ); captureClientEvent("data:export", { @@ -275,6 +296,15 @@ export function KeywordsTab({ } /> + {filtersOverBudget ? ( +
+ + Saved filters exceed this scope's {maxConditions}-condition + limit and were not applied. Open Filters to trim them. + +
+ ) : null} + setShowFilters((prev) => !prev)} @@ -311,11 +341,12 @@ export function KeywordsTab({ @@ -334,7 +365,7 @@ export function KeywordsTab({ } > ; @@ -54,7 +59,11 @@ const PAGE_RANGE_FILTERS = [ type Props = { projectId: string; - domain: string; + /** Research target as displayed: hostname, plus the path for URL scopes. */ + target: string; + /** Hostname only, for resolving relative result URLs. */ + hostname: string; + scope: ResearchScope; routeState: DomainOverviewRouteState; setSearchParams: (updates: SearchUpdate) => void; onSortClick: (sort: DomainSortMode) => void; @@ -64,7 +73,9 @@ type Props = { export function PagesTab({ projectId, - domain, + target, + hostname, + scope, routeState, setSearchParams, onSortClick, @@ -72,15 +83,18 @@ export function PagesTab({ onPageSizeChange, }: Props) { const [showFilters, setShowFilters] = useState(false); + // Scope filters consume part of DataForSEO's fixed filter budget. + const maxConditions = + MAX_DATAFORSEO_FILTER_CONDITIONS - RESEARCH_SCOPE_FILTER_SLOTS.pages[scope]; const filterPreferences = useDomainPageFilterPreferences( - `${projectId}:${domain}`, + `${projectId}:${target}`, ); const { filters: preferredFilters, save: savePreferredFilters, clear: clearPreferredFilters, } = filterPreferences; - const appliedPagesFilters = useMemo( + const restoredFilters = useMemo( () => routeState.hasAppliedPageFilters ? routeState.appliedPageFilters @@ -91,18 +105,26 @@ export function PagesTab({ routeState.hasAppliedPageFilters, ], ); + // Filters restored from the URL or saved preferences can exceed this + // scope's tighter budget; sending them would make the server reject the + // whole query, so hold them back and let the panel explain. + const filtersOverBudget = + countPageFilterConditions(restoredFilters) > maxConditions; + const appliedPagesFilters = filtersOverBudget + ? EMPTY_DOMAIN_FILTERS + : restoredFilters; const query = useDomainPagesQuery({ projectId, - domain, - includeSubdomains: routeState.subdomains, + domain: target, + scope, locationCode: routeState.sentLocationCode, page: routeState.page, pageSize: routeState.pageSize, sortMode: routeState.sort, sortOrder: routeState.order, appliedFilters: appliedPagesFilters, - enabled: Boolean(domain), + enabled: Boolean(target), }); const rows = query.data?.pages ?? EMPTY_PAGES_ROWS; @@ -124,14 +146,13 @@ export function PagesTab({ const applyFilters = useCallback( (values: PagesFilterValues) => { - if (countPageFilterConditions(values) > MAX_DATAFORSEO_FILTER_CONDITIONS) - return; + if (countPageFilterConditions(values) > maxConditions) return; const update = buildPagesSearchUpdate(values); debugDomain("PagesTab:apply-filters", { values, update }); savePreferredFilters(values); setSearchParams(update); }, - [savePreferredFilters, setSearchParams], + [maxConditions, savePreferredFilters, setSearchParams], ); const resetFilters = useCallback(() => { @@ -143,12 +164,12 @@ export function PagesTab({ const activeFilterCount = useMemo( () => - PAGE_FILTER_FIELDS.filter((k) => appliedPagesFilters[k].trim() !== "") - .length, - [appliedPagesFilters], + PAGE_FILTER_FIELDS.filter((k) => restoredFilters[k].trim() !== "").length, + [restoredFilters], ); const exportTable = useMemo(() => pagesToTable(rows), [rows]); + const fileNamePrefix = target.replaceAll("/", "-"); const handleCopy = async () => { await navigator.clipboard.writeText(JSON.stringify(rows, null, 2)); @@ -163,7 +184,7 @@ export function PagesTab({ }; const handleDownload = (extension: "csv" | "xls") => { downloadCsv( - `${domain}-pages.${extension}`, + `${fileNamePrefix}-pages.${extension}`, buildCsv(exportTable.headers, exportTable.rows), ); if (extension === "csv") { @@ -176,6 +197,15 @@ export function PagesTab({ return ( <> + {filtersOverBudget ? ( +
+ + Saved filters exceed this scope's {maxConditions}-condition + limit and were not applied. Open Filters to trim them. + +
+ ) : null} + setShowFilters((prev) => !prev)} @@ -212,11 +242,12 @@ export function PagesTab({ @@ -235,7 +266,7 @@ export function PagesTab({ } >
@@ -6,6 +14,7 @@ export function StatCard({ label, value }: { label: string; value: string }) { {label}

{value}

+ {hint ?

{hint}

: null}
); diff --git a/src/client/features/domain/domainRouteState.test.ts b/src/client/features/domain/domainRouteState.test.ts index 726a427..4e94f57 100644 --- a/src/client/features/domain/domainRouteState.test.ts +++ b/src/client/features/domain/domainRouteState.test.ts @@ -1,5 +1,41 @@ import { describe, expect, it } from "vitest"; import { getDomainRouteState } from "./domainRouteState"; +import { toScopeSearchParam } from "@/shared/researchScope"; + +describe("research scope resolution", () => { + it("derives the scope from the domain input when the URL omits it", () => { + expect(getDomainRouteState({ domain: "example.com" }).scope).toBe( + "subdomains", + ); + expect(getDomainRouteState({ domain: "example.com/blog" }).scope).toBe( + "subfolder", + ); + }); + + it("migrates the legacy subdomains param", () => { + expect( + getDomainRouteState({ domain: "example.com", subdomains: true }).scope, + ).toBe("subdomains"); + expect( + getDomainRouteState({ domain: "example.com", subdomains: false }).scope, + ).toBe("domain"); + }); + + it("ignores a scope the domain input cannot support", () => { + expect( + getDomainRouteState({ domain: "example.com", scope: "subfolder" }).scope, + ).toBe("subdomains"); + }); + + it("omits the scope param when it matches the input's default", () => { + expect(toScopeSearchParam("example.com", "subdomains")).toBeUndefined(); + expect(toScopeSearchParam("example.com", "domain")).toBe("domain"); + expect(toScopeSearchParam("example.com/blog", "subfolder")).toBeUndefined(); + expect(toScopeSearchParam("example.com/blog", "exact_url")).toBe( + "exact_url", + ); + }); +}); describe("getDomainRouteState", () => { it("uses a Labs-backed project market when the URL omits loc", () => { diff --git a/src/client/features/domain/domainRouteState.ts b/src/client/features/domain/domainRouteState.ts index 8180389..68acde6 100644 --- a/src/client/features/domain/domainRouteState.ts +++ b/src/client/features/domain/domainRouteState.ts @@ -21,11 +21,21 @@ import { PAGE_FILTER_FIELDS, PAGE_SEARCH_PARAM_BY_FIELD, } from "@/client/features/domain/domainFilterUtils"; -import { resolveSortOrder, toSortMode, toSortOrder } from "./utils"; +import { + defaultScopeForPath, + isScopeAllowedForInput, + type ResearchScope, +} from "@/shared/researchScope"; +import { + getResearchInputPath, + resolveSortOrder, + toSortMode, + toSortOrder, +} from "./utils"; export type DomainOverviewRouteState = { domain: string; - subdomains: boolean; + scope: ResearchScope; sort: DomainSortMode; order: SortOrder; tab: DomainActiveTab; @@ -40,6 +50,18 @@ export type DomainOverviewRouteState = { hasAppliedPageFilters: boolean; }; +function resolveScope(search: DomainSearchParams): ResearchScope { + const path = getResearchInputPath(search.domain ?? ""); + if (search.scope && isScopeAllowedForInput(search.scope, path)) { + return search.scope; + } + // Legacy param: pre-scope URLs encoded "Include subdomains" here. + if (search.subdomains != null) { + return search.subdomains ? "subdomains" : "domain"; + } + return defaultScopeForPath(path); +} + function numberToFilterString(value: number | undefined): string { if (value == null || !Number.isFinite(value)) return ""; return String(value); @@ -62,7 +84,7 @@ export function getDomainRouteState( return { domain: search.domain ?? "", - subdomains: search.subdomains ?? true, + scope: resolveScope(search), sort: normalizedSort, order: resolveSortOrder(normalizedSort, toSortOrder(search.order ?? null)), tab: search.tab ?? "keywords", diff --git a/src/client/features/domain/domainSearchValidation.ts b/src/client/features/domain/domainSearchValidation.ts index 1393b75..1e95a97 100644 --- a/src/client/features/domain/domainSearchValidation.ts +++ b/src/client/features/domain/domainSearchValidation.ts @@ -1,4 +1,4 @@ -import { normalizeDomainTarget } from "@/client/features/domain/utils"; +import { parseResearchTarget } from "@/shared/researchScope"; import { createFormValidationErrors } from "@/client/lib/forms"; import type { DomainControlsValues } from "@/client/features/domain/types"; @@ -11,10 +11,11 @@ export function getDomainSearchValidationErrors(value: DomainControlsValues) { }); } - if (!normalizeDomainTarget(value.domain)) { + const parsed = parseResearchTarget(value.domain, value.scope); + if (!parsed.ok) { return createFormValidationErrors({ fields: { - domain: "Please enter a valid URL or domain (e.g. example.com)", + domain: parsed.message, }, }); } diff --git a/src/client/features/domain/hooks/useDomainKeywordsQuery.ts b/src/client/features/domain/hooks/useDomainKeywordsQuery.ts index 1e69efb..d279b50 100644 --- a/src/client/features/domain/hooks/useDomainKeywordsQuery.ts +++ b/src/client/features/domain/hooks/useDomainKeywordsQuery.ts @@ -2,6 +2,7 @@ import { useEffect, useMemo } from "react"; import { useQuery } from "@tanstack/react-query"; import { getDomainKeywordsPage } from "@/serverFunctions/domain"; import { debugDomain } from "@/client/features/domain/domainDebug"; +import type { ResearchScope } from "@/shared/researchScope"; import type { DomainFilterValues, DomainSortMode, @@ -11,7 +12,7 @@ import type { type DomainKeywordsQueryInput = { projectId: string; domain: string; - includeSubdomains: boolean; + scope: ResearchScope; locationCode: number | undefined; page: number; pageSize: number; @@ -57,7 +58,7 @@ export function useDomainKeywordsQuery(input: DomainKeywordsQueryInput) { "domain-keywords", input.projectId, input.domain, - input.includeSubdomains, + input.scope, input.locationCode, input.page, input.pageSize, @@ -68,7 +69,7 @@ export function useDomainKeywordsQuery(input: DomainKeywordsQueryInput) { [ filtersPayload, input.domain, - input.includeSubdomains, + input.scope, input.locationCode, input.page, input.pageSize, @@ -93,7 +94,7 @@ export function useDomainKeywordsQuery(input: DomainKeywordsQueryInput) { data: { projectId: input.projectId, domain: input.domain, - includeSubdomains: input.includeSubdomains, + scope: input.scope, locationCode: input.locationCode, page: input.page, pageSize: input.pageSize, diff --git a/src/client/features/domain/hooks/useDomainOverviewQuery.ts b/src/client/features/domain/hooks/useDomainOverviewQuery.ts index a6cea65..424e422 100644 --- a/src/client/features/domain/hooks/useDomainOverviewQuery.ts +++ b/src/client/features/domain/hooks/useDomainOverviewQuery.ts @@ -1,10 +1,11 @@ import { useQuery } from "@tanstack/react-query"; import { getDomainOverview } from "@/serverFunctions/domain"; +import type { ResearchScope } from "@/shared/researchScope"; type Input = { projectId: string; domain: string; - includeSubdomains: boolean; + scope: ResearchScope; locationCode: number | undefined; }; @@ -17,7 +18,7 @@ export function useDomainOverviewQuery(input: Input) { "domain-overview", input.projectId, trimmedDomain, - input.includeSubdomains, + input.scope, input.locationCode, ], queryFn: () => @@ -25,7 +26,7 @@ export function useDomainOverviewQuery(input: Input) { data: { projectId: input.projectId, domain: trimmedDomain, - includeSubdomains: input.includeSubdomains, + scope: input.scope, locationCode: input.locationCode, }, }), diff --git a/src/client/features/domain/hooks/useDomainPagesQuery.ts b/src/client/features/domain/hooks/useDomainPagesQuery.ts index 28a433c..c276b1a 100644 --- a/src/client/features/domain/hooks/useDomainPagesQuery.ts +++ b/src/client/features/domain/hooks/useDomainPagesQuery.ts @@ -3,6 +3,7 @@ import { useQuery } from "@tanstack/react-query"; import { getDomainPagesPage } from "@/serverFunctions/domain"; import { debugDomain } from "@/client/features/domain/domainDebug"; import { toPageSortMode } from "@/client/features/domain/utils"; +import type { ResearchScope } from "@/shared/researchScope"; import type { DomainSortMode, PagesFilterValues, @@ -12,7 +13,7 @@ import type { type DomainPagesQueryInput = { projectId: string; domain: string; - includeSubdomains: boolean; + scope: ResearchScope; locationCode: number | undefined; page: number; pageSize: number; @@ -29,7 +30,7 @@ export function useDomainPagesQuery(input: DomainPagesQueryInput) { "domain-pages", input.projectId, input.domain, - input.includeSubdomains, + input.scope, input.locationCode, input.page, input.pageSize, @@ -40,7 +41,7 @@ export function useDomainPagesQuery(input: DomainPagesQueryInput) { [ input.appliedFilters, input.domain, - input.includeSubdomains, + input.scope, input.locationCode, input.page, input.pageSize, @@ -65,7 +66,7 @@ export function useDomainPagesQuery(input: DomainPagesQueryInput) { data: { projectId: input.projectId, domain: input.domain, - includeSubdomains: input.includeSubdomains, + scope: input.scope, locationCode: input.locationCode, page: input.page, pageSize: input.pageSize, diff --git a/src/client/features/domain/types.ts b/src/client/features/domain/types.ts index 6e77d30..cddebf7 100644 --- a/src/client/features/domain/types.ts +++ b/src/client/features/domain/types.ts @@ -1,3 +1,5 @@ +import type { ResearchScope } from "@/shared/researchScope"; + export type KeywordRow = { keyword: string; position: number | null; @@ -57,7 +59,7 @@ export type PageFilterKey = keyof PagesFilterValues; export type DomainControlsValues = { domain: string; - subdomains: boolean; + scope: ResearchScope; sort: "rank" | "traffic" | "volume" | "score" | "cpc"; locationCode: number; }; @@ -69,7 +71,7 @@ export type DomainActiveTab = "keywords" | "pages"; export type DomainHistoryItem = { timestamp: number; domain: string; - subdomains: boolean; + scope: ResearchScope; sort: DomainSortMode; tab: DomainActiveTab; search?: string; diff --git a/src/client/features/domain/utils.ts b/src/client/features/domain/utils.ts index 5ae3ff2..2e6c482 100644 --- a/src/client/features/domain/utils.ts +++ b/src/client/features/domain/utils.ts @@ -4,7 +4,7 @@ import type { PageRow, SortOrder, } from "@/client/features/domain/types"; -import { isValidDomainHost } from "@/types/schemas/domain"; +import { parseResearchTarget } from "@/shared/researchScope"; export function toSortMode(value: string | null): DomainSortMode | undefined { if ( @@ -55,26 +55,10 @@ export function toPageSortMode( return "traffic"; } -export function normalizeDomainTarget(input: string): string | null { - const value = input.trim(); - if (!value) return null; - - const withProtocol = /^[a-zA-Z][a-zA-Z\d+.-]*:\/\//.test(value) - ? value - : `https://${value}`; - - try { - const parsed = new URL(withProtocol); - const hostname = parsed.hostname.toLowerCase(); - if (!hostname || !hostname.includes(".")) return null; - if (!/^[a-z\d.-]+$/.test(hostname)) return null; - if (!isValidDomainHost(hostname)) return null; - - const path = parsed.pathname === "/" ? "" : parsed.pathname; - return `${hostname}${path}`; - } catch { - return null; - } +/** Normalized path of a domain input; `""` when it is a root or unparseable. */ +export function getResearchInputPath(input: string): string { + const parsed = parseResearchTarget(input); + return parsed.ok ? parsed.target.path : ""; } export function formatNumber(value: number | null | undefined) { diff --git a/src/client/features/search-tabs/SearchTabStrip.tsx b/src/client/features/search-tabs/SearchTabStrip.tsx index 00b00f8..fddd573 100644 --- a/src/client/features/search-tabs/SearchTabStrip.tsx +++ b/src/client/features/search-tabs/SearchTabStrip.tsx @@ -193,7 +193,7 @@ function getSearchTabQueryConfig( "domain-overview", projectId, trimmedDomain, - input.subdomains, + input.scope, input.locationCode, ], queryFn: () => @@ -201,7 +201,7 @@ function getSearchTabQueryConfig( data: { projectId, domain: trimmedDomain, - includeSubdomains: input.subdomains, + scope: input.scope, locationCode: input.locationCode, }, }), diff --git a/src/client/features/search-tabs/types.ts b/src/client/features/search-tabs/types.ts index bef8f60..997e077 100644 --- a/src/client/features/search-tabs/types.ts +++ b/src/client/features/search-tabs/types.ts @@ -2,18 +2,18 @@ import type { KeywordMode, ResultLimit, } from "@/client/features/keywords/keywordResearchTypes"; -import type { BacklinksTargetScope } from "@/types/schemas/backlinks"; +import type { ResearchScope } from "@/shared/researchScope"; export type BacklinksSearchTabInput = { type: "backlinks"; target: string; - scope: BacklinksTargetScope; + scope: ResearchScope; }; export type DomainSearchTabInput = { type: "domain"; domain: string; - subdomains: boolean; + scope: ResearchScope; locationCode?: number; }; diff --git a/src/client/features/search-tabs/useSearchTabs.test.ts b/src/client/features/search-tabs/useSearchTabs.test.ts index cfa3050..ab4ccaf 100644 --- a/src/client/features/search-tabs/useSearchTabs.test.ts +++ b/src/client/features/search-tabs/useSearchTabs.test.ts @@ -21,7 +21,7 @@ function searchTab(index: number): SearchTab { input: { type: "backlinks", target: `example-${index}.com`, - scope: "domain", + scope: "subdomains", }, }; } @@ -60,7 +60,7 @@ describe("parseStoredState", () => { persistedTab({ type: "domain", domain: "example.com", - subdomains: true, + scope: "subfolder", }), ], }); @@ -70,11 +70,59 @@ describe("parseStoredState", () => { expect(state.tabs[0].input).toEqual({ type: "domain", domain: "example.com", - subdomains: true, + scope: "subfolder", locationCode: undefined, }); }); + it("migrates domain tabs stored before research scopes", () => { + const state = parseStoredState({ + activeTabId: null, + tabs: [ + { + ...persistedTab({ + type: "domain", + domain: "a.com", + subdomains: true, + }), + id: "tab-1", + }, + { + ...persistedTab({ + type: "domain", + domain: "b.com", + subdomains: false, + }), + id: "tab-2", + }, + { + ...persistedTab({ + type: "backlinks", + target: "d.com/page", + scope: "page", + }), + id: "tab-3", + }, + ], + }); + + expect(state.tabs.map((tab) => tab.input)).toEqual([ + { + type: "domain", + domain: "a.com", + scope: "subdomains", + locationCode: undefined, + }, + { + type: "domain", + domain: "b.com", + scope: "domain", + locationCode: undefined, + }, + { type: "backlinks", target: "d.com/page", scope: "exact_url" }, + ]); + }); + it("keeps keyword tabs persisted without a locationCode (default location)", () => { const state = parseStoredState({ activeTabId: "tab-1", @@ -108,7 +156,7 @@ describe("parseStoredState", () => { persistedTab({ type: "domain", domain: "example.com", - subdomains: false, + scope: "domain", locationCode: 2840, }), ], @@ -125,7 +173,7 @@ describe("parseStoredState", () => { ...persistedTab({ type: "backlinks", target: `example-${index}.com`, - scope: "domain", + scope: "subdomains", }), id: `tab-${index}`, })), @@ -140,7 +188,7 @@ describe("parseStoredState", () => { const state = parseStoredState({ activeTabId: null, tabs: [ - persistedTab({ type: "domain", subdomains: true }), + persistedTab({ type: "domain", scope: "domain" }), persistedTab({ type: "keyword", keyword: "seo tools", @@ -150,7 +198,7 @@ describe("parseStoredState", () => { persistedTab({ type: "domain", domain: "example.com", - subdomains: true, + scope: "domain", locationCode: "us", }), persistedTab({ type: "unknown" }), diff --git a/src/client/features/search-tabs/useSearchTabs.ts b/src/client/features/search-tabs/useSearchTabs.ts index 4e63cd0..c915e57 100644 --- a/src/client/features/search-tabs/useSearchTabs.ts +++ b/src/client/features/search-tabs/useSearchTabs.ts @@ -1,4 +1,8 @@ import { useCallback, useMemo, useSyncExternalStore } from "react"; +import { + researchScopeSchema, + type ResearchScope, +} from "@/shared/researchScope"; import type { SearchTab, SearchTabInput } from "./types"; type TabsState = { @@ -24,21 +28,28 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } +function toResearchScope(value: unknown): ResearchScope | null { + const parsed = researchScopeSchema.safeParse(value); + return parsed.success ? parsed.data : null; +} + function parseTabInput(value: unknown): SearchTabInput | null { if (!isRecord(value)) return null; if (value.type === "backlinks") { if (typeof value.target !== "string" || value.target === "") return null; - if (value.scope !== "domain" && value.scope !== "page") return null; + // Legacy backlinks tabs stored "page"; "domain" survives as a valid scope. + const scope = + value.scope === "page" ? "exact_url" : toResearchScope(value.scope); + if (!scope) return null; return { type: "backlinks", target: value.target, - scope: value.scope, + scope, }; } if (value.type === "domain") { if (typeof value.domain !== "string" || value.domain === "") return null; - if (typeof value.subdomains !== "boolean") return null; // locationCode is optional in DomainSearchTabInput: tabs opened at the // default location persist no loc param, so accept a missing key. if ( @@ -47,10 +58,19 @@ function parseTabInput(value: unknown): SearchTabInput | null { ) { return null; } + // Legacy tabs stored an "include subdomains" boolean instead of a scope. + const scope = + toResearchScope(value.scope) ?? + (typeof value.subdomains === "boolean" + ? value.subdomains + ? "subdomains" + : "domain" + : null); + if (!scope) return null; return { type: "domain", domain: value.domain, - subdomains: value.subdomains, + scope, locationCode: typeof value.locationCode === "number" ? value.locationCode : undefined, }; diff --git a/src/client/hooks/useBacklinksSearchHistory.ts b/src/client/hooks/useBacklinksSearchHistory.ts index a5f14b6..45751e5 100644 --- a/src/client/hooks/useBacklinksSearchHistory.ts +++ b/src/client/hooks/useBacklinksSearchHistory.ts @@ -1,22 +1,63 @@ import { z } from "zod"; import { useLocalHistoryStore } from "@/client/hooks/useLocalHistoryStore"; import { jsonCodec } from "@/shared/json"; +import { + researchScopeSchema, + type ResearchScope, +} from "@/shared/researchScope"; export interface BacklinksSearchHistoryItem { target: string; - scope: "domain" | "page"; + scope: ResearchScope; timestamp: number; + /** Marks items written with research-scope values (see the codec below). */ + scopeVersion: typeof SCOPE_VERSION; } -type AddBacklinksSearchInput = Omit; +type AddBacklinksSearchInput = Omit< + BacklinksSearchHistoryItem, + "timestamp" | "scopeVersion" +>; const MAX_HISTORY = 20; +const SCOPE_VERSION = 2; -const backlinksSearchHistoryItemSchema = z.object({ - target: z.string(), - scope: z.enum(["domain", "page"]), - timestamp: z.number(), -}); +/** + * Before research scopes, "domain" meant the hostname plus its subdomains and + * "page" meant one URL. Both names survive with different meanings, so only + * items lacking `scopeVersion` get translated — a new "domain" pick must not + * be re-read as "subdomains". + */ +const LEGACY_SCOPES: Record = { + domain: "subdomains", + page: "exact_url", +}; + +const backlinksSearchHistoryItemSchema = z + .object({ + target: z.string(), + scope: z.string(), + scopeVersion: z.literal(SCOPE_VERSION).optional(), + timestamp: z.number(), + }) + .transform((item, ctx) => { + const scope = researchScopeSchema.safeParse( + item.scopeVersion === SCOPE_VERSION + ? item.scope + : LEGACY_SCOPES[item.scope], + ); + if (!scope.success) { + ctx.addIssue({ code: "custom", message: "Unknown backlinks scope" }); + return z.NEVER; + } + + return { + target: item.target, + scope: scope.data, + timestamp: item.timestamp, + scopeVersion: SCOPE_VERSION, + } satisfies BacklinksSearchHistoryItem; + }); const backlinksSearchHistorySchema = z.array(backlinksSearchHistoryItemSchema); const backlinksSearchHistoryCodec = jsonCodec(backlinksSearchHistorySchema); @@ -43,6 +84,7 @@ export function useBacklinksSearchHistory(projectId: string) { createItem: (item) => ({ ...item, timestamp: Date.now(), + scopeVersion: SCOPE_VERSION, }), getItemKey: (item) => item.timestamp, }); diff --git a/src/client/hooks/useBrandLookupSearchHistory.ts b/src/client/hooks/useBrandLookupSearchHistory.ts index 9ac41c4..8d6b440 100644 --- a/src/client/hooks/useBrandLookupSearchHistory.ts +++ b/src/client/hooks/useBrandLookupSearchHistory.ts @@ -1,10 +1,14 @@ import { z } from "zod"; import { useTimestampedSearchHistory } from "@/client/hooks/useTimestampedSearchHistory"; +import { researchScopeSchema } from "@/shared/researchScope"; const brandLookupSearchBodySchema = z.object({ query: z.string(), // Optional/defaulted so pre-existing history entries (query only) still parse. competitors: z.array(z.string()).optional().default([]), + // Absent on entries saved before scopes existed, and on lookups that used + // the query's default scope — both re-derive the default on restore. + scope: researchScopeSchema.optional(), }); type BrandLookupSearchBody = z.infer; @@ -21,6 +25,7 @@ export function useBrandLookupSearchHistory(projectId: string) { // the saved (already paid for) Share-of-Voice comparison of the same brand. isSame: (a, b) => a.query === b.query && - a.competitors.join(",") === b.competitors.join(","), + a.competitors.join(",") === b.competitors.join(",") && + a.scope === b.scope, }); } diff --git a/src/client/hooks/useDomainSearchHistory.ts b/src/client/hooks/useDomainSearchHistory.ts index 23d5ee1..1044248 100644 --- a/src/client/hooks/useDomainSearchHistory.ts +++ b/src/client/hooks/useDomainSearchHistory.ts @@ -1,13 +1,17 @@ import { z } from "zod"; import { useLocalHistoryStore } from "@/client/hooks/useLocalHistoryStore"; import { jsonCodec } from "@/shared/json"; +import { + researchScopeSchema, + type ResearchScope, +} from "@/shared/researchScope"; type DomainSortMode = "rank" | "traffic" | "volume" | "score" | "cpc"; type DomainTab = "keywords" | "pages"; export interface DomainSearchHistoryItem { domain: string; - subdomains: boolean; + scope: ResearchScope; sort: DomainSortMode; tab: DomainTab; locationCode?: number; @@ -18,14 +22,24 @@ type AddDomainSearchInput = Omit; const MAX_HISTORY = 20; -const domainSearchHistoryItemSchema = z.object({ - domain: z.string(), - subdomains: z.boolean(), - sort: z.enum(["rank", "traffic", "volume", "score", "cpc"]), - tab: z.enum(["keywords", "pages"]), - locationCode: z.number().int().positive().optional(), - timestamp: z.number(), -}); +const domainSearchHistoryItemSchema = z + .object({ + domain: z.string(), + scope: researchScopeSchema.optional(), + /** Legacy field: pre-scope history encoded "Include subdomains" here. */ + subdomains: z.boolean().optional(), + sort: z.enum(["rank", "traffic", "volume", "score", "cpc"]), + tab: z.enum(["keywords", "pages"]), + locationCode: z.number().int().positive().optional(), + timestamp: z.number(), + }) + .transform( + ({ subdomains, scope, ...item }): DomainSearchHistoryItem => ({ + ...item, + // Legacy items always carried the boolean, so scope-less means true. + scope: scope ?? (subdomains === false ? "domain" : "subdomains"), + }), + ); const domainSearchHistorySchema = z.array(domainSearchHistoryItemSchema); const domainSearchHistoryCodec = jsonCodec(domainSearchHistorySchema); @@ -36,7 +50,7 @@ function isSameSearch( ): boolean { return ( a.domain === b.domain && - a.subdomains === b.subdomains && + a.scope === b.scope && a.sort === b.sort && a.tab === b.tab && a.locationCode === b.locationCode diff --git a/src/routes/_project/p/$projectId/backlinks.tsx b/src/routes/_project/p/$projectId/backlinks.tsx index 7581577..5317d3c 100644 --- a/src/routes/_project/p/$projectId/backlinks.tsx +++ b/src/routes/_project/p/$projectId/backlinks.tsx @@ -1,10 +1,10 @@ import { createFileRoute, useNavigate } from "@tanstack/react-router"; import { BacklinksPage } from "@/client/features/backlinks/BacklinksPage"; -import { inferBacklinksSearchScopeFromTarget } from "@/client/features/backlinks/backlinksSearchScope"; import { DEFAULT_BACKLINKS_PAGE_SIZE, backlinksSearchSchema, } from "@/types/schemas/backlinks"; +import { defaultScopeForInput } from "@/shared/researchScope"; export const Route = createFileRoute("/_project/p/$projectId/backlinks")({ validateSearch: backlinksSearchSchema, @@ -24,7 +24,7 @@ function BacklinksRoute() { order, view, } = Route.useSearch(); - const scope = rawScope ?? inferBacklinksSearchScopeFromTarget(target); + const scope = rawScope ?? defaultScopeForInput(target); return ( { + initialScope={scope} + onSearchChange={(nextQuery, nextCompetitors, nextScope) => { void navigate({ search: (prev) => ({ ...prev, @@ -28,6 +29,9 @@ function BrandLookupRoute() { nextCompetitors.length > 0 ? nextCompetitors.join(",") : undefined, + // The page passes a scope only when it differs from the default + // derived from `q`. + scope: nextScope, }), replace: true, }); diff --git a/src/routes/_project/p/$projectId/domain.tsx b/src/routes/_project/p/$projectId/domain.tsx index 7bc53f9..abbbee9 100644 --- a/src/routes/_project/p/$projectId/domain.tsx +++ b/src/routes/_project/p/$projectId/domain.tsx @@ -13,7 +13,6 @@ import { useProjectMarket } from "@/client/features/projects/useProjectMarket"; const DEFAULT_DOMAIN_SEARCH = { domain: "", - subdomains: true, sort: "traffic", order: undefined, tab: "keywords", diff --git a/src/server/features/ai-search/services/brandLookup.test.ts b/src/server/features/ai-search/services/brandLookup.test.ts index da70be1..9222eaf 100644 --- a/src/server/features/ai-search/services/brandLookup.test.ts +++ b/src/server/features/ai-search/services/brandLookup.test.ts @@ -25,8 +25,18 @@ vi.mock("@/server/lib/dataforseo", () => { CHATGPT_LANGUAGE_CODE: "en", CHATGPT_LOCATION_CODE: 2840, buildLlmTarget: vi.fn( - ({ type, value }: { type: "domain" | "keyword"; value: string }) => - type === "domain" ? { domain: value } : { keyword: value }, + ({ + type, + value, + includeSubdomains, + }: { + type: "domain" | "keyword"; + value: string; + includeSubdomains?: boolean; + }) => + type === "domain" + ? { domain: value, include_subdomains: includeSubdomains ?? true } + : { keyword: value }, ), createDataforseoClient: vi.fn(() => dataforseoClientMock), }; @@ -89,6 +99,7 @@ function baseArgs(overrides: Partial): ShapeArgs { return { query: "acme", detected: { type: "keyword", value: "acme" }, + researchTarget: null, platformBundles: [ platformBundle("chat_gpt", 10, 100), platformBundle("google", 5, 50), @@ -208,6 +219,37 @@ describe("getBrandLookup", () => { dataforseoClientMock.aiSearch.aggregatedMetrics, ).not.toHaveBeenCalled(); }); + + it("drops subdomains and keys the cache on scope for a URL query", async () => { + resetBrandLookupMocks(); + + const result = await getBrandLookup( + { + projectId: "project_123", + query: "https://acme.com/blog", + competitors: [], + locationCode: 2840, + languageCode: "en", + }, + billingCustomer, + ); + + // No URL-level targeting exists upstream: the call is domain-only with + // subdomains excluded, and page rows are filtered in shaping. + expect( + dataforseoClientMock.aiSearch.aggregatedMetrics, + ).toHaveBeenCalledWith( + expect.objectContaining({ + target: { domain: "acme.com", include_subdomains: false }, + }), + ); + expect(cacheMock.buildCacheKey).toHaveBeenCalledWith( + "ai-search:brand-lookup", + expect.objectContaining({ scope: "subfolder", path: "/blog" }), + ); + expect(result.resolvedTarget).toBe("acme.com/blog"); + expect(result.aggregatesAreDomainLevel).toBe(true); + }); }); describe("resolveCompetitorGroups", () => { diff --git a/src/server/features/ai-search/services/brandLookup.ts b/src/server/features/ai-search/services/brandLookup.ts index 4964dd2..6ec3e9a 100644 --- a/src/server/features/ai-search/services/brandLookup.ts +++ b/src/server/features/ai-search/services/brandLookup.ts @@ -26,6 +26,10 @@ import { type BrandLookupResult, } from "@/types/schemas/ai-search"; import { detectTarget } from "@/shared/targetDetection"; +import { + parseResearchTarget, + type ResearchTarget, +} from "@/shared/researchScope"; /** * Brand Lookup is the AI-search analog of Domain Overview. The user types a @@ -49,6 +53,11 @@ export async function getBrandLookup( billingCustomer: BillingCustomerContext, ): Promise { const detected = detectTarget(input.query); + const researchTarget = resolveResearchTarget(input, detected); + // The LLM mentions API only scopes a domain target by subdomain inclusion; + // exact_url/subfolder are honored by post-filtering page rows in shaping. + const includeSubdomains = + researchTarget === null || researchTarget.scope === "subdomains"; const competitorGroups = resolveCompetitorGroups( detected.value, input.competitors, @@ -71,6 +80,16 @@ export async function getBrandLookup( .join("|"), locationCode: input.locationCode, languageCode: input.languageCode, + // Scope changes both the provider call (include_subdomains) and the + // page-level filtering, so it must not share a cache entry. The path only + // affects output under URL scopes — keying it for domain/subdomains would + // re-buy identical fan-outs for example.com vs example.com/blog. + scope: researchTarget?.scope ?? null, + path: + researchTarget?.scope === "exact_url" || + researchTarget?.scope === "subfolder" + ? researchTarget.path + : "", }); const cached = brandLookupResultSchema.safeParse(await getCached(cacheKey)); @@ -78,7 +97,7 @@ export async function getBrandLookup( return { ...cached.data, query: input.query, - resolvedTarget: detected.value, + resolvedTarget: researchTarget?.display ?? detected.value, }; } @@ -92,7 +111,13 @@ export async function getBrandLookup( for (const platform of PLATFORMS) { settled.push( await settle(() => - fetchPlatformData(platform, detected, input, dataforseo), + fetchPlatformData( + platform, + detected, + includeSubdomains, + input, + dataforseo, + ), ), ); } @@ -102,7 +127,13 @@ export async function getBrandLookup( const crossSettled = competitorGroups.length > 0 ? await settle(() => - fetchCrossAggregated(detected, competitorGroups, input, dataforseo), + fetchCrossAggregated( + detected, + competitorGroups, + includeSubdomains, + input, + dataforseo, + ), ) : ({ status: "fulfilled", value: [] } as PromiseFulfilledResult< CrossOutcome[] @@ -125,6 +156,7 @@ export async function getBrandLookup( const result = shapeResult({ query: input.query, detected, + researchTarget, platformBundles, crossOutcomes, competitorKeys: competitorGroups.map((g) => g.label), @@ -151,6 +183,26 @@ export async function getBrandLookup( return result; } +/** + * Scopes only apply to domain/URL queries — a brand keyword has no URL to + * narrow. A domain the parser rejects (fake TLD) keeps today's unscoped + * behavior and fails downstream with the provider's own validation error. + */ +function resolveResearchTarget( + input: BrandLookupInput, + detected: ReturnType, +): ResearchTarget | null { + if (detected.type !== "domain") return null; + const parsed = parseResearchTarget(input.query, input.scope); + if (!parsed.ok) { + // An explicit scope that doesn't fit the input (Subfolder without a path) + // must error, not silently run an unscoped lookup. + if (input.scope) throw new AppError("VALIDATION_ERROR", parsed.message); + return null; + } + return parsed.target; +} + async function settle( execute: () => Promise, ): Promise> { @@ -169,12 +221,14 @@ type PlatformFetchInput = Pick< async function fetchPlatformData( platform: LlmPlatform, detected: ReturnType, + includeSubdomains: boolean, input: PlatformFetchInput, dataforseo: ReturnType, ): Promise { const target = buildLlmTarget({ type: detected.type, value: detected.value, + includeSubdomains, }); // ChatGPT mentions DB only contains US/en data per DataForSEO docs. @@ -240,23 +294,33 @@ async function fetchPlatformData( * single failure doesn't discard the other — matching the per-platform * fan-out in {@link getBrandLookup}. The target's aggregation_key is the * resolved target value so SoV can flag the target row. + * + * Share of Voice always compares domain against domain — the provider has no + * URL-level targeting — so every group (target and competitors) uses the same + * subdomain rule and the UI labels the section domain-level under URL scopes. */ async function fetchCrossAggregated( detected: ReturnType, competitors: CompetitorGroup[], + includeSubdomains: boolean, input: PlatformFetchInput, dataforseo: ReturnType, ): Promise { const groups = [ { key: detected.value, - target: buildLlmTarget({ type: detected.type, value: detected.value }), + target: buildLlmTarget({ + type: detected.type, + value: detected.value, + includeSubdomains, + }), }, ...competitors.map((competitor) => ({ key: competitor.label, target: buildLlmTarget({ type: competitor.detected.type, value: competitor.detected.value, + includeSubdomains, }), })), ]; diff --git a/src/server/features/ai-search/services/brandLookupShaping.test.ts b/src/server/features/ai-search/services/brandLookupShaping.test.ts index 1ae368d..84e339f 100644 --- a/src/server/features/ai-search/services/brandLookupShaping.test.ts +++ b/src/server/features/ai-search/services/brandLookupShaping.test.ts @@ -6,6 +6,7 @@ import type { LlmTopPagesItem, } from "@/server/lib/dataforseoLlmSchemas"; import { brandLookupResultSchema } from "@/types/schemas/ai-search"; +import { parseResearchTarget } from "@/shared/researchScope"; function platformBundle( platform: "chat_gpt" | "google", @@ -46,6 +47,7 @@ function baseArgs(overrides: Partial = {}): ShapeArgs { return { query: "acme", detected: { type: "keyword", value: "acme" }, + researchTarget: null, platformBundles: [ platformBundle("chat_gpt", 10, 100), platformBundle("google", 5, 50), @@ -163,4 +165,64 @@ describe("shapeResult", () => { }); expect(brandLookupResultSchema.safeParse(result).success).toBe(true); }); + + it("keeps only in-scope page rows and prompts under a subfolder scope", () => { + const parsed = parseResearchTarget("acme.com/blog", "subfolder"); + if (!parsed.ok) throw new Error(parsed.message); + + const result = shapeResult( + baseArgs({ + detected: { type: "domain", value: "acme.com" }, + researchTarget: parsed.target, + platformBundles: [ + { + platform: "google", + status: "success", + bundle: { + aggregated: { + platform: [ + { key: "google", mentions: 12, ai_search_volume: 120 }, + ], + }, + topPages: [ + page("https://acme.com/blog/post"), + // Sibling path, and another domain cited alongside the brand. + page("https://acme.com/blogging"), + page("https://other.example/review"), + ], + mentions: [ + mention("in scope", "https://acme.com/blog/post"), + mention("out of scope", "https://acme.com/pricing"), + ], + complete: true, + }, + }, + ], + }), + ); + + expect(result.topPages.map((row) => row.url)).toEqual([ + "https://acme.com/blog/post", + ]); + expect(result.topQueries.map((row) => row.question)).toEqual(["in scope"]); + // Mentions can't be narrowed upstream, so they stay and get flagged. + expect(result.totalMentions).toBe(12); + expect(result.aggregatesAreDomainLevel).toBe(true); + expect(result.resolvedTarget).toBe("acme.com/blog"); + }); }); + +function page(url: string): LlmTopPagesItem { + return { + key: url, + platform: [{ key: "google", mentions: 2, ai_search_volume: 20 }], + }; +} + +function mention(question: string, sourceUrl: string): LlmMentionItem { + return { + question, + ai_search_volume: 10, + sources: [{ url: sourceUrl }], + }; +} diff --git a/src/server/features/ai-search/services/brandLookupShaping.ts b/src/server/features/ai-search/services/brandLookupShaping.ts index 2aee9c1..7d2df70 100644 --- a/src/server/features/ai-search/services/brandLookupShaping.ts +++ b/src/server/features/ai-search/services/brandLookupShaping.ts @@ -19,6 +19,10 @@ import { } from "@/server/features/ai-search/services/shareOfVoice"; import type { BrandLookupResult } from "@/types/schemas/ai-search"; import type { detectTarget } from "@/shared/targetDetection"; +import { + urlMatchesResearchTarget, + type ResearchTarget, +} from "@/shared/researchScope"; const TOP_QUERIES_PER_PLATFORM = 25; const TOP_SOURCES_PER_PLATFORM = 10; @@ -45,6 +49,8 @@ export type PlatformOutcome = { export type ShapeArgs = { query: string; detected: ReturnType; + /** Resolved research target for domain queries; null for brand keywords. */ + researchTarget: ResearchTarget | null; platformBundles: PlatformOutcome[]; crossOutcomes: CrossOutcome[]; /** Labels of the resolved competitor groups, as sent to cross_aggregated. */ @@ -54,6 +60,11 @@ export type ShapeArgs = { }; export function shapeResult(args: ShapeArgs): BrandLookupResult { + // Post-filter page URLs only for URL scopes; domain/subdomains were already + // scoped in the provider call, and brand-keyword queries have no target. + const scope = args.researchTarget?.scope; + const pageFilter = + scope === "exact_url" || scope === "subfolder" ? args.researchTarget : null; const successfulBundles = args.platformBundles.filter( (b): b is PlatformOutcome & { bundle: PlatformBundle } => b.status === "success" && b.bundle !== null, @@ -104,9 +115,10 @@ export function shapeResult(args: ShapeArgs): BrandLookupResult { sourcesPerPlatform: TOP_SOURCES_PER_PLATFORM, keywordsPerSource: KEYWORDS_PER_SOURCE, }, + pageFilter, ); - const topQueries = shapeTopQueries(successfulBundles); + const topQueries = shapeTopQueries(successfulBundles, pageFilter); const trendBundles = chatGptLocaleMatches ? successfulBundles : successfulBundles.filter((b) => b.platform !== "chat_gpt"); @@ -129,7 +141,9 @@ export function shapeResult(args: ShapeArgs): BrandLookupResult { return { query: args.query, detectedTargetType: args.detected.type, - resolvedTarget: args.detected.value, + resolvedTarget: args.researchTarget?.display ?? args.detected.value, + scope: args.researchTarget?.scope ?? null, + aggregatesAreDomainLevel: pageFilter !== null, fetchedAt: new Date().toISOString(), hasData, totalMentions, @@ -144,6 +158,7 @@ export function shapeResult(args: ShapeArgs): BrandLookupResult { function shapeTopQueries( bundles: Array, + pageFilter: ResearchTarget | null, ): BrandLookupResult["topQueries"] { return sortBy( bundles.flatMap((bundle) => @@ -153,6 +168,17 @@ function shapeTopQueries( (item): item is LlmMentionItem & { question: string } => typeof item.question === "string" && item.question.length > 0, ) + // Under a URL scope a prompt only counts when the answer actually + // cited a page in scope — a brand named in the answer text is + // domain-level evidence we must not attribute to the page. + .filter( + (item) => + pageFilter === null || + (item.sources ?? []).some((source) => { + const url = safeHttpUrl(source.url); + return url != null && urlMatchesResearchTarget(url, pageFilter); + }), + ) .map((item) => ({ question: truncate(item.question, MAX_QUESTION_LENGTH), platform: bundle.platform, diff --git a/src/server/features/ai-search/services/citedSources.ts b/src/server/features/ai-search/services/citedSources.ts index 8e9e6dd..5d18041 100644 --- a/src/server/features/ai-search/services/citedSources.ts +++ b/src/server/features/ai-search/services/citedSources.ts @@ -7,6 +7,10 @@ import type { import { safeHostname, safeHttpUrl } from "@/server/features/ai-search/safeUrl"; import { roundOrNull } from "@/server/features/ai-search/services/shareOfVoice"; import type { BrandLookupResult } from "@/types/schemas/ai-search"; +import { + urlMatchesResearchTarget, + type ResearchTarget, +} from "@/shared/researchScope"; type Bundle = { platform: LlmPlatform; @@ -24,10 +28,15 @@ const MAX_QUESTION_LENGTH = 500; * examples from the mentions sample when the exact cited URL appears there. * The page metrics stay authoritative while the prompt examples remain plainly * sample-based. + * + * `pageFilter` narrows the rows to a URL-scoped research target (the provider + * has no URL-level targeting). Filtering happens before the per-platform cap so + * in-scope pages can't be crowded out by out-of-scope ones. */ export function deriveCitedSources( bundles: Bundle[], limits: { sourcesPerPlatform: number; keywordsPerSource: number }, + pageFilter: ResearchTarget | null = null, ): BrandLookupResult["topPages"] { const promptExamples = buildPromptExamples(bundles); @@ -36,6 +45,9 @@ export function deriveCitedSources( .map((page) => { const url = safeHttpUrl(page.key); if (!url || url.length > MAX_URL_LENGTH) return null; + if (pageFilter && !urlMatchesResearchTarget(url, pageFilter)) { + return null; + } const platformGroup = page.platform?.find( (entry) => entry.key === bundle.platform, ); diff --git a/src/server/features/backlinks/services/BacklinksService.billing.test.ts b/src/server/features/backlinks/services/BacklinksService.billing.test.ts index 3bf8518..e230798 100644 --- a/src/server/features/backlinks/services/BacklinksService.billing.test.ts +++ b/src/server/features/backlinks/services/BacklinksService.billing.test.ts @@ -37,6 +37,19 @@ const billingCustomer = { userEmail: "team@example.com", }; +function mockTarget( + overrides: Partial> = {}, +) { + vi.mocked(normalizeBacklinksTarget).mockReturnValue({ + apiTarget: "example.com", + displayTarget: "example.com", + scope: "domain", + includeSubdomains: false, + path: "", + ...overrides, + }); +} + const pageInputDefaults = { projectId: "project_123", page: 1, @@ -63,11 +76,7 @@ beforeEach(() => { }); it("profiles only the summary and history for the overview and reuses cache on repeat", async () => { - vi.mocked(normalizeBacklinksTarget).mockReturnValue({ - apiTarget: "example.com", - displayTarget: "example.com", - scope: "domain", - }); + mockTarget(); backlinksSummaryMock.mockResolvedValue({ rank: 42, backlinks: 1200, @@ -115,11 +124,7 @@ it("profiles only the summary and history for the overview and reuses cache on r }); it("profiles backlink rows per page with offset and total count", async () => { - vi.mocked(normalizeBacklinksTarget).mockReturnValue({ - apiTarget: "example.com", - displayTarget: "example.com", - scope: "domain", - }); + mockTarget(); backlinksRowsMock.mockResolvedValue({ items: [ { @@ -172,11 +177,7 @@ it("profiles backlink rows per page with offset and total count", async () => { }); it("translates filters into DataForSEO conditions for backlink rows", async () => { - vi.mocked(normalizeBacklinksTarget).mockReturnValue({ - apiTarget: "example.com", - displayTarget: "example.com", - scope: "domain", - }); + mockTarget(); backlinksRowsMock.mockResolvedValue({ items: [], totalCount: 0 }); await service.profileBacklinksPage( @@ -211,10 +212,11 @@ it("translates filters into DataForSEO conditions for backlink rows", async () = }); it("profiles referring domains and top pages pages separately", async () => { - vi.mocked(normalizeBacklinksTarget).mockReturnValue({ + mockTarget({ apiTarget: "https://example.com/foo", displayTarget: "https://example.com/foo", - scope: "page", + scope: "exact_url", + includeSubdomains: true, }); referringDomainsMock.mockResolvedValue({ items: [ @@ -269,11 +271,7 @@ it("profiles referring domains and top pages pages separately", async () => { }); it("does not fall back to target spam score for referring domains", async () => { - vi.mocked(normalizeBacklinksTarget).mockReturnValue({ - apiTarget: "example.com", - displayTarget: "example.com", - scope: "domain", - }); + mockTarget(); referringDomainsMock.mockResolvedValue({ items: [ { @@ -304,12 +302,8 @@ it("does not fall back to target spam score for referring domains", async () => expect(domains.rows[0]?.spamScore).toBeNull(); }); -it("keeps page cache entries isolated per page and per organization", async () => { - vi.mocked(normalizeBacklinksTarget).mockReturnValue({ - apiTarget: "example.com", - displayTarget: "example.com", - scope: "domain", - }); +it("keeps page cache entries isolated per page, organization, and scope", async () => { + mockTarget(); backlinksRowsMock.mockResolvedValue({ items: [], totalCount: 0 }); const input = { @@ -331,6 +325,14 @@ it("keeps page cache entries isolated per page and per organization", async () = userEmail: "other@example.com", }); expect(backlinksRowsMock).toHaveBeenCalledTimes(3); + + // Same hostname, subdomains included: a different result set, not a cache hit. + mockTarget({ scope: "subdomains", includeSubdomains: true }); + await service.profileBacklinksPage( + { ...input, scope: "subdomains" }, + billingCustomer, + ); + expect(backlinksRowsMock).toHaveBeenCalledTimes(4); }); function parseCachedValue(raw: string): unknown { @@ -340,3 +342,49 @@ function parseCachedValue(raw: string): unknown { return null; } } + +it("builds subfolder overview totals from two filtered backlink counts", async () => { + mockTarget({ + displayTarget: "example.com/blog", + scope: "subfolder", + path: "/blog", + }); + backlinksRowsMock + .mockResolvedValueOnce({ items: [], totalCount: 2500 }) + .mockResolvedValueOnce({ items: [], totalCount: 180 }); + + const { overview } = await service.profileOverview( + { target: "example.com/blog", scope: "subfolder" }, + billingCustomer, + ); + + expect(overview.summary.backlinks).toBe(2500); + expect(overview.summary.referringDomains).toBe(180); + expect(overview.summary.rank).toBeNull(); + expect(overview.trends).toEqual([]); + expect(backlinksSummaryMock).not.toHaveBeenCalled(); + expect(backlinksHistoryMock).not.toHaveBeenCalled(); + expect(backlinksRowsMock).toHaveBeenCalledTimes(2); + expect(backlinksRowsMock).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + mode: "as_is", + limit: 1, + filters: [ + [ + ["url_to", "like", "%://example.com/blog"], + "or", + ["url_to", "like", "%://example.com/blog/%"], + "or", + ["url_to", "like", "%://www.example.com/blog"], + "or", + ["url_to", "like", "%://www.example.com/blog/%"], + ], + ], + }), + ); + expect(backlinksRowsMock).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ mode: "one_per_domain", limit: 1 }), + ); +}); diff --git a/src/server/features/backlinks/services/BacklinksService.ts b/src/server/features/backlinks/services/BacklinksService.ts index f7c2137..4fbd331 100644 --- a/src/server/features/backlinks/services/BacklinksService.ts +++ b/src/server/features/backlinks/services/BacklinksService.ts @@ -25,7 +25,7 @@ const defaultCache: BacklinksCache = { type BacklinksPageCacheInput = { target: string; - scope?: "domain" | "page"; + scope?: BacklinksLookupInput["scope"]; page: number; pageSize: number; sortField: string; @@ -124,6 +124,12 @@ function buildTargetCacheInput( organizationId: billingCustomer.organizationId, target: normalizedTarget.apiTarget, scope: normalizedTarget.scope, + // Subfolder scope keeps the hostname as the API target, so the path must + // separate cache entries. + path: normalizedTarget.path, + // Same hostname, different result set — and keeping it in the key retires + // entries written before scopes could exclude subdomains. + includeSubdomains: normalizedTarget.includeSubdomains, }; } diff --git a/src/server/features/backlinks/services/backlinksOverviewSchema.ts b/src/server/features/backlinks/services/backlinksOverviewSchema.ts index 071445a..07bf585 100644 --- a/src/server/features/backlinks/services/backlinksOverviewSchema.ts +++ b/src/server/features/backlinks/services/backlinksOverviewSchema.ts @@ -56,7 +56,7 @@ const backlinksNewLostTrendRowSchema = z.object({ export const backlinksOverviewSchema = z.object({ target: z.string(), displayTarget: z.string(), - scope: z.enum(["domain", "page"]), + scope: z.enum(["exact_url", "subfolder", "domain", "subdomains"]), summary: z.object({ rank: z.number().nullable(), backlinks: z.number().nullable(), diff --git a/src/server/features/backlinks/services/backlinksRowMappers.ts b/src/server/features/backlinks/services/backlinksRowMappers.ts new file mode 100644 index 0000000..15af56a --- /dev/null +++ b/src/server/features/backlinks/services/backlinksRowMappers.ts @@ -0,0 +1,73 @@ +import type { + BacklinksItem, + DomainPageSummaryItem, + ReferringDomainItem, +} from "@/server/lib/dataforseo"; + +export function normalizeHistoryDate(value: string | null | undefined) { + return value ? value.slice(0, 10) : null; +} + +export function mapBacklinksRows(rows: BacklinksItem[]) { + return rows.map((item) => ({ + domainFrom: item.domain_from ?? null, + urlFrom: item.url_from ?? null, + urlTo: item.url_to ?? null, + anchor: item.anchor ?? null, + itemType: item.item_type ?? null, + isDofollow: item.dofollow ?? null, + relAttributes: item.rel_attributes ?? item.attributes ?? [], + rank: item.rank ?? null, + domainFromRank: item.domain_from_rank ?? null, + pageFromRank: item.page_from_rank ?? null, + spamScore: item.backlink_spam_score ?? item.backlinks_spam_score ?? null, + firstSeen: item.first_seen ?? null, + lastSeen: item.lost_date ?? item.last_visited ?? null, + isLost: item.is_lost ?? Boolean(item.lost_date), + isBroken: item.is_broken ?? false, + linksCount: item.links_count ?? null, + })); +} + +export function mapReferringDomainsRows(rows: ReferringDomainItem[]) { + return rows.map((item) => ({ + domain: item.domain ?? null, + backlinks: item.backlinks ?? null, + referringPages: item.referring_pages ?? null, + rank: item.rank ?? null, + spamScore: item.backlinks_spam_score ?? null, + firstSeen: item.first_seen ?? null, + brokenBacklinks: item.broken_backlinks ?? null, + brokenPages: item.broken_pages ?? null, + })); +} + +export function mapTopPagesRows(rows: DomainPageSummaryItem[]) { + return rows.map((item) => ({ + page: item.page ?? item.url ?? null, + backlinks: item.backlinks ?? null, + referringDomains: item.referring_domains ?? null, + rank: item.rank ?? null, + brokenBacklinks: item.broken_backlinks ?? null, + })); +} + +export function buildPageResult( + input: { page: number; pageSize: number }, + offset: number, + data: { rows: TRow[]; totalCount: number | null }, +) { + const hasMore = + data.totalCount != null + ? offset + data.rows.length < data.totalCount + : data.rows.length === input.pageSize; + + return { + rows: data.rows, + totalCount: data.totalCount, + hasMore, + page: input.page, + pageSize: input.pageSize, + fetchedAt: new Date().toISOString(), + }; +} diff --git a/src/server/features/backlinks/services/backlinksServiceData.ts b/src/server/features/backlinks/services/backlinksServiceData.ts index 2f5ec52..5a37b8d 100644 --- a/src/server/features/backlinks/services/backlinksServiceData.ts +++ b/src/server/features/backlinks/services/backlinksServiceData.ts @@ -5,17 +5,15 @@ import { createDataforseoClient, normalizeBacklinksTarget, type BacklinksHistoryItem, - type BacklinksItem, type BacklinksSummaryItem, - type DomainPageSummaryItem, - type ReferringDomainItem, } from "@/server/lib/dataforseo"; -import type { - BacklinksLookupInput, - BacklinksRowsPageInput, - BacklinksSpamFilterOptions, - ReferringDomainsPageInput, - TopPagesPageInput, +import { + normalizeBacklinksSpamFilterOptions, + type BacklinksLookupInput, + type BacklinksRowsPageInput, + type BacklinksSpamFilterOptions, + type ReferringDomainsPageInput, + type TopPagesPageInput, } from "@/types/schemas/backlinks"; import { @@ -36,6 +34,21 @@ import { buildTopPagesApiFilters, buildTopPagesOrderBy, } from "@/server/features/backlinks/services/backlinksApiFilters"; +import { + buildBacklinksScopeFilter, + countExpressionConditions, + prependScopeClauses, +} from "@/server/lib/dataforseo/researchScopeFilters"; +import { buildSubfolderOverview } from "@/server/features/backlinks/services/backlinksSubfolderOverview"; +import { + buildPageResult, + mapBacklinksRows, + mapReferringDomainsRows, + mapTopPagesRows, + normalizeHistoryDate, +} from "@/server/features/backlinks/services/backlinksRowMappers"; +import { assertFilterConditionBudget } from "@/server/lib/dataforseo/filters"; +import { AppError } from "@/server/lib/errors"; // The page-request schemas carry projectId for the web middleware; the // service layer is organization-scoped and never reads it. @@ -61,10 +74,6 @@ export type BacklinksCache = { set(key: string, data: unknown, ttlSeconds: number): Promise; }; -type BacklinksOverviewProfile = { - overview: BacklinksOverviewResult; -}; - type BacklinksDateRange = { dateFrom: string; dateTo: string; @@ -76,7 +85,7 @@ export async function profileBacklinksOverview( input: BacklinksLookupInput, billingCustomer: BillingCustomerContext, creditFeature?: CreditFeature, -): Promise { +): Promise<{ overview: BacklinksOverviewResult }> { const cached = backlinksOverviewCacheSchema.safeParse( await cache.get(cacheKey), ); @@ -92,20 +101,40 @@ export async function profileBacklinksOverview( const normalizedTarget = normalizeBacklinksTarget(input.target, { scope: input.scope, }); + + if (normalizedTarget.scope === "subfolder") { + const overview = await buildSubfolderOverview( + dataforseo, + normalizedTarget, + now, + creditFeature, + ); + await cacheValue( + cache, + cacheKey, + { overview }, + BACKLINKS_OVERVIEW_TTL_SECONDS, + ); + return { overview }; + } + const dateRange = buildBacklinksDateRange(now); const [summary, history] = await Promise.all([ dataforseo.backlinks.summary({ target: normalizedTarget.apiTarget, + includeSubdomains: normalizedTarget.includeSubdomains, creditFeature, }), - normalizedTarget.scope === "domain" - ? dataforseo.backlinks.history({ + // history/live only accepts a hostname and has no include_subdomains field, + // so trends are unavailable for a page and subdomain-inclusive otherwise. + normalizedTarget.scope === "exact_url" + ? Promise.resolve([]) + : dataforseo.backlinks.history({ target: normalizedTarget.apiTarget, ...dateRange, creditFeature, - }) - : Promise.resolve([]), + }), ]); const overview = buildOverviewResult({ @@ -140,11 +169,22 @@ export async function profileBacklinksRowsPage( const dataforseo = createDataforseoClient(billingCustomer); const offset = (input.page - 1) * input.pageSize; - const filters = buildBacklinksRowsApiFilters(input.filters); + + const target = normalizeBacklinksTarget(input.target, { scope: input.scope }); + const scopeFilter = buildBacklinksScopeFilter("url_to", target); + const userFilters = buildBacklinksRowsApiFilters(input.filters); + // The scope group and the server-appended spam condition share the same + // 8-condition budget as user filters. + assertFilterConditionBudget( + scopeFilter.conditionCount + + countExpressionConditions(userFilters) + + (normalizeBacklinksSpamFilterOptions(spamOptions).hideSpam ? 1 : 0), + ); + const filters = prependScopeClauses(scopeFilter, userFilters); const response = await dataforseo.backlinks.rows({ - target: normalizeBacklinksTarget(input.target, { scope: input.scope }) - .apiTarget, + target: target.apiTarget, + includeSubdomains: target.includeSubdomains, limit: input.pageSize, offset, orderBy: buildBacklinksRowsOrderBy(input.sortField, input.sortOrder), @@ -180,9 +220,20 @@ export async function profileReferringDomainsPage( const offset = (input.page - 1) * input.pageSize; const filters = buildReferringDomainsApiFilters(input.filters); + const target = normalizeBacklinksTarget(input.target, { scope: input.scope }); + // referring_domains has no URL field to filter on, so subfolder scope has no + // accurate source for this breakdown (the count still comes from the + // overview's filtered totals). + if (target.scope === "subfolder") { + throw new AppError( + "VALIDATION_ERROR", + "Referring domains can't be broken down for a subfolder — use the Backlinks tab, or switch to Domain or Subdomains scope.", + ); + } + const response = await dataforseo.backlinks.referringDomains({ - target: normalizeBacklinksTarget(input.target, { scope: input.scope }) - .apiTarget, + target: target.apiTarget, + includeSubdomains: target.includeSubdomains, limit: input.pageSize, offset, orderBy: buildReferringDomainsOrderBy(input.sortField, input.sortOrder), @@ -212,11 +263,18 @@ export async function profileTopPagesPage( const dataforseo = createDataforseoClient(billingCustomer); const offset = (input.page - 1) * input.pageSize; - const filters = buildTopPagesApiFilters(input.filters); + + const target = normalizeBacklinksTarget(input.target, { scope: input.scope }); + const scopeFilter = buildBacklinksScopeFilter("url", target); + const userFilters = buildTopPagesApiFilters(input.filters); + assertFilterConditionBudget( + scopeFilter.conditionCount + countExpressionConditions(userFilters), + ); + const filters = prependScopeClauses(scopeFilter, userFilters); const response = await dataforseo.backlinks.domainPages({ - target: normalizeBacklinksTarget(input.target, { scope: input.scope }) - .apiTarget, + target: target.apiTarget, + includeSubdomains: target.includeSubdomains, limit: input.pageSize, offset, orderBy: buildTopPagesOrderBy(input.sortField, input.sortOrder), @@ -232,26 +290,6 @@ export async function profileTopPagesPage( return result; } -function buildPageResult( - input: { page: number; pageSize: number }, - offset: number, - data: { rows: TRow[]; totalCount: number | null }, -) { - const hasMore = - data.totalCount != null - ? offset + data.rows.length < data.totalCount - : data.rows.length === input.pageSize; - - return { - rows: data.rows, - totalCount: data.totalCount, - hasMore, - page: input.page, - pageSize: input.pageSize, - fetchedAt: new Date().toISOString(), - }; -} - function buildBacklinksDateRange(now: Date): BacklinksDateRange { const todayUtc = new Date( Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()), @@ -336,54 +374,6 @@ function buildOverviewResult(args: { }; } -function normalizeHistoryDate(value: string | null | undefined) { - return value ? value.slice(0, 10) : null; -} - -function mapBacklinksRows(rows: BacklinksItem[]) { - return rows.map((item) => ({ - domainFrom: item.domain_from ?? null, - urlFrom: item.url_from ?? null, - urlTo: item.url_to ?? null, - anchor: item.anchor ?? null, - itemType: item.item_type ?? null, - isDofollow: item.dofollow ?? null, - relAttributes: item.rel_attributes ?? item.attributes ?? [], - rank: item.rank ?? null, - domainFromRank: item.domain_from_rank ?? null, - pageFromRank: item.page_from_rank ?? null, - spamScore: item.backlink_spam_score ?? item.backlinks_spam_score ?? null, - firstSeen: item.first_seen ?? null, - lastSeen: item.lost_date ?? item.last_visited ?? null, - isLost: item.is_lost ?? Boolean(item.lost_date), - isBroken: item.is_broken ?? false, - linksCount: item.links_count ?? null, - })); -} - -function mapReferringDomainsRows(rows: ReferringDomainItem[]) { - return rows.map((item) => ({ - domain: item.domain ?? null, - backlinks: item.backlinks ?? null, - referringPages: item.referring_pages ?? null, - rank: item.rank ?? null, - spamScore: item.backlinks_spam_score ?? null, - firstSeen: item.first_seen ?? null, - brokenBacklinks: item.broken_backlinks ?? null, - brokenPages: item.broken_pages ?? null, - })); -} - -function mapTopPagesRows(rows: DomainPageSummaryItem[]) { - return rows.map((item) => ({ - page: item.page ?? item.url ?? null, - backlinks: item.backlinks ?? null, - referringDomains: item.referring_domains ?? null, - rank: item.rank ?? null, - brokenBacklinks: item.broken_backlinks ?? null, - })); -} - async function cacheValue( cache: BacklinksCache, key: string, diff --git a/src/server/features/backlinks/services/backlinksSubfolderOverview.ts b/src/server/features/backlinks/services/backlinksSubfolderOverview.ts new file mode 100644 index 0000000..de2e88b --- /dev/null +++ b/src/server/features/backlinks/services/backlinksSubfolderOverview.ts @@ -0,0 +1,63 @@ +import type { CreditFeature } from "@/shared/billing-credit-features"; +import type { + createDataforseoClient, + normalizeBacklinksTarget, +} from "@/server/lib/dataforseo"; +import { buildBacklinksScopeFilter } from "@/server/lib/dataforseo/researchScopeFilters"; +import type { BacklinksOverviewResult } from "@/server/features/backlinks/services/backlinksOverviewSchema"; + +/** + * The summary and history endpoints only take a whole target, so subfolder + * totals come from two filtered `total_count` reads of the backlinks list: + * every link (`as_is`) and one per referring domain. Rank, spam scores, + * new/lost, and trends have no filtered source and stay null/empty. + * Sequenced, not parallel: hosted billing checks balance per call. + */ +export async function buildSubfolderOverview( + dataforseo: ReturnType, + normalizedTarget: ReturnType, + now: Date, + creditFeature?: CreditFeature, +): Promise { + const filters = buildBacklinksScopeFilter("url_to", normalizedTarget).clauses; + + const allLinks = await dataforseo.backlinks.rows({ + target: normalizedTarget.apiTarget, + includeSubdomains: normalizedTarget.includeSubdomains, + limit: 1, + mode: "as_is", + filters, + creditFeature, + }); + const perDomain = await dataforseo.backlinks.rows({ + target: normalizedTarget.apiTarget, + includeSubdomains: normalizedTarget.includeSubdomains, + limit: 1, + mode: "one_per_domain", + filters, + creditFeature, + }); + + return { + target: normalizedTarget.apiTarget, + displayTarget: normalizedTarget.displayTarget, + scope: "subfolder", + summary: { + rank: null, + backlinks: allLinks.totalCount, + referringPages: null, + referringDomains: perDomain.totalCount, + brokenBacklinks: null, + brokenPages: null, + backlinksSpamScore: null, + targetSpamScore: null, + newBacklinks: null, + lostBacklinks: null, + newReferringDomains: null, + lostReferringDomains: null, + }, + trends: [], + newLostTrends: [], + fetchedAt: now.toISOString(), + }; +} diff --git a/src/server/features/dashboard/services/DashboardService.ts b/src/server/features/dashboard/services/DashboardService.ts index 5809f68..5497fa4 100644 --- a/src/server/features/dashboard/services/DashboardService.ts +++ b/src/server/features/dashboard/services/DashboardService.ts @@ -243,12 +243,14 @@ async function ensureBacklinkSnapshot(input: { return getBacklinkSummary(projectId, domain); } - const normalized = normalizeBacklinksTarget(domain, { scope: "domain" }); + // Dashboard totals cover the whole site, subdomains included. + const normalized = normalizeBacklinksTarget(domain, { scope: "subdomains" }); const dataforseo = createDataforseoClient(input.billingCustomer); try { const summary = await dataforseo.backlinks.summary({ target: normalized.apiTarget, + includeSubdomains: normalized.includeSubdomains, }); await BacklinkSnapshotRepository.insert({ projectId, diff --git a/src/server/features/domain/services/DomainService.ts b/src/server/features/domain/services/DomainService.ts index 2333851..9f5dcac 100644 --- a/src/server/features/domain/services/DomainService.ts +++ b/src/server/features/domain/services/DomainService.ts @@ -4,7 +4,10 @@ import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; import type { CreditFeature } from "@/shared/billing-credit-features"; import { createDataforseoClient } from "@/server/lib/dataforseo"; -import { normalizeDomainInput } from "@/server/lib/domainUtils"; +import { buildRankedKeywordsScopeFilter } from "@/server/lib/dataforseo/researchScopeFilters"; +import { joinClauses } from "@/server/lib/dataforseo/filters"; +import { parseResearchTargetOrThrow } from "@/server/lib/domainUtils"; +import type { ResearchScope } from "@/shared/researchScope"; import { mapKeywordItem } from "@/server/features/domain/services/domainKeywordMapper"; import { getKeywordsPage } from "@/server/features/domain/services/domainKeywordsPage"; import { getPagesPage } from "@/server/features/domain/services/domainPagesPage"; @@ -29,26 +32,33 @@ const domainOverviewResultSchema = z.object({ fetchedAt: z.string(), }); -type DomainOverviewResult = z.infer; +type DomainOverviewResult = z.infer & { + /** Requested research scope, echoed for display. */ + scope: ResearchScope; + displayTarget: string; +}; async function getOverview( input: { projectId: string; domain: string; - includeSubdomains: boolean; + scope?: ResearchScope; locationCode: number; languageCode: string; }, billingCustomer: BillingCustomerContext, metering: MeteringOverrides = {}, ): Promise { - const domain = normalizeDomainInput(input.domain, input.includeSubdomains); + const target = parseResearchTargetOrThrow(input.domain, input.scope); + const domain = target.hostname; + // domain_rank_overview has no filters and always covers the hostname plus + // all of its subdomains, so every scope shares one cache entry per hostname. + // Callers label the metrics as domain-wide for narrower scopes. const cacheKey = await buildCacheKey("domain:overview", { organizationId: billingCustomer.organizationId, projectId: input.projectId, domain, - includeSubdomains: input.includeSubdomains, locationCode: input.locationCode, languageCode: input.languageCode, }); @@ -56,7 +66,11 @@ async function getOverview( const cachedRaw = await getCached(cacheKey); const cached = domainOverviewResultSchema.safeParse(cachedRaw); if (cached.success && cached.data.hasData) { - return cached.data; + return { + ...cached.data, + scope: target.scope, + displayTarget: target.display, + }; } const nowIso = new Date().toISOString(); @@ -80,7 +94,7 @@ async function getOverview( ? Math.round(metrics.metrics.organic.count) : null; - const result: DomainOverviewResult = { + const stored: z.infer = { domain, organicTraffic, organicKeywords, @@ -90,11 +104,11 @@ async function getOverview( fetchedAt: nowIso, }; - if (result.hasData) { + if (stored.hasData) { // waitUntil, not void: workerd cancels unregistered pending I/O once the // response is sent, so a fire-and-forget put never persists the cache. waitUntil( - setCached(cacheKey, result, DOMAIN_OVERVIEW_TTL_SECONDS).catch( + setCached(cacheKey, stored, DOMAIN_OVERVIEW_TTL_SECONDS).catch( (error) => { console.error("domain.overview.cache-write failed:", error); }, @@ -102,12 +116,13 @@ async function getOverview( ); } - return result; + return { ...stored, scope: target.scope, displayTarget: target.display }; } async function getSuggestedKeywords( input: { domain: string; + scope?: ResearchScope; locationCode: number; languageCode: string; organizationId: string; @@ -125,12 +140,15 @@ async function getSuggestedKeywords( keywordDifficulty: number | null; }> > { - const domain = normalizeDomainInput(input.domain, true); + const target = parseResearchTargetOrThrow(input.domain, input.scope); + const scopeFilter = buildRankedKeywordsScopeFilter(target); const cacheKey = await buildCacheKey("domain:keyword-suggestions", { organizationId: billingCustomer.organizationId, projectId: input.projectId, - domain, + domain: target.hostname, + scope: target.scope, + path: target.path, locationCode: input.locationCode, languageCode: input.languageCode, }); @@ -155,11 +173,15 @@ async function getSuggestedKeywords( const dataforseo = createDataforseoClient(billingCustomer); const rankedKeywordsResponse = await dataforseo.domain.rankedKeywords({ - target: domain, + target: target.hostname, locationCode: input.locationCode, languageCode: input.languageCode, limit: 100, orderBy: ["ranked_serp_element.serp_item.etv,desc"], + filters: + scopeFilter.clauses.length > 0 + ? joinClauses(scopeFilter.clauses, "and") + : undefined, ...metering, }); diff --git a/src/server/features/domain/services/domainKeywordFilters.ts b/src/server/features/domain/services/domainKeywordFilters.ts index e54bb70..de24fb3 100644 --- a/src/server/features/domain/services/domainKeywordFilters.ts +++ b/src/server/features/domain/services/domainKeywordFilters.ts @@ -6,6 +6,7 @@ import { parseFilterTerms, type FilterClause, } from "@/server/lib/dataforseo/filters"; +import type { ScopeFilter } from "@/server/lib/dataforseo/researchScopeFilters"; import type { DomainKeywordsFilters } from "@/types/schemas/domain"; export type DomainKeywordsSortMode = @@ -34,12 +35,14 @@ export function buildOrderBy( /** * Each include/exclude term is one ilike clause; numeric ranges add one per * bound; the free-text search term adds one OR-group of two (keyword OR url). - * The client surfaces the same condition count and disables Apply when over - * the DataForSEO budget. + * Scope clauses (research scope narrowing) are ANDed in front and consume + * part of the same budget. The client surfaces the same condition count and + * disables Apply when over the DataForSEO budget. */ export function buildKeywordFilters( filters: DomainKeywordsFilters, searchTerm?: string, + scopeFilter?: ScopeFilter, ): unknown[] { const conditions: FilterClause[] = []; @@ -92,11 +95,20 @@ export function buildKeywordFilters( const trimmedSearch = searchTerm?.trim(); const searchGroup = trimmedSearch ? buildSearchGroup(trimmedSearch) : null; - // The search OR-group costs 2 slots; everything else is 1. - assertFilterConditionBudget(conditions.length + (searchGroup ? 2 : 0)); + // The search OR-group costs 2 slots; scope clauses cost their reported + // count; everything else is 1. + assertFilterConditionBudget( + (scopeFilter?.conditionCount ?? 0) + + conditions.length + + (searchGroup ? 2 : 0), + ); return joinClauses( - searchGroup ? [...conditions, searchGroup] : conditions, + [ + ...(scopeFilter?.clauses ?? []), + ...conditions, + ...(searchGroup ? [searchGroup] : []), + ], "and", ); } diff --git a/src/server/features/domain/services/domainKeywordsPage.ts b/src/server/features/domain/services/domainKeywordsPage.ts index dd1c60c..08be364 100644 --- a/src/server/features/domain/services/domainKeywordsPage.ts +++ b/src/server/features/domain/services/domainKeywordsPage.ts @@ -3,7 +3,9 @@ import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; import { createDataforseoClient } from "@/server/lib/dataforseo"; import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; -import { normalizeDomainInput } from "@/server/lib/domainUtils"; +import { parseResearchTargetOrThrow } from "@/server/lib/domainUtils"; +import { buildRankedKeywordsScopeFilter } from "@/server/lib/dataforseo/researchScopeFilters"; +import type { ResearchScope } from "@/shared/researchScope"; import { mapKeywordItem } from "@/server/features/domain/services/domainKeywordMapper"; import { computeHasMore } from "@/server/features/domain/services/pagination"; import { @@ -43,7 +45,7 @@ export async function getKeywordsPage( input: { projectId: string; domain: string; - includeSubdomains: boolean; + scope?: ResearchScope; locationCode: number; languageCode: string; page: number; @@ -55,16 +57,18 @@ export async function getKeywordsPage( }, billingCustomer: BillingCustomerContext, ): Promise { - const domain = normalizeDomainInput(input.domain, input.includeSubdomains); + const target = parseResearchTargetOrThrow(input.domain, input.scope); + const scopeFilter = buildRankedKeywordsScopeFilter(target); const offset = (input.page - 1) * input.pageSize; const orderBy = buildOrderBy(input.sortMode, input.sortOrder); - const filters = buildKeywordFilters(input.filters, input.search); + const filters = buildKeywordFilters(input.filters, input.search, scopeFilter); const cacheKey = await buildCacheKey("domain:keywords-page", { organizationId: billingCustomer.organizationId, projectId: input.projectId, - domain, - includeSubdomains: input.includeSubdomains, + domain: target.hostname, + scope: target.scope, + path: target.path, locationCode: input.locationCode, languageCode: input.languageCode, page: input.page, @@ -83,7 +87,7 @@ export async function getKeywordsPage( const dataforseo = createDataforseoClient(billingCustomer); const response = await dataforseo.domain.rankedKeywords({ - target: domain, + target: target.hostname, locationCode: input.locationCode, languageCode: input.languageCode, limit: input.pageSize, @@ -108,7 +112,7 @@ export async function getKeywordsPage( ); const result: DomainKeywordsPageResult = { - domain, + domain: target.hostname, page: input.page, pageSize: input.pageSize, totalCount, diff --git a/src/server/features/domain/services/domainPagesPage.ts b/src/server/features/domain/services/domainPagesPage.ts index a5967ab..edc4d9b 100644 --- a/src/server/features/domain/services/domainPagesPage.ts +++ b/src/server/features/domain/services/domainPagesPage.ts @@ -3,7 +3,16 @@ import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; import { createDataforseoClient } from "@/server/lib/dataforseo"; import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; -import { normalizeDomainInput, toRelativePath } from "@/server/lib/domainUtils"; +import { + parseResearchTargetOrThrow, + toRelativePath, +} from "@/server/lib/domainUtils"; +import { + buildRelevantPagesScopeFilter, + type ScopeFilter, +} from "@/server/lib/dataforseo/researchScopeFilters"; +import { assertFilterConditionBudget } from "@/server/lib/dataforseo/filters"; +import type { ResearchScope } from "@/shared/researchScope"; import type { RelevantPagesItem } from "@/server/lib/dataforseo"; import { computeHasMore } from "@/server/features/domain/services/pagination"; import type { DomainKeywordsFilters } from "@/types/schemas/domain"; @@ -71,7 +80,8 @@ function parseTerms(value: string | undefined): string[] { function buildPageFilters( filters: DomainKeywordsFilters, - searchTerm?: string, + searchTerm: string | undefined, + scopeFilter: ScopeFilter, ): unknown[] { const conditions: unknown[][] = []; @@ -100,8 +110,12 @@ function buildPageFilters( conditions.push(["page_address", "ilike", `%${escapeLikeTerm(trimmed)}%`]); } + assertFilterConditionBudget(scopeFilter.conditionCount + conditions.length); + const expressions: unknown[] = []; - for (const condition of conditions) pushAnd(expressions, condition); + for (const condition of [...scopeFilter.clauses, ...conditions]) { + pushAnd(expressions, condition); + } return expressions; } @@ -123,7 +137,7 @@ export async function getPagesPage( input: { projectId: string; domain: string; - includeSubdomains: boolean; + scope?: ResearchScope; locationCode: number; languageCode: string; page: number; @@ -135,16 +149,18 @@ export async function getPagesPage( }, billingCustomer: BillingCustomerContext, ): Promise { - const domain = normalizeDomainInput(input.domain, input.includeSubdomains); + const target = parseResearchTargetOrThrow(input.domain, input.scope); + const scopeFilter = buildRelevantPagesScopeFilter(target); const offset = (input.page - 1) * input.pageSize; const orderBy = [`${SORT_FIELD_BY_MODE[input.sortMode]},${input.sortOrder}`]; - const filters = buildPageFilters(input.filters, input.search); + const filters = buildPageFilters(input.filters, input.search, scopeFilter); const cacheKey = await buildCacheKey("domain:pages-page", { organizationId: billingCustomer.organizationId, projectId: input.projectId, - domain, - includeSubdomains: input.includeSubdomains, + domain: target.hostname, + scope: target.scope, + path: target.path, locationCode: input.locationCode, languageCode: input.languageCode, page: input.page, @@ -163,7 +179,7 @@ export async function getPagesPage( const dataforseo = createDataforseoClient(billingCustomer); const response = await dataforseo.domain.relevantPages({ - target: domain, + target: target.hostname, locationCode: input.locationCode, languageCode: input.languageCode, limit: input.pageSize, @@ -188,7 +204,7 @@ export async function getPagesPage( ); const result: DomainPagesPageResult = { - domain, + domain: target.hostname, page: input.page, pageSize: input.pageSize, totalCount, diff --git a/src/server/features/onboarding/onboardingChatTools.ts b/src/server/features/onboarding/onboardingChatTools.ts index d5d9e2c..626c1bd 100644 --- a/src/server/features/onboarding/onboardingChatTools.ts +++ b/src/server/features/onboarding/onboardingChatTools.ts @@ -144,7 +144,7 @@ function coreSiteTools(ctx: ToolContext): ToolSet { { projectId: project.id, domain: project.domain, - includeSubdomains: false, + scope: "domain", locationCode: project.locationCode, languageCode: project.languageCode, }, diff --git a/src/server/features/onboarding/onboardingMarketTools.ts b/src/server/features/onboarding/onboardingMarketTools.ts index 142458a..2a12e2c 100644 --- a/src/server/features/onboarding/onboardingMarketTools.ts +++ b/src/server/features/onboarding/onboardingMarketTools.ts @@ -36,7 +36,7 @@ export function marketTools(ctx: ToolContext): ToolSet { { projectId: project.id, domain, - includeSubdomains: false, + scope: "domain", locationCode: project.locationCode, languageCode: project.languageCode, }, @@ -216,7 +216,7 @@ export function marketTools(ctx: ToolContext): ToolSet { execute: async ({ domain }) => { try { const { overview } = await BacklinksService.profileOverview( - { target: domain, scope: "domain" }, + { target: domain, scope: "subdomains" }, billingCustomer, "onboarding", ); diff --git a/src/server/lib/dataforseo/backlinks.test.ts b/src/server/lib/dataforseo/backlinks.test.ts index 949f365..49df5ad 100644 --- a/src/server/lib/dataforseo/backlinks.test.ts +++ b/src/server/lib/dataforseo/backlinks.test.ts @@ -29,56 +29,96 @@ const billed = { result_count: 0, }; -describe("normalizeBacklinksTarget", () => { - it("treats explicit homepage URLs as page lookups", () => { - expect(normalizeBacklinksTarget("https://Example.com/")).toEqual({ - apiTarget: "https://example.com/", - displayTarget: "https://example.com/", - scope: "page", - }); - }); +function okResponse(result: unknown[]) { + return new Response( + JSON.stringify({ + status_code: 20000, + status_message: "Ok.", + tasks: [{ status_code: 20000, status_message: "Ok.", ...billed, result }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); +} - it("trims trailing slashes from non-root page URLs", () => { +describe("normalizeBacklinksTarget", () => { + it("defaults inputs with a path to a subfolder lookup", () => { expect( normalizeBacklinksTarget("https://github.com/every-app/open-seo/"), ).toEqual({ - apiTarget: "https://github.com/every-app/open-seo", - displayTarget: "https://github.com/every-app/open-seo", - scope: "page", + apiTarget: "github.com", + displayTarget: "github.com/every-app/open-seo", + scope: "subfolder", + includeSubdomains: false, + path: "/every-app/open-seo", }); }); - it("treats bare hostnames as domain lookups", () => { + it("strips query strings and fragments for subfolder lookups", () => { + expect( + normalizeBacklinksTarget("example.com/blog?utm_source=x#hero", { + scope: "subfolder", + }).path, + ).toBe("/blog"); + }); + + it("rejects subfolder scope without a path", () => { + expectValidationError(() => + normalizeBacklinksTarget("example.com", { scope: "subfolder" }), + ); + }); + + it("defaults bare hostnames to subdomains scope", () => { expect(normalizeBacklinksTarget("Example.com")).toEqual({ apiTarget: "example.com", displayTarget: "example.com", - scope: "domain", + scope: "subdomains", + includeSubdomains: true, + path: "", }); }); - it("lets callers force domain scope for full URLs", () => { + it("includes subdomains only for subdomains scope", () => { expect( normalizeBacklinksTarget("https://Example.com/pricing", { - scope: "domain", + scope: "subdomains", }), ).toEqual({ apiTarget: "example.com", displayTarget: "example.com", - scope: "domain", + scope: "subdomains", + includeSubdomains: true, + path: "", }); }); - it("lets callers force page scope for bare hostnames", () => { + it("lets callers force a page lookup for bare hostnames", () => { + expect( + normalizeBacklinksTarget("Example.com", { scope: "exact_url" }), + ).toEqual({ + apiTarget: "https://example.com/", + displayTarget: "https://example.com/", + scope: "exact_url", + includeSubdomains: true, + path: "", + }); + }); + + it("maps the legacy page scope onto exact_url", () => { expect(normalizeBacklinksTarget("Example.com", { scope: "page" })).toEqual({ apiTarget: "https://example.com/", displayTarget: "https://example.com/", - scope: "page", + scope: "exact_url", + includeSubdomains: true, + path: "", }); }); - it("rejects page targets with query strings or fragments", () => { + it("rejects exact-url targets with query strings or fragments", () => { expectValidationError(() => - normalizeBacklinksTarget("https://example.com/pricing?token=secret#hero"), + normalizeBacklinksTarget( + "https://example.com/pricing?token=secret#hero", + { scope: "exact_url" }, + ), ); }); @@ -136,23 +176,7 @@ describe("fetchBacklinksSummary", () => { }); it("treats null summary results as a valid zero-data response", async () => { - vi.mocked(fetch).mockResolvedValue( - new Response( - JSON.stringify({ - status_code: 20000, - status_message: "Ok.", - tasks: [ - { - status_code: 20000, - status_message: "Ok.", - ...billed, - result: [null], - }, - ], - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ), - ); + vi.mocked(fetch).mockResolvedValue(okResponse([null])); classifyBacklinksError.mockReturnValue(null); await expect( @@ -161,23 +185,7 @@ describe("fetchBacklinksSummary", () => { }); it("treats empty summary results as a valid zero-data response", async () => { - vi.mocked(fetch).mockResolvedValue( - new Response( - JSON.stringify({ - status_code: 20000, - status_message: "Ok.", - tasks: [ - { - status_code: 20000, - status_message: "Ok.", - ...billed, - result: [], - }, - ], - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ), - ); + vi.mocked(fetch).mockResolvedValue(okResponse([])); classifyBacklinksError.mockReturnValue(null); await expect( @@ -185,26 +193,28 @@ describe("fetchBacklinksSummary", () => { ).resolves.toMatchObject({ data: {} }); }); + it("asks DataForSEO to exclude subdomains for a domain-scoped target", async () => { + vi.mocked(fetch).mockResolvedValue(okResponse([])); + classifyBacklinksError.mockReturnValue(null); + + await fetchBacklinksSummary({ + target: "example.com", + includeSubdomains: false, + }); + + const body = vi.mocked(fetch).mock.calls[0]?.[1]?.body; + if (typeof body !== "string") { + throw new Error("Expected DataForSEO request body to be a string"); + } + expect(JSON.parse(body)).toMatchObject([ + { target: "example.com", include_subdomains: false }, + ]); + }); + it("treats empty backlinks rows and history results as valid empty arrays", async () => { - const emptyOk = () => - new Response( - JSON.stringify({ - status_code: 20000, - status_message: "Ok.", - tasks: [ - { - status_code: 20000, - status_message: "Ok.", - ...billed, - result: [], - }, - ], - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ); vi.mocked(fetch) - .mockResolvedValueOnce(emptyOk()) - .mockResolvedValueOnce(emptyOk()); + .mockResolvedValueOnce(okResponse([])) + .mockResolvedValueOnce(okResponse([])); classifyBacklinksError.mockReturnValue(null); await expect( diff --git a/src/server/lib/dataforseo/backlinks.ts b/src/server/lib/dataforseo/backlinks.ts index a57ed82..06708e1 100644 --- a/src/server/lib/dataforseo/backlinks.ts +++ b/src/server/lib/dataforseo/backlinks.ts @@ -21,7 +21,15 @@ import { type DataforseoApiResponse, } from "@/server/lib/dataforseo/envelope"; -type BacklinksRequest = { target: string }; +type BacklinksRequest = { + target: string; + /** + * Whether the target's subdomains count. Defaults to DataForSEO's `true`. + * The API ignores it for page targets; `backlinks/history/live` has no such + * field, so a domain-scoped history is always subdomain-inclusive. + */ + includeSubdomains?: boolean; +}; type BacklinksListRequest = BacklinksRequest & BacklinksSpamFilterOptions & { limit?: number; @@ -140,7 +148,7 @@ export const backlinksHistoryItemSchema = z function buildCommonPayload(input: BacklinksRequest) { return { target: input.target, - include_subdomains: true, + include_subdomains: input.includeSubdomains ?? true, include_indirect_links: true, exclude_internal_backlinks: true, backlinks_status_type: "live", diff --git a/src/server/lib/dataforseo/labs.ts b/src/server/lib/dataforseo/labs.ts index 2eb97ff..895527b 100644 --- a/src/server/lib/dataforseo/labs.ts +++ b/src/server/lib/dataforseo/labs.ts @@ -211,8 +211,10 @@ export async function fetchRankedKeywords(input: { orderBy?: string[]; filters?: unknown[]; itemTypes?: DataforseoLabsItemType[]; - includeSubdomains?: boolean; }): Promise> { + // Note: ranked_keywords has no include_subdomains parameter — a domain + // target always covers the hostname plus its subdomains. Narrower scopes + // are expressed through `filters` (see researchScopeFilters.ts). const response = await labsApi().googleRankedKeywordsLive([ new DataforseoLabsGoogleRankedKeywordsLiveRequestInfo({ target: input.target, @@ -223,7 +225,6 @@ export async function fetchRankedKeywords(input: { order_by: input.orderBy, filters: input.filters, item_types: input.itemTypes, - include_subdomains: input.includeSubdomains, }), ]); const task = assertOk(response); diff --git a/src/server/lib/dataforseo/researchScopeFilters.test.ts b/src/server/lib/dataforseo/researchScopeFilters.test.ts new file mode 100644 index 0000000..62ee99d --- /dev/null +++ b/src/server/lib/dataforseo/researchScopeFilters.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { + BACKLINKS_SUBFOLDER_FILTER_CONDITIONS, + parseResearchTarget, + RESEARCH_SCOPES, + RESEARCH_SCOPE_FILTER_SLOTS, +} from "@/shared/researchScope"; +import { + buildBacklinksScopeFilter, + buildRankedKeywordsScopeFilter, + buildRelevantPagesScopeFilter, +} from "./researchScopeFilters"; + +function target( + input: string, + scope: Parameters[1], +) { + const parsed = parseResearchTarget(input, scope); + if (!parsed.ok) throw new Error(parsed.message); + return parsed.target; +} + +describe("buildRankedKeywordsScopeFilter", () => { + it("adds nothing for subdomains scope", () => { + expect( + buildRankedKeywordsScopeFilter(target("example.com", "subdomains")), + ).toEqual({ clauses: [], conditionCount: 0 }); + }); + + it("pins the exact hostname (plus www) for domain scope", () => { + const filter = buildRankedKeywordsScopeFilter( + target("example.com", "domain"), + ); + expect(filter.clauses).toEqual([ + [ + "ranked_serp_element.serp_item.domain", + "in", + ["example.com", "www.example.com"], + ], + ]); + expect(filter.conditionCount).toBe(1); + }); + + it("matches the subfolder, its children, and query variants — not siblings", () => { + const filter = buildRankedKeywordsScopeFilter( + target("example.com/blog", "subfolder"), + ); + expect(filter.clauses[1]).toEqual([ + ["ranked_serp_element.serp_item.relative_url", "=", "/blog"], + "or", + ["ranked_serp_element.serp_item.relative_url", "like", "/blog/%"], + "or", + ["ranked_serp_element.serp_item.relative_url", "like", "/blog?%"], + ]); + expect(filter.conditionCount).toBe(4); + }); + + it("escapes like wildcards in the path but not equality values", () => { + const filter = buildRankedKeywordsScopeFilter( + target("example.com/100%25_deals", "subfolder"), + ); + expect(filter.clauses[1]).toEqual([ + ["ranked_serp_element.serp_item.relative_url", "=", "/100%25_deals"], + "or", + [ + "ranked_serp_element.serp_item.relative_url", + "like", + "/100\\%25\\_deals/%", + ], + "or", + [ + "ranked_serp_element.serp_item.relative_url", + "like", + "/100\\%25\\_deals?%", + ], + ]); + }); + + it("covers trailing-slash and query variants for exact_url scope", () => { + const filter = buildRankedKeywordsScopeFilter( + target("example.com/pricing", "exact_url"), + ); + expect(filter.clauses[1]).toEqual([ + [ + "ranked_serp_element.serp_item.relative_url", + "in", + ["/pricing", "/pricing/"], + ], + "or", + ["ranked_serp_element.serp_item.relative_url", "like", "/pricing?%"], + "or", + ["ranked_serp_element.serp_item.relative_url", "like", "/pricing/?%"], + ]); + }); +}); + +describe("buildRelevantPagesScopeFilter", () => { + it("pins both host variants for domain scope", () => { + const filter = buildRelevantPagesScopeFilter( + target("example.com", "domain"), + ); + expect(filter.clauses).toEqual([ + [ + ["page_address", "like", "%://example.com/%"], + "or", + ["page_address", "like", "%://www.example.com/%"], + ], + ]); + expect(filter.conditionCount).toBe(2); + }); + + it("builds absolute prefix patterns for subfolder scope", () => { + const filter = buildRelevantPagesScopeFilter( + target("example.com/blog", "subfolder"), + ); + expect(filter.clauses[0]).toEqual([ + ["page_address", "like", "%://example.com/blog"], + "or", + ["page_address", "like", "%://example.com/blog/%"], + "or", + ["page_address", "like", "%://www.example.com/blog"], + "or", + ["page_address", "like", "%://www.example.com/blog/%"], + ]); + expect(filter.conditionCount).toBe(4); + }); +}); + +it("keeps the shared filter-slot constants in sync with the built clauses", () => { + for (const scope of RESEARCH_SCOPES) { + const parsed = target("example.com/blog", scope); + expect(buildRankedKeywordsScopeFilter(parsed).conditionCount).toBe( + RESEARCH_SCOPE_FILTER_SLOTS.keywords[parsed.scope], + ); + expect(buildRelevantPagesScopeFilter(parsed).conditionCount).toBe( + RESEARCH_SCOPE_FILTER_SLOTS.pages[parsed.scope], + ); + } + expect( + buildBacklinksScopeFilter("url_to", { + scope: "subfolder", + apiTarget: "example.com", + path: "/blog", + }).conditionCount, + ).toBe(BACKLINKS_SUBFOLDER_FILTER_CONDITIONS); +}); diff --git a/src/server/lib/dataforseo/researchScopeFilters.ts b/src/server/lib/dataforseo/researchScopeFilters.ts new file mode 100644 index 0000000..47110cb --- /dev/null +++ b/src/server/lib/dataforseo/researchScopeFilters.ts @@ -0,0 +1,173 @@ +import { + escapeLikeTerm, + joinClauses, + type FilterClause, +} from "@/server/lib/dataforseo/filters"; +import type { ResearchScope, ResearchTarget } from "@/shared/researchScope"; + +/** + * Provider-side scope filters for the DataForSEO Labs endpoints. A Labs domain + * target rolls up the hostname AND all of its subdomains (verified against the + * live API), so every scope narrower than `subdomains` needs filter conditions: + * + * - ranked_keywords rows expose `serp_item.domain` (exact result hostname, + * which reports `www.` for www-canonical sites) and `serp_item.relative_url` + * (path + query string). + * - relevant_pages rows only expose `page_address` (absolute URL), so both the + * host pin and the path prefix ride on `like` patterns. `regex` is unusable: + * the API rejects any pattern containing `/`. + * + * Query-string URL variants (`/path?x=y`) are covered for ranked_keywords but + * not for relevant_pages, where query-string page addresses are rare and each + * extra pattern costs one of the 8 filter conditions. + */ + +export type ScopeFilter = { + /** Clauses to AND in front of user filters. Empty for subdomains scope. */ + clauses: FilterClause[]; + /** How many of the 8-condition budget the clauses consume. */ + conditionCount: number; +}; + +const NO_FILTER: ScopeFilter = { clauses: [], conditionCount: 0 }; + +/** + * Leaf conditions in a clause list or joined expression, counting through + * nested OR/AND groups and skipping "and"/"or" separators. + * RESEARCH_SCOPE_FILTER_SLOTS mirrors these counts for the client; a test + * pins the two together. + */ +export function countExpressionConditions(clauses: readonly unknown[]): number { + let count = 0; + for (const clause of clauses) { + if (!Array.isArray(clause)) continue; + count += Array.isArray(clause[0]) ? countExpressionConditions(clause) : 1; + } + return count; +} + +function scopeFilter(clauses: FilterClause[]): ScopeFilter { + return { clauses, conditionCount: countExpressionConditions(clauses) }; +} + +function subfolderUrlClauses( + field: string, + hostname: string, + path: string, +): FilterClause { + const hosts = [hostname, `www.${hostname}`].map(escapeLikeTerm); + const escapedPath = escapeLikeTerm(path); + return joinClauses( + hosts.flatMap((host) => [ + [field, "like", `%://${host}${escapedPath}`], + [field, "like", `%://${host}${escapedPath}/%`], + ]), + "or", + ); +} + +/** + * Scope filter on an absolute-URL field of the Backlinks API (`url_to` for + * backlink rows, `url` for domain pages). Only subfolder scope needs filters — + * the API has no prefix targeting — every other scope rides on the target. + */ +export function buildBacklinksScopeFilter( + field: "url_to" | "url", + target: { scope: ResearchScope; apiTarget: string; path: string }, +): ScopeFilter { + if (target.scope !== "subfolder") return NO_FILTER; + // For subfolder scope apiTarget is the bare hostname. + return scopeFilter([ + subfolderUrlClauses(field, target.apiTarget, target.path), + ]); +} + +/** ANDs scope clauses in front of an already-joined flat filter expression. */ +export function prependScopeClauses( + scope: ScopeFilter, + expression: unknown[], +): unknown[] { + if (scope.clauses.length === 0) return expression; + const joined = joinClauses(scope.clauses, "and"); + return expression.length === 0 ? joined : [...joined, "and", ...expression]; +} + +function hostPin(target: ResearchTarget): FilterClause { + return [ + "ranked_serp_element.serp_item.domain", + "in", + [target.hostname, `www.${target.hostname}`], + ]; +} + +export function buildRankedKeywordsScopeFilter( + target: ResearchTarget, +): ScopeFilter { + const relativeUrl = "ranked_serp_element.serp_item.relative_url"; + const path = target.path || "/"; + const escapedPath = escapeLikeTerm(path); + + switch (target.scope) { + case "subdomains": + return NO_FILTER; + case "domain": + return scopeFilter([hostPin(target)]); + case "subfolder": + return scopeFilter([ + hostPin(target), + joinClauses( + [ + [relativeUrl, "=", path], + [relativeUrl, "like", `${escapedPath}/%`], + [relativeUrl, "like", `${escapedPath}?%`], + ], + "or", + ), + ]); + case "exact_url": + return scopeFilter([ + hostPin(target), + joinClauses( + [ + [relativeUrl, "in", [path, `${path}/`]], + [relativeUrl, "like", `${escapedPath}?%`], + [relativeUrl, "like", `${escapedPath}/?%`], + ], + "or", + ), + ]); + } +} + +export function buildRelevantPagesScopeFilter( + target: ResearchTarget, +): ScopeFilter { + const hosts = [target.hostname, `www.${target.hostname}`].map(escapeLikeTerm); + const path = escapeLikeTerm(target.path || "/"); + + switch (target.scope) { + case "subdomains": + return NO_FILTER; + case "domain": + return scopeFilter([ + joinClauses( + hosts.map((host) => ["page_address", "like", `%://${host}/%`]), + "or", + ), + ]); + case "subfolder": + return scopeFilter([ + subfolderUrlClauses("page_address", target.hostname, target.path), + ]); + case "exact_url": + return scopeFilter([ + joinClauses( + hosts.flatMap((host) => [ + ["page_address", "like", `%://${host}${path}`], + ["page_address", "like", `%://${host}${path}/`], + ]), + "or", + ), + ]); + } +} diff --git a/src/server/lib/dataforseo/shared.ts b/src/server/lib/dataforseo/shared.ts index 215191d..63de3d4 100644 --- a/src/server/lib/dataforseo/shared.ts +++ b/src/server/lib/dataforseo/shared.ts @@ -31,11 +31,18 @@ export type LlmTarget = export function buildLlmTarget(input: { type: "domain" | "keyword"; value: string; + /** + * Domain targets only. Defaults to the historical behavior (subdomains + * included); research scopes narrower than `subdomains` pass `false`. There + * is no URL/path-level targeting in this API — page-level scoping happens by + * post-filtering the returned page URLs. + */ + includeSubdomains?: boolean; }): LlmTarget { if (input.type === "domain") { return { domain: input.value, - include_subdomains: true, + include_subdomains: input.includeSubdomains ?? true, search_filter: "include", search_scope: ["any"], }; diff --git a/src/server/lib/dataforseoBacklinksTarget.ts b/src/server/lib/dataforseoBacklinksTarget.ts index 3fcc452..8fcb0c6 100644 --- a/src/server/lib/dataforseoBacklinksTarget.ts +++ b/src/server/lib/dataforseoBacklinksTarget.ts @@ -1,118 +1,79 @@ import { AppError } from "@/server/lib/errors"; -import type { BacklinksLookupInput } from "@/types/schemas/backlinks"; -import { parse as parseTld } from "tldts"; +import { + resolveBacklinksScope, + type BacklinksScopeWithLegacy, +} from "@/types/schemas/backlinks"; +import { + parseResearchTarget, + type ResearchScope, +} from "@/shared/researchScope"; type NormalizedBacklinkTarget = { apiTarget: string; displayTarget: string; - scope: "domain" | "page"; + scope: ResearchScope; + /** DataForSEO `include_subdomains`; ignored by the API for page targets. */ + includeSubdomains: boolean; + /** + * Subfolder scope only: the normalized path driving the url_to/url prefix + * filters (the API itself has no prefix targeting). `""` for other scopes. + */ + path: string; }; type NormalizeBacklinksTargetOptions = { - scope?: BacklinksLookupInput["scope"]; + scope?: BacklinksScopeWithLegacy; }; -function normalizePageTargetUrl(url: URL, hostname: string): string { - const normalizedUrl = new URL(url.toString()); - normalizedUrl.hostname = hostname; - - if (normalizedUrl.pathname.length > 1) { - normalizedUrl.pathname = normalizedUrl.pathname.replace(/\/+$/, ""); - } - - return normalizedUrl.toString(); -} - +/** + * Backlinks-flavored wrapper over the shared research-target parser. The + * backlinks-specific rules: an exact-URL target is sent as an absolute URL + * (preserving an explicit http:// scheme, since url matching is exact) and + * rejects query strings/fragments instead of silently stripping them. + */ export function normalizeBacklinksTarget( input: string, options: NormalizeBacklinksTargetOptions = {}, ): NormalizedBacklinkTarget { const trimmed = input.trim(); - if (!trimmed) { - throw new AppError("VALIDATION_ERROR", "Target is required"); + const requestedScope = options.scope + ? resolveBacklinksScope(options.scope) + : undefined; + + const parsed = parseResearchTarget(trimmed, requestedScope); + if (!parsed.ok) { + throw new AppError("VALIDATION_ERROR", parsed.message); } + const target = parsed.target; - const hasExplicitProtocol = /^[a-zA-Z][a-zA-Z\d+.-]*:\/\//.test(trimmed); - const withProtocol = hasExplicitProtocol ? trimmed : `https://${trimmed}`; - - let parsed: URL; - try { - parsed = new URL(withProtocol); - } catch { - throw new AppError("VALIDATION_ERROR", "Target is invalid"); - } - - const exactHostname = parsed.hostname.toLowerCase(); - const domainHostname = exactHostname.replace(/^www\./, ""); - if (!domainHostname || !domainHostname.includes(".")) { - throw new AppError("VALIDATION_ERROR", "Target is invalid"); - } - - const parsedHostname = parseTld(domainHostname, { - allowPrivateDomains: true, - }); - if ( - parsedHostname.isIp || - !parsedHostname.publicSuffix || - (parsedHostname.isIcann !== true && parsedHostname.isPrivate !== true) - ) { - throw new AppError("VALIDATION_ERROR", "Target is invalid"); - } - - if (parsed.username || parsed.password) { - throw new AppError( - "VALIDATION_ERROR", - "Page URLs with embedded credentials are not supported", - ); - } - - const hasMeaningfulPath = parsed.pathname !== "/"; - const requestedScope = options.scope; - - if (requestedScope === "domain") { + if (target.scope !== "exact_url") { return { - apiTarget: domainHostname, - displayTarget: domainHostname, - scope: "domain", + apiTarget: target.hostname, + displayTarget: + target.scope === "subfolder" ? target.display : target.hostname, + scope: target.scope, + includeSubdomains: target.scope === "subdomains", + path: target.scope === "subfolder" ? target.path : "", }; } - if (parsed.search || parsed.hash) { + // Query strings and fragments would target a different page than the one + // the user sees, so a page lookup rejects them rather than dropping them. + if (/[?#]/.test(trimmed)) { throw new AppError( "VALIDATION_ERROR", "Page URLs with query strings or fragments are not supported", ); } - if (requestedScope === "page") { - const normalizedUrl = new URL(parsed.toString()); - if (!hasExplicitProtocol && !hasMeaningfulPath) { - normalizedUrl.pathname = "/"; - } - - const normalizedTarget = normalizePageTargetUrl( - normalizedUrl, - exactHostname, - ); - return { - apiTarget: normalizedTarget, - displayTarget: normalizedTarget, - scope: "page", - }; - } - - if (hasExplicitProtocol || hasMeaningfulPath) { - const normalizedTarget = normalizePageTargetUrl(parsed, exactHostname); - return { - apiTarget: normalizedTarget, - displayTarget: normalizedTarget, - scope: "page", - }; - } - + const protocol = /^http:\/\//i.test(trimmed) ? "http" : "https"; + const pageUrl = `${protocol}://${target.urlHostname}${target.path || "/"}`; return { - apiTarget: domainHostname, - displayTarget: domainHostname, - scope: "domain", + apiTarget: pageUrl, + displayTarget: pageUrl, + scope: "exact_url", + // Irrelevant for a page target; kept true so the payload is unchanged. + includeSubdomains: true, + path: "", }; } diff --git a/src/server/lib/domainUtils.test.ts b/src/server/lib/domainUtils.test.ts index 1ad12d2..a6009ee 100644 --- a/src/server/lib/domainUtils.test.ts +++ b/src/server/lib/domainUtils.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { normalizeDomainInput } from "@/server/lib/domainUtils"; -import { isValidDomainHost } from "@/types/schemas/domain"; +import { isValidDomainHost } from "@/shared/researchScope"; describe("isValidDomainHost", () => { it("accepts real registrable domains", () => { diff --git a/src/server/lib/domainUtils.ts b/src/server/lib/domainUtils.ts index f3ad828..d540443 100644 --- a/src/server/lib/domainUtils.ts +++ b/src/server/lib/domainUtils.ts @@ -1,6 +1,22 @@ import { getDomain } from "tldts"; import { AppError } from "@/server/lib/errors"; -import { isValidDomainHost } from "@/types/schemas/domain"; +import { + isValidDomainHost, + parseResearchTarget, + type ResearchScope, + type ResearchTarget, +} from "@/shared/researchScope"; + +export function parseResearchTargetOrThrow( + input: string, + scope?: ResearchScope, +): ResearchTarget { + const parsed = parseResearchTarget(input, scope); + if (!parsed.ok) { + throw new AppError("VALIDATION_ERROR", parsed.message); + } + return parsed.target; +} export function toRelativePath(url: string | null | undefined): string | null { if (!url) return null; diff --git a/src/server/mcp/tools/dataforseo-research-tools.test.ts b/src/server/mcp/tools/dataforseo-research-tools.test.ts index 9bf929c..862da1f 100644 --- a/src/server/mcp/tools/dataforseo-research-tools.test.ts +++ b/src/server/mcp/tools/dataforseo-research-tools.test.ts @@ -184,33 +184,6 @@ describe("DataForSEO research MCP tools", () => { expect(textContent(result)).toContain("Do you serve breakfast?"); }); - it("passes only explicit brand exclusions to ranked keyword filters", async () => { - const rankedKeywords = vi.fn().mockResolvedValue({ - items: [], - totalCount: 0, - }); - - mocks.createDataforseoClient.mockReturnValue({ - domain: { rankedKeywords }, - }); - const { getRankedKeywordsTool } = researchTools; - - await getRankedKeywordsTool.handler( - { - projectId: "project_1", - target: "acmeexample.com", - excludeBrandTerms: ["acme"], - }, - toolContext, - ); - - expect(rankedKeywords).toHaveBeenCalledWith( - expect.objectContaining({ - filters: [["keyword_data.keyword", "not_ilike", "%acme%"]], - }), - ); - }); - it("filters SERP competitors only by explicit excluded domains", async () => { const serpCompetitors = vi.fn().mockResolvedValue([ { domain: "directory.example", visibility: 10 }, @@ -382,3 +355,67 @@ describe("DataForSEO research MCP tools", () => { expect(rows[0]).not.toHaveProperty("monthly_searches"); }); }); + +describe("get_ranked_keywords scope handling", () => { + const rankedKeywords = + vi.fn< + (input: { + filters?: unknown[]; + }) => Promise<{ items: unknown[]; totalCount: number }> + >(); + + beforeEach(() => { + mocks.getProjectForOrganization.mockResolvedValue(usProjectRow); + rankedKeywords.mockResolvedValue({ items: [], totalCount: 0 }); + mocks.createDataforseoClient.mockReturnValue({ + domain: { rankedKeywords }, + }); + }); + + it("passes only explicit brand exclusions to ranked keyword filters", async () => { + await researchTools.getRankedKeywordsTool.handler( + { + projectId: "project_1", + target: "acmeexample.com", + scope: "subdomains", + excludeBrandTerms: ["acme"], + }, + toolContext, + ); + + expect(rankedKeywords).toHaveBeenCalledWith( + expect.objectContaining({ + filters: [["keyword_data.keyword", "not_ilike", "%acme%"]], + }), + ); + }); + + it("defaults a bare domain to subdomains scope with no scope filters", async () => { + const result = await researchTools.getRankedKeywordsTool.handler( + { projectId: "project_1", target: "acmeexample.com" }, + toolContext, + ); + + expect(rankedKeywords).toHaveBeenCalledWith( + expect.objectContaining({ + target: "acmeexample.com", + filters: undefined, + }), + ); + expect(result.structuredContent).toMatchObject({ + target: "acmeexample.com", + scope: "subdomains", + }); + }); + + // The clause shape itself is pinned in researchScopeFilters.test.ts; here + // only "the scope filter reaches the API call" is the invariant. + it("sends scope filters for an explicit narrower scope", async () => { + await researchTools.getRankedKeywordsTool.handler( + { projectId: "project_1", target: "acmeexample.com", scope: "domain" }, + toolContext, + ); + + expect(Array.isArray(rankedKeywords.mock.calls[0]?.[0].filters)).toBe(true); + }); +}); diff --git a/src/server/mcp/tools/dataforseo-research-tools.ts b/src/server/mcp/tools/dataforseo-research-tools.ts index a5c5389..c4cca3d 100644 --- a/src/server/mcp/tools/dataforseo-research-tools.ts +++ b/src/server/mcp/tools/dataforseo-research-tools.ts @@ -28,6 +28,16 @@ import { locationCodeSchema, projectIdSchema, } from "@/server/mcp/schemas"; +import { assertFilterConditionBudget } from "@/server/lib/dataforseo/filters"; +import { + buildRankedKeywordsScopeFilter, + type ScopeFilter, +} from "@/server/lib/dataforseo/researchScopeFilters"; +import { parseResearchTargetOrThrow } from "@/server/lib/domainUtils"; +import { + RESEARCH_SCOPE_PARAM_DESCRIPTION, + researchScopeSchema, +} from "@/shared/researchScope"; const rankedResultTypeSchema = z.enum([ "organic", @@ -128,6 +138,9 @@ const getRankedKeywordsInputSchema = { target: rankedTargetSchema.describe( "Domain (no protocol/www) or absolute page URL to list ranked keywords for.", ), + scope: researchScopeSchema + .optional() + .describe(RESEARCH_SCOPE_PARAM_DESCRIPTION), market: marketSchema, locationCode: locationCodeSchema .optional() @@ -148,9 +161,7 @@ const getRankedKeywordsInputSchema = { includeSubdomains: z .boolean() .optional() - .describe( - "Include subdomains of the target. Defaults to true for domains, false for page URLs.", - ), + .describe("Deprecated: use scope ('subdomains' or 'domain') instead."), minSearchVolume: z .number() .int() @@ -446,18 +457,27 @@ function pushAnd(filters: unknown[], condition: unknown[]) { filters.push(condition); } -function buildRankedKeywordFilters(args: { - minSearchVolume?: number; - maxRank?: number; - excludeBrandTerms?: string[]; -}) { +function buildRankedKeywordFilters( + args: { + minSearchVolume?: number; + maxRank?: number; + excludeBrandTerms?: string[]; + }, + scopeFilter?: ScopeFilter, +) { const filters: unknown[] = []; + let conditionCount = 0; + if (scopeFilter) { + for (const clause of scopeFilter.clauses) pushAnd(filters, clause); + conditionCount += scopeFilter.conditionCount; + } if (args.minSearchVolume != null) { pushAnd(filters, [ "keyword_data.keyword_info.search_volume", ">=", args.minSearchVolume, ]); + conditionCount += 1; } if (args.maxRank != null) { pushAnd(filters, [ @@ -465,12 +485,15 @@ function buildRankedKeywordFilters(args: { "<=", args.maxRank, ]); + conditionCount += 1; } if (args.excludeBrandTerms != null) { for (const term of args.excludeBrandTerms) { pushAnd(filters, ["keyword_data.keyword", "not_ilike", `%${term}%`]); } + conditionCount += args.excludeBrandTerms.length; } + assertFilterConditionBudget(conditionCount); return filters.length > 0 ? filters : undefined; } @@ -638,6 +661,8 @@ export const getRankedKeywordsTool = { outputSchema: { keywords: z.array(looseObjectOutputSchema), totalCount: z.number().nullable(), + target: z.string().optional(), + scope: researchScopeSchema.optional(), ...optionalMetaOutputSchema, }, annotations: { @@ -648,39 +673,61 @@ export const getRankedKeywordsTool = { }, handler: withMcpProjectAuth(async (args: GetRankedKeywordsArgs, context) => { const client = createDataforseoClient(context.billing); - const targetIsPage = /^https?:\/\//.test(args.target); + // Legacy includeSubdomains only ever selected between whole-host scopes + // for bare domains; for page URLs it meant exact-page results. Mapping it + // to domain/subdomains for a URL target would silently drop the path. + const parsedDefault = parseResearchTargetOrThrow(args.target); + const legacyScope = + args.includeSubdomains == null + ? undefined + : parsedDefault.path === "" + ? args.includeSubdomains + ? "subdomains" + : "domain" + : "exact_url"; + const requestedScope = args.scope ?? legacyScope; + const target = requestedScope + ? parseResearchTargetOrThrow(args.target, requestedScope) + : parsedDefault; + const scopeFilter = buildRankedKeywordsScopeFilter(target); const market = resolveMarketSelector(args, context.project); const keywords = await client.domain.rankedKeywords({ - target: args.target, + target: target.hostname, locationCode: market.locationCode, languageCode: market.languageCode, limit: args.limit ?? 50, offset: args.offset, orderBy: sortOrderByRankedMode(args.sortBy), - filters: buildRankedKeywordFilters({ - minSearchVolume: args.minSearchVolume, - maxRank: args.maxRank, - excludeBrandTerms: args.excludeBrandTerms, - }), + filters: buildRankedKeywordFilters( + { + minSearchVolume: args.minSearchVolume, + maxRank: args.maxRank, + excludeBrandTerms: args.excludeBrandTerms, + }, + scopeFilter, + ), itemTypes: args.resultTypes, - includeSubdomains: args.includeSubdomains ?? !targetIsPage, }); const rankedRows = keywords.items.map(toRankedKeywordRow); + const targetLabel = `${target.display} (scope: ${target.scope})`; const text = rankedRows.length === 0 - ? `No ranked keyword rows for ${args.target}.` - : `Found ${rankedRows.length} ranked keyword rows for ${args.target}${keywords.totalCount != null ? ` (of ${keywords.totalCount} total)` : ""}:\n${formatMcpTable(rankedRows, RANKED_KEYWORD_COLUMNS)}`; + ? `No ranked keyword rows for ${targetLabel}.` + : `Found ${rankedRows.length} ranked keyword rows for ${targetLabel}${keywords.totalCount != null ? ` (of ${keywords.totalCount} total)` : ""}:\n${formatMcpTable(rankedRows, RANKED_KEYWORD_COLUMNS)}`; return mcpResponse({ text, meta: buildProjectMeta( context, args.projectId, `/p/${args.projectId}/domain`, + { domain: target.display, scope: target.scope }, ), structuredContent: { keywords: keywords.items, totalCount: keywords.totalCount, + target: target.display, + scope: target.scope, }, }); }), diff --git a/src/server/mcp/tools/get-backlinks-overview.ts b/src/server/mcp/tools/get-backlinks-overview.ts index d1d46c0..e29c1ed 100644 --- a/src/server/mcp/tools/get-backlinks-overview.ts +++ b/src/server/mcp/tools/get-backlinks-overview.ts @@ -13,6 +13,13 @@ import { type McpTableColumn, } from "@/server/mcp/table"; import { projectIdSchema } from "@/server/mcp/schemas"; +import { + BACKLINKS_SCOPE_DESCRIPTION, + backlinksScopeWithLegacySchema, + resolveBacklinksScope, +} from "@/types/schemas/backlinks"; +import { normalizeBacklinksTarget } from "@/server/lib/dataforseo"; +import { researchScopeSchema } from "@/shared/researchScope"; const REFERRING_DOMAIN_COLUMNS: McpTableColumn[] = [ { header: "domain", value: (row) => readPath(row, "domain") }, @@ -32,12 +39,9 @@ const inputSchema = { .describe( "Domain or URL to analyze (e.g. 'example.com' or 'https://example.com/blog').", ), - scope: z - .enum(["domain", "page"]) + scope: backlinksScopeWithLegacySchema .optional() - .describe( - "'domain' analyzes the whole domain; 'page' analyzes a specific URL. Defaults to 'domain'.", - ), + .describe(BACKLINKS_SCOPE_DESCRIPTION), hideSpam: z .boolean() .optional() @@ -55,11 +59,14 @@ export const getBacklinksOverviewTool = { config: { title: "Get backlinks overview", description: - "Returns a backlinks profile summary (total backlinks, referring domains, top referring domains). Charges credits (~50 typical for a domain, ~25 for a single page). Self-hosted deployments need the Backlinks API enabled on their DataForSEO account.", + "Returns a backlinks profile summary (total backlinks, referring domains, top referring domains). Charges credits (~50 typical for a domain, ~25 for a single page). Note: bare domains default to scope 'subdomains'; pass scope 'domain' to exclude subdomains from the totals. Targets with a path default to 'subfolder', whose counts come from filtered backlink totals (no rank/trends/referring-domain breakdown). Trend data always includes subdomains (provider limitation). Self-hosted deployments need the Backlinks API enabled on their DataForSEO account.", inputSchema, outputSchema: { + target: z.string(), + scope: researchScopeSchema, + scopeNote: z.string().optional(), overview: looseObjectOutputSchema, - referringDomains: looseObjectOutputSchema, + referringDomains: looseObjectOutputSchema.optional(), ...optionalMetaOutputSchema, }, annotations: { @@ -69,35 +76,58 @@ export const getBacklinksOverviewTool = { }, }, handler: withMcpProjectAuth(async (args: Args, context) => { - const lookup = { target: args.target, scope: args.scope }; + const lookup = { + target: args.target, + scope: args.scope ? resolveBacklinksScope(args.scope) : undefined, + }; const spamOptions = { hideSpam: args.hideSpam ?? true }; + // referring_domains has no URL filter, so the per-domain breakdown is + // skipped for subfolder scope (the referring-domain count in the summary + // is still subfolder-accurate). + const resolvedScope = normalizeBacklinksTarget(args.target, { + scope: lookup.scope, + }).scope; const [overview, refDomains] = await Promise.all([ BacklinksService.profileOverview(lookup, context.billing), - BacklinksService.profileReferringDomainsPage( - { - ...lookup, - page: 1, - pageSize: 100, - sortField: "backlinks", - sortOrder: "desc", - filters: {}, - }, - context.billing, - spamOptions, - ), + resolvedScope === "subfolder" + ? Promise.resolve(null) + : BacklinksService.profileReferringDomainsPage( + { + ...lookup, + page: 1, + pageSize: 100, + sortField: "backlinks", + sortOrder: "desc", + filters: {}, + }, + context.billing, + spamOptions, + ), ]); - const topDomains = refDomains.rows ?? []; + const topDomains = refDomains?.rows ?? []; const summary = overview.overview.summary; + const { displayTarget, scope } = overview.overview; + // backlinks/history has no include_subdomains, so trend series stay + // subdomain-inclusive even when the summary excludes subdomains. + const scopeNote = + scope === "domain" + ? "Summary excludes subdomains; trend data includes subdomains (provider limitation)." + : scope === "subfolder" + ? "Counts are computed from filtered backlink totals; rank, trends, and the referring-domains breakdown aren't available for subfolders." + : undefined; const text = [ - `Backlinks profile for ${args.target} (${args.scope ?? "domain"}):`, + `Backlinks profile for ${displayTarget} (scope: ${scope}):`, + ...(scopeNote ? [`Note: ${scopeNote}`] : []), `- backlinks: ${formatMetric(summary.backlinks)}`, `- referring domains: ${formatMetric(summary.referringDomains)}`, `- referring pages: ${formatMetric(summary.referringPages)}`, `- rank: ${formatMetric(summary.rank)}`, "", - topDomains.length === 0 - ? "No referring domains found." - : `Referring domains (${topDomains.length}):\n${formatMcpTable(topDomains, REFERRING_DOMAIN_COLUMNS)}`, + refDomains === null + ? "Referring-domains breakdown unavailable for subfolder scope." + : topDomains.length === 0 + ? "No referring domains found." + : `Referring domains (${topDomains.length}):\n${formatMcpTable(topDomains, REFERRING_DOMAIN_COLUMNS)}`, ].join("\n"); return mcpResponse({ text, @@ -105,9 +135,15 @@ export const getBacklinksOverviewTool = { context, args.projectId, `/p/${args.projectId}/backlinks`, - { target: args.target }, + { target: args.target, scope }, ), - structuredContent: { overview, referringDomains: refDomains }, + structuredContent: { + target: displayTarget, + scope, + scopeNote, + overview, + referringDomains: refDomains ?? undefined, + }, }); }), }; diff --git a/src/server/mcp/tools/get-backlinks-profile.ts b/src/server/mcp/tools/get-backlinks-profile.ts index 0018685..0511c04 100644 --- a/src/server/mcp/tools/get-backlinks-profile.ts +++ b/src/server/mcp/tools/get-backlinks-profile.ts @@ -12,13 +12,17 @@ import { projectIdSchema } from "@/server/mcp/schemas"; import { BACKLINKS_DEFAULT_SORT, BACKLINKS_PAGE_SIZES, + BACKLINKS_SCOPE_DESCRIPTION, DEFAULT_BACKLINKS_PAGE_SIZE, backlinksRowsFiltersSchema, backlinksRowsModeSchema, backlinksRowsSortFieldSchema, + backlinksScopeWithLegacySchema, backlinksSortOrderSchema, - backlinksTargetScopeSchema, + resolveBacklinksScope, } from "@/types/schemas/backlinks"; +import { researchScopeSchema } from "@/shared/researchScope"; +import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget"; const inputSchema = { projectId: projectIdSchema, @@ -29,11 +33,9 @@ const inputSchema = { .describe( "Domain or URL to analyze (e.g. 'example.com' or 'https://example.com/blog').", ), - scope: backlinksTargetScopeSchema + scope: backlinksScopeWithLegacySchema .optional() - .describe( - "'domain' analyzes the whole domain; 'page' analyzes a specific URL. Defaults to 'domain'.", - ), + .describe(BACKLINKS_SCOPE_DESCRIPTION), page: z .number() .int() @@ -124,6 +126,8 @@ export const getBacklinksProfileTool = { "Returns one bounded page of detailed backlink rows for a domain or page: linking URLs, target URLs, anchors, dofollow/nofollow, authority/spam signals, and lost/broken status. Supports filters, sorting, one_per_domain/as_is mode, and pagination. Charges credits (~30 per page typical). Self-hosted deployments need the Backlinks API enabled on their DataForSEO account.", inputSchema, outputSchema: { + target: z.string(), + scope: researchScopeSchema, backlinks: backlinksProfileOutputSchema, ...optionalMetaOutputSchema, }, @@ -138,7 +142,7 @@ export const getBacklinksProfileTool = { // backlinksRowsPageRequestSchema), so pass them straight through. const request = { target: args.target, - scope: args.scope, + scope: args.scope ? resolveBacklinksScope(args.scope) : undefined, page: args.page, pageSize: args.pageSize, sortField: args.sortField, @@ -147,13 +151,18 @@ export const getBacklinksProfileTool = { mode: args.mode, }; + // The page result carries only rows, so resolve the target here to report + // the scope the rows were actually fetched with. + const target = normalizeBacklinksTarget(request.target, { + scope: request.scope, + }); const backlinks = await BacklinksService.profileBacklinksPage( request, context.billing, { hideSpam: args.hideSpam ?? true }, ); const text = [ - `Backlinks profile for ${request.target} (${request.scope ?? "domain"}):`, + `Backlinks profile for ${target.displayTarget} (scope: ${target.scope}):`, `- page: ${backlinks.page}`, `- page size: ${backlinks.pageSize}`, `- rows returned: ${backlinks.rows.length}`, @@ -171,9 +180,13 @@ export const getBacklinksProfileTool = { context, args.projectId, `/p/${args.projectId}/backlinks`, - { target: request.target, scope: request.scope }, + { target: request.target, scope: target.scope }, ), - structuredContent: { backlinks }, + structuredContent: { + target: target.displayTarget, + scope: target.scope, + backlinks, + }, }); }), }; diff --git a/src/server/mcp/tools/get-domain-keyword-suggestions.ts b/src/server/mcp/tools/get-domain-keyword-suggestions.ts index 36b42dc..c69bb21 100644 --- a/src/server/mcp/tools/get-domain-keyword-suggestions.ts +++ b/src/server/mcp/tools/get-domain-keyword-suggestions.ts @@ -22,6 +22,11 @@ import { locationCodeSchema, projectIdSchema, } from "@/server/mcp/schemas"; +import { + RESEARCH_SCOPE_PARAM_DESCRIPTION, + researchScopeSchema, +} from "@/shared/researchScope"; +import { parseResearchTargetOrThrow } from "@/server/lib/domainUtils"; const SUGGESTION_COLUMNS: McpTableColumn[] = [ { header: "keyword", value: (row) => readPath(row, "keyword") }, @@ -35,7 +40,13 @@ const inputSchema = { domain: z .string() .min(1) - .describe("Competitor or reference domain to extract keywords from."), + .max(2048) + .describe( + "Competitor or reference domain or URL to extract keywords from.", + ), + scope: researchScopeSchema + .optional() + .describe(RESEARCH_SCOPE_PARAM_DESCRIPTION), locationCode: locationCodeSchema.optional(), languageCode: languageCodeSchema.optional(), } as const; @@ -51,6 +62,8 @@ export const getDomainKeywordSuggestionsTool = { inputSchema, outputSchema: { keywords: z.array(looseObjectOutputSchema), + target: z.string().optional(), + scope: researchScopeSchema.optional(), ...optionalMetaOutputSchema, }, annotations: { @@ -69,6 +82,7 @@ export const getDomainKeywordSuggestionsTool = { const keywords = await DomainService.getSuggestedKeywords( { domain: args.domain, + scope: args.scope, locationCode, languageCode, organizationId: context.auth.organizationId, @@ -76,10 +90,15 @@ export const getDomainKeywordSuggestionsTool = { }, context.billing, ); + // The service already parsed the same input, so this cannot throw here. + const parsed = parseResearchTargetOrThrow(args.domain, args.scope); + const target = parsed.display; + const scope = parsed.scope; + const targetLabel = `${target} (scope: ${scope})`; const text = keywords.length === 0 - ? `No ranked keywords found for ${args.domain}.` - : `Keywords for ${args.domain} (${keywords.length}):\n${formatMcpTable(keywords, SUGGESTION_COLUMNS)}`; + ? `No ranked keywords found for ${targetLabel}.` + : `Keywords for ${targetLabel} (${keywords.length}):\n${formatMcpTable(keywords, SUGGESTION_COLUMNS)}`; return mcpResponse({ text, meta: buildProjectMeta( @@ -87,10 +106,11 @@ export const getDomainKeywordSuggestionsTool = { args.projectId, `/p/${args.projectId}/domain`, { - domain: args.domain, + domain: target, + ...(scope ? { scope } : {}), }, ), - structuredContent: { keywords }, + structuredContent: { keywords, target, scope }, }); }), }; diff --git a/src/server/mcp/tools/get-domain-overview.ts b/src/server/mcp/tools/get-domain-overview.ts index d8ed693..6c79a69 100644 --- a/src/server/mcp/tools/get-domain-overview.ts +++ b/src/server/mcp/tools/get-domain-overview.ts @@ -14,15 +14,27 @@ import { locationCodeSchema, projectIdSchema, } from "@/server/mcp/schemas"; +import { + RESEARCH_SCOPE_PARAM_DESCRIPTION, + researchScopeSchema, +} from "@/shared/researchScope"; const inputSchema = { projectId: projectIdSchema, - domain: z.string().min(1).describe("Domain to analyze (e.g. 'example.com')."), + domain: z + .string() + .min(1) + .max(2048) + .describe("Domain or URL to analyze (e.g. 'example.com')."), + scope: researchScopeSchema + .optional() + .describe( + `${RESEARCH_SCOPE_PARAM_DESCRIPTION} Overview metrics always cover the hostname plus subdomains; narrower scopes are labeled accordingly — use get_ranked_keywords with a scope for scoped keyword data.`, + ), includeSubdomains: z .boolean() .optional() - .default(false) - .describe("Include subdomains in the domain's metrics. Defaults to false."), + .describe("Deprecated: use scope ('subdomains' or 'domain') instead."), locationCode: locationCodeSchema.optional(), languageCode: languageCodeSchema.optional(), } as const; @@ -39,6 +51,8 @@ export const getDomainOverviewTool = { outputSchema: z .object({ domain: z.string().optional(), + scope: researchScopeSchema.optional(), + displayTarget: z.string().optional(), organicTraffic: z.number().nullable().optional(), organicKeywords: z.number().nullable().optional(), backlinks: z.number().nullable().optional(), @@ -59,22 +73,34 @@ export const getDomainOverviewTool = { ); assertLabsLocationCode(locationCode); assertLanguageForLocation(locationCode, languageCode); + const scope = + args.scope ?? + (args.includeSubdomains == null + ? undefined + : args.includeSubdomains + ? "subdomains" + : "domain"); const result = await DomainService.getOverview( { projectId: args.projectId, domain: args.domain, - includeSubdomains: args.includeSubdomains, + scope, locationCode, languageCode, }, context.billing, ); const text = [ - `Domain: ${result.domain}`, + `Target: ${result.displayTarget} (scope: ${result.scope})`, `Organic traffic: ${result.organicTraffic ?? "?"}`, `Organic keywords: ${result.organicKeywords ?? "?"}`, `Backlinks: ${result.backlinks ?? "?"}`, `Referring domains: ${result.referringDomains ?? "?"}`, + ...(result.scope === "subdomains" + ? [] + : [ + "Note: overview metrics cover the whole domain including subdomains; use get_ranked_keywords with this scope for scoped keyword data.", + ]), ].join("\n"); return mcpResponse({ text, diff --git a/src/server/mcp/tools/output-schema-validation.test.ts b/src/server/mcp/tools/output-schema-validation.test.ts index 1c333ca..4b4640b 100644 --- a/src/server/mcp/tools/output-schema-validation.test.ts +++ b/src/server/mcp/tools/output-schema-validation.test.ts @@ -113,6 +113,8 @@ describe("DataForSEO research tool output schemas", () => { const schema = objectSchema(getBacklinksProfileTool.config.outputSchema); const result = await schema.safeParseAsync({ + target: "example.com", + scope: "domain", backlinks: backlinkPage, meta: { organizationId: "org_123", diff --git a/src/server/mcp/tools/tool-text-output.test.ts b/src/server/mcp/tools/tool-text-output.test.ts index 74475b4..60c28e8 100644 --- a/src/server/mcp/tools/tool-text-output.test.ts +++ b/src/server/mcp/tools/tool-text-output.test.ts @@ -7,6 +7,7 @@ import { getRankTrackerTool } from "./get-rank-tracker"; import { getSerpResultsTool } from "./get-serp-results"; import { researchKeywordsTool } from "./research-keywords"; import { makeToolContext, textContent } from "./tool-test-support"; +import type * as backlinksTargetModule from "@/server/lib/dataforseoBacklinksTarget"; // Verifies that each tool renders its actual row data into the text content // block (not just a count), across the tools whose data comes from OpenSEO @@ -29,9 +30,17 @@ const mocks = vi.hoisted(() => ({ })); vi.mock("cloudflare:workers", () => ({ env: {} })); -vi.mock("@/server/lib/dataforseo", () => ({ - createDataforseoClient: mocks.createDataforseoClient, -})); +vi.mock("@/server/lib/dataforseo", async () => { + // Real target normalizer (pure, leaf module) so scope resolution in the + // backlinks tools matches production. + const targets = await vi.importActual( + "@/server/lib/dataforseoBacklinksTarget", + ); + return { + createDataforseoClient: mocks.createDataforseoClient, + normalizeBacklinksTarget: targets.normalizeBacklinksTarget, + }; +}); vi.mock("@/server/features/projects/services/ProjectService", () => ({ ProjectService: { getProjectForOrganization: mocks.getProjectForOrganization, diff --git a/src/shared/researchScope.test.ts b/src/shared/researchScope.test.ts new file mode 100644 index 0000000..d77c0a1 --- /dev/null +++ b/src/shared/researchScope.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; +import { + defaultScopeForPath, + isScopeAllowedForInput, + parseResearchTarget, + urlMatchesResearchTarget, +} from "./researchScope"; + +function parseOk( + input: string, + scope?: Parameters[1], +) { + const result = parseResearchTarget(input, scope); + if (!result.ok) throw new Error(`expected ok parse, got: ${result.message}`); + return result.target; +} + +describe("parseResearchTarget", () => { + it("defaults a root domain to subdomains scope", () => { + const target = parseOk("Example.com"); + expect(target).toMatchObject({ + scope: "subdomains", + hostname: "example.com", + path: "", + display: "example.com", + }); + }); + + it("defaults a URL with a path to subfolder scope", () => { + const target = parseOk("example.com/commercial-insurance/"); + expect(target).toMatchObject({ + scope: "subfolder", + path: "/commercial-insurance", + display: "example.com/commercial-insurance", + }); + }); + + it("strips query strings and fragments without failing", () => { + const target = parseOk("https://example.com/blog?utm_source=x#section"); + expect(target.path).toBe("/blog"); + }); + + it("preserves path casing and percent encoding", () => { + const target = parseOk("example.com/Docs/%7Euser"); + expect(target.path).toBe("/Docs/%7Euser"); + }); + + it("strips www from hostname but keeps it for page URLs", () => { + const target = parseOk("www.example.com/blog", "exact_url"); + expect(target.hostname).toBe("example.com"); + expect(target.urlHostname).toBe("www.example.com"); + }); + + it("keeps subdomain hostnames intact", () => { + const target = parseOk("blog.example.com", "subdomains"); + expect(target.hostname).toBe("blog.example.com"); + }); + + it("rejects subfolder for a root input instead of silently rescoping", () => { + const result = parseResearchTarget("example.com", "subfolder"); + expect(result).toEqual({ + ok: false, + message: "Add a path to use Subfolder (e.g. example.com/blog)", + }); + }); + + it("allows exact_url for a root input", () => { + expect(parseOk("example.com", "exact_url").scope).toBe("exact_url"); + }); + + it("rejects invalid hosts and credentials", () => { + expect(parseResearchTarget("example.por").ok).toBe(false); + expect(parseResearchTarget("").ok).toBe(false); + expect(parseResearchTarget("my_site.com").ok).toBe(false); + expect(parseResearchTarget("https://user:pw@example.com/x").ok).toBe(false); + }); +}); + +describe("scope helpers", () => { + it("computes defaults from the path", () => { + expect(defaultScopeForPath("")).toBe("subdomains"); + expect(defaultScopeForPath("/blog")).toBe("subfolder"); + }); + + it("only disallows subfolder without a path", () => { + expect(isScopeAllowedForInput("subfolder", "")).toBe(false); + expect(isScopeAllowedForInput("subfolder", "/blog")).toBe(true); + expect(isScopeAllowedForInput("exact_url", "")).toBe(true); + expect(isScopeAllowedForInput("domain", "/blog")).toBe(true); + }); +}); + +describe("urlMatchesResearchTarget", () => { + const subfolder = parseOk("example.com/blog", "subfolder"); + + it("matches the subfolder itself and its children", () => { + expect( + urlMatchesResearchTarget("https://example.com/blog", subfolder), + ).toBe(true); + expect( + urlMatchesResearchTarget("https://example.com/blog/post?x=1", subfolder), + ).toBe(true); + expect( + urlMatchesResearchTarget("https://www.example.com/blog/", subfolder), + ).toBe(true); + }); + + it("excludes similarly named sibling paths and other hosts", () => { + expect( + urlMatchesResearchTarget("https://example.com/blogging", subfolder), + ).toBe(false); + expect( + urlMatchesResearchTarget("https://sub.example.com/blog/post", subfolder), + ).toBe(false); + }); + + it("matches exact URLs ignoring trailing slash, query, and fragment", () => { + const exact = parseOk("example.com/pricing", "exact_url"); + expect( + urlMatchesResearchTarget("https://example.com/pricing/", exact), + ).toBe(true); + expect( + urlMatchesResearchTarget("https://example.com/pricing?ref=x#top", exact), + ).toBe(true); + expect( + urlMatchesResearchTarget("https://example.com/pricing/plans", exact), + ).toBe(false); + }); + + it("separates domain scope from subdomains scope", () => { + const domain = parseOk("example.com", "domain"); + const subs = parseOk("example.com", "subdomains"); + expect(urlMatchesResearchTarget("https://blog.example.com/x", domain)).toBe( + false, + ); + expect(urlMatchesResearchTarget("https://blog.example.com/x", subs)).toBe( + true, + ); + expect(urlMatchesResearchTarget("https://notexample.com/x", subs)).toBe( + false, + ); + }); +}); diff --git a/src/shared/researchScope.ts b/src/shared/researchScope.ts new file mode 100644 index 0000000..8980925 --- /dev/null +++ b/src/shared/researchScope.ts @@ -0,0 +1,234 @@ +import { parse as parseTld } from "tldts"; +import { z } from "zod"; + +/** + * True when `host` resolves to a real registrable domain (public-suffix list), + * rejecting IPs and fake TLDs like `example.por` before they reach DataForSEO. + */ +export function isValidDomainHost(host: string): boolean { + const parsed = parseTld(host, { allowPrivateDomains: true }); + return ( + !parsed.isIp && + !!parsed.publicSuffix && + (parsed.isIcann === true || parsed.isPrivate === true) + ); +} + +/** + * Research scope for any URL/domain input: + * - exact_url: one normalized page URL only + * - subfolder: the selected path and its children (not similarly named siblings) + * - domain: the selected hostname, excluding its subdomains + * - subdomains: the selected hostname and all of its subdomains + */ +export const RESEARCH_SCOPES = [ + "exact_url", + "subfolder", + "domain", + "subdomains", +] as const; + +export type ResearchScope = (typeof RESEARCH_SCOPES)[number]; + +export const researchScopeSchema = z.enum(RESEARCH_SCOPES); + +export const RESEARCH_SCOPE_LABELS: Record = { + exact_url: "Exact URL", + subfolder: "Subfolder", + domain: "Domain", + subdomains: "Subdomains", +}; + +/** Base wording for MCP `scope` params; tools append their own caveats. */ +export const RESEARCH_SCOPE_PARAM_DESCRIPTION = + "Research scope: 'domain' (hostname without subdomains), 'subdomains' (hostname plus all subdomains), 'subfolder' (path and its children), or 'exact_url' (one page). Defaults to 'subdomains' for root inputs and 'subfolder' when the input has a path."; + +/** One-line explanations shown in the scope dropdown. */ +export const RESEARCH_SCOPE_DESCRIPTIONS: Record = { + exact_url: "One page only", + subfolder: "The path and everything under it", + domain: "The hostname, without subdomains", + subdomains: "The domain plus all its subdomains", +}; + +/** Wildcard-style pattern examples shown under each scope option. */ +export const RESEARCH_SCOPE_EXAMPLES: Record = { + exact_url: "example.com/path", + subfolder: "example.com/path/*", + domain: "example.com/*", + subdomains: "*.example.com/*", +}; + +export type ResearchTarget = { + scope: ResearchScope; + /** Lowercased hostname with a leading `www.` stripped. */ + hostname: string; + /** Hostname as entered (lowercased, `www.` preserved) for building page URLs. */ + urlHostname: string; + /** + * Normalized path: `""` for the root, otherwise `/like/This` — casing and + * percent-encoding preserved, trailing slashes / query / fragment stripped. + */ + path: string; + /** What to show users: hostname, plus the path for URL-scoped research. */ + display: string; +}; + +type ParseResearchTargetResult = + | { ok: true; target: ResearchTarget } + | { ok: false; message: string }; + +function normalizePath(pathname: string): string { + if (pathname === "/") return ""; + const trimmed = pathname.replace(/\/+$/, ""); + return trimmed === "" ? "" : trimmed; +} + +/** A subfolder needs a non-root path; every other scope works for any input. */ +export function isScopeAllowedForInput( + scope: ResearchScope, + path: string, +): boolean { + return scope !== "subfolder" || path !== ""; +} + +/** Root inputs default to subdomains scope; inputs with a path to subfolder. */ +export function defaultScopeForPath(path: string): ResearchScope { + return path === "" ? "subdomains" : "subfolder"; +} + +/** Scope implied by the input itself; unparseable input falls back to domain. */ +export function defaultScopeForInput(input: string): ResearchScope { + const parsed = parseResearchTarget(input); + return parsed.ok ? parsed.target.scope : "domain"; +} + +/** + * Scope as persisted in URLs and history: omitted when it matches the input's + * implied default, so shared links re-derive the same scope. + */ +export function toScopeSearchParam( + input: string, + scope: ResearchScope, +): ResearchScope | undefined { + return scope === defaultScopeForInput(input) ? undefined : scope; +} + +export function parseResearchTarget( + input: string, + requestedScope?: ResearchScope, +): ParseResearchTargetResult { + const trimmed = input.trim(); + if (!trimmed) { + return { ok: false, message: "Enter a domain or URL" }; + } + + const withProtocol = /^[a-zA-Z][a-zA-Z\d+.-]*:\/\//.test(trimmed) + ? trimmed + : `https://${trimmed}`; + + let parsed: URL; + try { + parsed = new URL(withProtocol); + } catch { + return { ok: false, message: "Enter a valid domain like example.com" }; + } + + if (parsed.username || parsed.password) { + return { + ok: false, + message: "URLs with embedded credentials are not supported", + }; + } + + const urlHostname = parsed.hostname.toLowerCase(); + const hostname = urlHostname.replace(/^www\./, ""); + // The charset check rejects hosts like my_site.com that URL() and tldts + // accept but DataForSEO bills and fails with an opaque "Invalid Field". + if ( + !hostname || + !hostname.includes(".") || + !/^[a-z\d.-]+$/.test(hostname) || + !isValidDomainHost(hostname) + ) { + return { ok: false, message: "Enter a valid domain like example.com" }; + } + + // Query strings and fragments never create separate research scopes. + const path = normalizePath(parsed.pathname); + + if (requestedScope === "subfolder" && path === "") { + return { + ok: false, + message: "Add a path to use Subfolder (e.g. example.com/blog)", + }; + } + + const scope = requestedScope ?? defaultScopeForPath(path); + + const usesPath = scope === "exact_url" || scope === "subfolder"; + return { + ok: true, + target: { + scope, + hostname, + urlHostname, + path, + display: usesPath ? `${hostname}${path}` : hostname, + }, + }; +} + +/** + * How many of DataForSEO's 8 filter conditions each scope consumes on the + * Labs endpoints (see buildRankedKeywordsScopeFilter / buildRelevantPagesScopeFilter). + * The client uses this to shrink the user-facing filter budget. + */ +export const RESEARCH_SCOPE_FILTER_SLOTS: Record< + "keywords" | "pages", + Record +> = { + keywords: { exact_url: 4, subfolder: 4, domain: 1, subdomains: 0 }, + pages: { exact_url: 4, subfolder: 4, domain: 2, subdomains: 0 }, +}; + +/** Conditions the backlinks subfolder url_to/url prefix group consumes. */ +export const BACKLINKS_SUBFOLDER_FILTER_CONDITIONS = 4; + +function hostMatches(candidateHost: string, target: ResearchTarget): boolean { + const host = candidateHost.toLowerCase().replace(/^www\./, ""); + if (target.scope === "subdomains") { + return host === target.hostname || host.endsWith(`.${target.hostname}`); + } + return host === target.hostname; +} + +/** + * Whether a result URL belongs to the research target. Used to post-filter + * provider rows that cannot be scoped provider-side. Subfolder matching + * includes the path and its children but excludes similarly named siblings + * (`/blog` matches `/blog/post`, not `/blogging`). + */ +export function urlMatchesResearchTarget( + url: string, + target: ResearchTarget, +): boolean { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + + if (!hostMatches(parsed.hostname, target)) return false; + + const path = normalizePath(parsed.pathname); + switch (target.scope) { + case "exact_url": + return path === target.path; + case "subfolder": + return path === target.path || path.startsWith(`${target.path}/`); + default: + return true; + } +} diff --git a/src/types/schemas/ai-search.ts b/src/types/schemas/ai-search.ts index f8228d4..cf81c69 100644 --- a/src/types/schemas/ai-search.ts +++ b/src/types/schemas/ai-search.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { researchScopeSchema } from "@/shared/researchScope"; /** * Input + output schemas for the AI Search feature (Brand Lookup + Prompt @@ -42,6 +43,10 @@ export const brandLookupInputSchema = z.object({ .array(z.string().trim().min(1).max(BRAND_LOOKUP_MAX_INPUT_LENGTH)) .max(BRAND_LOOKUP_MAX_COMPETITORS) .default([]), + // Research scope for domain/URL queries. Ignored for brand keywords, which + // have no URL to scope. Omitted = derive from the query (root → domain, path + // → subfolder). + scope: researchScopeSchema.optional(), locationCode: z.number().int().positive().default(2840), languageCode: z.string().min(2).max(8).default("en"), }); @@ -114,7 +119,18 @@ const brandMonthlyVolumeSchema = z.object({ export const brandLookupResultSchema = z.object({ query: z.string(), detectedTargetType: z.enum(["domain", "keyword"]), + /** Hostname for domain scopes, hostname + path for URL scopes. */ resolvedTarget: z.string(), + // Resolved scope, or null for keyword lookups. Defaulted so cache entries + // written before scopes existed still parse. + scope: researchScopeSchema.nullable().default(null), + /** + * True under exact_url/subfolder scope: the LLM mentions API has no + * URL-level targeting, so totals, per-platform counts, monthly volume and + * Share of Voice stay domain-wide and the UI must say so. Page-level rows + * are filtered to the scope. + */ + aggregatesAreDomainLevel: z.boolean().default(false), fetchedAt: z.string(), hasData: z.boolean(), totalMentions: z.number().int().nonnegative().nullable(), @@ -255,10 +271,12 @@ export type PromptExplorerResult = z.infer; * is a comma-joined competitor list (route + page treat the parsed result as an * opaque string array). `c` accepts a raw string (from the URL) OR an array * (TanStack Router re-validates its own transformed output on navigate) — same - * union pattern as `models` below. + * union pattern as `models` below. `scope` is only present when it differs + * from the scope derived from `q`. */ export const brandLookupSearchSchema = z.object({ q: z.string().optional(), + scope: researchScopeSchema.optional().catch(undefined), c: z .union([z.string(), z.array(z.string())]) .optional() diff --git a/src/types/schemas/backlinks.ts b/src/types/schemas/backlinks.ts index e65cfc7..820834f 100644 --- a/src/types/schemas/backlinks.ts +++ b/src/types/schemas/backlinks.ts @@ -1,7 +1,45 @@ import { z } from "zod"; +import { + RESEARCH_SCOPES, + RESEARCH_SCOPE_PARAM_DESCRIPTION, + type ResearchScope, +} from "@/shared/researchScope"; export const backlinksTabSchema = z.enum(["backlinks", "domains", "pages"]); -export const backlinksTargetScopeSchema = z.enum(["domain", "page"]); + +/** + * Backlinks uses the shared research scopes. DataForSEO has no prefix + * targeting, so subfolder rides on url_to/url prefix filters: backlink rows + * and top pages are provider-filtered, summary counts come from filtered + * backlink totals, and rank/trends/referring-domains stay unavailable. + */ +export type BacklinksTargetScope = ResearchScope; + +/** + * Scope as it arrives from persisted or external state (URLs, MCP clients). + * "page" is the pre-research-scope name for exact_url and is still accepted. + */ +export const backlinksScopeWithLegacySchema = z.enum([ + ...RESEARCH_SCOPES, + "page", +]); + +export type BacklinksScopeWithLegacy = z.infer< + typeof backlinksScopeWithLegacySchema +>; + +export function resolveBacklinksScope( + scope: BacklinksScopeWithLegacy, +): ResearchScope { + return scope === "page" ? "exact_url" : scope; +} + +export const backlinksScopeParamSchema = + backlinksScopeWithLegacySchema.transform(resolveBacklinksScope); + +/** Shared wording for the MCP tools that take a backlinks scope. */ +export const BACKLINKS_SCOPE_DESCRIPTION = `${RESEARCH_SCOPE_PARAM_DESCRIPTION} 'page' is a deprecated alias of 'exact_url'. Subfolder counts are computed from filtered backlink totals; rank, trends, and the referring-domains breakdown are unavailable for subfolders.`; + const DEFAULT_BACKLINKS_SPAM_THRESHOLD = 40; function normalizeBacklinksSpamThreshold(value: number) { @@ -33,7 +71,7 @@ export function normalizeBacklinksSpamFilterOptions( } export const backlinksLookupSchema = z.object({ target: z.string().min(1, "Target is required").max(2048), - scope: backlinksTargetScopeSchema.optional(), + scope: backlinksScopeParamSchema.optional(), }); export const backlinksOverviewInputSchema = backlinksLookupSchema.extend({ @@ -182,7 +220,7 @@ export const topPagesPageRequestSchema = backlinksPageRequestBase.extend({ export const backlinksSearchSchema = z.object({ target: z.string().optional(), - scope: backlinksTargetScopeSchema.optional(), + scope: backlinksScopeParamSchema.optional().catch(undefined), tab: backlinksTabSchema.optional(), page: z.coerce.number().int().positive().optional().catch(undefined), size: z.coerce @@ -204,7 +242,6 @@ export const backlinksSearchSchema = z.object({ export type BacklinksLookupInput = z.infer; export type BacklinksTab = z.infer; -export type BacklinksTargetScope = z.infer; export type BacklinksSortOrder = z.infer; export type BacklinksRowsSortField = z.infer< typeof backlinksRowsSortFieldSchema diff --git a/src/types/schemas/domain.ts b/src/types/schemas/domain.ts index c1e6db3..e848b80 100644 --- a/src/types/schemas/domain.ts +++ b/src/types/schemas/domain.ts @@ -1,5 +1,5 @@ -import { parse as parseTld } from "tldts"; import { z } from "zod"; +import { isValidDomainHost, researchScopeSchema } from "@/shared/researchScope"; /** * Extract and validate a bare hostname from user input that may be a full URL. @@ -13,19 +13,6 @@ export function normalizeDomain(input: string): string { return hostname.replace(/^www\./, ""); } -/** - * True when `host` resolves to a real registrable domain (public-suffix list), - * rejecting IPs and fake TLDs like `example.por` before they reach DataForSEO. - */ -export function isValidDomainHost(host: string): boolean { - const parsed = parseTld(host, { allowPrivateDomains: true }); - return ( - !parsed.isIp && - !!parsed.publicSuffix && - (parsed.isIcann === true || parsed.isPrivate === true) - ); -} - /** Zod field: accepts a bare domain or full URL, outputs a clean hostname. */ export const domainField = z .string() @@ -57,8 +44,8 @@ export const booleanSearchParamSchema = z export const domainOverviewSchema = z.object({ projectId: z.string().uuid(), - domain: z.string().min(1, "Domain is required").max(255), - includeSubdomains: z.boolean().default(true), + domain: z.string().min(1, "Domain is required").max(2048), + scope: researchScopeSchema.optional(), locationCode: z.number().int().positive().optional(), languageCode: z.string().min(2).max(8).optional(), }); @@ -73,7 +60,8 @@ const domainTabs = ["keywords", "pages"] as const; export const domainKeywordSuggestionsSchema = z.object({ projectId: z.string().uuid(), - domain: domainField, + domain: z.string().min(1, "Domain is required").max(2048), + scope: researchScopeSchema.optional(), locationCode: z.number().int().positive().optional(), languageCode: z.string().min(2).max(8).optional(), }); @@ -117,8 +105,8 @@ export type DomainKeywordsFilters = z.infer; export const domainKeywordsPageRequestSchema = z.object({ projectId: z.string().uuid(), - domain: z.string().min(1).max(255), - includeSubdomains: z.boolean().default(true), + domain: z.string().min(1).max(2048), + scope: researchScopeSchema.optional(), locationCode: z.number().int().positive().optional(), languageCode: z.string().min(2).max(8).optional(), page: z.number().int().positive().default(1), @@ -139,8 +127,8 @@ const domainPagesSortModes = ["traffic", "keywords"] as const; export const domainPagesPageRequestSchema = z.object({ projectId: z.string().uuid(), - domain: z.string().min(1).max(255), - includeSubdomains: z.boolean().default(true), + domain: z.string().min(1).max(2048), + scope: researchScopeSchema.optional(), locationCode: z.number().int().positive().optional(), languageCode: z.string().min(2).max(8).optional(), page: z.number().int().positive().default(1), @@ -169,6 +157,8 @@ const filterNumberParam = optionalSearchNumberParam; export const domainSearchSchema = z.object({ domain: z.string().optional(), + scope: researchScopeSchema.optional().catch(undefined), + /** Legacy param: pre-scope URLs encoded "Include subdomains" here. */ subdomains: booleanSearchParamSchema.optional(), sort: z.enum(domainSortModes).optional(), order: z.enum(domainSortOrders).optional(),