feat: hide spam backlinks by default (#89)

* feat: hide spam backlinks by default

* feat: add configurable backlinks spam filtering

* refactor: simplify backlinks spam filtering flow

Reduce duplicated spam-filter state and normalization so the backlinks page has one source of truth and client/server caching stays aligned.

* refactor: simplify backlinks spam filter semantics

* fix: keep spam filtering scoped to backlink rows

* feat: show backlinks spam score
This commit is contained in:
Ben Senescu 2026-04-07 18:21:58 -04:00 committed by Ben Senescu
parent 673459877e
commit a10f4d82a0
12 changed files with 402 additions and 116 deletions

View File

@ -6,6 +6,7 @@ import {
navigateToBacklinksTab, navigateToBacklinksTab,
useBacklinksPageData, useBacklinksPageData,
} from "./useBacklinksPageData"; } from "./useBacklinksPageData";
import { useBacklinksSpamPreferences } from "./useBacklinksSpamPreferences";
import { getStandardErrorMessage } from "@/client/lib/error-messages"; import { getStandardErrorMessage } from "@/client/lib/error-messages";
export function BacklinksPage({ export function BacklinksPage({
@ -13,6 +14,8 @@ export function BacklinksPage({
searchState, searchState,
navigate, navigate,
}: BacklinksPageProps) { }: BacklinksPageProps) {
const { hideSpam, setHideSpam, setSpamThreshold, spamThreshold } =
useBacklinksSpamPreferences();
const { const {
accessStatus, accessStatus,
accessStatusErrorMessage, accessStatusErrorMessage,
@ -26,7 +29,10 @@ export function BacklinksPage({
searchCardInitialValues, searchCardInitialValues,
testAccessMutation, testAccessMutation,
topPagesQuery, topPagesQuery,
} = useBacklinksPageData({ projectId, searchState }); } = useBacklinksPageData({
projectId,
searchState,
});
return ( return (
<div className="px-4 py-4 pb-24 overflow-auto md:px-6 md:py-6 md:pb-8"> <div className="px-4 py-4 pb-24 overflow-auto md:px-6 md:py-6 md:pb-8">
@ -81,10 +87,14 @@ export function BacklinksPage({
} }
testIsPending={testAccessMutation.isPending} testIsPending={testAccessMutation.isPending}
topPages={topPagesQuery.data} topPages={topPagesQuery.data}
hideSpam={hideSpam}
spamThreshold={spamThreshold}
onRetryAccess={() => void accessStatusQuery.refetch()} onRetryAccess={() => void accessStatusQuery.refetch()}
onSetActiveTab={(tab) => navigateToBacklinksTab(navigate, tab)} onSetActiveTab={(tab) => navigateToBacklinksTab(navigate, tab)}
onRetryOverview={() => void overviewQuery.refetch()} onRetryOverview={() => void overviewQuery.refetch()}
onTestAccess={() => testAccessMutation.mutate()} onTestAccess={() => testAccessMutation.mutate()}
onSetHideSpam={setHideSpam}
onSetSpamThreshold={setSpamThreshold}
/> />
</div> </div>
</div> </div>

View File

@ -25,11 +25,13 @@ type BacklinksBodyProps = {
backlinksDisabledByError: boolean; backlinksDisabledByError: boolean;
backlinksEnabled: boolean; backlinksEnabled: boolean;
isAccessStatusLoading: boolean; isAccessStatusLoading: boolean;
hideSpam: boolean;
overviewData: BacklinksOverviewData | undefined; overviewData: BacklinksOverviewData | undefined;
overviewError: string | null; overviewError: string | null;
overviewLoading: boolean; overviewLoading: boolean;
referringDomains: BacklinksReferringDomainsData | undefined; referringDomains: BacklinksReferringDomainsData | undefined;
searchState: BacklinksSearchState; searchState: BacklinksSearchState;
spamThreshold: number;
tabErrorMessage: string | null; tabErrorMessage: string | null;
tabLoading: boolean; tabLoading: boolean;
testError: string | null; testError: string | null;
@ -39,6 +41,8 @@ type BacklinksBodyProps = {
onSetActiveTab: (tab: BacklinksSearchState["tab"]) => void; onSetActiveTab: (tab: BacklinksSearchState["tab"]) => void;
onRetryOverview: () => void; onRetryOverview: () => void;
onTestAccess: () => void; onTestAccess: () => void;
onSetHideSpam: (hideSpam: boolean) => void;
onSetSpamThreshold: (threshold: number) => void;
}; };
export function BacklinksBody({ export function BacklinksBody({
@ -47,11 +51,13 @@ export function BacklinksBody({
backlinksDisabledByError, backlinksDisabledByError,
backlinksEnabled, backlinksEnabled,
isAccessStatusLoading, isAccessStatusLoading,
hideSpam,
overviewData, overviewData,
overviewError, overviewError,
overviewLoading, overviewLoading,
referringDomains, referringDomains,
searchState, searchState,
spamThreshold,
tabErrorMessage, tabErrorMessage,
tabLoading, tabLoading,
testError, testError,
@ -61,7 +67,36 @@ export function BacklinksBody({
onSetActiveTab, onSetActiveTab,
onRetryOverview, onRetryOverview,
onTestAccess, onTestAccess,
onSetHideSpam,
onSetSpamThreshold,
}: BacklinksBodyProps) { }: BacklinksBodyProps) {
const [filterText, setFilterText] = useState("");
useEffect(() => {
setFilterText("");
}, [searchState.target, searchState.tab]);
const mergedData = useMemo(
() => mergeTabData(overviewData, referringDomains, topPages),
[overviewData, referringDomains, topPages],
);
const normalizedFilter = filterText.trim().toLowerCase();
const filteredData = useMemo(
() =>
filterOverviewData(
mergedData,
normalizedFilter,
searchState.tab,
hideSpam,
spamThreshold,
),
[hideSpam, mergedData, normalizedFilter, searchState.tab, spamThreshold],
);
const summaryStats = useMemo(
() => buildSummaryStats(mergedData),
[mergedData],
);
if (isAccessStatusLoading) { if (isAccessStatusLoading) {
return <BacklinksAccessLoadingState />; return <BacklinksAccessLoadingState />;
} }
@ -86,76 +121,20 @@ export function BacklinksBody({
); );
} }
return (
<BacklinksContent
data={overviewData}
errorMessage={overviewError}
isLoading={overviewLoading}
referringDomains={referringDomains}
searchState={searchState}
tabErrorMessage={tabErrorMessage}
tabLoading={tabLoading}
topPages={topPages}
onSetActiveTab={onSetActiveTab}
onRetry={onRetryOverview}
/>
);
}
function BacklinksContent({
data,
errorMessage,
isLoading,
referringDomains,
searchState,
tabErrorMessage,
tabLoading,
topPages,
onSetActiveTab,
onRetry,
}: {
data: BacklinksOverviewData | undefined;
errorMessage: string | null;
isLoading: boolean;
referringDomains: BacklinksReferringDomainsData | undefined;
searchState: BacklinksSearchState;
tabErrorMessage: string | null;
tabLoading: boolean;
topPages: BacklinksTopPagesData | undefined;
onSetActiveTab: (tab: BacklinksSearchState["tab"]) => void;
onRetry: () => void;
}) {
const [filterText, setFilterText] = useState("");
useEffect(() => {
setFilterText("");
}, [searchState.target, searchState.tab]);
const mergedData = useMemo(
() => mergeTabData(data, referringDomains, topPages),
[data, referringDomains, topPages],
);
const normalizedFilter = filterText.trim().toLowerCase();
const filteredData = useMemo(
() => filterOverviewData(mergedData, normalizedFilter),
[mergedData, normalizedFilter],
);
const summaryStats = useMemo(
() => buildSummaryStats(mergedData),
[mergedData],
);
if (!searchState.target) { if (!searchState.target) {
return <BacklinksEmptyState />; return <BacklinksEmptyState />;
} }
if (isLoading) { if (overviewLoading) {
return <BacklinksLoadingState />; return <BacklinksLoadingState />;
} }
if (!mergedData) { if (!mergedData) {
return ( return (
<BacklinksErrorState errorMessage={errorMessage} onRetry={onRetry} /> <BacklinksErrorState
errorMessage={overviewError}
onRetry={onRetryOverview}
/>
); );
} }
@ -166,12 +145,16 @@ function BacklinksContent({
activeTab={searchState.tab} activeTab={searchState.tab}
filteredData={filteredData} filteredData={filteredData}
filterText={filterText} filterText={filterText}
hideSpam={hideSpam}
spamThreshold={spamThreshold}
isTabLoading={searchState.tab !== "backlinks" && tabLoading} isTabLoading={searchState.tab !== "backlinks" && tabLoading}
tabErrorMessage={ tabErrorMessage={
searchState.tab !== "backlinks" ? tabErrorMessage : null searchState.tab !== "backlinks" ? tabErrorMessage : null
} }
onFilterTextChange={setFilterText} onFilterTextChange={setFilterText}
onSetActiveTab={onSetActiveTab} onSetActiveTab={onSetActiveTab}
onSetHideSpam={onSetHideSpam}
onSetSpamThreshold={onSetSpamThreshold}
/> />
</> </>
); );
@ -196,13 +179,23 @@ function mergeTabData(
function filterOverviewData( function filterOverviewData(
data: BacklinksOverviewData | undefined, data: BacklinksOverviewData | undefined,
normalizedFilter: string, normalizedFilter: string,
activeTab: BacklinksSearchState["tab"],
hideSpam: boolean,
spamThreshold: number,
) { ) {
if (!data) { if (!data) {
return { backlinks: [], referringDomains: [], topPages: [] }; return { backlinks: [], referringDomains: [], topPages: [] };
} }
const backlinksRows =
activeTab === "backlinks" && hideSpam
? data.backlinks.filter(
(row) => row.spamScore == null || row.spamScore <= spamThreshold,
)
: data.backlinks;
return { return {
backlinks: data.backlinks.filter((row) => { backlinks: backlinksRows.filter((row) => {
if (!normalizedFilter) return true; if (!normalizedFilter) return true;
return [row.domainFrom, row.urlFrom, row.urlTo, row.anchor, row.itemType] return [row.domainFrom, row.urlFrom, row.urlTo, row.anchor, row.itemType]
.filter((value): value is string => Boolean(value)) .filter((value): value is string => Boolean(value))

View File

@ -1,3 +1,4 @@
import { useEffect, useState } from "react";
import { HeaderHelpLabel } from "@/client/features/keywords/components"; import { HeaderHelpLabel } from "@/client/features/keywords/components";
import { Search } from "lucide-react"; import { Search } from "lucide-react";
import { import {
@ -49,10 +50,14 @@ export function BacklinksResultsCard({
activeTab, activeTab,
filteredData, filteredData,
filterText, filterText,
hideSpam,
spamThreshold,
isTabLoading, isTabLoading,
tabErrorMessage, tabErrorMessage,
onFilterTextChange, onFilterTextChange,
onSetActiveTab, onSetActiveTab,
onSetHideSpam,
onSetSpamThreshold,
}: { }: {
activeTab: BacklinksSearchState["tab"]; activeTab: BacklinksSearchState["tab"];
filteredData: { filteredData: {
@ -61,10 +66,14 @@ export function BacklinksResultsCard({
topPages: BacklinksOverviewData["topPages"]; topPages: BacklinksOverviewData["topPages"];
}; };
filterText: string; filterText: string;
hideSpam: boolean;
spamThreshold: number;
isTabLoading: boolean; isTabLoading: boolean;
tabErrorMessage: string | null; tabErrorMessage: string | null;
onFilterTextChange: (value: string) => void; onFilterTextChange: (value: string) => void;
onSetActiveTab: (tab: BacklinksSearchState["tab"]) => void; onSetActiveTab: (tab: BacklinksSearchState["tab"]) => void;
onSetHideSpam: (hideSpam: boolean) => void;
onSetSpamThreshold: (threshold: number) => void;
}) { }) {
return ( return (
<div className="card bg-base-100 border border-base-300"> <div className="card bg-base-100 border border-base-300">
@ -72,8 +81,12 @@ export function BacklinksResultsCard({
<ResultsHeader <ResultsHeader
activeTab={activeTab} activeTab={activeTab}
filterText={filterText} filterText={filterText}
hideSpam={hideSpam}
spamThreshold={spamThreshold}
onFilterTextChange={onFilterTextChange} onFilterTextChange={onFilterTextChange}
onSetActiveTab={onSetActiveTab} onSetActiveTab={onSetActiveTab}
onSetHideSpam={onSetHideSpam}
onSetSpamThreshold={onSetSpamThreshold}
/> />
{tabErrorMessage ? ( {tabErrorMessage ? (
<div className="alert alert-error"> <div className="alert alert-error">
@ -122,14 +135,38 @@ function OverviewGrid({
function ResultsHeader({ function ResultsHeader({
activeTab, activeTab,
filterText, filterText,
hideSpam,
spamThreshold,
onFilterTextChange, onFilterTextChange,
onSetActiveTab, onSetActiveTab,
onSetHideSpam,
onSetSpamThreshold,
}: { }: {
activeTab: BacklinksSearchState["tab"]; activeTab: BacklinksSearchState["tab"];
filterText: string; filterText: string;
hideSpam: boolean;
spamThreshold: number;
onFilterTextChange: (value: string) => void; onFilterTextChange: (value: string) => void;
onSetActiveTab: (tab: BacklinksSearchState["tab"]) => void; onSetActiveTab: (tab: BacklinksSearchState["tab"]) => void;
onSetHideSpam: (hideSpam: boolean) => void;
onSetSpamThreshold: (threshold: number) => void;
}) { }) {
const [draftSpamThreshold, setDraftSpamThreshold] = useState(
String(spamThreshold),
);
useEffect(() => {
setDraftSpamThreshold(String(spamThreshold));
}, [spamThreshold]);
function commitSpamThreshold() {
onSetSpamThreshold(
draftSpamThreshold.trim() === ""
? spamThreshold
: Number(draftSpamThreshold),
);
}
return ( return (
<div className="flex flex-col gap-3 lg:flex-row lg:items-start lg:justify-between"> <div className="flex flex-col gap-3 lg:flex-row lg:items-start lg:justify-between">
<div className="space-y-2"> <div className="space-y-2">
@ -157,14 +194,49 @@ function ResultsHeader({
</p> </p>
</div> </div>
<label className="input input-bordered input-sm flex w-full max-w-xs items-center gap-2"> <div className="flex flex-col items-end gap-2">
<Search className="size-4 text-base-content/60" /> <label className="input input-bordered input-sm flex w-full max-w-xs items-center gap-2">
<input <Search className="size-4 text-base-content/60" />
placeholder="Filter current tab" <input
value={filterText} placeholder="Filter current tab"
onChange={(event) => onFilterTextChange(event.target.value)} value={filterText}
/> onChange={(event) => onFilterTextChange(event.target.value)}
</label> />
</label>
{activeTab === "backlinks" ? (
<div className="flex items-center gap-3 text-sm">
<label className="flex cursor-pointer items-center gap-2">
<input
type="checkbox"
className="checkbox checkbox-xs"
checked={hideSpam}
onChange={(event) => onSetHideSpam(event.target.checked)}
/>
<span className="text-base-content/70">Hide spam</span>
</label>
<label className="flex items-center gap-2 text-base-content/70">
<span>Max spam</span>
<input
type="number"
min={0}
max={100}
step={1}
value={draftSpamThreshold}
disabled={!hideSpam}
className="input input-bordered input-xs w-20"
onBlur={commitSpamThreshold}
onChange={(event) => setDraftSpamThreshold(event.target.value)}
onKeyDown={(event) => {
if (event.key === "Enter") {
commitSpamThreshold();
event.currentTarget.blur();
}
}}
/>
</label>
</div>
) : null}
</div>
</div> </div>
); );
} }

View File

@ -294,6 +294,33 @@ export const backlinksColumns: ColumnDef<GroupedBacklinkDomain>[] = [
}, },
sortingFn: "basic", sortingFn: "basic",
}, },
{
id: "spamScore",
accessorKey: "spamScore",
header: ({ column }) => (
<SortableHeader
column={column}
label="Spam"
helpText="Estimated spam risk for the linking domain or backlink. Higher scores are more likely to be manipulative or low quality."
align="right"
/>
),
size: 70,
minSize: 50,
cell: ({ row }) => {
const value =
row.depth > 0
? row.original._backlink?.spamScore
: row.original.spamScore;
return (
<div className="text-right tabular-nums text-sm">
{value != null && value > 0 ? Math.round(value) : null}
</div>
);
},
sortingFn: "basic",
},
{ {
id: "firstSeen", id: "firstSeen",
accessorKey: "firstSeen", accessorKey: "firstSeen",

View File

@ -6,7 +6,6 @@ import {
type SortDirection, type SortDirection,
type TopPagesTableSort, type TopPagesTableSort,
} from "./backlinksTableSorting"; } from "./backlinksTableSorting";
export function ReferringDomainsTableHeader({ export function ReferringDomainsTableHeader({
sort, sort,
onSortChange, onSortChange,

View File

@ -1,7 +1,6 @@
import type { BacklinksOverviewData } from "./backlinksPageTypes"; import type { BacklinksOverviewData } from "./backlinksPageTypes";
export type SortDirection = "asc" | "desc"; export type SortDirection = "asc" | "desc";
export type ReferringDomainsTableSortField = export type ReferringDomainsTableSortField =
| "domain" | "domain"
| "backlinks" | "backlinks"
@ -51,7 +50,6 @@ export function getNextSort<TField extends string>(
direction: current.direction === "asc" ? "desc" : "asc", direction: current.direction === "asc" ? "desc" : "asc",
}; };
} }
export function sortReferringDomainRows( export function sortReferringDomainRows(
rows: BacklinksOverviewData["referringDomains"], rows: BacklinksOverviewData["referringDomains"],
sort: ReferringDomainsTableSort, sort: ReferringDomainsTableSort,

View File

@ -52,10 +52,7 @@ export function useBacklinksPageData({
) )
: null; : null;
const backlinksEnabled = accessStatus?.enabled ?? false; const backlinksEnabled = accessStatus?.enabled ?? false;
const requestInput = useMemo( const requestInput = buildBacklinksRequestInput(projectId, searchState);
() => buildBacklinksRequestInput(projectId, searchState),
[projectId, searchState],
);
const searchCardInitialValues = useMemo( const searchCardInitialValues = useMemo(
() => ({ () => ({
target: searchState.target, target: searchState.target,
@ -71,20 +68,19 @@ export function useBacklinksPageData({
}, },
}); });
const queryKeyParts = [ const baseQueryKeyParts = [
projectId, projectId,
searchState.scope, searchState.scope,
searchState.target, searchState.target,
] as const; ] as const;
const overviewQuery = useQuery({ const overviewQuery = useQuery({
queryKey: ["backlinksOverview", ...queryKeyParts], queryKey: ["backlinksOverview", ...baseQueryKeyParts],
enabled: backlinksEnabled && Boolean(searchState.target), enabled: backlinksEnabled && Boolean(searchState.target),
queryFn: () => getBacklinksOverview({ data: requestInput }), queryFn: () => getBacklinksOverview({ data: requestInput }),
}); });
const referringDomainsQuery = useQuery({ const referringDomainsQuery = useQuery({
queryKey: ["backlinksReferringDomains", ...queryKeyParts], queryKey: ["backlinksReferringDomains", ...baseQueryKeyParts],
enabled: enabled:
backlinksEnabled && backlinksEnabled &&
Boolean(searchState.target) && Boolean(searchState.target) &&
@ -93,7 +89,7 @@ export function useBacklinksPageData({
}); });
const topPagesQuery = useQuery({ const topPagesQuery = useQuery({
queryKey: ["backlinksTopPages", ...queryKeyParts], queryKey: ["backlinksTopPages", ...baseQueryKeyParts],
enabled: enabled:
backlinksEnabled && backlinksEnabled &&
Boolean(searchState.target) && Boolean(searchState.target) &&

View File

@ -0,0 +1,100 @@
import { useEffect, useState } from "react";
import {
DEFAULT_BACKLINKS_SPAM_THRESHOLD,
normalizeBacklinksSpamThreshold,
} from "@/types/schemas/backlinks";
const STORAGE_KEY = "backlinks-spam-preferences";
type BacklinksSpamPreferences = {
hideSpam: boolean;
spamThreshold: number;
};
const DEFAULT_PREFERENCES: BacklinksSpamPreferences = {
hideSpam: true,
spamThreshold: DEFAULT_BACKLINKS_SPAM_THRESHOLD,
};
function normalizeBacklinksSpamPreferences(
value: unknown,
): BacklinksSpamPreferences {
if (!value || typeof value !== "object") {
return DEFAULT_PREFERENCES;
}
const preferences = value as {
hideSpam?: unknown;
spamThreshold?: unknown;
};
return {
hideSpam:
typeof preferences.hideSpam === "boolean"
? preferences.hideSpam
: DEFAULT_PREFERENCES.hideSpam,
spamThreshold: normalizeBacklinksSpamThreshold(
typeof preferences.spamThreshold === "number"
? preferences.spamThreshold
: DEFAULT_PREFERENCES.spamThreshold,
),
};
}
function loadBacklinksSpamPreferences() {
if (typeof window === "undefined") {
return DEFAULT_PREFERENCES;
}
try {
const raw = localStorage.getItem(STORAGE_KEY);
if (!raw) return DEFAULT_PREFERENCES;
return normalizeBacklinksSpamPreferences(JSON.parse(raw));
} catch {
return DEFAULT_PREFERENCES;
}
}
function saveBacklinksSpamPreferences(preferences: BacklinksSpamPreferences) {
if (typeof window === "undefined") {
return;
}
try {
localStorage.setItem(STORAGE_KEY, JSON.stringify(preferences));
} catch {
// storage full or unavailable - silently ignore
}
}
export function useBacklinksSpamPreferences() {
const [preferences, setPreferences] = useState<BacklinksSpamPreferences>(
loadBacklinksSpamPreferences,
);
useEffect(() => {
saveBacklinksSpamPreferences(preferences);
}, [preferences]);
function setHideSpam(nextHideSpam: boolean) {
setPreferences((current) => ({
...current,
hideSpam: nextHideSpam,
}));
}
function setSpamThreshold(nextSpamThreshold: number) {
setPreferences((current) => ({
...current,
spamThreshold: normalizeBacklinksSpamThreshold(nextSpamThreshold),
}));
}
return {
hideSpam: preferences.hideSpam,
setHideSpam,
setSpamThreshold,
spamThreshold: preferences.spamThreshold,
};
}

View File

@ -1,5 +1,9 @@
import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinks"; import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinks";
import {
normalizeBacklinksSpamFilterOptions,
type BacklinksSpamFilterOptions,
} from "@/types/schemas/backlinks";
import { import {
profileBacklinksOverview, profileBacklinksOverview,
profileReferringDomainsRows, profileReferringDomainsRows,
@ -19,19 +23,33 @@ function createBacklinksService(cache: BacklinksCache = defaultCache) {
async profileOverview( async profileOverview(
input: BacklinksLookupInput, input: BacklinksLookupInput,
billingCustomer: BillingCustomerContext, billingCustomer: BillingCustomerContext,
options?: BacklinksSpamFilterOptions,
) { ) {
const cacheKey = await buildOverviewCacheKey(input, billingCustomer); const cacheKey = await buildBacklinksCacheKey(
"backlinks:overview",
input,
billingCustomer,
options,
);
return profileBacklinksOverview(cache, cacheKey, input, billingCustomer); return profileBacklinksOverview(
cache,
cacheKey,
input,
billingCustomer,
options,
);
}, },
async profileReferringDomains( async profileReferringDomains(
input: BacklinksLookupInput, input: BacklinksLookupInput,
billingCustomer: BillingCustomerContext, billingCustomer: BillingCustomerContext,
options?: BacklinksSpamFilterOptions,
) { ) {
const cacheKey = await buildTabCacheKey( const cacheKey = await buildBacklinksCacheKey(
"backlinks:referring-domains", "backlinks:referring-domains",
input, input,
billingCustomer, billingCustomer,
options,
); );
return profileReferringDomainsRows( return profileReferringDomainsRows(
@ -39,13 +57,14 @@ function createBacklinksService(cache: BacklinksCache = defaultCache) {
cacheKey, cacheKey,
input, input,
billingCustomer, billingCustomer,
options,
); );
}, },
async profileTopPages( async profileTopPages(
input: BacklinksLookupInput, input: BacklinksLookupInput,
billingCustomer: BillingCustomerContext, billingCustomer: BillingCustomerContext,
) { ) {
const cacheKey = await buildTabCacheKey( const cacheKey = await buildBacklinksCacheKey(
"backlinks:top-pages", "backlinks:top-pages",
input, input,
billingCustomer, billingCustomer,
@ -56,32 +75,33 @@ function createBacklinksService(cache: BacklinksCache = defaultCache) {
} as const; } as const;
} }
async function buildOverviewCacheKey( async function buildBacklinksCacheKey(
input: BacklinksLookupInput,
billingCustomer: BillingCustomerContext,
): Promise<string> {
const normalizedTarget = normalizeBacklinksTarget(input.target, {
scope: input.scope,
});
return buildCacheKey("backlinks:overview", {
organizationId: billingCustomer.organizationId,
target: normalizedTarget.apiTarget,
scope: normalizedTarget.scope,
});
}
async function buildTabCacheKey(
prefix: string, prefix: string,
input: BacklinksLookupInput, input: BacklinksLookupInput,
billingCustomer: BillingCustomerContext, billingCustomer: BillingCustomerContext,
options?: BacklinksSpamFilterOptions,
): Promise<string> { ): Promise<string> {
const normalizedTarget = normalizeBacklinksTarget(input.target, { const normalizedTarget = normalizeBacklinksTarget(input.target, {
scope: input.scope, scope: input.scope,
}); });
return buildCacheKey(prefix, { const cacheKeyInput = {
organizationId: billingCustomer.organizationId, organizationId: billingCustomer.organizationId,
target: normalizedTarget.apiTarget, target: normalizedTarget.apiTarget,
scope: normalizedTarget.scope, scope: normalizedTarget.scope,
};
if (!options) {
return buildCacheKey(prefix, cacheKeyInput);
}
const spamFilterOptions = normalizeBacklinksSpamFilterOptions(options);
return buildCacheKey(prefix, {
...cacheKeyInput,
hideSpam: String(spamFilterOptions.hideSpam),
...(spamFilterOptions.hideSpam
? { spamThreshold: String(spamFilterOptions.spamThreshold) }
: {}),
}); });
} }

View File

@ -10,6 +10,10 @@ import {
normalizeBacklinksTarget, normalizeBacklinksTarget,
} from "@/server/lib/dataforseoBacklinks"; } from "@/server/lib/dataforseoBacklinks";
import { createDataforseoClient } from "@/server/lib/dataforseoClient"; import { createDataforseoClient } from "@/server/lib/dataforseoClient";
import {
normalizeBacklinksSpamFilterOptions,
type BacklinksSpamFilterOptions,
} from "@/types/schemas/backlinks";
import { import {
backlinksOverviewSchema, backlinksOverviewSchema,
referringDomainRowSchema, referringDomainRowSchema,
@ -58,6 +62,7 @@ export async function profileBacklinksOverview(
cacheKey: string, cacheKey: string,
input: BacklinksLookupInput, input: BacklinksLookupInput,
billingCustomer: BillingCustomerContext, billingCustomer: BillingCustomerContext,
options?: BacklinksSpamFilterOptions,
): Promise<BacklinksOverviewProfile> { ): Promise<BacklinksOverviewProfile> {
const cachedRaw = await cache.get(cacheKey); const cachedRaw = await cache.get(cacheKey);
const cached = backlinksOverviewCacheSchema.safeParse(cachedRaw); const cached = backlinksOverviewCacheSchema.safeParse(cachedRaw);
@ -73,12 +78,16 @@ export async function profileBacklinksOverview(
const normalizedTarget = normalizeBacklinksTarget(input.target, { const normalizedTarget = normalizeBacklinksTarget(input.target, {
scope: input.scope, scope: input.scope,
}); });
const request = buildBacklinksRequest(normalizedTarget.apiTarget); const request = buildBacklinksListRequest(
normalizedTarget.apiTarget,
100,
options,
);
const dateRange = buildBacklinksDateRange(now); const dateRange = buildBacklinksDateRange(now);
const [summary, backlinks, history] = await Promise.all([ const [summary, backlinks, history] = await Promise.all([
dataforseo.backlinks.summary(request), dataforseo.backlinks.summary({ target: request.target }),
dataforseo.backlinks.rows({ ...request, limit: 100 }), dataforseo.backlinks.rows(request),
normalizedTarget.scope === "domain" normalizedTarget.scope === "domain"
? dataforseo.backlinks.history({ ? dataforseo.backlinks.history({
target: normalizedTarget.apiTarget, target: normalizedTarget.apiTarget,
@ -109,6 +118,7 @@ export async function profileReferringDomainsRows(
cacheKey: string, cacheKey: string,
input: BacklinksLookupInput, input: BacklinksLookupInput,
billingCustomer: BillingCustomerContext, billingCustomer: BillingCustomerContext,
options?: BacklinksSpamFilterOptions,
): Promise<ReferringDomainsProfile> { ): Promise<ReferringDomainsProfile> {
const cachedRaw = await cache.get(cacheKey); const cachedRaw = await cache.get(cacheKey);
const cached = referringDomainsCacheSchema.safeParse(cachedRaw); const cached = referringDomainsCacheSchema.safeParse(cachedRaw);
@ -120,13 +130,12 @@ export async function profileReferringDomainsRows(
const dataforseo = createDataforseoClient(billingCustomer); const dataforseo = createDataforseoClient(billingCustomer);
const request = buildBacklinksRequest( const request = buildBacklinksListRequest(
normalizeBacklinksTarget(input.target, { scope: input.scope }).apiTarget, normalizeBacklinksTarget(input.target, { scope: input.scope }).apiTarget,
100,
options,
); );
const response = await dataforseo.backlinks.referringDomains({ const response = await dataforseo.backlinks.referringDomains(request);
...request,
limit: 100,
});
const rows = mapReferringDomainsRows(response); const rows = mapReferringDomainsRows(response);
await cacheValue(cache, cacheKey, { rows }, BACKLINKS_TAB_TTL_SECONDS); await cacheValue(cache, cacheKey, { rows }, BACKLINKS_TAB_TTL_SECONDS);
@ -168,6 +177,18 @@ function buildBacklinksRequest(target: string): BacklinksRequest {
return { target }; return { target };
} }
function buildBacklinksListRequest(
target: string,
limit: number,
options?: BacklinksSpamFilterOptions,
) {
return {
...buildBacklinksRequest(target),
limit,
...normalizeBacklinksSpamFilterOptions(options),
};
}
function buildBacklinksDateRange(now: Date): BacklinksDateRange { function buildBacklinksDateRange(now: Date): BacklinksDateRange {
const todayUtc = new Date( const todayUtc = new Date(
Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()), Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()),

View File

@ -1,4 +1,8 @@
import { AppError } from "@/server/lib/errors"; import { AppError } from "@/server/lib/errors";
import {
normalizeBacklinksSpamFilterOptions,
type BacklinksSpamFilterOptions,
} from "@/types/schemas/backlinks";
import type { import type {
DataforseoApiCallCost, DataforseoApiCallCost,
DataforseoApiResponse, DataforseoApiResponse,
@ -24,9 +28,10 @@ export type BacklinksRequest = {
target: string; target: string;
}; };
export type BacklinksListRequest = BacklinksRequest & { export type BacklinksListRequest = BacklinksRequest &
limit?: number; BacklinksSpamFilterOptions & {
}; limit?: number;
};
export type BacklinksTimeseriesRequest = { export type BacklinksTimeseriesRequest = {
target: string; target: string;
@ -186,11 +191,16 @@ export async function fetchBacklinksSummaryRaw(input: BacklinksRequest) {
} }
export async function fetchBacklinksRowsRaw(input: BacklinksListRequest) { export async function fetchBacklinksRowsRaw(input: BacklinksListRequest) {
const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input);
const filters = spamFilterOptions.hideSpam
? [["backlink_spam_score", "<=", spamFilterOptions.spamThreshold]]
: undefined;
const response = await postBacklinks("/v3/backlinks/backlinks/live", [ const response = await postBacklinks("/v3/backlinks/backlinks/live", [
{ {
...buildCommonPayload(input), ...buildCommonPayload(input),
limit: input.limit ?? 100, limit: input.limit ?? 100,
order_by: ["rank,desc"], order_by: ["rank,desc"],
...(filters ? { filters } : {}),
}, },
]); ]);
const data = parseItems( const data = parseItems(
@ -205,11 +215,16 @@ export async function fetchBacklinksRowsRaw(input: BacklinksListRequest) {
} }
export async function fetchReferringDomainsRaw(input: BacklinksListRequest) { export async function fetchReferringDomainsRaw(input: BacklinksListRequest) {
const spamFilterOptions = normalizeBacklinksSpamFilterOptions(input);
const filters = spamFilterOptions.hideSpam
? [["backlinks_spam_score", "<=", spamFilterOptions.spamThreshold]]
: undefined;
const response = await postBacklinks("/v3/backlinks/referring_domains/live", [ const response = await postBacklinks("/v3/backlinks/referring_domains/live", [
{ {
...buildCommonPayload(input), ...buildCommonPayload(input),
limit: input.limit ?? 100, limit: input.limit ?? 100,
order_by: ["backlinks,desc"], order_by: ["backlinks,desc"],
...(filters ? { filters } : {}),
}, },
]); ]);
const data = parseItems( const data = parseItems(

View File

@ -2,7 +2,35 @@ import { z } from "zod";
export const backlinksTabSchema = z.enum(["backlinks", "domains", "pages"]); export const backlinksTabSchema = z.enum(["backlinks", "domains", "pages"]);
export const backlinksTargetScopeSchema = z.enum(["domain", "page"]); export const backlinksTargetScopeSchema = z.enum(["domain", "page"]);
export const DEFAULT_BACKLINKS_SPAM_THRESHOLD = 40;
export function normalizeBacklinksSpamThreshold(value: number) {
if (!Number.isFinite(value)) {
return DEFAULT_BACKLINKS_SPAM_THRESHOLD;
}
return Math.min(100, Math.max(0, Math.trunc(value)));
}
export type BacklinksSpamFilterOptions = {
hideSpam?: boolean;
spamThreshold?: number;
};
export function normalizeBacklinksSpamFilterOptions(
options?: BacklinksSpamFilterOptions,
) {
const hideSpam = options?.hideSpam ?? true;
return {
hideSpam,
spamThreshold: hideSpam
? normalizeBacklinksSpamThreshold(
options?.spamThreshold ?? DEFAULT_BACKLINKS_SPAM_THRESHOLD,
)
: undefined,
};
}
export const backlinksLookupSchema = z.object({ export const backlinksLookupSchema = z.object({
target: z.string().min(1, "Target is required").max(2048), target: z.string().min(1, "Target is required").max(2048),
scope: backlinksTargetScopeSchema.optional(), scope: backlinksTargetScopeSchema.optional(),
@ -12,10 +40,17 @@ export const backlinksProjectSchema = z.object({
projectId: z.string().min(1), projectId: z.string().min(1),
}); });
export const backlinksOverviewInputSchema = backlinksLookupSchema.extend({ const backlinksSpamFilterSchema = z.object({
projectId: z.string().min(1), hideSpam: z.boolean().optional(),
spamThreshold: z.number().int().min(0).max(100).optional(),
}); });
export const backlinksOverviewInputSchema = backlinksLookupSchema
.extend({
projectId: z.string().min(1),
})
.merge(backlinksSpamFilterSchema);
export const backlinksSearchSchema = z.object({ export const backlinksSearchSchema = z.object({
target: z.string().optional(), target: z.string().optional(),
scope: backlinksTargetScopeSchema.optional(), scope: backlinksTargetScopeSchema.optional(),