Shrink eager worker bundle: lazy boundaries at module seams + build-enforced guard (#366)

This commit is contained in:
Ben Senescu 2026-07-06 22:53:47 -04:00 committed by Ben Senescu
parent a337f0ac08
commit 7caaebbbac
19 changed files with 507 additions and 211 deletions

View File

@ -38,6 +38,14 @@ jobs:
- name: Run tests - name: Run tests
run: pnpm run test:ci run: pnpm run test:ci
# Runs the leanWorkerBundle generateBundle assertion: fails if a
# denylisted package (dataforseo-client, autumn-js, ...) re-enters the
# worker's eager startup graph. See vite-plugin-lean-worker-bundle.ts.
- name: Build worker (eager-bundle guard)
run: pnpm vite build
env:
NODE_OPTIONS: --max-old-space-size=4096
- name: Install website dependencies - name: Install website dependencies
run: pnpm --dir web install --frozen-lockfile run: pnpm --dir web install --frozen-lockfile

View File

@ -18,6 +18,10 @@
"src/db/pg/schema.ts", "src/db/pg/schema.ts",
// Standalone CLI/dev scripts, invoked via package.json scripts // Standalone CLI/dev scripts, invoked via package.json scripts
"scripts/**", "scripts/**",
// DataForSEO section barrel consumed via the dynamic import in
// client.ts (loadDataforseoSections) + property access, which knip
// can't trace
"src/server/lib/dataforseo/sections.ts",
], ],
"project": ["**/*.{js,mjs,ts,tsx}", "!src/routeTree.gen.ts", "!web/**"], "project": ["**/*.{js,mjs,ts,tsx}", "!src/routeTree.gen.ts", "!web/**"],
"ignore": ["drizzle-prod.config.ts"], "ignore": ["drizzle-prod.config.ts"],

View File

@ -1,14 +1,16 @@
import process from "node:process"; import process from "node:process";
import { import {
buildLlmTarget,
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
fetchLlmAggregatedMetrics, fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics, fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch, fetchLlmMentionsSearch,
fetchLlmTopPages, fetchLlmTopPages,
type LlmPlatform,
} from "@/server/lib/dataforseo/ai"; } from "@/server/lib/dataforseo/ai";
import {
buildLlmTarget,
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
type LlmPlatform,
} from "@/server/lib/dataforseo/shared";
import { applyBillingMarkupUsd } from "@/shared/billing"; import { applyBillingMarkupUsd } from "@/shared/billing";
import { resolveCompetitorGroups } from "@/server/features/ai-search/services/shareOfVoice"; import { resolveCompetitorGroups } from "@/server/features/ai-search/services/shareOfVoice";
import { parseCompetitorList } from "@/types/schemas/ai-search"; import { parseCompetitorList } from "@/types/schemas/ai-search";

View File

@ -1,10 +1,17 @@
import { createFileRoute } from "@tanstack/react-router"; import { createFileRoute } from "@tanstack/react-router";
import { autumnHandler } from "autumn-js/fetch"; import type { autumnHandler } from "autumn-js/fetch";
import { env } from "cloudflare:workers"; import { env } from "cloudflare:workers";
import { isHostedAuthMode } from "@/lib/auth-mode"; import { isHostedAuthMode } from "@/lib/auth-mode";
import { resolveHostedContext } from "@/middleware/ensure-user/hosted"; import { resolveHostedContext } from "@/middleware/ensure-user/hosted";
const handler = autumnHandler({ let handlerPromise: Promise<ReturnType<typeof autumnHandler>> | undefined;
// Lazy: keeps autumn-js/fetch out of the eager isolate startup graph;
// resolves instantly after the first request.
function loadHandler() {
return (handlerPromise ??= import("autumn-js/fetch").then(
({ autumnHandler }) =>
autumnHandler({
identify: async (request) => { identify: async (request) => {
const context = await resolveHostedContext(request.headers); const context = await resolveHostedContext(request.headers);
@ -12,16 +19,18 @@ const handler = autumnHandler({
customerId: context.organizationId, customerId: context.organizationId,
}; };
}, },
}); }),
));
}
function handleAutumnRequest(request: Request) { async function handleAutumnRequest(request: Request) {
if (!isHostedAuthMode(env.AUTH_MODE)) { if (!isHostedAuthMode(env.AUTH_MODE)) {
return new Response("Not found", { return new Response("Not found", {
status: 404, status: 404,
}); });
} }
return handler(request); return (await loadHandler())(request);
} }
export const Route = createFileRoute("/api/autumn/$")({ export const Route = createFileRoute("/api/autumn/$")({

View File

@ -1,18 +1,26 @@
import { Autumn } from "autumn-js"; import type { Autumn } from "autumn-js";
import { getRequiredEnvValue } from "@/server/lib/runtime-env"; import { getRequiredEnvValue } from "@/server/lib/runtime-env";
export const autumn = new Autumn({ let autumnPromise: Promise<Autumn> | undefined;
// Lazy: keeps the ~450 kB autumn-js SDK out of the eager isolate startup
// graph (self-hosted deployments never load it at all); resolves instantly
// after the first call.
function loadAutumn(): Promise<Autumn> {
return (autumnPromise ??= import("autumn-js").then(
({ Autumn }) =>
new Autumn({
secretKey: () => getRequiredEnvValue("AUTUMN_SECRET_KEY"), secretKey: () => getRequiredEnvValue("AUTUMN_SECRET_KEY"),
// Retries 429/500/502/503/504 (per-operation retryCodes) plus connection // Retries 429/500/502/503/504 (per-operation retryCodes) plus
// errors. Cloudflare 52x statuses are not in the SDK's retry list, so those // connection errors. Cloudflare 52x statuses are not in the SDK's
// still surface immediately. // retry list, so those still surface immediately.
// //
// These reads/gates (customers.getOrCreate, check) sit on the request hot // These reads/gates (customers.getOrCreate, check) sit on the request
// path, so keep the total retry window short: when Autumn is rate-limiting or // hot path, so keep the total retry window short: when Autumn is
// slow, an 8s backoff window held the isolate open for seconds on every // rate-limiting or slow, an 8s backoff window held the isolate open
// request and cascaded into region-wide congestion (incident 2026-07-06). // for seconds on every request and cascaded into region-wide
// Fail fast instead — a caller that can't gate surfaces an error rather than // congestion (incident 2026-07-06). Fail fast instead — a caller that
// hanging. // can't gate surfaces an error rather than hanging.
retryConfig: { retryConfig: {
strategy: "backoff", strategy: "backoff",
backoff: { backoff: {
@ -23,7 +31,23 @@ export const autumn = new Autumn({
}, },
retryConnectionErrors: true, retryConnectionErrors: true,
}, },
}); }),
));
}
/** Shape-preserving lazy facade over the SDK client: call sites keep the
* plain `autumn.check(...)` form. Covers only the methods we use add a
* line here when adopting a new one. */
export const autumn = {
check: (...args: Parameters<Autumn["check"]>) =>
loadAutumn().then((client) => client.check(...args)),
track: (...args: Parameters<Autumn["track"]>) =>
loadAutumn().then((client) => client.track(...args)),
customers: {
getOrCreate: (...args: Parameters<Autumn["customers"]["getOrCreate"]>) =>
loadAutumn().then((client) => client.customers.getOrCreate(...args)),
},
};
// track() has no idempotency key, so replaying a deduction Autumn already // track() has no idempotency key, so replaying a deduction Autumn already
// processed (5xx after a successful write, dropped connection) would // processed (5xx after a successful write, dropped connection) would

View File

@ -3,7 +3,7 @@ import {
CHATGPT_LANGUAGE_CODE, CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE, CHATGPT_LOCATION_CODE,
type LlmPlatform, type LlmPlatform,
} from "@/server/lib/dataforseo/ai"; } from "@/server/lib/dataforseo/shared";
import type { import type {
LlmAggregatedTotal, LlmAggregatedTotal,
LlmMentionItem, LlmMentionItem,

View File

@ -28,6 +28,7 @@ import {
import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification"; import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification";
import { AppError } from "@/server/lib/errors"; import { AppError } from "@/server/lib/errors";
import { aiOptimizationApi } from "@/server/lib/dataforseo/core"; import { aiOptimizationApi } from "@/server/lib/dataforseo/core";
import type { LlmPlatform, LlmTarget } from "@/server/lib/dataforseo/shared";
import { import {
assertOk, assertOk,
buildTaskBilling, buildTaskBilling,
@ -36,12 +37,6 @@ import {
type DataforseoTaskLike, type DataforseoTaskLike,
} from "@/server/lib/dataforseo/envelope"; } from "@/server/lib/dataforseo/envelope";
// ChatGPT mention/response data is only available for US/en per DataForSEO docs.
export const CHATGPT_LOCATION_CODE = 2840;
export const CHATGPT_LANGUAGE_CODE = "en";
export type LlmPlatform = "chat_gpt" | "google";
const classifyAiSearchError = createDataforseoBillingClassifier({ const classifyAiSearchError = createDataforseoBillingClassifier({
pathPrefix: "/ai_optimization/", pathPrefix: "/ai_optimization/",
billingIssueCode: "AI_SEARCH_BILLING_ISSUE", billingIssueCode: "AI_SEARCH_BILLING_ISSUE",
@ -52,45 +47,6 @@ const classifyAiSearchError = createDataforseoBillingClassifier({
const assertOptions = (path: string) => const assertOptions = (path: string) =>
({ classify: classifyAiSearchError, classifyPath: path }) as const; ({ classify: classifyAiSearchError, classifyPath: path }) as const;
// ---------------------------------------------------------------------------
// Target builders — DataForSEO's `target` array accepts domain OR keyword
// entries. We always pass exactly one target per call.
// ---------------------------------------------------------------------------
type LlmTarget =
| {
domain: string;
include_subdomains?: boolean;
search_filter?: "include" | "exclude";
search_scope?: string[];
}
| {
keyword: string;
search_filter?: "include" | "exclude";
search_scope?: string[];
match_type?: "word_match" | "partial_match";
};
export function buildLlmTarget(input: {
type: "domain" | "keyword";
value: string;
}): LlmTarget {
if (input.type === "domain") {
return {
domain: input.value,
include_subdomains: true,
search_filter: "include",
search_scope: ["any"],
};
}
return {
keyword: input.value,
search_filter: "include",
search_scope: ["any", "brand_entities"],
match_type: "word_match",
};
}
function clampLimit(value: number, min: number, max: number): number { function clampLimit(value: number, min: number, max: number): number {
return Math.min(max, Math.max(min, Math.floor(value))); return Math.min(max, Math.max(min, Math.floor(value)));
} }

View File

@ -19,8 +19,8 @@ import {
fetchBacklinksHistory, fetchBacklinksHistory,
fetchBacklinksRows, fetchBacklinksRows,
fetchBacklinksSummary, fetchBacklinksSummary,
normalizeBacklinksTarget,
} from "@/server/lib/dataforseo/backlinks"; } from "@/server/lib/dataforseo/backlinks";
import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
// A successful DataForSEO task always carries billing metadata (path + cost). // A successful DataForSEO task always carries billing metadata (path + cost).
const billed = { const billed = {

View File

@ -21,8 +21,6 @@ import {
type DataforseoApiResponse, type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope"; } from "@/server/lib/dataforseo/envelope";
export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
type BacklinksRequest = { target: string }; type BacklinksRequest = { target: string };
type BacklinksListRequest = BacklinksRequest & type BacklinksListRequest = BacklinksRequest &
BacklinksSpamFilterOptions & { BacklinksSpamFilterOptions & {

View File

@ -8,45 +8,9 @@ import {
trackUsageCreditSpend, trackUsageCreditSpend,
} from "@/server/billing/subscription"; } from "@/server/billing/subscription";
import type { BillingCustomerContext } from "@/server/billing/subscription"; import type { BillingCustomerContext } from "@/server/billing/subscription";
import { // Type-only namespace import: erased at compile, so the section modules (and
fetchBusinessListingsSearch, // the SDK they pull in) still only load through loadDataforseoSections below.
fetchQuestionsAnswers, import type * as sections from "@/server/lib/dataforseo/sections";
} from "@/server/lib/dataforseo/business";
import {
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
fetchDomainPagesSummary,
fetchReferringDomains,
} from "@/server/lib/dataforseo/backlinks";
import {
fetchDomainRankOverview,
fetchKeywordIdeas,
fetchKeywordOverview,
fetchKeywordSuggestions,
fetchRankedKeywords,
fetchRelatedKeywords,
fetchRelevantPages,
fetchSerpCompetitors,
} from "@/server/lib/dataforseo/labs";
import {
fetchAdsKeywordIdeas,
fetchAdsSearchVolume,
} from "@/server/lib/dataforseo/google-ads";
import {
fetchLiveSerp,
fetchLocalSerp,
fetchRankCheckSerp,
postRankCheckTasks,
} from "@/server/lib/dataforseo/serp";
import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
import {
fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";
import { import {
DataforseoChargedTaskError, DataforseoChargedTaskError,
type DataforseoApiCallCost, type DataforseoApiCallCost,
@ -57,10 +21,24 @@ import { AppError } from "@/server/lib/errors";
export { mapDataforseoPathToCreditFeature }; export { mapDataforseoPathToCreditFeature };
/** The section-fetcher barrel (sections.ts), as a type for `meter` pickers. */
export type DataforseoSections = typeof sections;
let sectionsPromise: Promise<DataforseoSections> | undefined;
/** Single lazy boundary for the DataForSEO subtree: the section fetchers and
* the ~3 MB dataforseo-client SDK they statically import stay out of the
* eager isolate startup graph and load once, on the first API call. */
export function loadDataforseoSections(): Promise<DataforseoSections> {
return (sectionsPromise ??= import("@/server/lib/dataforseo/sections"));
}
/** /**
* Wraps a section fetcher with billing metering. Each entry on the client is * Wraps a section fetcher with billing metering. Each entry on the client is
* `meter(customer, fetcher, defaultFeature?)`, which returns a function with the * `meter(customer, (s) => s.fetchX, defaultFeature?)`, which returns a function
* fetcher's own input type and resolves to its unwrapped `.data`. * with the fetcher's own input type and resolves to its unwrapped `.data`. The
* picker indirection (rather than the fetcher itself) keeps the section
* modules behind loadDataforseoSections.
* *
* `defaultFeature` is the fallback credit feature; a caller can override it per * `defaultFeature` is the fallback credit feature; a caller can override it per
* call by passing `creditFeature` in the input (e.g. an MCP tool attributing * call by passing `creditFeature` in the input (e.g. an MCP tool attributing
@ -69,13 +47,15 @@ export { mapDataforseoPathToCreditFeature };
*/ */
function meter<I, T>( function meter<I, T>(
customer: BillingCustomerContext, customer: BillingCustomerContext,
fetcher: (input: I) => Promise<DataforseoApiResponse<T>>, pick: (
sections: DataforseoSections,
) => (input: I) => Promise<DataforseoApiResponse<T>>,
defaultFeature?: CreditFeature, defaultFeature?: CreditFeature,
): (input: I & { creditFeature?: CreditFeature }) => Promise<T> { ): (input: I & { creditFeature?: CreditFeature }) => Promise<T> {
return (input) => return (input) =>
meterDataforseoCall( meterDataforseoCall(
customer, customer,
() => fetcher(input), async () => pick(await loadDataforseoSections())(input),
input.creditFeature ?? defaultFeature, input.creditFeature ?? defaultFeature,
); );
} }
@ -85,56 +65,71 @@ export function createDataforseoClient(customer: BillingCustomerContext) {
business: { business: {
businessListings: meter( businessListings: meter(
customer, customer,
fetchBusinessListingsSearch, (s) => s.fetchBusinessListingsSearch,
"local_seo",
),
questionsAnswers: meter(
customer,
(s) => s.fetchQuestionsAnswers,
"local_seo", "local_seo",
), ),
questionsAnswers: meter(customer, fetchQuestionsAnswers, "local_seo"),
}, },
backlinks: { backlinks: {
summary: meter(customer, fetchBacklinksSummary), summary: meter(customer, (s) => s.fetchBacklinksSummary),
rows: meter(customer, fetchBacklinksRows), rows: meter(customer, (s) => s.fetchBacklinksRows),
referringDomains: meter(customer, fetchReferringDomains), referringDomains: meter(customer, (s) => s.fetchReferringDomains),
domainPages: meter(customer, fetchDomainPagesSummary), domainPages: meter(customer, (s) => s.fetchDomainPagesSummary),
history: meter(customer, fetchBacklinksHistory), history: meter(customer, (s) => s.fetchBacklinksHistory),
}, },
keywords: { keywords: {
related: meter(customer, fetchRelatedKeywords), related: meter(customer, (s) => s.fetchRelatedKeywords),
suggestions: meter(customer, fetchKeywordSuggestions), suggestions: meter(customer, (s) => s.fetchKeywordSuggestions),
ideas: meter(customer, fetchKeywordIdeas), ideas: meter(customer, (s) => s.fetchKeywordIdeas),
// Google Ads endpoints for countries Labs doesn't support. // Google Ads endpoints for countries Labs doesn't support.
adsIdeas: meter(customer, fetchAdsKeywordIdeas), adsIdeas: meter(customer, (s) => s.fetchAdsKeywordIdeas),
adsSearchVolume: meter(customer, fetchAdsSearchVolume), adsSearchVolume: meter(customer, (s) => s.fetchAdsSearchVolume),
}, },
domain: { domain: {
rankOverview: meter(customer, fetchDomainRankOverview), rankOverview: meter(customer, (s) => s.fetchDomainRankOverview),
rankedKeywords: meter(customer, fetchRankedKeywords), rankedKeywords: meter(customer, (s) => s.fetchRankedKeywords),
relevantPages: meter(customer, fetchRelevantPages), relevantPages: meter(customer, (s) => s.fetchRelevantPages),
}, },
serp: { serp: {
live: meter(customer, fetchLiveSerp), live: meter(customer, (s) => s.fetchLiveSerp),
rankCheck: meter(customer, fetchRankCheckSerp, "rank_tracking"), rankCheck: meter(customer, (s) => s.fetchRankCheckSerp, "rank_tracking"),
// Posts up to 100 queued rank check tasks; one metered charge covers the // Posts up to 100 queued rank check tasks; one metered charge covers the
// whole batch (DataForSEO bills task_post at post time, collection is // whole batch (DataForSEO bills task_post at post time, collection is
// free). // free).
rankCheckTaskPost: meter(customer, postRankCheckTasks, "rank_tracking"), rankCheckTaskPost: meter(
local: meter(customer, fetchLocalSerp, "local_seo"), customer,
(s) => s.postRankCheckTasks,
"rank_tracking",
),
local: meter(customer, (s) => s.fetchLocalSerp, "local_seo"),
}, },
labs: { labs: {
// Callers (e.g. the keyword-metrics MCP tool) can attribute the spend to // Callers (e.g. the keyword-metrics MCP tool) can attribute the spend to
// their own feature by passing `creditFeature` in the input; defaults to // their own feature by passing `creditFeature` in the input; defaults to
// rank_tracking when omitted. // rank_tracking when omitted.
keywordOverview: meter(customer, fetchKeywordOverview, "rank_tracking"), keywordOverview: meter(
serpCompetitors: meter(customer, fetchSerpCompetitors), customer,
(s) => s.fetchKeywordOverview,
"rank_tracking",
),
serpCompetitors: meter(customer, (s) => s.fetchSerpCompetitors),
}, },
lighthouse: { lighthouse: {
live: meter(customer, fetchLighthouseResult), live: meter(customer, (s) => s.fetchLighthouseResult),
}, },
aiSearch: { aiSearch: {
mentionsSearch: meter(customer, fetchLlmMentionsSearch), mentionsSearch: meter(customer, (s) => s.fetchLlmMentionsSearch),
aggregatedMetrics: meter(customer, fetchLlmAggregatedMetrics), aggregatedMetrics: meter(customer, (s) => s.fetchLlmAggregatedMetrics),
topPages: meter(customer, fetchLlmTopPages), topPages: meter(customer, (s) => s.fetchLlmTopPages),
crossAggregatedMetrics: meter(customer, fetchLlmCrossAggregatedMetrics), crossAggregatedMetrics: meter(
llmResponse: meter(customer, fetchLlmResponse), customer,
(s) => s.fetchLlmCrossAggregatedMetrics,
),
llmResponse: meter(customer, (s) => s.fetchLlmResponse),
}, },
} as const; } as const;
} }

View File

@ -6,13 +6,13 @@ vi.mock("@/server/lib/runtime-env", () => ({
import { fetchQuestionsAnswers } from "@/server/lib/dataforseo/business"; import { fetchQuestionsAnswers } from "@/server/lib/dataforseo/business";
import { import {
buildLlmTarget,
fetchLlmAggregatedMetrics, fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics, fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch, fetchLlmMentionsSearch,
fetchLlmResponse, fetchLlmResponse,
fetchLlmTopPages, fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai"; } from "@/server/lib/dataforseo/ai";
import { buildLlmTarget } from "@/server/lib/dataforseo/shared";
function parseDataforseoRequestBody(init: RequestInit | undefined): unknown { function parseDataforseoRequestBody(init: RequestInit | undefined): unknown {
const body = init?.body; const body = init?.body;

View File

@ -1,7 +1,15 @@
// Public surface of the DataForSEO integration. Internals live in the // Public surface of the DataForSEO integration. Internals live in the
// per-section files (labs / serp / business / backlinks / ai / // per-section files (labs / serp / business / backlinks / ai / lighthouse),
// lighthouse); everything funnels through envelope.ts (status + billing) and is // which sit behind the single dynamic import in client.ts so the ~3 MB SDK
// metered in client.ts. // loads lazily; everything funnels through envelope.ts (status + billing) and
// is metered in client.ts. Runtime values re-exported here must be SDK-free
// (shared.ts) or lazy — a static value re-export from a section file would
// drag the SDK back into the eager isolate startup graph.
import {
loadDataforseoSections,
type DataforseoSections,
} from "@/server/lib/dataforseo/client";
export { createDataforseoClient } from "@/server/lib/dataforseo/client"; export { createDataforseoClient } from "@/server/lib/dataforseo/client";
@ -10,35 +18,41 @@ export {
type KeywordMetricRow, type KeywordMetricRow,
} from "@/server/lib/dataforseo/keyword-metrics"; } from "@/server/lib/dataforseo/keyword-metrics";
export {
type LabsKeywordDataItem,
type DomainRankedKeywordItem,
type RelevantPagesItem,
} from "@/server/lib/dataforseo/labs";
export { type AdsKeywordIdeaItem } from "@/server/lib/dataforseo/google-ads";
export {
fetchRankCheckTaskResult,
MAX_TASKS_PER_POST,
type SerpLiveItem,
type RankCheckResult,
type RankCheckTaskInput,
type PostedRankCheckTask,
} from "@/server/lib/dataforseo/serp";
export {
normalizeBacklinksTarget,
type BacklinksSummaryItem,
type BacklinksItem,
type ReferringDomainItem,
type DomainPageSummaryItem,
type BacklinksHistoryItem,
} from "@/server/lib/dataforseo/backlinks";
export { export {
buildLlmTarget, buildLlmTarget,
CHATGPT_LANGUAGE_CODE, CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE, CHATGPT_LOCATION_CODE,
MAX_TASKS_PER_POST,
type LlmPlatform, type LlmPlatform,
} from "@/server/lib/dataforseo/ai"; } from "@/server/lib/dataforseo/shared";
export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
/** Lazy wrapper for the one section fetcher called outside the metered client
* (rank-check task collection is free at DataForSEO, so it skips metering). */
export const fetchRankCheckTaskResult: DataforseoSections["fetchRankCheckTaskResult"] =
async (input) =>
(await loadDataforseoSections()).fetchRankCheckTaskResult(input);
export type {
LabsKeywordDataItem,
DomainRankedKeywordItem,
RelevantPagesItem,
} from "@/server/lib/dataforseo/labs";
export type { AdsKeywordIdeaItem } from "@/server/lib/dataforseo/google-ads";
export type {
SerpLiveItem,
RankCheckResult,
RankCheckTaskInput,
PostedRankCheckTask,
} from "@/server/lib/dataforseo/serp";
export type {
BacklinksSummaryItem,
BacklinksItem,
ReferringDomainItem,
DomainPageSummaryItem,
BacklinksHistoryItem,
} from "@/server/lib/dataforseo/backlinks";

View File

@ -0,0 +1,54 @@
// Root of the lazily loaded DataForSEO subtree. The section fetchers — and
// the ~3 MB dataforseo-client SDK they statically import — are reached only
// through the single dynamic import in client.ts (loadDataforseoSections), so
// the whole subtree lands in one lazy chunk outside the eager isolate startup
// graph. Never import this barrel or a section file statically from eager
// server code; the leanWorkerBundle vite plugin fails the build if the SDK
// re-enters the eager graph. SDK-free values live in shared.ts instead.
export {
fetchBusinessListingsSearch,
fetchQuestionsAnswers,
} from "@/server/lib/dataforseo/business";
export {
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
fetchDomainPagesSummary,
fetchReferringDomains,
} from "@/server/lib/dataforseo/backlinks";
export {
fetchDomainRankOverview,
fetchKeywordIdeas,
fetchKeywordOverview,
fetchKeywordSuggestions,
fetchRankedKeywords,
fetchRelatedKeywords,
fetchRelevantPages,
fetchSerpCompetitors,
} from "@/server/lib/dataforseo/labs";
export {
fetchAdsKeywordIdeas,
fetchAdsSearchVolume,
} from "@/server/lib/dataforseo/google-ads";
export {
fetchLiveSerp,
fetchLocalSerp,
fetchRankCheckSerp,
fetchRankCheckTaskResult,
postRankCheckTasks,
} from "@/server/lib/dataforseo/serp";
export { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
export {
fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";

View File

@ -7,6 +7,7 @@ import {
SerpGoogleOrganicTaskPostRequestInfo, SerpGoogleOrganicTaskPostRequestInfo,
} from "dataforseo-client"; } from "dataforseo-client";
import { serpApi } from "@/server/lib/dataforseo/core"; import { serpApi } from "@/server/lib/dataforseo/core";
import { MAX_TASKS_PER_POST } from "@/server/lib/dataforseo/shared";
import { import {
assertOk, assertOk,
buildTaskBilling, buildTaskBilling,
@ -184,9 +185,6 @@ export async function fetchRankCheckSerp(input: {
// stragglers, orchestrated by the rank check workflow. // stragglers, orchestrated by the rank check workflow.
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
/** Max tasks DataForSEO accepts in a single task_post request. */
export const MAX_TASKS_PER_POST = 100;
export interface RankCheckTaskInput { export interface RankCheckTaskInput {
keyword: string; keyword: string;
keywordId: string; keywordId: string;

View File

@ -0,0 +1,50 @@
// SDK-free constants and target builders shared between eager server code
// (features, workflows, MCP tools) and the lazily loaded section fetchers.
// Keep this module free of dataforseo-client and section-file imports —
// anything imported from here must be safe to evaluate in the eager isolate
// startup graph.
// ChatGPT mention/response data is only available for US/en per DataForSEO docs.
export const CHATGPT_LOCATION_CODE = 2840;
export const CHATGPT_LANGUAGE_CODE = "en";
export type LlmPlatform = "chat_gpt" | "google";
/** Max tasks DataForSEO accepts in a single task_post request. */
export const MAX_TASKS_PER_POST = 100;
// DataForSEO's LLM-mentions `target` array accepts domain OR keyword entries.
// We always pass exactly one target per call.
export type LlmTarget =
| {
domain: string;
include_subdomains?: boolean;
search_filter?: "include" | "exclude";
search_scope?: string[];
}
| {
keyword: string;
search_filter?: "include" | "exclude";
search_scope?: string[];
match_type?: "word_match" | "partial_match";
};
export function buildLlmTarget(input: {
type: "domain" | "keyword";
value: string;
}): LlmTarget {
if (input.type === "domain") {
return {
domain: input.value,
include_subdomains: true,
search_filter: "include",
search_scope: ["any"],
};
}
return {
keyword: input.value,
search_filter: "include",
search_scope: ["any", "brand_entities"],
match_type: "word_match",
};
}

View File

@ -1,5 +1,6 @@
/** /**
* Build-time stand-in for `just-bash`, wired up via the vite.config.ts alias. * Build-time stand-in for `just-bash`, wired up via the leanWorkerBundle
* plugin's alias (vite-plugin-lean-worker-bundle.ts).
* *
* `@cloudflare/think` eagerly imports just-bash (~21 MB of source, plus * `@cloudflare/think` eagerly imports just-bash (~21 MB of source, plus
* turndown and the 8.6 MB @mixmark-io/domino DOM implementation) at module * turndown and the 8.6 MB @mixmark-io/domino DOM implementation) at module
@ -14,7 +15,8 @@
* @cloudflare/think loads just-bash lazily. * @cloudflare/think loads just-bash lazily.
*/ */
const STUBBED_MESSAGE = const STUBBED_MESSAGE =
"just-bash is stubbed out of the worker bundle (see vite.config.ts); " + "just-bash is stubbed out of the worker bundle (see " +
"vite-plugin-lean-worker-bundle.ts); " +
"the workspace bash tool is disabled for this deployment"; "the workspace bash tool is disabled for this deployment";
// Covers every named import in the dependency graph: `Bash` (Think's // Covers every named import in the dependency graph: `Bash` (Think's

View File

@ -0,0 +1,30 @@
/**
* Build-time stand-in for `workers-ai-provider` (and its /anthropic and
* /openai subpaths), wired up via the leanWorkerBundle vite plugin.
*
* `@cloudflare/think` eagerly imports all three (~540 kB with the @ai-sdk/
* openai + @ai-sdk/anthropic providers they pull in) solely to build its lazy
* default provider in `resolveModel()` which only runs when `getModel()`
* returns a bare model-id string. Both our chat agents (SamChatAgent,
* OnboardingChatAgent) return a constructed OpenRouter LanguageModel instead,
* so the default provider is dead code; this stub keeps it out of the eager
* worker bundle. Same pattern as just-bash-stub.ts.
*/
const STUBBED_MESSAGE =
"workers-ai-provider is stubbed out of the worker bundle to keep it out of " +
"the eager isolate startup graph (see vite-plugin-lean-worker-bundle.ts); " +
"SamChatAgent/OnboardingChatAgent override getModel with OpenRouter";
// Covers every named import in the dependency graph: `createWorkersAI` (root),
// `anthropic` (/anthropic) and `openai` (/openai).
export function createWorkersAI(): never {
throw new Error(STUBBED_MESSAGE);
}
export function anthropic(): never {
throw new Error(STUBBED_MESSAGE);
}
export function openai(): never {
throw new Error(STUBBED_MESSAGE);
}

View File

@ -0,0 +1,168 @@
import { fileURLToPath } from "node:url";
import type { Plugin } from "vite";
// Literal `new URL(..., import.meta.url)` per stub (rather than a path
// helper) so knip sees the stub files as used.
const JUST_BASH_STUB = fileURLToPath(
new URL("./src/server/lib/just-bash-stub.ts", import.meta.url),
);
const WORKERS_AI_PROVIDER_STUB = fileURLToPath(
new URL("./src/server/lib/workers-ai-provider-stub.ts", import.meta.url),
);
/**
* Dependencies that must never be reachable from the worker's eager startup
* module graph. The 128 MB isolate limit is shared by everything evaluated at
* startup (production OOM bursts trace back to baseline heap, not leaks), so
* each of these is either loaded lazily behind a dynamic import or stubbed
* out. `generateBundle` below fails the build if one sneaks back in via a
* static import chain e.g. an eager `import { fetchLiveSerp } from
* "@/server/lib/dataforseo/serp"` instead of going through the metered client.
*/
const EAGER_DENYLIST: Array<{ pattern: RegExp; expected: string }> = [
{
pattern: /node_modules\/dataforseo-client\//,
expected:
"lazy-loaded behind loadDataforseoSections() — eager code must go " +
"through the metered client or src/server/lib/dataforseo/shared.ts",
},
{
pattern: /node_modules\/autumn-js\//,
expected:
"lazy-loaded behind the facade in src/server/billing/autumn.ts and " +
"the /api/autumn route's lazy handler",
},
{
pattern:
/node_modules\/(workers-ai-provider|@ai-sdk\/(openai|anthropic))\//,
expected:
"aliased to workers-ai-provider-stub.ts (@cloudflare/think's default " +
"provider path is dead code — our agents construct OpenRouter models)",
},
{
pattern: /node_modules\/just-bash\//,
expected:
"aliased to just-bash-stub.ts (Think's workspace bash tool is disabled)",
},
{
// The barrel (index.*) is allowed — the load hook rewrites its content to
// re-export English only — as is en.* itself. Every other locale module
// must stay out; if the load-hook swap ever regresses, the real barrel
// pulls the individual locale files back in and they match here.
pattern: /node_modules\/zod\/v4\/locales\/(?!en\.|index\.)/,
expected:
"replaced with an en-only barrel via the load hook below (we never " +
"localize zod errors)",
},
{
// Pre-existing boundary from the site-audit engine, guarded here too.
pattern: /node_modules\/cheerio\//,
expected:
"lazy-loaded behind the page-analyzer dynamic import " +
"(site-audit-workflow-helpers.ts)",
},
];
/**
* Keeps the worker's eager server bundle lean, in three parts:
*
* 1. `resolve.alias` stubs for packages that are pure dead weight (dead code
* paths in @cloudflare/think).
* 2. A `load`-hook swap of zod v4's all-languages locales barrel for an
* en-only barrel (the barrel is imported via relative specifiers inside
* zod itself, and pre-enforced resolvers in this plugin stack win the
* resolveId race, so matching the resolved file path in `load` is the
* reliable seam).
* 3. A `generateBundle` assertion that walks the static-import closure of the
* worker entry chunk and fails the build if any EAGER_DENYLIST module is
* reachable turning "we verified the chunk by grepping a sourcemap once"
* into a permanent regression test.
*/
export function leanWorkerBundle(): Plugin {
return {
name: "lean-worker-bundle",
config() {
return {
resolve: {
alias: {
// Rationale for each stub lives in its docblock. TODO: remove the
// just-bash workaround once @cloudflare/think stops eagerly
// importing it (https://github.com/cloudflare/agents/issues/1673).
"just-bash": JUST_BASH_STUB,
// Subpaths must precede the bare specifier, which would otherwise
// prefix-match them.
"workers-ai-provider/anthropic": WORKERS_AI_PROVIDER_STUB,
"workers-ai-provider/openai": WORKERS_AI_PROVIDER_STUB,
"workers-ai-provider": WORKERS_AI_PROVIDER_STUB,
},
},
};
},
load(id) {
// zod v4's core/classic entrypoints re-export the full locales barrel
// (`export * as locales from "../locales/index.js"`, every language,
// ~208 kB) into the eager bundle. Zod's default English error map
// imports `../locales/en.js` directly and bypasses the barrel, so only
// `z.locales.<lang>` consumers need it — and we never localize zod
// errors. Serve an en-only barrel in its place so `z.locales.en` keeps
// working and the other ~40 languages tree-shake away. (`./en.js`
// resolves relative to the real barrel path, so no stub file needed.)
const barrel = id.match(
/node_modules\/zod\/v4\/locales\/index\.(js|cjs)$/,
);
if (barrel) {
return `export { default as en } from "./en.${barrel[1]}";`;
}
},
generateBundle(_options, bundle) {
// Only the worker build matters for isolate memory; the client bundle
// never contains these packages (and the zod swap applies everywhere).
if (this.environment.name !== "ssr") return;
// Bundle keys are chunk fileNames already.
const chunkAt = (fileName: string) => {
const output = bundle[fileName];
return output?.type === "chunk" ? output : undefined;
};
// Static-import closure from the entry chunks: everything here is
// evaluated at isolate startup. Dynamic imports are excluded — landing
// there is the point of the lazy boundaries.
const eager = new Set<string>();
const queue = Object.values(bundle)
.filter((output) => output.type === "chunk" && output.isEntry)
.map((chunk) => chunk.fileName);
while (queue.length > 0) {
const fileName = queue.pop();
if (fileName === undefined || eager.has(fileName)) continue;
eager.add(fileName);
queue.push(...(chunkAt(fileName)?.imports ?? []));
}
const violations: string[] = [];
for (const rule of EAGER_DENYLIST) {
for (const fileName of eager) {
const hits =
chunkAt(fileName)?.moduleIds.filter((id) =>
rule.pattern.test(id),
) ?? [];
if (hits.length > 0) {
violations.push(
`${hits[0]}${hits.length > 1 ? ` (+${hits.length - 1} more modules)` : ""}\n` +
` reached the eager startup graph via chunk ${fileName}\n` +
` expected: ${rule.expected}`,
);
}
}
}
if (violations.length > 0) {
this.error(
`[lean-worker-bundle] denylisted module(s) in the worker's eager ` +
`startup graph:\n\n${violations.join("\n\n")}\n\nRestore the ` +
`lazy/stub boundary, or update EAGER_DENYLIST in ` +
`vite-plugin-lean-worker-bundle.ts if this is intentional.`,
);
}
},
};
}

View File

@ -1,4 +1,3 @@
import { fileURLToPath } from "node:url";
import { tanstackStart } from "@tanstack/react-start/plugin/vite"; import { tanstackStart } from "@tanstack/react-start/plugin/vite";
import { defineConfig, loadEnv } from "vite"; import { defineConfig, loadEnv } from "vite";
import tsConfigPaths from "vite-tsconfig-paths"; import tsConfigPaths from "vite-tsconfig-paths";
@ -6,6 +5,7 @@ import viteReact from "@vitejs/plugin-react";
import tailwindcss from "@tailwindcss/vite"; import tailwindcss from "@tailwindcss/vite";
import { cloudflare } from "@cloudflare/vite-plugin"; import { cloudflare } from "@cloudflare/vite-plugin";
import { devtools } from "@tanstack/devtools-vite"; import { devtools } from "@tanstack/devtools-vite";
import { leanWorkerBundle } from "./vite-plugin-lean-worker-bundle";
export default defineConfig(({ mode }) => { export default defineConfig(({ mode }) => {
const env = loadEnv(mode, process.cwd(), ""); const env = loadEnv(mode, process.cwd(), "");
@ -22,23 +22,6 @@ export default defineConfig(({ mode }) => {
const emitSourcemaps = env.POSTHOG_SOURCEMAPS === "true"; const emitSourcemaps = env.POSTHOG_SOURCEMAPS === "true";
return { return {
resolve: {
alias: {
// TODO: Remove this workaround once @cloudflare/think stops eagerly
// importing just-bash at module init
// (https://github.com/cloudflare/agents/issues/1673).
//
// just-bash (plus its turndown → @mixmark-io/domino chain, ~30 MB of
// source) is only used by Think's workspace bash tool, which SAM
// disables — but the eager import drags it into the main worker's
// startup module graph, inflating every isolate's baseline heap
// toward the 128 MB limit (production OOM bursts on unrelated
// routes). Alias it to a throwing stub so it never ships.
"just-bash": fileURLToPath(
new URL("./src/server/lib/just-bash-stub.ts", import.meta.url),
),
},
},
envPrefix: [ envPrefix: [
"VITE_", "VITE_",
"AUTH_MODE", "AUTH_MODE",
@ -60,6 +43,7 @@ export default defineConfig(({ mode }) => {
outDir: emitSourcemaps ? "dist-sourcemaps" : "dist", outDir: emitSourcemaps ? "dist-sourcemaps" : "dist",
}, },
plugins: [ plugins: [
leanWorkerBundle(),
showDevtools showDevtools
? devtools({ ? devtools({
consolePiping: { consolePiping: {