admin/metatron-open-seo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Shrink eager worker bundle: lazy boundaries at module seams + build-enforced guard (#366)

Browse Source
This commit is contained in:
Ben Senescu 2026-07-06 22:53:47 -04:00 committed by Ben Senescu
parent a337f0ac08
commit 7caaebbbac
19 changed files with 507 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/ci.yml vendored
View File

@ -38,6 +38,14 @@ jobs:
- name: Run tests
run: pnpm run test:ci
# Runs the leanWorkerBundle generateBundle assertion: fails if a
# denylisted package (dataforseo-client, autumn-js, ...) re-enters the
# worker's eager startup graph. See vite-plugin-lean-worker-bundle.ts.
- name: Build worker (eager-bundle guard)
run: pnpm vite build
env:
NODE_OPTIONS: --max-old-space-size=4096
- name: Install website dependencies
run: pnpm --dir web install --frozen-lockfile

4
knip.jsonc
View File

@ -18,6 +18,10 @@
"src/db/pg/schema.ts",
// Standalone CLI/dev scripts, invoked via package.json scripts
"scripts/**",
// DataForSEO section barrel — consumed via the dynamic import in
// client.ts (loadDataforseoSections) + property access, which knip
// can't trace
"src/server/lib/dataforseo/sections.ts",
],
"project": ["**/*.{js,mjs,ts,tsx}", "!src/routeTree.gen.ts", "!web/**"],
"ignore": ["drizzle-prod.config.ts"],

10
scripts/brand-lookup-cost-profile.ts
View File

@ -1,14 +1,16 @@
import process from "node:process";
import {
buildLlmTarget,
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmTopPages,
type LlmPlatform,
} from "@/server/lib/dataforseo/ai";
import {
buildLlmTarget,
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
type LlmPlatform,
} from "@/server/lib/dataforseo/shared";
import { applyBillingMarkupUsd } from "@/shared/billing";
import { resolveCompetitorGroups } from "@/server/features/ai-search/services/shareOfVoice";
import { parseCompetitorList } from "@/types/schemas/ai-search";

31
src/routes/api/autumn/$.ts
View File

@ -1,27 +1,36 @@
import { createFileRoute } from "@tanstack/react-router";
import { autumnHandler } from "autumn-js/fetch";
import type { autumnHandler } from "autumn-js/fetch";
import { env } from "cloudflare:workers";
import { isHostedAuthMode } from "@/lib/auth-mode";
import { resolveHostedContext } from "@/middleware/ensure-user/hosted";
const handler = autumnHandler({
identify: async (request) => {
const context = await resolveHostedContext(request.headers);
let handlerPromise: Promise<ReturnType<typeof autumnHandler>> | undefined;
return {
customerId: context.organizationId,
};
},
});
// Lazy: keeps autumn-js/fetch out of the eager isolate startup graph;
// resolves instantly after the first request.
function loadHandler() {
return (handlerPromise ??= import("autumn-js/fetch").then(
({ autumnHandler }) =>
autumnHandler({
identify: async (request) => {
const context = await resolveHostedContext(request.headers);
function handleAutumnRequest(request: Request) {
return {
customerId: context.organizationId,
};
},
}),
));
}
async function handleAutumnRequest(request: Request) {
if (!isHostedAuthMode(env.AUTH_MODE)) {
return new Response("Not found", {
status: 404,
});
}
return handler(request);
return (await loadHandler())(request);
}
export const Route = createFileRoute("/api/autumn/$")({

70
src/server/billing/autumn.ts
View File

@ -1,29 +1,53 @@
import { Autumn } from "autumn-js";
import type { Autumn } from "autumn-js";
import { getRequiredEnvValue } from "@/server/lib/runtime-env";
export const autumn = new Autumn({
secretKey: () => getRequiredEnvValue("AUTUMN_SECRET_KEY"),
// Retries 429/500/502/503/504 (per-operation retryCodes) plus connection
// errors. Cloudflare 52x statuses are not in the SDK's retry list, so those
// still surface immediately.
//
// These reads/gates (customers.getOrCreate, check) sit on the request hot
// path, so keep the total retry window short: when Autumn is rate-limiting or
// slow, an 8s backoff window held the isolate open for seconds on every
// request and cascaded into region-wide congestion (incident 2026-07-06).
// Fail fast instead — a caller that can't gate surfaces an error rather than
// hanging.
retryConfig: {
strategy: "backoff",
backoff: {
initialInterval: 250,
maxInterval: 1000,
exponent: 1.5,
maxElapsedTime: 2500,
},
retryConnectionErrors: true,
let autumnPromise: Promise<Autumn> | undefined;
// Lazy: keeps the ~450 kB autumn-js SDK out of the eager isolate startup
// graph (self-hosted deployments never load it at all); resolves instantly
// after the first call.
function loadAutumn(): Promise<Autumn> {
return (autumnPromise ??= import("autumn-js").then(
({ Autumn }) =>
new Autumn({
secretKey: () => getRequiredEnvValue("AUTUMN_SECRET_KEY"),
// Retries 429/500/502/503/504 (per-operation retryCodes) plus
// connection errors. Cloudflare 52x statuses are not in the SDK's
// retry list, so those still surface immediately.
//
// These reads/gates (customers.getOrCreate, check) sit on the request
// hot path, so keep the total retry window short: when Autumn is
// rate-limiting or slow, an 8s backoff window held the isolate open
// for seconds on every request and cascaded into region-wide
// congestion (incident 2026-07-06). Fail fast instead — a caller that
// can't gate surfaces an error rather than hanging.
retryConfig: {
strategy: "backoff",
backoff: {
initialInterval: 250,
maxInterval: 1000,
exponent: 1.5,
maxElapsedTime: 2500,
},
retryConnectionErrors: true,
},
}),
));
}
/** Shape-preserving lazy facade over the SDK client: call sites keep the
* plain `autumn.check(...)` form. Covers only the methods we use — add a
* line here when adopting a new one. */
export const autumn = {
check: (...args: Parameters<Autumn["check"]>) =>
loadAutumn().then((client) => client.check(...args)),
track: (...args: Parameters<Autumn["track"]>) =>
loadAutumn().then((client) => client.track(...args)),
customers: {
getOrCreate: (...args: Parameters<Autumn["customers"]["getOrCreate"]>) =>
loadAutumn().then((client) => client.customers.getOrCreate(...args)),
},
});
};
// track() has no idempotency key, so replaying a deduction Autumn already
// processed (5xx after a successful write, dropped connection) would

2
src/server/features/ai-search/services/brandLookupShaping.ts
View File

@ -3,7 +3,7 @@ import {
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
type LlmPlatform,
} from "@/server/lib/dataforseo/ai";
} from "@/server/lib/dataforseo/shared";
import type {
LlmAggregatedTotal,
LlmMentionItem,

46
src/server/lib/dataforseo/ai.ts
View File

@ -28,6 +28,7 @@ import {
import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification";
import { AppError } from "@/server/lib/errors";
import { aiOptimizationApi } from "@/server/lib/dataforseo/core";
import type { LlmPlatform, LlmTarget } from "@/server/lib/dataforseo/shared";
import {
assertOk,
buildTaskBilling,
@ -36,12 +37,6 @@ import {
type DataforseoTaskLike,
} from "@/server/lib/dataforseo/envelope";
// ChatGPT mention/response data is only available for US/en per DataForSEO docs.
export const CHATGPT_LOCATION_CODE = 2840;
export const CHATGPT_LANGUAGE_CODE = "en";
export type LlmPlatform = "chat_gpt" | "google";
const classifyAiSearchError = createDataforseoBillingClassifier({
pathPrefix: "/ai_optimization/",
billingIssueCode: "AI_SEARCH_BILLING_ISSUE",
@ -52,45 +47,6 @@ const classifyAiSearchError = createDataforseoBillingClassifier({
const assertOptions = (path: string) =>
({ classify: classifyAiSearchError, classifyPath: path }) as const;
// ---------------------------------------------------------------------------
// Target builders — DataForSEO's `target` array accepts domain OR keyword
// entries. We always pass exactly one target per call.
// ---------------------------------------------------------------------------
type LlmTarget =
| {
domain: string;
include_subdomains?: boolean;
search_filter?: "include" | "exclude";
search_scope?: string[];
}
| {
keyword: string;
search_filter?: "include" | "exclude";
search_scope?: string[];
match_type?: "word_match" | "partial_match";
};
export function buildLlmTarget(input: {
type: "domain" | "keyword";
value: string;
}): LlmTarget {
if (input.type === "domain") {
return {
domain: input.value,
include_subdomains: true,
search_filter: "include",
search_scope: ["any"],
};
}
return {
keyword: input.value,
search_filter: "include",
search_scope: ["any", "brand_entities"],
match_type: "word_match",
};
}
function clampLimit(value: number, min: number, max: number): number {
return Math.min(max, Math.max(min, Math.floor(value)));
}

2
src/server/lib/dataforseo/backlinks.test.ts
View File

@ -19,8 +19,8 @@ import {
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
normalizeBacklinksTarget,
} from "@/server/lib/dataforseo/backlinks";
import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
// A successful DataForSEO task always carries billing metadata (path + cost).
const billed = {

2
src/server/lib/dataforseo/backlinks.ts
View File

@ -21,8 +21,6 @@ import {
type DataforseoApiResponse,
} from "@/server/lib/dataforseo/envelope";
export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
type BacklinksRequest = { target: string };
type BacklinksListRequest = BacklinksRequest &
BacklinksSpamFilterOptions & {

135
src/server/lib/dataforseo/client.ts
View File

@ -8,45 +8,9 @@ import {
trackUsageCreditSpend,
} from "@/server/billing/subscription";
import type { BillingCustomerContext } from "@/server/billing/subscription";
import {
fetchBusinessListingsSearch,
fetchQuestionsAnswers,
} from "@/server/lib/dataforseo/business";
import {
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
fetchDomainPagesSummary,
fetchReferringDomains,
} from "@/server/lib/dataforseo/backlinks";
import {
fetchDomainRankOverview,
fetchKeywordIdeas,
fetchKeywordOverview,
fetchKeywordSuggestions,
fetchRankedKeywords,
fetchRelatedKeywords,
fetchRelevantPages,
fetchSerpCompetitors,
} from "@/server/lib/dataforseo/labs";
import {
fetchAdsKeywordIdeas,
fetchAdsSearchVolume,
} from "@/server/lib/dataforseo/google-ads";
import {
fetchLiveSerp,
fetchLocalSerp,
fetchRankCheckSerp,
postRankCheckTasks,
} from "@/server/lib/dataforseo/serp";
import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
import {
fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";
// Type-only namespace import: erased at compile, so the section modules (and
// the SDK they pull in) still only load through loadDataforseoSections below.
import type * as sections from "@/server/lib/dataforseo/sections";
import {
DataforseoChargedTaskError,
type DataforseoApiCallCost,
@ -57,10 +21,24 @@ import { AppError } from "@/server/lib/errors";
export { mapDataforseoPathToCreditFeature };
/** The section-fetcher barrel (sections.ts), as a type for `meter` pickers. */
export type DataforseoSections = typeof sections;
let sectionsPromise: Promise<DataforseoSections> | undefined;
/** Single lazy boundary for the DataForSEO subtree: the section fetchers and
* the ~3 MB dataforseo-client SDK they statically import stay out of the
* eager isolate startup graph and load once, on the first API call. */
export function loadDataforseoSections(): Promise<DataforseoSections> {
return (sectionsPromise ??= import("@/server/lib/dataforseo/sections"));
}
/**
* Wraps a section fetcher with billing metering. Each entry on the client is
* `meter(customer, fetcher, defaultFeature?)`, which returns a function with the
* fetcher's own input type and resolves to its unwrapped `.data`.
* `meter(customer, (s) => s.fetchX, defaultFeature?)`, which returns a function
* with the fetcher's own input type and resolves to its unwrapped `.data`. The
* picker indirection (rather than the fetcher itself) keeps the section
* modules behind loadDataforseoSections.
*
* `defaultFeature` is the fallback credit feature; a caller can override it per
* call by passing `creditFeature` in the input (e.g. an MCP tool attributing
@ -69,13 +47,15 @@ export { mapDataforseoPathToCreditFeature };
*/
function meter<I, T>(
customer: BillingCustomerContext,
fetcher: (input: I) => Promise<DataforseoApiResponse<T>>,
pick: (
sections: DataforseoSections,
) => (input: I) => Promise<DataforseoApiResponse<T>>,
defaultFeature?: CreditFeature,
): (input: I & { creditFeature?: CreditFeature }) => Promise<T> {
return (input) =>
meterDataforseoCall(
customer,
() => fetcher(input),
async () => pick(await loadDataforseoSections())(input),
input.creditFeature ?? defaultFeature,
);
}
@ -85,56 +65,71 @@ export function createDataforseoClient(customer: BillingCustomerContext) {
business: {
businessListings: meter(
customer,
fetchBusinessListingsSearch,
(s) => s.fetchBusinessListingsSearch,
"local_seo",
),
questionsAnswers: meter(
customer,
(s) => s.fetchQuestionsAnswers,
"local_seo",
),
questionsAnswers: meter(customer, fetchQuestionsAnswers, "local_seo"),
},
backlinks: {
summary: meter(customer, fetchBacklinksSummary),
rows: meter(customer, fetchBacklinksRows),
referringDomains: meter(customer, fetchReferringDomains),
domainPages: meter(customer, fetchDomainPagesSummary),
history: meter(customer, fetchBacklinksHistory),
summary: meter(customer, (s) => s.fetchBacklinksSummary),
rows: meter(customer, (s) => s.fetchBacklinksRows),
referringDomains: meter(customer, (s) => s.fetchReferringDomains),
domainPages: meter(customer, (s) => s.fetchDomainPagesSummary),
history: meter(customer, (s) => s.fetchBacklinksHistory),
},
keywords: {
related: meter(customer, fetchRelatedKeywords),
suggestions: meter(customer, fetchKeywordSuggestions),
ideas: meter(customer, fetchKeywordIdeas),
related: meter(customer, (s) => s.fetchRelatedKeywords),
suggestions: meter(customer, (s) => s.fetchKeywordSuggestions),
ideas: meter(customer, (s) => s.fetchKeywordIdeas),
// Google Ads endpoints for countries Labs doesn't support.
adsIdeas: meter(customer, fetchAdsKeywordIdeas),
adsSearchVolume: meter(customer, fetchAdsSearchVolume),
adsIdeas: meter(customer, (s) => s.fetchAdsKeywordIdeas),
adsSearchVolume: meter(customer, (s) => s.fetchAdsSearchVolume),
},
domain: {
rankOverview: meter(customer, fetchDomainRankOverview),
rankedKeywords: meter(customer, fetchRankedKeywords),
relevantPages: meter(customer, fetchRelevantPages),
rankOverview: meter(customer, (s) => s.fetchDomainRankOverview),
rankedKeywords: meter(customer, (s) => s.fetchRankedKeywords),
relevantPages: meter(customer, (s) => s.fetchRelevantPages),
},
serp: {
live: meter(customer, fetchLiveSerp),
rankCheck: meter(customer, fetchRankCheckSerp, "rank_tracking"),
live: meter(customer, (s) => s.fetchLiveSerp),
rankCheck: meter(customer, (s) => s.fetchRankCheckSerp, "rank_tracking"),
// Posts up to 100 queued rank check tasks; one metered charge covers the
// whole batch (DataForSEO bills task_post at post time, collection is
// free).
rankCheckTaskPost: meter(customer, postRankCheckTasks, "rank_tracking"),
local: meter(customer, fetchLocalSerp, "local_seo"),
rankCheckTaskPost: meter(
customer,
(s) => s.postRankCheckTasks,
"rank_tracking",
),
local: meter(customer, (s) => s.fetchLocalSerp, "local_seo"),
},
labs: {
// Callers (e.g. the keyword-metrics MCP tool) can attribute the spend to
// their own feature by passing `creditFeature` in the input; defaults to
// rank_tracking when omitted.
keywordOverview: meter(customer, fetchKeywordOverview, "rank_tracking"),
serpCompetitors: meter(customer, fetchSerpCompetitors),
keywordOverview: meter(
customer,
(s) => s.fetchKeywordOverview,
"rank_tracking",
),
serpCompetitors: meter(customer, (s) => s.fetchSerpCompetitors),
},
lighthouse: {
live: meter(customer, fetchLighthouseResult),
live: meter(customer, (s) => s.fetchLighthouseResult),
},
aiSearch: {
mentionsSearch: meter(customer, fetchLlmMentionsSearch),
aggregatedMetrics: meter(customer, fetchLlmAggregatedMetrics),
topPages: meter(customer, fetchLlmTopPages),
crossAggregatedMetrics: meter(customer, fetchLlmCrossAggregatedMetrics),
llmResponse: meter(customer, fetchLlmResponse),
mentionsSearch: meter(customer, (s) => s.fetchLlmMentionsSearch),
aggregatedMetrics: meter(customer, (s) => s.fetchLlmAggregatedMetrics),
topPages: meter(customer, (s) => s.fetchLlmTopPages),
crossAggregatedMetrics: meter(
customer,
(s) => s.fetchLlmCrossAggregatedMetrics,
),
llmResponse: meter(customer, (s) => s.fetchLlmResponse),
},
} as const;
}

2
src/server/lib/dataforseo/endpoints.test.ts
View File

@ -6,13 +6,13 @@ vi.mock("@/server/lib/runtime-env", () => ({
import { fetchQuestionsAnswers } from "@/server/lib/dataforseo/business";
import {
buildLlmTarget,
fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";
import { buildLlmTarget } from "@/server/lib/dataforseo/shared";
function parseDataforseoRequestBody(init: RequestInit | undefined): unknown {
const body = init?.body;

74
src/server/lib/dataforseo/index.ts
View File

@ -1,7 +1,15 @@
// Public surface of the DataForSEO integration. Internals live in the
// per-section files (labs / serp / business / backlinks / ai /
// lighthouse); everything funnels through envelope.ts (status + billing) and is
// metered in client.ts.
// per-section files (labs / serp / business / backlinks / ai / lighthouse),
// which sit behind the single dynamic import in client.ts so the ~3 MB SDK
// loads lazily; everything funnels through envelope.ts (status + billing) and
// is metered in client.ts. Runtime values re-exported here must be SDK-free
// (shared.ts) or lazy — a static value re-export from a section file would
// drag the SDK back into the eager isolate startup graph.
import {
loadDataforseoSections,
type DataforseoSections,
} from "@/server/lib/dataforseo/client";
export { createDataforseoClient } from "@/server/lib/dataforseo/client";
@ -10,35 +18,41 @@ export {
type KeywordMetricRow,
} from "@/server/lib/dataforseo/keyword-metrics";
export {
type LabsKeywordDataItem,
type DomainRankedKeywordItem,
type RelevantPagesItem,
} from "@/server/lib/dataforseo/labs";
export { type AdsKeywordIdeaItem } from "@/server/lib/dataforseo/google-ads";
export {
fetchRankCheckTaskResult,
MAX_TASKS_PER_POST,
type SerpLiveItem,
type RankCheckResult,
type RankCheckTaskInput,
type PostedRankCheckTask,
} from "@/server/lib/dataforseo/serp";
export {
normalizeBacklinksTarget,
type BacklinksSummaryItem,
type BacklinksItem,
type ReferringDomainItem,
type DomainPageSummaryItem,
type BacklinksHistoryItem,
} from "@/server/lib/dataforseo/backlinks";
export {
buildLlmTarget,
CHATGPT_LANGUAGE_CODE,
CHATGPT_LOCATION_CODE,
MAX_TASKS_PER_POST,
type LlmPlatform,
} from "@/server/lib/dataforseo/ai";
} from "@/server/lib/dataforseo/shared";
export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";
/** Lazy wrapper for the one section fetcher called outside the metered client
* (rank-check task collection is free at DataForSEO, so it skips metering). */
export const fetchRankCheckTaskResult: DataforseoSections["fetchRankCheckTaskResult"] =
async (input) =>
(await loadDataforseoSections()).fetchRankCheckTaskResult(input);
export type {
LabsKeywordDataItem,
DomainRankedKeywordItem,
RelevantPagesItem,
} from "@/server/lib/dataforseo/labs";
export type { AdsKeywordIdeaItem } from "@/server/lib/dataforseo/google-ads";
export type {
SerpLiveItem,
RankCheckResult,
RankCheckTaskInput,
PostedRankCheckTask,
} from "@/server/lib/dataforseo/serp";
export type {
BacklinksSummaryItem,
BacklinksItem,
ReferringDomainItem,
DomainPageSummaryItem,
BacklinksHistoryItem,
} from "@/server/lib/dataforseo/backlinks";

54
src/server/lib/dataforseo/sections.ts Normal file
View File

@ -0,0 +1,54 @@
// Root of the lazily loaded DataForSEO subtree. The section fetchers — and
// the ~3 MB dataforseo-client SDK they statically import — are reached only
// through the single dynamic import in client.ts (loadDataforseoSections), so
// the whole subtree lands in one lazy chunk outside the eager isolate startup
// graph. Never import this barrel or a section file statically from eager
// server code; the leanWorkerBundle vite plugin fails the build if the SDK
// re-enters the eager graph. SDK-free values live in shared.ts instead.
export {
fetchBusinessListingsSearch,
fetchQuestionsAnswers,
} from "@/server/lib/dataforseo/business";
export {
fetchBacklinksHistory,
fetchBacklinksRows,
fetchBacklinksSummary,
fetchDomainPagesSummary,
fetchReferringDomains,
} from "@/server/lib/dataforseo/backlinks";
export {
fetchDomainRankOverview,
fetchKeywordIdeas,
fetchKeywordOverview,
fetchKeywordSuggestions,
fetchRankedKeywords,
fetchRelatedKeywords,
fetchRelevantPages,
fetchSerpCompetitors,
} from "@/server/lib/dataforseo/labs";
export {
fetchAdsKeywordIdeas,
fetchAdsSearchVolume,
} from "@/server/lib/dataforseo/google-ads";
export {
fetchLiveSerp,
fetchLocalSerp,
fetchRankCheckSerp,
fetchRankCheckTaskResult,
postRankCheckTasks,
} from "@/server/lib/dataforseo/serp";
export { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse";
export {
fetchLlmAggregatedMetrics,
fetchLlmCrossAggregatedMetrics,
fetchLlmMentionsSearch,
fetchLlmResponse,
fetchLlmTopPages,
} from "@/server/lib/dataforseo/ai";

4
src/server/lib/dataforseo/serp.ts
View File

@ -7,6 +7,7 @@ import {
SerpGoogleOrganicTaskPostRequestInfo,
} from "dataforseo-client";
import { serpApi } from "@/server/lib/dataforseo/core";
import { MAX_TASKS_PER_POST } from "@/server/lib/dataforseo/shared";
import {
assertOk,
buildTaskBilling,
@ -184,9 +185,6 @@ export async function fetchRankCheckSerp(input: {
// stragglers, orchestrated by the rank check workflow.
// ---------------------------------------------------------------------------
/** Max tasks DataForSEO accepts in a single task_post request. */
export const MAX_TASKS_PER_POST = 100;
export interface RankCheckTaskInput {
keyword: string;
keywordId: string;

50
src/server/lib/dataforseo/shared.ts Normal file
View File

@ -0,0 +1,50 @@
// SDK-free constants and target builders shared between eager server code
// (features, workflows, MCP tools) and the lazily loaded section fetchers.
// Keep this module free of dataforseo-client and section-file imports —
// anything imported from here must be safe to evaluate in the eager isolate
// startup graph.
// ChatGPT mention/response data is only available for US/en per DataForSEO docs.
export const CHATGPT_LOCATION_CODE = 2840;
export const CHATGPT_LANGUAGE_CODE = "en";
export type LlmPlatform = "chat_gpt" | "google";
/** Max tasks DataForSEO accepts in a single task_post request. */
export const MAX_TASKS_PER_POST = 100;
// DataForSEO's LLM-mentions `target` array accepts domain OR keyword entries.
// We always pass exactly one target per call.
export type LlmTarget =
| {
domain: string;
include_subdomains?: boolean;
search_filter?: "include" | "exclude";
search_scope?: string[];
}
| {
keyword: string;
search_filter?: "include" | "exclude";
search_scope?: string[];
match_type?: "word_match" | "partial_match";
};
export function buildLlmTarget(input: {
type: "domain" | "keyword";
value: string;
}): LlmTarget {
if (input.type === "domain") {
return {
domain: input.value,
include_subdomains: true,
search_filter: "include",
search_scope: ["any"],
};
}
return {
keyword: input.value,
search_filter: "include",
search_scope: ["any", "brand_entities"],
match_type: "word_match",
};
}

6
src/server/lib/just-bash-stub.ts
View File

@ -1,5 +1,6 @@
/**
* Build-time stand-in for `just-bash`, wired up via the vite.config.ts alias.
* Build-time stand-in for `just-bash`, wired up via the leanWorkerBundle
* plugin's alias (vite-plugin-lean-worker-bundle.ts).
*
* `@cloudflare/think` eagerly imports just-bash (~21 MB of source, plus
* turndown and the 8.6 MB @mixmark-io/domino DOM implementation) at module
@ -14,7 +15,8 @@
* @cloudflare/think loads just-bash lazily.
*/
const STUBBED_MESSAGE =
"just-bash is stubbed out of the worker bundle (see vite.config.ts); " +
"just-bash is stubbed out of the worker bundle (see " +
"vite-plugin-lean-worker-bundle.ts); " +
"the workspace bash tool is disabled for this deployment";
// Covers every named import in the dependency graph: `Bash` (Think's

30
src/server/lib/workers-ai-provider-stub.ts Normal file
View File

@ -0,0 +1,30 @@
/**
* Build-time stand-in for `workers-ai-provider` (and its /anthropic and
* /openai subpaths), wired up via the leanWorkerBundle vite plugin.
*
* `@cloudflare/think` eagerly imports all three (~540 kB with the @ai-sdk/
* openai + @ai-sdk/anthropic providers they pull in) solely to build its lazy
* default provider in `resolveModel()` — which only runs when `getModel()`
* returns a bare model-id string. Both our chat agents (SamChatAgent,
* OnboardingChatAgent) return a constructed OpenRouter LanguageModel instead,
* so the default provider is dead code; this stub keeps it out of the eager
* worker bundle. Same pattern as just-bash-stub.ts.
*/
const STUBBED_MESSAGE =
"workers-ai-provider is stubbed out of the worker bundle to keep it out of " +
"the eager isolate startup graph (see vite-plugin-lean-worker-bundle.ts); " +
"SamChatAgent/OnboardingChatAgent override getModel with OpenRouter";
// Covers every named import in the dependency graph: `createWorkersAI` (root),
// `anthropic` (/anthropic) and `openai` (/openai).
export function createWorkersAI(): never {
throw new Error(STUBBED_MESSAGE);
}
export function anthropic(): never {
throw new Error(STUBBED_MESSAGE);
}
export function openai(): never {
throw new Error(STUBBED_MESSAGE);
}

168
vite-plugin-lean-worker-bundle.ts Normal file
View File

@ -0,0 +1,168 @@
import { fileURLToPath } from "node:url";
import type { Plugin } from "vite";
// Literal `new URL(..., import.meta.url)` per stub (rather than a path
// helper) so knip sees the stub files as used.
const JUST_BASH_STUB = fileURLToPath(
new URL("./src/server/lib/just-bash-stub.ts", import.meta.url),
);
const WORKERS_AI_PROVIDER_STUB = fileURLToPath(
new URL("./src/server/lib/workers-ai-provider-stub.ts", import.meta.url),
);
/**
* Dependencies that must never be reachable from the worker's eager startup
* module graph. The 128 MB isolate limit is shared by everything evaluated at
* startup (production OOM bursts trace back to baseline heap, not leaks), so
* each of these is either loaded lazily behind a dynamic import or stubbed
* out. `generateBundle` below fails the build if one sneaks back in via a
* static import chain — e.g. an eager `import { fetchLiveSerp } from
* "@/server/lib/dataforseo/serp"` instead of going through the metered client.
*/
const EAGER_DENYLIST: Array<{ pattern: RegExp; expected: string }> = [
{
pattern: /node_modules\/dataforseo-client\//,
expected:
"lazy-loaded behind loadDataforseoSections() — eager code must go " +
"through the metered client or src/server/lib/dataforseo/shared.ts",
},
{
pattern: /node_modules\/autumn-js\//,
expected:
"lazy-loaded behind the facade in src/server/billing/autumn.ts and " +
"the /api/autumn route's lazy handler",
},
{
pattern:
/node_modules\/(workers-ai-provider|@ai-sdk\/(openai|anthropic))\//,
expected:
"aliased to workers-ai-provider-stub.ts (@cloudflare/think's default " +
"provider path is dead code — our agents construct OpenRouter models)",
},
{
pattern: /node_modules\/just-bash\//,
expected:
"aliased to just-bash-stub.ts (Think's workspace bash tool is disabled)",
},
{
// The barrel (index.*) is allowed — the load hook rewrites its content to
// re-export English only — as is en.* itself. Every other locale module
// must stay out; if the load-hook swap ever regresses, the real barrel
// pulls the individual locale files back in and they match here.
pattern: /node_modules\/zod\/v4\/locales\/(?!en\.|index\.)/,
expected:
"replaced with an en-only barrel via the load hook below (we never " +
"localize zod errors)",
},
{
// Pre-existing boundary from the site-audit engine, guarded here too.
pattern: /node_modules\/cheerio\//,
expected:
"lazy-loaded behind the page-analyzer dynamic import " +
"(site-audit-workflow-helpers.ts)",
},
];
/**
* Keeps the worker's eager server bundle lean, in three parts:
*
* 1. `resolve.alias` stubs for packages that are pure dead weight (dead code
* paths in @cloudflare/think).
* 2. A `load`-hook swap of zod v4's all-languages locales barrel for an
* en-only barrel (the barrel is imported via relative specifiers inside
* zod itself, and pre-enforced resolvers in this plugin stack win the
* resolveId race, so matching the resolved file path in `load` is the
* reliable seam).
* 3. A `generateBundle` assertion that walks the static-import closure of the
* worker entry chunk and fails the build if any EAGER_DENYLIST module is
* reachable — turning "we verified the chunk by grepping a sourcemap once"
* into a permanent regression test.
*/
export function leanWorkerBundle(): Plugin {
return {
name: "lean-worker-bundle",
config() {
return {
resolve: {
alias: {
// Rationale for each stub lives in its docblock. TODO: remove the
// just-bash workaround once @cloudflare/think stops eagerly
// importing it (https://github.com/cloudflare/agents/issues/1673).
"just-bash": JUST_BASH_STUB,
// Subpaths must precede the bare specifier, which would otherwise
// prefix-match them.
"workers-ai-provider/anthropic": WORKERS_AI_PROVIDER_STUB,
"workers-ai-provider/openai": WORKERS_AI_PROVIDER_STUB,
"workers-ai-provider": WORKERS_AI_PROVIDER_STUB,
},
},
};
},
load(id) {
// zod v4's core/classic entrypoints re-export the full locales barrel
// (`export * as locales from "../locales/index.js"`, every language,
// ~208 kB) into the eager bundle. Zod's default English error map
// imports `../locales/en.js` directly and bypasses the barrel, so only
// `z.locales.<lang>` consumers need it — and we never localize zod
// errors. Serve an en-only barrel in its place so `z.locales.en` keeps
// working and the other ~40 languages tree-shake away. (`./en.js`
// resolves relative to the real barrel path, so no stub file needed.)
const barrel = id.match(
/node_modules\/zod\/v4\/locales\/index\.(js|cjs)$/,
);
if (barrel) {
return `export { default as en } from "./en.${barrel[1]}";`;
}
},
generateBundle(_options, bundle) {
// Only the worker build matters for isolate memory; the client bundle
// never contains these packages (and the zod swap applies everywhere).
if (this.environment.name !== "ssr") return;
// Bundle keys are chunk fileNames already.
const chunkAt = (fileName: string) => {
const output = bundle[fileName];
return output?.type === "chunk" ? output : undefined;
};
// Static-import closure from the entry chunks: everything here is
// evaluated at isolate startup. Dynamic imports are excluded — landing
// there is the point of the lazy boundaries.
const eager = new Set<string>();
const queue = Object.values(bundle)
.filter((output) => output.type === "chunk" && output.isEntry)
.map((chunk) => chunk.fileName);
while (queue.length > 0) {
const fileName = queue.pop();
if (fileName === undefined || eager.has(fileName)) continue;
eager.add(fileName);
queue.push(...(chunkAt(fileName)?.imports ?? []));
}
const violations: string[] = [];
for (const rule of EAGER_DENYLIST) {
for (const fileName of eager) {
const hits =
chunkAt(fileName)?.moduleIds.filter((id) =>
rule.pattern.test(id),
) ?? [];
if (hits.length > 0) {
violations.push(
`${hits[0]}${hits.length > 1 ? ` (+${hits.length - 1} more modules)` : ""}\n` +
` reached the eager startup graph via chunk ${fileName}\n` +
` expected: ${rule.expected}`,
);
}
}
}
if (violations.length > 0) {
this.error(
`[lean-worker-bundle] denylisted module(s) in the worker's eager ` +
`startup graph:\n\n${violations.join("\n\n")}\n\nRestore the ` +
`lazy/stub boundary, or update EAGER_DENYLIST in ` +
`vite-plugin-lean-worker-bundle.ts if this is intentional.`,
);
}
},
};
}

20
vite.config.ts
View File

@ -1,4 +1,3 @@
import { fileURLToPath } from "node:url";
import { tanstackStart } from "@tanstack/react-start/plugin/vite";
import { defineConfig, loadEnv } from "vite";
import tsConfigPaths from "vite-tsconfig-paths";
@ -6,6 +5,7 @@ import viteReact from "@vitejs/plugin-react";
import tailwindcss from "@tailwindcss/vite";
import { cloudflare } from "@cloudflare/vite-plugin";
import { devtools } from "@tanstack/devtools-vite";
import { leanWorkerBundle } from "./vite-plugin-lean-worker-bundle";
export default defineConfig(({ mode }) => {
const env = loadEnv(mode, process.cwd(), "");
@ -22,23 +22,6 @@ export default defineConfig(({ mode }) => {
const emitSourcemaps = env.POSTHOG_SOURCEMAPS === "true";
return {
resolve: {
alias: {
// TODO: Remove this workaround once @cloudflare/think stops eagerly
// importing just-bash at module init
// (https://github.com/cloudflare/agents/issues/1673).
//
// just-bash (plus its turndown → @mixmark-io/domino chain, ~30 MB of
// source) is only used by Think's workspace bash tool, which SAM
// disables — but the eager import drags it into the main worker's
// startup module graph, inflating every isolate's baseline heap
// toward the 128 MB limit (production OOM bursts on unrelated
// routes). Alias it to a throwing stub so it never ships.
"just-bash": fileURLToPath(
new URL("./src/server/lib/just-bash-stub.ts", import.meta.url),
),
},
},
envPrefix: [
"VITE_",
"AUTH_MODE",
@ -60,6 +43,7 @@ export default defineConfig(({ mode }) => {
outDir: emitSourcemaps ? "dist-sourcemaps" : "dist",
},
plugins: [
leanWorkerBundle(),
showDevtools
? devtools({
consolePiping: {