diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f25c4ba..afff891 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,14 @@ jobs: - name: Run tests run: pnpm run test:ci + # Runs the leanWorkerBundle generateBundle assertion: fails if a + # denylisted package (dataforseo-client, autumn-js, ...) re-enters the + # worker's eager startup graph. See vite-plugin-lean-worker-bundle.ts. + - name: Build worker (eager-bundle guard) + run: pnpm vite build + env: + NODE_OPTIONS: --max-old-space-size=4096 + - name: Install website dependencies run: pnpm --dir web install --frozen-lockfile diff --git a/knip.jsonc b/knip.jsonc index e9df798..9ccb98d 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -18,6 +18,10 @@ "src/db/pg/schema.ts", // Standalone CLI/dev scripts, invoked via package.json scripts "scripts/**", + // DataForSEO section barrel — consumed via the dynamic import in + // client.ts (loadDataforseoSections) + property access, which knip + // can't trace + "src/server/lib/dataforseo/sections.ts", ], "project": ["**/*.{js,mjs,ts,tsx}", "!src/routeTree.gen.ts", "!web/**"], "ignore": ["drizzle-prod.config.ts"], diff --git a/scripts/brand-lookup-cost-profile.ts b/scripts/brand-lookup-cost-profile.ts index 7de8ea7..fc2dae2 100644 --- a/scripts/brand-lookup-cost-profile.ts +++ b/scripts/brand-lookup-cost-profile.ts @@ -1,14 +1,16 @@ import process from "node:process"; import { - buildLlmTarget, - CHATGPT_LANGUAGE_CODE, - CHATGPT_LOCATION_CODE, fetchLlmAggregatedMetrics, fetchLlmCrossAggregatedMetrics, fetchLlmMentionsSearch, fetchLlmTopPages, - type LlmPlatform, } from "@/server/lib/dataforseo/ai"; +import { + buildLlmTarget, + CHATGPT_LANGUAGE_CODE, + CHATGPT_LOCATION_CODE, + type LlmPlatform, +} from "@/server/lib/dataforseo/shared"; import { applyBillingMarkupUsd } from "@/shared/billing"; import { resolveCompetitorGroups } from "@/server/features/ai-search/services/shareOfVoice"; import { parseCompetitorList } from "@/types/schemas/ai-search"; diff --git a/src/routes/api/autumn/$.ts b/src/routes/api/autumn/$.ts index d794578..47aaaff 100644 --- a/src/routes/api/autumn/$.ts +++ b/src/routes/api/autumn/$.ts @@ -1,27 +1,36 @@ import { createFileRoute } from "@tanstack/react-router"; -import { autumnHandler } from "autumn-js/fetch"; +import type { autumnHandler } from "autumn-js/fetch"; import { env } from "cloudflare:workers"; import { isHostedAuthMode } from "@/lib/auth-mode"; import { resolveHostedContext } from "@/middleware/ensure-user/hosted"; -const handler = autumnHandler({ - identify: async (request) => { - const context = await resolveHostedContext(request.headers); +let handlerPromise: Promise> | undefined; - return { - customerId: context.organizationId, - }; - }, -}); +// Lazy: keeps autumn-js/fetch out of the eager isolate startup graph; +// resolves instantly after the first request. +function loadHandler() { + return (handlerPromise ??= import("autumn-js/fetch").then( + ({ autumnHandler }) => + autumnHandler({ + identify: async (request) => { + const context = await resolveHostedContext(request.headers); -function handleAutumnRequest(request: Request) { + return { + customerId: context.organizationId, + }; + }, + }), + )); +} + +async function handleAutumnRequest(request: Request) { if (!isHostedAuthMode(env.AUTH_MODE)) { return new Response("Not found", { status: 404, }); } - return handler(request); + return (await loadHandler())(request); } export const Route = createFileRoute("/api/autumn/$")({ diff --git a/src/server/billing/autumn.ts b/src/server/billing/autumn.ts index be9dbe6..b92119f 100644 --- a/src/server/billing/autumn.ts +++ b/src/server/billing/autumn.ts @@ -1,29 +1,53 @@ -import { Autumn } from "autumn-js"; +import type { Autumn } from "autumn-js"; import { getRequiredEnvValue } from "@/server/lib/runtime-env"; -export const autumn = new Autumn({ - secretKey: () => getRequiredEnvValue("AUTUMN_SECRET_KEY"), - // Retries 429/500/502/503/504 (per-operation retryCodes) plus connection - // errors. Cloudflare 52x statuses are not in the SDK's retry list, so those - // still surface immediately. - // - // These reads/gates (customers.getOrCreate, check) sit on the request hot - // path, so keep the total retry window short: when Autumn is rate-limiting or - // slow, an 8s backoff window held the isolate open for seconds on every - // request and cascaded into region-wide congestion (incident 2026-07-06). - // Fail fast instead — a caller that can't gate surfaces an error rather than - // hanging. - retryConfig: { - strategy: "backoff", - backoff: { - initialInterval: 250, - maxInterval: 1000, - exponent: 1.5, - maxElapsedTime: 2500, - }, - retryConnectionErrors: true, +let autumnPromise: Promise | undefined; + +// Lazy: keeps the ~450 kB autumn-js SDK out of the eager isolate startup +// graph (self-hosted deployments never load it at all); resolves instantly +// after the first call. +function loadAutumn(): Promise { + return (autumnPromise ??= import("autumn-js").then( + ({ Autumn }) => + new Autumn({ + secretKey: () => getRequiredEnvValue("AUTUMN_SECRET_KEY"), + // Retries 429/500/502/503/504 (per-operation retryCodes) plus + // connection errors. Cloudflare 52x statuses are not in the SDK's + // retry list, so those still surface immediately. + // + // These reads/gates (customers.getOrCreate, check) sit on the request + // hot path, so keep the total retry window short: when Autumn is + // rate-limiting or slow, an 8s backoff window held the isolate open + // for seconds on every request and cascaded into region-wide + // congestion (incident 2026-07-06). Fail fast instead — a caller that + // can't gate surfaces an error rather than hanging. + retryConfig: { + strategy: "backoff", + backoff: { + initialInterval: 250, + maxInterval: 1000, + exponent: 1.5, + maxElapsedTime: 2500, + }, + retryConnectionErrors: true, + }, + }), + )); +} + +/** Shape-preserving lazy facade over the SDK client: call sites keep the + * plain `autumn.check(...)` form. Covers only the methods we use — add a + * line here when adopting a new one. */ +export const autumn = { + check: (...args: Parameters) => + loadAutumn().then((client) => client.check(...args)), + track: (...args: Parameters) => + loadAutumn().then((client) => client.track(...args)), + customers: { + getOrCreate: (...args: Parameters) => + loadAutumn().then((client) => client.customers.getOrCreate(...args)), }, -}); +}; // track() has no idempotency key, so replaying a deduction Autumn already // processed (5xx after a successful write, dropped connection) would diff --git a/src/server/features/ai-search/services/brandLookupShaping.ts b/src/server/features/ai-search/services/brandLookupShaping.ts index 236a99f..2aee9c1 100644 --- a/src/server/features/ai-search/services/brandLookupShaping.ts +++ b/src/server/features/ai-search/services/brandLookupShaping.ts @@ -3,7 +3,7 @@ import { CHATGPT_LANGUAGE_CODE, CHATGPT_LOCATION_CODE, type LlmPlatform, -} from "@/server/lib/dataforseo/ai"; +} from "@/server/lib/dataforseo/shared"; import type { LlmAggregatedTotal, LlmMentionItem, diff --git a/src/server/lib/dataforseo/ai.ts b/src/server/lib/dataforseo/ai.ts index 657024f..21675ea 100644 --- a/src/server/lib/dataforseo/ai.ts +++ b/src/server/lib/dataforseo/ai.ts @@ -28,6 +28,7 @@ import { import { createDataforseoBillingClassifier } from "@/server/lib/dataforseoBillingClassification"; import { AppError } from "@/server/lib/errors"; import { aiOptimizationApi } from "@/server/lib/dataforseo/core"; +import type { LlmPlatform, LlmTarget } from "@/server/lib/dataforseo/shared"; import { assertOk, buildTaskBilling, @@ -36,12 +37,6 @@ import { type DataforseoTaskLike, } from "@/server/lib/dataforseo/envelope"; -// ChatGPT mention/response data is only available for US/en per DataForSEO docs. -export const CHATGPT_LOCATION_CODE = 2840; -export const CHATGPT_LANGUAGE_CODE = "en"; - -export type LlmPlatform = "chat_gpt" | "google"; - const classifyAiSearchError = createDataforseoBillingClassifier({ pathPrefix: "/ai_optimization/", billingIssueCode: "AI_SEARCH_BILLING_ISSUE", @@ -52,45 +47,6 @@ const classifyAiSearchError = createDataforseoBillingClassifier({ const assertOptions = (path: string) => ({ classify: classifyAiSearchError, classifyPath: path }) as const; -// --------------------------------------------------------------------------- -// Target builders — DataForSEO's `target` array accepts domain OR keyword -// entries. We always pass exactly one target per call. -// --------------------------------------------------------------------------- - -type LlmTarget = - | { - domain: string; - include_subdomains?: boolean; - search_filter?: "include" | "exclude"; - search_scope?: string[]; - } - | { - keyword: string; - search_filter?: "include" | "exclude"; - search_scope?: string[]; - match_type?: "word_match" | "partial_match"; - }; - -export function buildLlmTarget(input: { - type: "domain" | "keyword"; - value: string; -}): LlmTarget { - if (input.type === "domain") { - return { - domain: input.value, - include_subdomains: true, - search_filter: "include", - search_scope: ["any"], - }; - } - return { - keyword: input.value, - search_filter: "include", - search_scope: ["any", "brand_entities"], - match_type: "word_match", - }; -} - function clampLimit(value: number, min: number, max: number): number { return Math.min(max, Math.max(min, Math.floor(value))); } diff --git a/src/server/lib/dataforseo/backlinks.test.ts b/src/server/lib/dataforseo/backlinks.test.ts index 1c7294d..949f365 100644 --- a/src/server/lib/dataforseo/backlinks.test.ts +++ b/src/server/lib/dataforseo/backlinks.test.ts @@ -19,8 +19,8 @@ import { fetchBacklinksHistory, fetchBacklinksRows, fetchBacklinksSummary, - normalizeBacklinksTarget, } from "@/server/lib/dataforseo/backlinks"; +import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget"; // A successful DataForSEO task always carries billing metadata (path + cost). const billed = { diff --git a/src/server/lib/dataforseo/backlinks.ts b/src/server/lib/dataforseo/backlinks.ts index c4b30d6..a57ed82 100644 --- a/src/server/lib/dataforseo/backlinks.ts +++ b/src/server/lib/dataforseo/backlinks.ts @@ -21,8 +21,6 @@ import { type DataforseoApiResponse, } from "@/server/lib/dataforseo/envelope"; -export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget"; - type BacklinksRequest = { target: string }; type BacklinksListRequest = BacklinksRequest & BacklinksSpamFilterOptions & { diff --git a/src/server/lib/dataforseo/client.ts b/src/server/lib/dataforseo/client.ts index 8b272fe..8065ad9 100644 --- a/src/server/lib/dataforseo/client.ts +++ b/src/server/lib/dataforseo/client.ts @@ -8,45 +8,9 @@ import { trackUsageCreditSpend, } from "@/server/billing/subscription"; import type { BillingCustomerContext } from "@/server/billing/subscription"; -import { - fetchBusinessListingsSearch, - fetchQuestionsAnswers, -} from "@/server/lib/dataforseo/business"; -import { - fetchBacklinksHistory, - fetchBacklinksRows, - fetchBacklinksSummary, - fetchDomainPagesSummary, - fetchReferringDomains, -} from "@/server/lib/dataforseo/backlinks"; -import { - fetchDomainRankOverview, - fetchKeywordIdeas, - fetchKeywordOverview, - fetchKeywordSuggestions, - fetchRankedKeywords, - fetchRelatedKeywords, - fetchRelevantPages, - fetchSerpCompetitors, -} from "@/server/lib/dataforseo/labs"; -import { - fetchAdsKeywordIdeas, - fetchAdsSearchVolume, -} from "@/server/lib/dataforseo/google-ads"; -import { - fetchLiveSerp, - fetchLocalSerp, - fetchRankCheckSerp, - postRankCheckTasks, -} from "@/server/lib/dataforseo/serp"; -import { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse"; -import { - fetchLlmAggregatedMetrics, - fetchLlmCrossAggregatedMetrics, - fetchLlmMentionsSearch, - fetchLlmResponse, - fetchLlmTopPages, -} from "@/server/lib/dataforseo/ai"; +// Type-only namespace import: erased at compile, so the section modules (and +// the SDK they pull in) still only load through loadDataforseoSections below. +import type * as sections from "@/server/lib/dataforseo/sections"; import { DataforseoChargedTaskError, type DataforseoApiCallCost, @@ -57,10 +21,24 @@ import { AppError } from "@/server/lib/errors"; export { mapDataforseoPathToCreditFeature }; +/** The section-fetcher barrel (sections.ts), as a type for `meter` pickers. */ +export type DataforseoSections = typeof sections; + +let sectionsPromise: Promise | undefined; + +/** Single lazy boundary for the DataForSEO subtree: the section fetchers and + * the ~3 MB dataforseo-client SDK they statically import stay out of the + * eager isolate startup graph and load once, on the first API call. */ +export function loadDataforseoSections(): Promise { + return (sectionsPromise ??= import("@/server/lib/dataforseo/sections")); +} + /** * Wraps a section fetcher with billing metering. Each entry on the client is - * `meter(customer, fetcher, defaultFeature?)`, which returns a function with the - * fetcher's own input type and resolves to its unwrapped `.data`. + * `meter(customer, (s) => s.fetchX, defaultFeature?)`, which returns a function + * with the fetcher's own input type and resolves to its unwrapped `.data`. The + * picker indirection (rather than the fetcher itself) keeps the section + * modules behind loadDataforseoSections. * * `defaultFeature` is the fallback credit feature; a caller can override it per * call by passing `creditFeature` in the input (e.g. an MCP tool attributing @@ -69,13 +47,15 @@ export { mapDataforseoPathToCreditFeature }; */ function meter( customer: BillingCustomerContext, - fetcher: (input: I) => Promise>, + pick: ( + sections: DataforseoSections, + ) => (input: I) => Promise>, defaultFeature?: CreditFeature, ): (input: I & { creditFeature?: CreditFeature }) => Promise { return (input) => meterDataforseoCall( customer, - () => fetcher(input), + async () => pick(await loadDataforseoSections())(input), input.creditFeature ?? defaultFeature, ); } @@ -85,56 +65,71 @@ export function createDataforseoClient(customer: BillingCustomerContext) { business: { businessListings: meter( customer, - fetchBusinessListingsSearch, + (s) => s.fetchBusinessListingsSearch, + "local_seo", + ), + questionsAnswers: meter( + customer, + (s) => s.fetchQuestionsAnswers, "local_seo", ), - questionsAnswers: meter(customer, fetchQuestionsAnswers, "local_seo"), }, backlinks: { - summary: meter(customer, fetchBacklinksSummary), - rows: meter(customer, fetchBacklinksRows), - referringDomains: meter(customer, fetchReferringDomains), - domainPages: meter(customer, fetchDomainPagesSummary), - history: meter(customer, fetchBacklinksHistory), + summary: meter(customer, (s) => s.fetchBacklinksSummary), + rows: meter(customer, (s) => s.fetchBacklinksRows), + referringDomains: meter(customer, (s) => s.fetchReferringDomains), + domainPages: meter(customer, (s) => s.fetchDomainPagesSummary), + history: meter(customer, (s) => s.fetchBacklinksHistory), }, keywords: { - related: meter(customer, fetchRelatedKeywords), - suggestions: meter(customer, fetchKeywordSuggestions), - ideas: meter(customer, fetchKeywordIdeas), + related: meter(customer, (s) => s.fetchRelatedKeywords), + suggestions: meter(customer, (s) => s.fetchKeywordSuggestions), + ideas: meter(customer, (s) => s.fetchKeywordIdeas), // Google Ads endpoints for countries Labs doesn't support. - adsIdeas: meter(customer, fetchAdsKeywordIdeas), - adsSearchVolume: meter(customer, fetchAdsSearchVolume), + adsIdeas: meter(customer, (s) => s.fetchAdsKeywordIdeas), + adsSearchVolume: meter(customer, (s) => s.fetchAdsSearchVolume), }, domain: { - rankOverview: meter(customer, fetchDomainRankOverview), - rankedKeywords: meter(customer, fetchRankedKeywords), - relevantPages: meter(customer, fetchRelevantPages), + rankOverview: meter(customer, (s) => s.fetchDomainRankOverview), + rankedKeywords: meter(customer, (s) => s.fetchRankedKeywords), + relevantPages: meter(customer, (s) => s.fetchRelevantPages), }, serp: { - live: meter(customer, fetchLiveSerp), - rankCheck: meter(customer, fetchRankCheckSerp, "rank_tracking"), + live: meter(customer, (s) => s.fetchLiveSerp), + rankCheck: meter(customer, (s) => s.fetchRankCheckSerp, "rank_tracking"), // Posts up to 100 queued rank check tasks; one metered charge covers the // whole batch (DataForSEO bills task_post at post time, collection is // free). - rankCheckTaskPost: meter(customer, postRankCheckTasks, "rank_tracking"), - local: meter(customer, fetchLocalSerp, "local_seo"), + rankCheckTaskPost: meter( + customer, + (s) => s.postRankCheckTasks, + "rank_tracking", + ), + local: meter(customer, (s) => s.fetchLocalSerp, "local_seo"), }, labs: { // Callers (e.g. the keyword-metrics MCP tool) can attribute the spend to // their own feature by passing `creditFeature` in the input; defaults to // rank_tracking when omitted. - keywordOverview: meter(customer, fetchKeywordOverview, "rank_tracking"), - serpCompetitors: meter(customer, fetchSerpCompetitors), + keywordOverview: meter( + customer, + (s) => s.fetchKeywordOverview, + "rank_tracking", + ), + serpCompetitors: meter(customer, (s) => s.fetchSerpCompetitors), }, lighthouse: { - live: meter(customer, fetchLighthouseResult), + live: meter(customer, (s) => s.fetchLighthouseResult), }, aiSearch: { - mentionsSearch: meter(customer, fetchLlmMentionsSearch), - aggregatedMetrics: meter(customer, fetchLlmAggregatedMetrics), - topPages: meter(customer, fetchLlmTopPages), - crossAggregatedMetrics: meter(customer, fetchLlmCrossAggregatedMetrics), - llmResponse: meter(customer, fetchLlmResponse), + mentionsSearch: meter(customer, (s) => s.fetchLlmMentionsSearch), + aggregatedMetrics: meter(customer, (s) => s.fetchLlmAggregatedMetrics), + topPages: meter(customer, (s) => s.fetchLlmTopPages), + crossAggregatedMetrics: meter( + customer, + (s) => s.fetchLlmCrossAggregatedMetrics, + ), + llmResponse: meter(customer, (s) => s.fetchLlmResponse), }, } as const; } diff --git a/src/server/lib/dataforseo/endpoints.test.ts b/src/server/lib/dataforseo/endpoints.test.ts index 47b4974..3046ce9 100644 --- a/src/server/lib/dataforseo/endpoints.test.ts +++ b/src/server/lib/dataforseo/endpoints.test.ts @@ -6,13 +6,13 @@ vi.mock("@/server/lib/runtime-env", () => ({ import { fetchQuestionsAnswers } from "@/server/lib/dataforseo/business"; import { - buildLlmTarget, fetchLlmAggregatedMetrics, fetchLlmCrossAggregatedMetrics, fetchLlmMentionsSearch, fetchLlmResponse, fetchLlmTopPages, } from "@/server/lib/dataforseo/ai"; +import { buildLlmTarget } from "@/server/lib/dataforseo/shared"; function parseDataforseoRequestBody(init: RequestInit | undefined): unknown { const body = init?.body; diff --git a/src/server/lib/dataforseo/index.ts b/src/server/lib/dataforseo/index.ts index 2fe98ac..7258fa2 100644 --- a/src/server/lib/dataforseo/index.ts +++ b/src/server/lib/dataforseo/index.ts @@ -1,7 +1,15 @@ // Public surface of the DataForSEO integration. Internals live in the -// per-section files (labs / serp / business / backlinks / ai / -// lighthouse); everything funnels through envelope.ts (status + billing) and is -// metered in client.ts. +// per-section files (labs / serp / business / backlinks / ai / lighthouse), +// which sit behind the single dynamic import in client.ts so the ~3 MB SDK +// loads lazily; everything funnels through envelope.ts (status + billing) and +// is metered in client.ts. Runtime values re-exported here must be SDK-free +// (shared.ts) or lazy — a static value re-export from a section file would +// drag the SDK back into the eager isolate startup graph. + +import { + loadDataforseoSections, + type DataforseoSections, +} from "@/server/lib/dataforseo/client"; export { createDataforseoClient } from "@/server/lib/dataforseo/client"; @@ -10,35 +18,41 @@ export { type KeywordMetricRow, } from "@/server/lib/dataforseo/keyword-metrics"; -export { - type LabsKeywordDataItem, - type DomainRankedKeywordItem, - type RelevantPagesItem, -} from "@/server/lib/dataforseo/labs"; - -export { type AdsKeywordIdeaItem } from "@/server/lib/dataforseo/google-ads"; - -export { - fetchRankCheckTaskResult, - MAX_TASKS_PER_POST, - type SerpLiveItem, - type RankCheckResult, - type RankCheckTaskInput, - type PostedRankCheckTask, -} from "@/server/lib/dataforseo/serp"; - -export { - normalizeBacklinksTarget, - type BacklinksSummaryItem, - type BacklinksItem, - type ReferringDomainItem, - type DomainPageSummaryItem, - type BacklinksHistoryItem, -} from "@/server/lib/dataforseo/backlinks"; - export { buildLlmTarget, CHATGPT_LANGUAGE_CODE, CHATGPT_LOCATION_CODE, + MAX_TASKS_PER_POST, type LlmPlatform, -} from "@/server/lib/dataforseo/ai"; +} from "@/server/lib/dataforseo/shared"; + +export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget"; + +/** Lazy wrapper for the one section fetcher called outside the metered client + * (rank-check task collection is free at DataForSEO, so it skips metering). */ +export const fetchRankCheckTaskResult: DataforseoSections["fetchRankCheckTaskResult"] = + async (input) => + (await loadDataforseoSections()).fetchRankCheckTaskResult(input); + +export type { + LabsKeywordDataItem, + DomainRankedKeywordItem, + RelevantPagesItem, +} from "@/server/lib/dataforseo/labs"; + +export type { AdsKeywordIdeaItem } from "@/server/lib/dataforseo/google-ads"; + +export type { + SerpLiveItem, + RankCheckResult, + RankCheckTaskInput, + PostedRankCheckTask, +} from "@/server/lib/dataforseo/serp"; + +export type { + BacklinksSummaryItem, + BacklinksItem, + ReferringDomainItem, + DomainPageSummaryItem, + BacklinksHistoryItem, +} from "@/server/lib/dataforseo/backlinks"; diff --git a/src/server/lib/dataforseo/sections.ts b/src/server/lib/dataforseo/sections.ts new file mode 100644 index 0000000..987ce6c --- /dev/null +++ b/src/server/lib/dataforseo/sections.ts @@ -0,0 +1,54 @@ +// Root of the lazily loaded DataForSEO subtree. The section fetchers — and +// the ~3 MB dataforseo-client SDK they statically import — are reached only +// through the single dynamic import in client.ts (loadDataforseoSections), so +// the whole subtree lands in one lazy chunk outside the eager isolate startup +// graph. Never import this barrel or a section file statically from eager +// server code; the leanWorkerBundle vite plugin fails the build if the SDK +// re-enters the eager graph. SDK-free values live in shared.ts instead. + +export { + fetchBusinessListingsSearch, + fetchQuestionsAnswers, +} from "@/server/lib/dataforseo/business"; + +export { + fetchBacklinksHistory, + fetchBacklinksRows, + fetchBacklinksSummary, + fetchDomainPagesSummary, + fetchReferringDomains, +} from "@/server/lib/dataforseo/backlinks"; + +export { + fetchDomainRankOverview, + fetchKeywordIdeas, + fetchKeywordOverview, + fetchKeywordSuggestions, + fetchRankedKeywords, + fetchRelatedKeywords, + fetchRelevantPages, + fetchSerpCompetitors, +} from "@/server/lib/dataforseo/labs"; + +export { + fetchAdsKeywordIdeas, + fetchAdsSearchVolume, +} from "@/server/lib/dataforseo/google-ads"; + +export { + fetchLiveSerp, + fetchLocalSerp, + fetchRankCheckSerp, + fetchRankCheckTaskResult, + postRankCheckTasks, +} from "@/server/lib/dataforseo/serp"; + +export { fetchLighthouseResult } from "@/server/lib/dataforseo/lighthouse"; + +export { + fetchLlmAggregatedMetrics, + fetchLlmCrossAggregatedMetrics, + fetchLlmMentionsSearch, + fetchLlmResponse, + fetchLlmTopPages, +} from "@/server/lib/dataforseo/ai"; diff --git a/src/server/lib/dataforseo/serp.ts b/src/server/lib/dataforseo/serp.ts index 8bae7ba..9ed6908 100644 --- a/src/server/lib/dataforseo/serp.ts +++ b/src/server/lib/dataforseo/serp.ts @@ -7,6 +7,7 @@ import { SerpGoogleOrganicTaskPostRequestInfo, } from "dataforseo-client"; import { serpApi } from "@/server/lib/dataforseo/core"; +import { MAX_TASKS_PER_POST } from "@/server/lib/dataforseo/shared"; import { assertOk, buildTaskBilling, @@ -184,9 +185,6 @@ export async function fetchRankCheckSerp(input: { // stragglers, orchestrated by the rank check workflow. // --------------------------------------------------------------------------- -/** Max tasks DataForSEO accepts in a single task_post request. */ -export const MAX_TASKS_PER_POST = 100; - export interface RankCheckTaskInput { keyword: string; keywordId: string; diff --git a/src/server/lib/dataforseo/shared.ts b/src/server/lib/dataforseo/shared.ts new file mode 100644 index 0000000..b692eea --- /dev/null +++ b/src/server/lib/dataforseo/shared.ts @@ -0,0 +1,50 @@ +// SDK-free constants and target builders shared between eager server code +// (features, workflows, MCP tools) and the lazily loaded section fetchers. +// Keep this module free of dataforseo-client and section-file imports — +// anything imported from here must be safe to evaluate in the eager isolate +// startup graph. + +// ChatGPT mention/response data is only available for US/en per DataForSEO docs. +export const CHATGPT_LOCATION_CODE = 2840; +export const CHATGPT_LANGUAGE_CODE = "en"; + +export type LlmPlatform = "chat_gpt" | "google"; + +/** Max tasks DataForSEO accepts in a single task_post request. */ +export const MAX_TASKS_PER_POST = 100; + +// DataForSEO's LLM-mentions `target` array accepts domain OR keyword entries. +// We always pass exactly one target per call. +export type LlmTarget = + | { + domain: string; + include_subdomains?: boolean; + search_filter?: "include" | "exclude"; + search_scope?: string[]; + } + | { + keyword: string; + search_filter?: "include" | "exclude"; + search_scope?: string[]; + match_type?: "word_match" | "partial_match"; + }; + +export function buildLlmTarget(input: { + type: "domain" | "keyword"; + value: string; +}): LlmTarget { + if (input.type === "domain") { + return { + domain: input.value, + include_subdomains: true, + search_filter: "include", + search_scope: ["any"], + }; + } + return { + keyword: input.value, + search_filter: "include", + search_scope: ["any", "brand_entities"], + match_type: "word_match", + }; +} diff --git a/src/server/lib/just-bash-stub.ts b/src/server/lib/just-bash-stub.ts index 22325a8..a5007dc 100644 --- a/src/server/lib/just-bash-stub.ts +++ b/src/server/lib/just-bash-stub.ts @@ -1,5 +1,6 @@ /** - * Build-time stand-in for `just-bash`, wired up via the vite.config.ts alias. + * Build-time stand-in for `just-bash`, wired up via the leanWorkerBundle + * plugin's alias (vite-plugin-lean-worker-bundle.ts). * * `@cloudflare/think` eagerly imports just-bash (~21 MB of source, plus * turndown and the 8.6 MB @mixmark-io/domino DOM implementation) at module @@ -14,7 +15,8 @@ * @cloudflare/think loads just-bash lazily. */ const STUBBED_MESSAGE = - "just-bash is stubbed out of the worker bundle (see vite.config.ts); " + + "just-bash is stubbed out of the worker bundle (see " + + "vite-plugin-lean-worker-bundle.ts); " + "the workspace bash tool is disabled for this deployment"; // Covers every named import in the dependency graph: `Bash` (Think's diff --git a/src/server/lib/workers-ai-provider-stub.ts b/src/server/lib/workers-ai-provider-stub.ts new file mode 100644 index 0000000..de14bd7 --- /dev/null +++ b/src/server/lib/workers-ai-provider-stub.ts @@ -0,0 +1,30 @@ +/** + * Build-time stand-in for `workers-ai-provider` (and its /anthropic and + * /openai subpaths), wired up via the leanWorkerBundle vite plugin. + * + * `@cloudflare/think` eagerly imports all three (~540 kB with the @ai-sdk/ + * openai + @ai-sdk/anthropic providers they pull in) solely to build its lazy + * default provider in `resolveModel()` — which only runs when `getModel()` + * returns a bare model-id string. Both our chat agents (SamChatAgent, + * OnboardingChatAgent) return a constructed OpenRouter LanguageModel instead, + * so the default provider is dead code; this stub keeps it out of the eager + * worker bundle. Same pattern as just-bash-stub.ts. + */ +const STUBBED_MESSAGE = + "workers-ai-provider is stubbed out of the worker bundle to keep it out of " + + "the eager isolate startup graph (see vite-plugin-lean-worker-bundle.ts); " + + "SamChatAgent/OnboardingChatAgent override getModel with OpenRouter"; + +// Covers every named import in the dependency graph: `createWorkersAI` (root), +// `anthropic` (/anthropic) and `openai` (/openai). +export function createWorkersAI(): never { + throw new Error(STUBBED_MESSAGE); +} + +export function anthropic(): never { + throw new Error(STUBBED_MESSAGE); +} + +export function openai(): never { + throw new Error(STUBBED_MESSAGE); +} diff --git a/vite-plugin-lean-worker-bundle.ts b/vite-plugin-lean-worker-bundle.ts new file mode 100644 index 0000000..271fa8d --- /dev/null +++ b/vite-plugin-lean-worker-bundle.ts @@ -0,0 +1,168 @@ +import { fileURLToPath } from "node:url"; +import type { Plugin } from "vite"; + +// Literal `new URL(..., import.meta.url)` per stub (rather than a path +// helper) so knip sees the stub files as used. +const JUST_BASH_STUB = fileURLToPath( + new URL("./src/server/lib/just-bash-stub.ts", import.meta.url), +); +const WORKERS_AI_PROVIDER_STUB = fileURLToPath( + new URL("./src/server/lib/workers-ai-provider-stub.ts", import.meta.url), +); +/** + * Dependencies that must never be reachable from the worker's eager startup + * module graph. The 128 MB isolate limit is shared by everything evaluated at + * startup (production OOM bursts trace back to baseline heap, not leaks), so + * each of these is either loaded lazily behind a dynamic import or stubbed + * out. `generateBundle` below fails the build if one sneaks back in via a + * static import chain — e.g. an eager `import { fetchLiveSerp } from + * "@/server/lib/dataforseo/serp"` instead of going through the metered client. + */ +const EAGER_DENYLIST: Array<{ pattern: RegExp; expected: string }> = [ + { + pattern: /node_modules\/dataforseo-client\//, + expected: + "lazy-loaded behind loadDataforseoSections() — eager code must go " + + "through the metered client or src/server/lib/dataforseo/shared.ts", + }, + { + pattern: /node_modules\/autumn-js\//, + expected: + "lazy-loaded behind the facade in src/server/billing/autumn.ts and " + + "the /api/autumn route's lazy handler", + }, + { + pattern: + /node_modules\/(workers-ai-provider|@ai-sdk\/(openai|anthropic))\//, + expected: + "aliased to workers-ai-provider-stub.ts (@cloudflare/think's default " + + "provider path is dead code — our agents construct OpenRouter models)", + }, + { + pattern: /node_modules\/just-bash\//, + expected: + "aliased to just-bash-stub.ts (Think's workspace bash tool is disabled)", + }, + { + // The barrel (index.*) is allowed — the load hook rewrites its content to + // re-export English only — as is en.* itself. Every other locale module + // must stay out; if the load-hook swap ever regresses, the real barrel + // pulls the individual locale files back in and they match here. + pattern: /node_modules\/zod\/v4\/locales\/(?!en\.|index\.)/, + expected: + "replaced with an en-only barrel via the load hook below (we never " + + "localize zod errors)", + }, + { + // Pre-existing boundary from the site-audit engine, guarded here too. + pattern: /node_modules\/cheerio\//, + expected: + "lazy-loaded behind the page-analyzer dynamic import " + + "(site-audit-workflow-helpers.ts)", + }, +]; + +/** + * Keeps the worker's eager server bundle lean, in three parts: + * + * 1. `resolve.alias` stubs for packages that are pure dead weight (dead code + * paths in @cloudflare/think). + * 2. A `load`-hook swap of zod v4's all-languages locales barrel for an + * en-only barrel (the barrel is imported via relative specifiers inside + * zod itself, and pre-enforced resolvers in this plugin stack win the + * resolveId race, so matching the resolved file path in `load` is the + * reliable seam). + * 3. A `generateBundle` assertion that walks the static-import closure of the + * worker entry chunk and fails the build if any EAGER_DENYLIST module is + * reachable — turning "we verified the chunk by grepping a sourcemap once" + * into a permanent regression test. + */ +export function leanWorkerBundle(): Plugin { + return { + name: "lean-worker-bundle", + config() { + return { + resolve: { + alias: { + // Rationale for each stub lives in its docblock. TODO: remove the + // just-bash workaround once @cloudflare/think stops eagerly + // importing it (https://github.com/cloudflare/agents/issues/1673). + "just-bash": JUST_BASH_STUB, + // Subpaths must precede the bare specifier, which would otherwise + // prefix-match them. + "workers-ai-provider/anthropic": WORKERS_AI_PROVIDER_STUB, + "workers-ai-provider/openai": WORKERS_AI_PROVIDER_STUB, + "workers-ai-provider": WORKERS_AI_PROVIDER_STUB, + }, + }, + }; + }, + load(id) { + // zod v4's core/classic entrypoints re-export the full locales barrel + // (`export * as locales from "../locales/index.js"`, every language, + // ~208 kB) into the eager bundle. Zod's default English error map + // imports `../locales/en.js` directly and bypasses the barrel, so only + // `z.locales.` consumers need it — and we never localize zod + // errors. Serve an en-only barrel in its place so `z.locales.en` keeps + // working and the other ~40 languages tree-shake away. (`./en.js` + // resolves relative to the real barrel path, so no stub file needed.) + const barrel = id.match( + /node_modules\/zod\/v4\/locales\/index\.(js|cjs)$/, + ); + if (barrel) { + return `export { default as en } from "./en.${barrel[1]}";`; + } + }, + generateBundle(_options, bundle) { + // Only the worker build matters for isolate memory; the client bundle + // never contains these packages (and the zod swap applies everywhere). + if (this.environment.name !== "ssr") return; + + // Bundle keys are chunk fileNames already. + const chunkAt = (fileName: string) => { + const output = bundle[fileName]; + return output?.type === "chunk" ? output : undefined; + }; + + // Static-import closure from the entry chunks: everything here is + // evaluated at isolate startup. Dynamic imports are excluded — landing + // there is the point of the lazy boundaries. + const eager = new Set(); + const queue = Object.values(bundle) + .filter((output) => output.type === "chunk" && output.isEntry) + .map((chunk) => chunk.fileName); + while (queue.length > 0) { + const fileName = queue.pop(); + if (fileName === undefined || eager.has(fileName)) continue; + eager.add(fileName); + queue.push(...(chunkAt(fileName)?.imports ?? [])); + } + + const violations: string[] = []; + for (const rule of EAGER_DENYLIST) { + for (const fileName of eager) { + const hits = + chunkAt(fileName)?.moduleIds.filter((id) => + rule.pattern.test(id), + ) ?? []; + if (hits.length > 0) { + violations.push( + `${hits[0]}${hits.length > 1 ? ` (+${hits.length - 1} more modules)` : ""}\n` + + ` reached the eager startup graph via chunk ${fileName}\n` + + ` expected: ${rule.expected}`, + ); + } + } + } + + if (violations.length > 0) { + this.error( + `[lean-worker-bundle] denylisted module(s) in the worker's eager ` + + `startup graph:\n\n${violations.join("\n\n")}\n\nRestore the ` + + `lazy/stub boundary, or update EAGER_DENYLIST in ` + + `vite-plugin-lean-worker-bundle.ts if this is intentional.`, + ); + } + }, + }; +} diff --git a/vite.config.ts b/vite.config.ts index 26856d1..c5221eb 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,4 +1,3 @@ -import { fileURLToPath } from "node:url"; import { tanstackStart } from "@tanstack/react-start/plugin/vite"; import { defineConfig, loadEnv } from "vite"; import tsConfigPaths from "vite-tsconfig-paths"; @@ -6,6 +5,7 @@ import viteReact from "@vitejs/plugin-react"; import tailwindcss from "@tailwindcss/vite"; import { cloudflare } from "@cloudflare/vite-plugin"; import { devtools } from "@tanstack/devtools-vite"; +import { leanWorkerBundle } from "./vite-plugin-lean-worker-bundle"; export default defineConfig(({ mode }) => { const env = loadEnv(mode, process.cwd(), ""); @@ -22,23 +22,6 @@ export default defineConfig(({ mode }) => { const emitSourcemaps = env.POSTHOG_SOURCEMAPS === "true"; return { - resolve: { - alias: { - // TODO: Remove this workaround once @cloudflare/think stops eagerly - // importing just-bash at module init - // (https://github.com/cloudflare/agents/issues/1673). - // - // just-bash (plus its turndown → @mixmark-io/domino chain, ~30 MB of - // source) is only used by Think's workspace bash tool, which SAM - // disables — but the eager import drags it into the main worker's - // startup module graph, inflating every isolate's baseline heap - // toward the 128 MB limit (production OOM bursts on unrelated - // routes). Alias it to a throwing stub so it never ships. - "just-bash": fileURLToPath( - new URL("./src/server/lib/just-bash-stub.ts", import.meta.url), - ), - }, - }, envPrefix: [ "VITE_", "AUTH_MODE", @@ -60,6 +43,7 @@ export default defineConfig(({ mode }) => { outDir: emitSourcemaps ? "dist-sourcemaps" : "dist", }, plugins: [ + leanWorkerBundle(), showDevtools ? devtools({ consolePiping: {