diff --git a/docs/SELF_HOSTING_CLOUDFLARE.md b/docs/SELF_HOSTING_CLOUDFLARE.md index a5e89d4..ea591f8 100644 --- a/docs/SELF_HOSTING_CLOUDFLARE.md +++ b/docs/SELF_HOSTING_CLOUDFLARE.md @@ -32,7 +32,19 @@ In the Cloudflare dashboard: - `TEAM_DOMAIN` (domain from `JWKS_URL`, for example `https://your-team.cloudflareaccess.com`) - `DATAFORSEO_API_KEY` -### 3) Validate setup +### 3) Optional: add an R2 lifecycle rule + +DataForSEO API responses are cached in R2 under the `dataforseo-cache/` prefix. This step is optional, but recommended to automatically clean up expired cache objects: + +```bash +npx wrangler r2 bucket lifecycle add open-seo dataforseo-cache-expiry dataforseo-cache/ --expire-days 7 +``` + +If you changed the R2 bucket name during deploy, replace `open-seo` with your bucket name. + +Without a lifecycle rule, cached objects under `dataforseo-cache/` will accumulate indefinitely and increase storage costs over time. + +### 4) Validate setup 1. Open your Worker URL again. 2. Sign in with Cloudflare Access. diff --git a/src/server/features/backlinks/services/BacklinksService.billing.test.ts b/src/server/features/backlinks/services/BacklinksService.billing.test.ts index 5f8899c..67d9180 100644 --- a/src/server/features/backlinks/services/BacklinksService.billing.test.ts +++ b/src/server/features/backlinks/services/BacklinksService.billing.test.ts @@ -7,6 +7,15 @@ const domainPagesMock = vi.fn(); const timeseriesSummaryMock = vi.fn(); const newLostTimeseriesMock = vi.fn(); +vi.mock("@/server/lib/r2-cache", () => ({ + buildCacheKey: vi.fn( + async (prefix: string, params: Record) => + `${prefix}:${JSON.stringify(params)}`, + ), + getCached: vi.fn(async () => null), + setCached: vi.fn(async () => undefined), +})); + vi.mock("@/server/lib/dataforseoBacklinks", () => ({ normalizeBacklinksTarget: vi.fn(), })); diff --git a/src/server/features/backlinks/services/BacklinksService.ts b/src/server/features/backlinks/services/BacklinksService.ts index 5d9cc78..118c76b 100644 --- a/src/server/features/backlinks/services/BacklinksService.ts +++ b/src/server/features/backlinks/services/BacklinksService.ts @@ -1,4 +1,4 @@ -import { buildCacheKey, getCached, setCached } from "@/server/lib/kv-cache"; +import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinks"; import { profileBacklinksOverview, @@ -20,20 +20,23 @@ function createBacklinksService(cache: BacklinksCache = defaultCache) { input: BacklinksLookupInput, billingCustomer: BillingCustomerContext, ) { - return profileBacklinksOverview( - cache, - buildOverviewCacheKey(input, billingCustomer), - input, - billingCustomer, - ); + const cacheKey = await buildOverviewCacheKey(input, billingCustomer); + + return profileBacklinksOverview(cache, cacheKey, input, billingCustomer); }, async profileReferringDomains( input: BacklinksLookupInput, billingCustomer: BillingCustomerContext, ) { + const cacheKey = await buildTabCacheKey( + "backlinks:referring-domains", + input, + billingCustomer, + ); + return profileReferringDomainsRows( cache, - buildTabCacheKey("backlinks:referring-domains", input, billingCustomer), + cacheKey, input, billingCustomer, ); @@ -42,20 +45,21 @@ function createBacklinksService(cache: BacklinksCache = defaultCache) { input: BacklinksLookupInput, billingCustomer: BillingCustomerContext, ) { - return profileTopPagesRows( - cache, - buildTabCacheKey("backlinks:top-pages", input, billingCustomer), + const cacheKey = await buildTabCacheKey( + "backlinks:top-pages", input, billingCustomer, ); + + return profileTopPagesRows(cache, cacheKey, input, billingCustomer); }, } as const; } -function buildOverviewCacheKey( +async function buildOverviewCacheKey( input: BacklinksLookupInput, billingCustomer: BillingCustomerContext, -) { +): Promise { const normalizedTarget = normalizeBacklinksTarget(input.target, { scope: input.scope, }); @@ -70,11 +74,11 @@ function buildOverviewCacheKey( }); } -function buildTabCacheKey( +async function buildTabCacheKey( prefix: string, input: BacklinksLookupInput, billingCustomer: BillingCustomerContext, -) { +): Promise { const normalizedTarget = normalizeBacklinksTarget(input.target, { scope: input.scope, }); diff --git a/src/server/features/domain/services/DomainService.ts b/src/server/features/domain/services/DomainService.ts index b4e8747..54d7a0d 100644 --- a/src/server/features/domain/services/DomainService.ts +++ b/src/server/features/domain/services/DomainService.ts @@ -4,7 +4,7 @@ import { type DomainRankedKeywordItem, } from "@/server/lib/dataforseo"; import { sortBy } from "remeda"; -import { buildCacheKey, getCached, setCached } from "@/server/lib/kv-cache"; +import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; import { createDataforseoClient } from "@/server/lib/dataforseoClient"; @@ -81,8 +81,7 @@ async function getOverview( ): Promise { const domain = normalizeDomainInput(input.domain, input.includeSubdomains); - // --- KV cache check --- - const cacheKey = buildCacheKey("domain:overview", { + const cacheKey = await buildCacheKey("domain:overview", { organizationId: billingCustomer.organizationId, domain, includeSubdomains: input.includeSubdomains, @@ -148,7 +147,6 @@ async function getOverview( fetchedAt: nowIso, }; - // Persist to KV (fire-and-forget; don't block response) if (result.hasData) { void setCached(cacheKey, result, DOMAIN_OVERVIEW_TTL_SECONDS).catch( (error) => { diff --git a/src/server/features/keywords/services/research/research.ts b/src/server/features/keywords/services/research/research.ts index e4d6a67..05fcbeb 100644 --- a/src/server/features/keywords/services/research/research.ts +++ b/src/server/features/keywords/services/research/research.ts @@ -5,7 +5,7 @@ import { buildCacheKey, getCached, setCached, -} from "@/server/lib/kv-cache"; +} from "@/server/lib/r2-cache"; import { KeywordResearchRepository } from "@/server/features/keywords/repositories/KeywordResearchRepository"; import type { KeywordResearchRow } from "@/types/keywords"; import type { ResearchKeywordsInput } from "@/types/schemas/keywords"; @@ -201,12 +201,12 @@ function isUsableCachedResult(cached: CachedResult): boolean { return true; } -function buildResearchCacheKey( +async function buildResearchCacheKey( input: ResearchKeywordsInput, normalizedKeywords: string[], mode: KeywordMode, billingCustomer: BillingCustomerContext, -): string { +): Promise { return buildCacheKey("kw:research", { cacheVersion: CACHE_VERSION, organizationId: billingCustomer.organizationId, @@ -255,7 +255,7 @@ export async function research( const seedKeyword = uniqueKeywords[0]; const mode = getMode(input); - const cacheKey = buildResearchCacheKey( + const cacheKey = await buildResearchCacheKey( input, uniqueKeywords, mode, diff --git a/src/server/features/keywords/services/research/serp.ts b/src/server/features/keywords/services/research/serp.ts index fad4a0a..f815ce2 100644 --- a/src/server/features/keywords/services/research/serp.ts +++ b/src/server/features/keywords/services/research/serp.ts @@ -1,5 +1,5 @@ import { type SerpLiveItem } from "@/server/lib/dataforseoClient"; -import { buildCacheKey, getCached, setCached } from "@/server/lib/kv-cache"; +import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache"; import type { SerpResultItem } from "@/types/keywords"; import { z } from "zod"; import type { BillingCustomerContext } from "@/server/billing/subscription"; @@ -64,7 +64,7 @@ async function getSerpLiveAnalysis( ): Promise { const keyword = normalizeKeyword(input.keyword); - const cacheKey = buildCacheKey("serp:analysis", { + const cacheKey = await buildCacheKey("serp:analysis", { organizationId: billingCustomer.organizationId, keyword, locationCode: input.locationCode, diff --git a/src/server/lib/kv-cache.ts b/src/server/lib/kv-cache.ts deleted file mode 100644 index 3ceaf29..0000000 --- a/src/server/lib/kv-cache.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { sortBy } from "remeda"; -import { z } from "zod"; -import { jsonCodec } from "@/shared/json"; -import { getWorkersBinding } from "@/server/lib/runtime-env"; - -/** - * Cache TTL constants in seconds. - */ -export const CACHE_TTL = { - /** Related keyword research results */ - researchResult: 86400, -} as const; - -const jsonUnknownCodec = jsonCodec(z.unknown()); - -/** - * Build a deterministic cache key from an endpoint slug and input params. - * Uses FNV-1a hash for compactness. - */ -export function buildCacheKey( - prefix: string, - params: Record, -): string { - const raw = JSON.stringify( - params, - sortBy(Object.keys(params), (key) => key), - ); - return `${prefix}:${fnv1a(raw)}`; -} - -/** - * Get a cached JSON value from KV. Returns null on miss. - */ -export async function getCached(key: string): Promise { - const kv = await getKvNamespace(); - const value = await kv.get(key, "text"); - if (value === null) return null; - const parsed = jsonUnknownCodec.safeParse(value); - return parsed.success ? parsed.data : null; -} - -/** - * Store a JSON value in KV with a TTL in seconds. - */ -export async function setCached( - key: string, - data: T, - ttlSeconds: number, -): Promise { - const kv = await getKvNamespace(); - await kv.put(key, JSON.stringify(data), { - expirationTtl: ttlSeconds, - }); -} - -async function getKvNamespace(): Promise { - const binding = await getWorkersBinding("KV"); - if (isKvNamespace(binding)) { - return binding; - } - - throw new Error("KV binding is not configured correctly"); -} - -function isKvNamespace(value: unknown): value is KVNamespace { - return ( - typeof value === "object" && - value !== null && - "get" in value && - typeof value.get === "function" && - "put" in value && - typeof value.put === "function" - ); -} - -/** - * FNV-1a hash — fast, good distribution for cache keys. - */ -function fnv1a(input: string): string { - let hash = 2166136261; - for (let i = 0; i < input.length; i++) { - hash ^= input.charCodeAt(i); - hash = Math.imul(hash, 16777619); - } - return (hash >>> 0).toString(36); -} diff --git a/src/server/lib/r2-cache.ts b/src/server/lib/r2-cache.ts new file mode 100644 index 0000000..f3a0ef0 --- /dev/null +++ b/src/server/lib/r2-cache.ts @@ -0,0 +1,74 @@ +import { env } from "cloudflare:workers"; +import { sortBy } from "remeda"; + +/** + * Cache TTL constants in seconds. + */ +export const CACHE_TTL = { + /** Related keyword research results */ + researchResult: 86400, +} as const; + +const CACHE_PREFIX = "dataforseo-cache/"; + +/** + * Build a deterministic cache key from an endpoint slug and input params. + * Uses a SHA-256 digest for stability across runtimes. + */ +export async function buildCacheKey( + prefix: string, + params: Record, +): Promise { + const raw = JSON.stringify( + Object.fromEntries(sortBy(Object.entries(params), ([key]) => key)), + ); + + return `${prefix}:${await sha256Hex(raw)}`; +} + +/** + * Get a cached JSON value from R2. Returns null on miss or expiry. + */ +export async function getCached(key: string): Promise { + const obj = await env.R2.get(`${CACHE_PREFIX}${key}`); + if (!obj) return null; + + const expiresAt = obj.customMetadata?.expiresAt; + if (expiresAt && Date.parse(expiresAt) < Date.now()) return null; + + try { + return JSON.parse(await obj.text()); + } catch { + return null; + } +} + +/** + * Store a JSON value in R2 with a soft TTL via custom metadata. + */ +export async function setCached( + key: string, + data: T, + ttlSeconds: number, +): Promise { + await env.R2.put(`${CACHE_PREFIX}${key}`, JSON.stringify(data), { + httpMetadata: { contentType: "application/json" }, + customMetadata: { + expiresAt: new Date(Date.now() + ttlSeconds * 1000).toISOString(), + }, + }); +} + +/** + * Compute a deterministic SHA-256 digest for cache keys. + */ +async function sha256Hex(input: string): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(input), + ); + + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); +}