refactor: move DataForSEO response cache from KV to R2 (#46)

* refactor: move lighthouse audits to dataforseo (#43)

* refactor: move lighthouse audits to dataforseo

* chore: remove obsolete audit settings modal

* refactor: rename psi flows to lighthouse

* save

* refactor: simplify audit lighthouse storage flow

* fix: separate lighthouse metrics from actionable audits

* refactor: remove redundant audit project inputs

* feat: redesign lighthouse issues screen with score gauges and table layout

Replace flat score cards with circular SVG gauges, condense metrics into
a compact grid, and switch issue list from cards to an expandable table
with fixed column widths.


* test: harden lighthouse regression coverage

* fix: restore project-scoped audit inputs

* refactor: simplify lighthouse payload handling

* refactor: inline lighthouse server handlers

* refactor: share audit workflow types

* refactor: simplify lighthouse payload flows

* save

* refactor: drop project pagespeed api key

* fix: restore lighthouse issues loading with resilient project context

* fix: restore audit issues back navigation

* refactor: simplify project context and lighthouse error handling

* fix: tolerate DataForSEO lighthouse payload drift

* refactor: route audit lighthouse through dataforseo client

---------


* refactor: move DataForSEO response cache from KV to R2

KV TTL-based expiry is imprecise for cache freshness. Switch to R2 with
soft TTL via custom metadata (expiresAt) and a 7-day lifecycle rule for
cleanup. Cache objects live under the `dataforseo-cache/` prefix,
separate from durable audit payloads in `site-audit/`.


* refactor: use Workers crypto for R2 cache keys

Keep the cache helper aligned with the Cloudflare runtime and avoid pulling Worker bindings into Vitest. Also clarify that R2 lifecycle cleanup is optional but recommended to control storage growth.

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

docs/SELF_HOSTING_CLOUDFLARE.md

@@ -32,7 +32,19 @@ In the Cloudflare dashboard:
    - `TEAM_DOMAIN` (domain from `JWKS_URL`, for example `https://your-team.cloudflareaccess.com`)
    - `DATAFORSEO_API_KEY`
 
-### 3) Validate setup
+### 3) Optional: add an R2 lifecycle rule
+
+DataForSEO API responses are cached in R2 under the `dataforseo-cache/` prefix. This step is optional, but recommended to automatically clean up expired cache objects:
+
+```bash
+npx wrangler r2 bucket lifecycle add open-seo dataforseo-cache-expiry dataforseo-cache/ --expire-days 7
+```
+
+If you changed the R2 bucket name during deploy, replace `open-seo` with your bucket name.
+
+Without a lifecycle rule, cached objects under `dataforseo-cache/` will accumulate indefinitely and increase storage costs over time.
+
+### 4) Validate setup
 
 1. Open your Worker URL again.
 2. Sign in with Cloudflare Access.

src/server/features/backlinks/services/BacklinksService.billing.test.ts

@@ -7,6 +7,15 @@ const domainPagesMock = vi.fn();
 const timeseriesSummaryMock = vi.fn();
 const newLostTimeseriesMock = vi.fn();
 
+vi.mock("@/server/lib/r2-cache", () => ({
+  buildCacheKey: vi.fn(
+    async (prefix: string, params: Record<string, unknown>) =>
+      `${prefix}:${JSON.stringify(params)}`,
+  ),
+  getCached: vi.fn(async () => null),
+  setCached: vi.fn(async () => undefined),
+}));
+
 vi.mock("@/server/lib/dataforseoBacklinks", () => ({
   normalizeBacklinksTarget: vi.fn(),
 }));

src/server/features/backlinks/services/BacklinksService.ts

@@ -1,4 +1,4 @@
-import { buildCacheKey, getCached, setCached } from "@/server/lib/kv-cache";
+import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
 import { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinks";
 import {
   profileBacklinksOverview,
@@ -20,20 +20,23 @@ function createBacklinksService(cache: BacklinksCache = defaultCache) {
       input: BacklinksLookupInput,
       billingCustomer: BillingCustomerContext,
     ) {
-      return profileBacklinksOverview(
-        cache,
-        buildOverviewCacheKey(input, billingCustomer),
-        input,
-        billingCustomer,
-      );
+      const cacheKey = await buildOverviewCacheKey(input, billingCustomer);
+
+      return profileBacklinksOverview(cache, cacheKey, input, billingCustomer);
     },
     async profileReferringDomains(
       input: BacklinksLookupInput,
       billingCustomer: BillingCustomerContext,
     ) {
+      const cacheKey = await buildTabCacheKey(
+        "backlinks:referring-domains",
+        input,
+        billingCustomer,
+      );
+
       return profileReferringDomainsRows(
         cache,
-        buildTabCacheKey("backlinks:referring-domains", input, billingCustomer),
+        cacheKey,
         input,
         billingCustomer,
       );
@@ -42,20 +45,21 @@ function createBacklinksService(cache: BacklinksCache = defaultCache) {
       input: BacklinksLookupInput,
       billingCustomer: BillingCustomerContext,
     ) {
-      return profileTopPagesRows(
-        cache,
-        buildTabCacheKey("backlinks:top-pages", input, billingCustomer),
+      const cacheKey = await buildTabCacheKey(
+        "backlinks:top-pages",
         input,
         billingCustomer,
       );
+
+      return profileTopPagesRows(cache, cacheKey, input, billingCustomer);
     },
   } as const;
 }
 
-function buildOverviewCacheKey(
+async function buildOverviewCacheKey(
   input: BacklinksLookupInput,
   billingCustomer: BillingCustomerContext,
-) {
+): Promise<string> {
   const normalizedTarget = normalizeBacklinksTarget(input.target, {
     scope: input.scope,
   });
@@ -70,11 +74,11 @@ function buildOverviewCacheKey(
   });
 }
 
-function buildTabCacheKey(
+async function buildTabCacheKey(
   prefix: string,
   input: BacklinksLookupInput,
   billingCustomer: BillingCustomerContext,
-) {
+): Promise<string> {
   const normalizedTarget = normalizeBacklinksTarget(input.target, {
     scope: input.scope,
   });

src/server/features/domain/services/DomainService.ts

@@ -4,7 +4,7 @@ import {
   type DomainRankedKeywordItem,
 } from "@/server/lib/dataforseo";
 import { sortBy } from "remeda";
-import { buildCacheKey, getCached, setCached } from "@/server/lib/kv-cache";
+import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
 import { z } from "zod";
 import type { BillingCustomerContext } from "@/server/billing/subscription";
 import { createDataforseoClient } from "@/server/lib/dataforseoClient";
@@ -81,8 +81,7 @@ async function getOverview(
 ): Promise<DomainOverviewResult> {
   const domain = normalizeDomainInput(input.domain, input.includeSubdomains);
 
-  // --- KV cache check ---
-  const cacheKey = buildCacheKey("domain:overview", {
+  const cacheKey = await buildCacheKey("domain:overview", {
     organizationId: billingCustomer.organizationId,
     domain,
     includeSubdomains: input.includeSubdomains,
@@ -148,7 +147,6 @@ async function getOverview(
     fetchedAt: nowIso,
   };
 
-  // Persist to KV (fire-and-forget; don't block response)
   if (result.hasData) {
     void setCached(cacheKey, result, DOMAIN_OVERVIEW_TTL_SECONDS).catch(
       (error) => {

src/server/features/keywords/services/research/research.ts

@@ -5,7 +5,7 @@ import {
   buildCacheKey,
   getCached,
   setCached,
-} from "@/server/lib/kv-cache";
+} from "@/server/lib/r2-cache";
 import { KeywordResearchRepository } from "@/server/features/keywords/repositories/KeywordResearchRepository";
 import type { KeywordResearchRow } from "@/types/keywords";
 import type { ResearchKeywordsInput } from "@/types/schemas/keywords";
@@ -201,12 +201,12 @@ function isUsableCachedResult(cached: CachedResult): boolean {
   return true;
 }
 
-function buildResearchCacheKey(
+async function buildResearchCacheKey(
   input: ResearchKeywordsInput,
   normalizedKeywords: string[],
   mode: KeywordMode,
   billingCustomer: BillingCustomerContext,
-): string {
+): Promise<string> {
   return buildCacheKey("kw:research", {
     cacheVersion: CACHE_VERSION,
     organizationId: billingCustomer.organizationId,
@@ -255,7 +255,7 @@ export async function research(
 
   const seedKeyword = uniqueKeywords[0];
   const mode = getMode(input);
-  const cacheKey = buildResearchCacheKey(
+  const cacheKey = await buildResearchCacheKey(
     input,
     uniqueKeywords,
     mode,

src/server/features/keywords/services/research/serp.ts

@@ -1,5 +1,5 @@
 import { type SerpLiveItem } from "@/server/lib/dataforseoClient";
-import { buildCacheKey, getCached, setCached } from "@/server/lib/kv-cache";
+import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";
 import type { SerpResultItem } from "@/types/keywords";
 import { z } from "zod";
 import type { BillingCustomerContext } from "@/server/billing/subscription";
@@ -64,7 +64,7 @@ async function getSerpLiveAnalysis(
 ): Promise<SerpAnalysisResult> {
   const keyword = normalizeKeyword(input.keyword);
 
-  const cacheKey = buildCacheKey("serp:analysis", {
+  const cacheKey = await buildCacheKey("serp:analysis", {
     organizationId: billingCustomer.organizationId,
     keyword,
     locationCode: input.locationCode,

src/server/lib/kv-cache.ts

@@ -1,86 +0,0 @@
-import { sortBy } from "remeda";
-import { z } from "zod";
-import { jsonCodec } from "@/shared/json";
-import { getWorkersBinding } from "@/server/lib/runtime-env";
-
-/**
- * Cache TTL constants in seconds.
- */
-export const CACHE_TTL = {
-  /** Related keyword research results */
-  researchResult: 86400,
-} as const;
-
-const jsonUnknownCodec = jsonCodec(z.unknown());
-
-/**
- * Build a deterministic cache key from an endpoint slug and input params.
- * Uses FNV-1a hash for compactness.
- */
-export function buildCacheKey(
-  prefix: string,
-  params: Record<string, unknown>,
-): string {
-  const raw = JSON.stringify(
-    params,
-    sortBy(Object.keys(params), (key) => key),
-  );
-  return `${prefix}:${fnv1a(raw)}`;
-}
-
-/**
- * Get a cached JSON value from KV. Returns null on miss.
- */
-export async function getCached(key: string): Promise<unknown> {
-  const kv = await getKvNamespace();
-  const value = await kv.get(key, "text");
-  if (value === null) return null;
-  const parsed = jsonUnknownCodec.safeParse(value);
-  return parsed.success ? parsed.data : null;
-}
-
-/**
- * Store a JSON value in KV with a TTL in seconds.
- */
-export async function setCached<T>(
-  key: string,
-  data: T,
-  ttlSeconds: number,
-): Promise<void> {
-  const kv = await getKvNamespace();
-  await kv.put(key, JSON.stringify(data), {
-    expirationTtl: ttlSeconds,
-  });
-}
-
-async function getKvNamespace(): Promise<KVNamespace> {
-  const binding = await getWorkersBinding("KV");
-  if (isKvNamespace(binding)) {
-    return binding;
-  }
-
-  throw new Error("KV binding is not configured correctly");
-}
-
-function isKvNamespace(value: unknown): value is KVNamespace {
-  return (
-    typeof value === "object" &&
-    value !== null &&
-    "get" in value &&
-    typeof value.get === "function" &&
-    "put" in value &&
-    typeof value.put === "function"
-  );
-}
-
-/**
- * FNV-1a hash — fast, good distribution for cache keys.
- */
-function fnv1a(input: string): string {
-  let hash = 2166136261;
-  for (let i = 0; i < input.length; i++) {
-    hash ^= input.charCodeAt(i);
-    hash = Math.imul(hash, 16777619);
-  }
-  return (hash >>> 0).toString(36);
-}

src/server/lib/r2-cache.ts

@@ -0,0 +1,74 @@
+import { env } from "cloudflare:workers";
+import { sortBy } from "remeda";
+
+/**
+ * Cache TTL constants in seconds.
+ */
+export const CACHE_TTL = {
+  /** Related keyword research results */
+  researchResult: 86400,
+} as const;
+
+const CACHE_PREFIX = "dataforseo-cache/";
+
+/**
+ * Build a deterministic cache key from an endpoint slug and input params.
+ * Uses a SHA-256 digest for stability across runtimes.
+ */
+export async function buildCacheKey(
+  prefix: string,
+  params: Record<string, unknown>,
+): Promise<string> {
+  const raw = JSON.stringify(
+    Object.fromEntries(sortBy(Object.entries(params), ([key]) => key)),
+  );
+
+  return `${prefix}:${await sha256Hex(raw)}`;
+}
+
+/**
+ * Get a cached JSON value from R2. Returns null on miss or expiry.
+ */
+export async function getCached(key: string): Promise<unknown> {
+  const obj = await env.R2.get(`${CACHE_PREFIX}${key}`);
+  if (!obj) return null;
+
+  const expiresAt = obj.customMetadata?.expiresAt;
+  if (expiresAt && Date.parse(expiresAt) < Date.now()) return null;
+
+  try {
+    return JSON.parse(await obj.text());
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Store a JSON value in R2 with a soft TTL via custom metadata.
+ */
+export async function setCached<T>(
+  key: string,
+  data: T,
+  ttlSeconds: number,
+): Promise<void> {
+  await env.R2.put(`${CACHE_PREFIX}${key}`, JSON.stringify(data), {
+    httpMetadata: { contentType: "application/json" },
+    customMetadata: {
+      expiresAt: new Date(Date.now() + ttlSeconds * 1000).toISOString(),
+    },
+  });
+}
+
+/**
+ * Compute a deterministic SHA-256 digest for cache keys.
+ */
+async function sha256Hex(input: string): Promise<string> {
+  const digest = await crypto.subtle.digest(
+    "SHA-256",
+    new TextEncoder().encode(input),
+  );
+
+  return Array.from(new Uint8Array(digest), (byte) =>
+    byte.toString(16).padStart(2, "0"),
+  ).join("");
+}