admin/metatron-open-seo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bound concurrent D1 upserts in saved-keyword metric refresh (#338)

Browse Source 
refreshSavedKeywordMetrics fanned out one Promise per keyword across an
entire location/language group, so a project with thousands of saved
keywords could trigger thousands of simultaneous D1 upserts, pressuring
Worker memory/CPU and D1 concurrency (potential OOM / availability DoS).

Restore bounded DB-write batching: upsert in chunks of 100 so concurrent
writes stay capped regardless of group size.
This commit is contained in:
Ben Senescu 2026-07-05 17:35:58 -04:00 committed by GitHub
parent 1a74904b67
commit c993ef4931
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 26 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/server/features/keywords/services/research/refresh-metrics.ts
View File

@ -7,6 +7,11 @@ import {
import type { BillingCustomerContext } from "@/server/billing/subscription"; import type { BillingCustomerContext } from "@/server/billing/subscription";
import type { RefreshSavedKeywordMetricsInput } from "@/types/schemas/keywords"; import type { RefreshSavedKeywordMetricsInput } from "@/types/schemas/keywords";
// Cap concurrent D1 upserts per group. A project can accumulate thousands of
// saved keywords in one location/language, and fanning out one promise each
// would flood D1/Worker resources; write in bounded chunks instead.
const REFRESH_UPSERT_BATCH_SIZE = 100;
export async function refreshSavedKeywordMetrics( export async function refreshSavedKeywordMetrics(
input: RefreshSavedKeywordMetricsInput, input: RefreshSavedKeywordMetricsInput,
billingCustomer: BillingCustomerContext, billingCustomer: BillingCustomerContext,
@ -41,24 +46,27 @@ export async function refreshSavedKeywordMetrics(
metrics.map((metric) => [metric.keyword.toLowerCase(), metric]), metrics.map((metric) => [metric.keyword.toLowerCase(), metric]),
); );
await Promise.all( for (let i = 0; i < groupRows.length; i += REFRESH_UPSERT_BATCH_SIZE) {
groupRows.map((r) => { const chunk = groupRows.slice(i, i + REFRESH_UPSERT_BATCH_SIZE);
const metric = byKeyword.get(r.row.keyword.toLowerCase()); await Promise.all(
if (!metric) return Promise.resolve(); chunk.map((r) => {
return KeywordResearchRepository.upsertKeywordMetric({ const metric = byKeyword.get(r.row.keyword.toLowerCase());
projectId: input.projectId, if (!metric) return Promise.resolve();
keyword: r.row.keyword, return KeywordResearchRepository.upsertKeywordMetric({
locationCode, projectId: input.projectId,
languageCode, keyword: r.row.keyword,
searchVolume: metric.searchVolume, locationCode,
cpc: metric.cpc, languageCode,
competition: metric.competition, searchVolume: metric.searchVolume,
keywordDifficulty: metric.keywordDifficulty, cpc: metric.cpc,
intent: normalizeIntent(metric.intent), competition: metric.competition,
monthlySearchesJson: JSON.stringify(metric.monthlySearches), keywordDifficulty: metric.keywordDifficulty,
}); intent: normalizeIntent(metric.intent),
}), monthlySearchesJson: JSON.stringify(metric.monthlySearches),
); });
}),
);
}
updated += byKeyword.size; updated += byKeyword.size;
} }