admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix retrorides 409 errors — treat as rate-limit, drop concurrency to 1

Browse Source 
HTTP 409 Conflict from the site WAF when requests run in parallel.
- 409 now retried with backoff (same as 429), up to 5 attempts at 5s increments
- Default concurrency changed from 3 to 1 (sequential fetches)
- RETRORIDES_CONCURRENCY env var to override if needed

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-06-25 19:36:59 +05:30
parent f48bc39751
commit 6938c113ae
2 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/business-logic/import-pipeline/sources/retrorides/index.js
View File

@ -50,7 +50,7 @@ async function fetchWebsiteData({ paths: runPaths }) {
} }
} }
const concurrency = Number(process.env.RETRORIDES_CONCURRENCY ?? 3); const concurrency = Number(process.env.RETRORIDES_CONCURRENCY ?? 1);
const now = new Date().toISOString(); const now = new Date().toISOString();
console.log(`[RETRORIDES] Scraping all products from retrorides.co.in (concurrency=${concurrency})...`); console.log(`[RETRORIDES] Scraping all products from retrorides.co.in (concurrency=${concurrency})...`);

13
src/business-logic/import-pipeline/sources/retrorides/scraper.js
View File

@ -22,13 +22,14 @@ function sleep(ms) {
} }
async function fetchText(url, maxAttempts) { async function fetchText(url, maxAttempts) {
maxAttempts = maxAttempts || 4; maxAttempts = maxAttempts || 5;
for (let attempt = 1; attempt <= maxAttempts; attempt++) { for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try { try {
const response = await fetch(url, { headers: FETCH_HEADERS }); const response = await fetch(url, { headers: FETCH_HEADERS });
if (response.status === 429) { // 429 = rate limit, 409 = WAF conflict from concurrent requests — both need backoff + retry
const waitMs = attempt * 4000; if (response.status === 429 || response.status === 409) {
console.log("[RETRY] " + url + " HTTP 429, retrying in " + waitMs + "ms..."); const waitMs = attempt * 5000;
console.log("[RETRY] " + url + " HTTP " + response.status + ", retrying in " + waitMs + "ms...");
await sleep(waitMs); await sleep(waitMs);
continue; continue;
} }
@ -38,7 +39,7 @@ async function fetchText(url, maxAttempts) {
return await response.text(); return await response.text();
} catch (err) { } catch (err) {
if (attempt === maxAttempts) throw err; if (attempt === maxAttempts) throw err;
await sleep(attempt * 1500); await sleep(attempt * 2000);
} }
} }
throw new Error("fetchText: exhausted retries"); throw new Error("fetchText: exhausted retries");
@ -186,7 +187,7 @@ async function mapWithConcurrency(items, concurrency, fn) {
} }
async function scrapeRetroRidesProducts(options) { async function scrapeRetroRidesProducts(options) {
const concurrency = (options && options.concurrency) || 3; const concurrency = (options && options.concurrency) || 1;
const urls = await getAllProductUrls(); const urls = await getAllProductUrls();
console.log("[RETRORIDES] Found " + urls.length + " product URLs from sitemap"); console.log("[RETRORIDES] Found " + urls.length + " product URLs from sitemap");
const products = await mapWithConcurrency(urls, concurrency, async function(url, index) { const products = await mapWithConcurrency(urls, concurrency, async function(url, index) {