diff --git a/src/business-logic/import-pipeline/sources/retrorides/index.js b/src/business-logic/import-pipeline/sources/retrorides/index.js index f9b6622..42ed347 100644 --- a/src/business-logic/import-pipeline/sources/retrorides/index.js +++ b/src/business-logic/import-pipeline/sources/retrorides/index.js @@ -50,7 +50,7 @@ async function fetchWebsiteData({ paths: runPaths }) { } } - const concurrency = Number(process.env.RETRORIDES_CONCURRENCY ?? 3); + const concurrency = Number(process.env.RETRORIDES_CONCURRENCY ?? 1); const now = new Date().toISOString(); console.log(`[RETRORIDES] Scraping all products from retrorides.co.in (concurrency=${concurrency})...`); diff --git a/src/business-logic/import-pipeline/sources/retrorides/scraper.js b/src/business-logic/import-pipeline/sources/retrorides/scraper.js index f13dda4..0d15945 100644 --- a/src/business-logic/import-pipeline/sources/retrorides/scraper.js +++ b/src/business-logic/import-pipeline/sources/retrorides/scraper.js @@ -22,13 +22,14 @@ function sleep(ms) { } async function fetchText(url, maxAttempts) { - maxAttempts = maxAttempts || 4; + maxAttempts = maxAttempts || 5; for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { const response = await fetch(url, { headers: FETCH_HEADERS }); - if (response.status === 429) { - const waitMs = attempt * 4000; - console.log("[RETRY] " + url + " HTTP 429, retrying in " + waitMs + "ms..."); + // 429 = rate limit, 409 = WAF conflict from concurrent requests — both need backoff + retry + if (response.status === 429 || response.status === 409) { + const waitMs = attempt * 5000; + console.log("[RETRY] " + url + " HTTP " + response.status + ", retrying in " + waitMs + "ms..."); await sleep(waitMs); continue; } @@ -38,7 +39,7 @@ async function fetchText(url, maxAttempts) { return await response.text(); } catch (err) { if (attempt === maxAttempts) throw err; - await sleep(attempt * 1500); + await sleep(attempt * 2000); } } throw new Error("fetchText: exhausted retries"); @@ -186,7 +187,7 @@ async function mapWithConcurrency(items, concurrency, fn) { } async function scrapeRetroRidesProducts(options) { - const concurrency = (options && options.concurrency) || 3; + const concurrency = (options && options.concurrency) || 1; const urls = await getAllProductUrls(); console.log("[RETRORIDES] Found " + urls.length + " product URLs from sitemap"); const products = await mapWithConcurrency(urls, concurrency, async function(url, index) {