From a8adf0073230999936c6812c478aab6aaf684435 Mon Sep 17 00:00:00 2001 From: MOHAN Date: Wed, 22 Jul 2026 18:13:29 +0530 Subject: [PATCH] fix(bikegear): reduce browser concurrency to 1 and add 1.5s delay between requests Cloudflare 429s when running 2+ concurrent browser pages against bikegear.in. Browser mode now runs 1 page at a time with a 1.5s polite delay after each load. HTTP mode keeps its original concurrency (3 listing / 2 detail). Override with BIKEGEAR_LISTING_CONCURRENCY / BIKEGEAR_DETAIL_CONCURRENCY / BIKEGEAR_REQUEST_DELAY. Co-Authored-By: Claude Sonnet 4.6 --- .../import-pipeline/sources/bikegear/scraper.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index 98a793e..9998ccb 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -14,8 +14,11 @@ const axios = require("axios"); const { BRANDS } = require("./brands"); const BASE_URL = "https://bikegear.in"; -const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 2); -const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2); +// Browser mode must run sequentially to avoid Cloudflare 429s — 1 page at a time +const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1)); +const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1)); +// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit +const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 1500); const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null; // Default to browser mode — it bypasses Cloudflare without needing a proxy const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; @@ -73,9 +76,11 @@ async function fetchHtmlWithBrowser(url, attempt = 1) { } if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`); - // Wait a moment for any JS-rendered content (Cloudflare challenge resolution) + // Wait for Cloudflare JS challenge to resolve, then add polite delay await page.waitForTimeout(1500); - return await page.content(); + const html = await page.content(); + if (BROWSER_REQUEST_DELAY > 0) await sleep(BROWSER_REQUEST_DELAY); + return html; } catch (err) { if (attempt <= 4 && (err.message?.includes("timeout") || err.message?.includes("net::"))) { await sleep(attempt * 2000);