From e4b3292533abc0df56d33d202538bb04ae6a2651 Mon Sep 17 00:00:00 2001
From: MOHAN <mohan326856@gmail.com>
Date: Fri, 24 Jul 2026 17:02:57 +0530
Subject: [PATCH] fix(bikegear): always delay between brands and tighten
 anti-ban settings

- Between-brand delay now fires for skipped brands too (was only firing for
  brands with products), fixing rapid-fire requests that caused 403 on detail pages
- Default request delay increased 1500ms -> 2500ms
- Browser context recycle interval tightened 20 -> 10 brands

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../sources/bikegear/scraper.js               | 55 +++++++++----------
 1 file changed, 27 insertions(+), 28 deletions(-)

diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js
index 5653437..cccb043 100644
--- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js
+++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js
@@ -18,7 +18,7 @@ const BASE_URL = "https://bikegear.in";
 const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1));
 const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
 // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
-const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 1500);
+const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
 const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
 // Default to browser mode — it bypasses Cloudflare without needing a proxy
 const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
@@ -407,7 +407,7 @@ async function scrapeProductDetail(productUrl, brandName) {
 }
 
 // Recycle the browser context every N brands to prevent memory accumulation
-const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 20);
+const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 10);
 
 /** Main entry point: scrape all configured brands and return flat product array. */
 async function scrapeBikeGear() {
@@ -417,39 +417,38 @@ async function scrapeBikeGear() {
 
   for (const brand of BRANDS) {
     const productUrls = await scrapeBrandProductUrls(brand);
+    brandsDone++;
 
     if (!productUrls.length) {
       console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`);
-      brandsDone++;
-      continue;
+    } else {
+      console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`);
+
+      let done = 0;
+      const products = await runConcurrent(productUrls, async (url) => {
+        const product = await scrapeProductDetail(url, brand.name);
+        done++;
+        if (product.scrapeError) {
+          console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);
+        } else {
+          console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`);
+        }
+        return product;
+      }, DETAIL_CONCURRENCY);
+
+      const ok = products.filter((p) => !p.scrapeError).length;
+      const fail = products.length - ok;
+      console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
+      allProducts.push(...products);
     }
 
-    console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`);
-
-    let done = 0;
-    const products = await runConcurrent(productUrls, async (url, i) => {
-      const product = await scrapeProductDetail(url, brand.name);
-      done++;
-      if (product.scrapeError) {
-        console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);
+    // Always delay/recycle between brands — even skipped ones — to avoid rapid-fire requests
+    if (brandsDone < totalBrands) {
+      if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0) {
+        await recycleBrowserContext();
       } else {
-        console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`);
+        await sleep(1000);
       }
-      return product;
-    }, DETAIL_CONCURRENCY);
-
-    const ok = products.filter((p) => !p.scrapeError).length;
-    const fail = products.length - ok;
-    console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
-
-    allProducts.push(...products);
-    brandsDone++;
-
-    // Recycle browser context every N brands to free memory
-    if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0 && brandsDone < totalBrands) {
-      await recycleBrowserContext();
-    } else if (brandsDone < totalBrands) {
-      await sleep(1000);
     }
   }