admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): reduce browser concurrency to 1 and add 1.5s delay between requests

Browse Source 
Cloudflare 429s when running 2+ concurrent browser pages against bikegear.in.
Browser mode now runs 1 page at a time with a 1.5s polite delay after each load.
HTTP mode keeps its original concurrency (3 listing / 2 detail).
Override with BIKEGEAR_LISTING_CONCURRENCY / BIKEGEAR_DETAIL_CONCURRENCY / BIKEGEAR_REQUEST_DELAY.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-07-22 18:13:29 +05:30
parent 0fbe75dd3d
commit a8adf00732
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -14,8 +14,11 @@ const axios = require("axios");
const { BRANDS } = require("./brands");
const BASE_URL = "https://bikegear.in";
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 2);
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2);
// Browser mode must run sequentially to avoid Cloudflare 429s — 1 page at a time
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1));
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 1500);
const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
// Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
@ -73,9 +76,11 @@ async function fetchHtmlWithBrowser(url, attempt = 1) {
}
if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`);
// Wait a moment for any JS-rendered content (Cloudflare challenge resolution)
// Wait for Cloudflare JS challenge to resolve, then add polite delay
await page.waitForTimeout(1500);
return await page.content();
const html = await page.content();
if (BROWSER_REQUEST_DELAY > 0) await sleep(BROWSER_REQUEST_DELAY);
return html;
} catch (err) {
if (attempt <= 4 && (err.message?.includes("timeout") || err.message?.includes("net::"))) {
await sleep(attempt * 2000);