admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): serialize browser fetches and slow 429 retries

Browse Source
This commit is contained in:
MOHAN 2026-08-01 18:52:58 +05:30
parent d4b12fdd69
commit aef5000539
1 changed files with 51 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -19,16 +19,23 @@ const axios = require("axios");
const { BRANDS } = require("./brands");
const BASE_URL = "https://bikegear.in";
const IS_HTTP_MODE = process.env.BIKEGEAR_FETCH_MODE === "http";
// Browser mode must run sequentially to avoid Cloudflare 429s — 1 page at a time
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1));
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
const LISTING_CONCURRENCY = IS_HTTP_MODE
? Math.max(1, Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3))
: 1;
const DETAIL_CONCURRENCY = IS_HTTP_MODE
? Math.max(1, Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2))
: 1;
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
const BROWSER_429_DELAY = Number(process.env.BIKEGEAR_429_DELAY ?? 30000);
const PROXY_URL = getProxyUrl();
// Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`);
console.log(`[BIKEGEAR] Concurrency: listing=${LISTING_CONCURRENCY} detail=${DETAIL_CONCURRENCY} delay=${BROWSER_REQUEST_DELAY}ms rateLimitDelay=${BROWSER_429_DELAY}ms`);
function getProxyUrl() {
if (process.env.BIKEGEAR_PROXY_URL) {
@ -97,6 +104,7 @@ const STEALTH_SCRIPT = `
`;
let _context = null; // launchPersistentContext = browser + context in one
let _browserFetchQueue = Promise.resolve();
async function getBrowserContext() {
if (_context) return _context;
@ -152,38 +160,52 @@ async function recycleBrowserContext() {
}
}
async function fetchHtmlWithBrowser(url, attempt = 1) {
async function fetchHtmlWithBrowserQueued(url) {
const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url));
_browserFetchQueue = run.catch(() => {});
return run;
}
async function fetchHtmlWithBrowser(url) {
const context = await getBrowserContext();
const page = await context.newPage();
try {
const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
const status = response?.status() ?? 0;
if (status === 429 || status >= 500) {
if (attempt <= 4) {
const wait = attempt * 3000;
console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
await sleep(wait);
return fetchHtmlWithBrowser(url, attempt + 1);
for (let attempt = 1; attempt <= 5; attempt += 1) {
const page = await context.newPage();
try {
const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45000 });
const status = response?.status() ?? 0;
if (status === 429 || status >= 500) {
if (attempt < 5) {
const wait = status === 429 ? attempt * BROWSER_429_DELAY : attempt * 5000;
console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/5)`);
await page.close().catch(() => {});
await sleep(wait);
continue;
}
throw new Error(`HTTP ${status} after retries`);
}
throw new Error(`HTTP ${status} after retries`);
}
if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`);
if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`);
// Wait for Cloudflare JS challenge to resolve, then add polite delay
await page.waitForTimeout(1500);
const html = await page.content();
if (BROWSER_REQUEST_DELAY > 0) await sleep(BROWSER_REQUEST_DELAY);
return html;
} catch (err) {
if (attempt <= 4 && (err.message?.includes("timeout") || err.message?.includes("net::"))) {
await sleep(attempt * 2000);
return fetchHtmlWithBrowser(url, attempt + 1);
// Wait for Cloudflare JS challenge to resolve, then add polite delay
await page.waitForTimeout(2500);
const html = await page.content();
if (BROWSER_REQUEST_DELAY > 0) await sleep(BROWSER_REQUEST_DELAY);
return html;
} catch (err) {
if (attempt < 5 && (err.message?.includes("timeout") || err.message?.includes("net::"))) {
const wait = attempt * 5000;
console.log(`[BIKEGEAR] RETRY(browser) ${url} ${err.message} — waiting ${wait}ms (${attempt}/5)`);
await sleep(wait);
continue;
}
throw err;
} finally {
await page.close().catch(() => {});
}
throw err;
} finally {
await page.close().catch(() => {});
}
throw new Error("Browser fetch failed after retries");
}
// ── HTTP fetch (axios) ─────────────────────────────────────────────────────
@ -244,7 +266,7 @@ function sleep(ms) {
function fetchHtml(url, attempt = 1) {
return FETCH_MODE === "browser"
? fetchHtmlWithBrowser(url, attempt)
? fetchHtmlWithBrowserQueued(url)
: fetchHtmlWithHttp(url, attempt);
}