admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): always delay between brands and tighten anti-ban settings

Browse Source 
- Between-brand delay now fires for skipped brands too (was only firing for
  brands with products), fixing rapid-fire requests that caused 403 on detail pages
- Default request delay increased 1500ms -> 2500ms
- Browser context recycle interval tightened 20 -> 10 brands

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-07-24 17:02:57 +05:30
parent 46d018c56e
commit e4b3292533
1 changed files with 27 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -18,7 +18,7 @@ const BASE_URL = "https://bikegear.in";
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1));
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 1500);
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
// Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
@ -407,7 +407,7 @@ async function scrapeProductDetail(productUrl, brandName) {
}
// Recycle the browser context every N brands to prevent memory accumulation
const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 20);
const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 10);
/** Main entry point: scrape all configured brands and return flat product array. */
async function scrapeBikeGear() {
@ -417,17 +417,15 @@ async function scrapeBikeGear() {
for (const brand of BRANDS) {
const productUrls = await scrapeBrandProductUrls(brand);
brandsDone++;
if (!productUrls.length) {
console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`);
brandsDone++;
continue;
}
} else {
console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`);
let done = 0;
const products = await runConcurrent(productUrls, async (url, i) => {
const products = await runConcurrent(productUrls, async (url) => {
const product = await scrapeProductDetail(url, brand.name);
done++;
if (product.scrapeError) {
@ -441,17 +439,18 @@ async function scrapeBikeGear() {
const ok = products.filter((p) => !p.scrapeError).length;
const fail = products.length - ok;
console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
allProducts.push(...products);
brandsDone++;
}
// Recycle browser context every N brands to free memory
if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0 && brandsDone < totalBrands) {
// Always delay/recycle between brands — even skipped ones — to avoid rapid-fire requests
if (brandsDone < totalBrands) {
if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0) {
await recycleBrowserContext();
} else if (brandsDone < totalBrands) {
} else {
await sleep(1000);
}
}
}
return allProducts;
}