admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): always delay between brands and tighten anti-ban settings

Browse Source 
- Between-brand delay now fires for skipped brands too (was only firing for
  brands with products), fixing rapid-fire requests that caused 403 on detail pages
- Default request delay increased 1500ms -> 2500ms
- Browser context recycle interval tightened 20 -> 10 brands

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-07-24 17:02:57 +05:30
parent 46d018c56e
commit e4b3292533
1 changed files with 27 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -18,7 +18,7 @@ const BASE_URL = "https://bikegear.in";
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1)); const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1));
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1)); const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 1500); const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null; const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
// Default to browser mode — it bypasses Cloudflare without needing a proxy // Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
@ -407,7 +407,7 @@ async function scrapeProductDetail(productUrl, brandName) {
} }
// Recycle the browser context every N brands to prevent memory accumulation // Recycle the browser context every N brands to prevent memory accumulation
const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 20); const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 10);
/** Main entry point: scrape all configured brands and return flat product array. */ /** Main entry point: scrape all configured brands and return flat product array. */
async function scrapeBikeGear() { async function scrapeBikeGear() {
@ -417,39 +417,38 @@ async function scrapeBikeGear() {
for (const brand of BRANDS) { for (const brand of BRANDS) {
const productUrls = await scrapeBrandProductUrls(brand); const productUrls = await scrapeBrandProductUrls(brand);
brandsDone++;
if (!productUrls.length) { if (!productUrls.length) {
console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`); console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`);
brandsDone++; } else {
continue; console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`);
let done = 0;
const products = await runConcurrent(productUrls, async (url) => {
const product = await scrapeProductDetail(url, brand.name);
done++;
if (product.scrapeError) {
console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);
} else {
console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`);
}
return product;
}, DETAIL_CONCURRENCY);
const ok = products.filter((p) => !p.scrapeError).length;
const fail = products.length - ok;
console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
allProducts.push(...products);
} }
console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`); // Always delay/recycle between brands — even skipped ones — to avoid rapid-fire requests
if (brandsDone < totalBrands) {
let done = 0; if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0) {
const products = await runConcurrent(productUrls, async (url, i) => { await recycleBrowserContext();
const product = await scrapeProductDetail(url, brand.name);
done++;
if (product.scrapeError) {
console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);
} else { } else {
console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`); await sleep(1000);
} }
return product;
}, DETAIL_CONCURRENCY);
const ok = products.filter((p) => !p.scrapeError).length;
const fail = products.length - ok;
console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
allProducts.push(...products);
brandsDone++;
// Recycle browser context every N brands to free memory
if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0 && brandsDone < totalBrands) {
await recycleBrowserContext();
} else if (brandsDone < totalBrands) {
await sleep(1000);
} }
} }