admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): always delay between brands and tighten anti-ban settings

Browse Source 
- Between-brand delay now fires for skipped brands too (was only firing for
  brands with products), fixing rapid-fire requests that caused 403 on detail pages
- Default request delay increased 1500ms -> 2500ms
- Browser context recycle interval tightened 20 -> 10 brands

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-07-24 17:02:57 +05:30
parent 46d018c56e
commit e4b3292533
1 changed files with 27 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -18,7 +18,7 @@ const BASE_URL = "https://bikegear.in";
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1)); const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1));
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1)); const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 1500); const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null; const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
// Default to browser mode — it bypasses Cloudflare without needing a proxy // Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
@ -407,7 +407,7 @@ async function scrapeProductDetail(productUrl, brandName) {
} }
// Recycle the browser context every N brands to prevent memory accumulation // Recycle the browser context every N brands to prevent memory accumulation
const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 20); const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 10);
/** Main entry point: scrape all configured brands and return flat product array. */ /** Main entry point: scrape all configured brands and return flat product array. */
async function scrapeBikeGear() { async function scrapeBikeGear() {
@ -417,17 +417,15 @@ async function scrapeBikeGear() {
for (const brand of BRANDS) { for (const brand of BRANDS) {
const productUrls = await scrapeBrandProductUrls(brand); const productUrls = await scrapeBrandProductUrls(brand);
brandsDone++;
if (!productUrls.length) { if (!productUrls.length) {
console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`); console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`);
brandsDone++; } else {
continue;
}
console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`); console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`);
let done = 0; let done = 0;
const products = await runConcurrent(productUrls, async (url, i) => { const products = await runConcurrent(productUrls, async (url) => {
const product = await scrapeProductDetail(url, brand.name); const product = await scrapeProductDetail(url, brand.name);
done++; done++;
if (product.scrapeError) { if (product.scrapeError) {
@ -441,17 +439,18 @@ async function scrapeBikeGear() {
const ok = products.filter((p) => !p.scrapeError).length; const ok = products.filter((p) => !p.scrapeError).length;
const fail = products.length - ok; const fail = products.length - ok;
console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`); console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
allProducts.push(...products); allProducts.push(...products);
brandsDone++; }
// Recycle browser context every N brands to free memory // Always delay/recycle between brands — even skipped ones — to avoid rapid-fire requests
if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0 && brandsDone < totalBrands) { if (brandsDone < totalBrands) {
if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0) {
await recycleBrowserContext(); await recycleBrowserContext();
} else if (brandsDone < totalBrands) { } else {
await sleep(1000); await sleep(1000);
} }
} }
}
return allProducts; return allProducts;
} }