diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index 5653437..cccb043 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -18,7 +18,7 @@ const BASE_URL = "https://bikegear.in"; const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1)); const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1)); // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit -const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 1500); +const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500); const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null; // Default to browser mode — it bypasses Cloudflare without needing a proxy const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; @@ -407,7 +407,7 @@ async function scrapeProductDetail(productUrl, brandName) { } // Recycle the browser context every N brands to prevent memory accumulation -const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 20); +const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 10); /** Main entry point: scrape all configured brands and return flat product array. */ async function scrapeBikeGear() { @@ -417,39 +417,38 @@ async function scrapeBikeGear() { for (const brand of BRANDS) { const productUrls = await scrapeBrandProductUrls(brand); + brandsDone++; if (!productUrls.length) { console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`); - brandsDone++; - continue; + } else { + console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`); + + let done = 0; + const products = await runConcurrent(productUrls, async (url) => { + const product = await scrapeProductDetail(url, brand.name); + done++; + if (product.scrapeError) { + console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`); + } else { + console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`); + } + return product; + }, DETAIL_CONCURRENCY); + + const ok = products.filter((p) => !p.scrapeError).length; + const fail = products.length - ok; + console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`); + allProducts.push(...products); } - console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`); - - let done = 0; - const products = await runConcurrent(productUrls, async (url, i) => { - const product = await scrapeProductDetail(url, brand.name); - done++; - if (product.scrapeError) { - console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`); + // Always delay/recycle between brands — even skipped ones — to avoid rapid-fire requests + if (brandsDone < totalBrands) { + if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0) { + await recycleBrowserContext(); } else { - console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`); + await sleep(1000); } - return product; - }, DETAIL_CONCURRENCY); - - const ok = products.filter((p) => !p.scrapeError).length; - const fail = products.length - ok; - console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`); - - allProducts.push(...products); - brandsDone++; - - // Recycle browser context every N brands to free memory - if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0 && brandsDone < totalBrands) { - await recycleBrowserContext(); - } else if (brandsDone < totalBrands) { - await sleep(1000); } }