diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index 7edd7e8..48699ec 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -19,16 +19,23 @@ const axios = require("axios"); const { BRANDS } = require("./brands"); const BASE_URL = "https://bikegear.in"; +const IS_HTTP_MODE = process.env.BIKEGEAR_FETCH_MODE === "http"; // Browser mode must run sequentially to avoid Cloudflare 429s — 1 page at a time -const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1)); -const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1)); +const LISTING_CONCURRENCY = IS_HTTP_MODE + ? Math.max(1, Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3)) + : 1; +const DETAIL_CONCURRENCY = IS_HTTP_MODE + ? Math.max(1, Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2)) + : 1; // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500); +const BROWSER_429_DELAY = Number(process.env.BIKEGEAR_429_DELAY ?? 30000); const PROXY_URL = getProxyUrl(); // Default to browser mode — it bypasses Cloudflare without needing a proxy const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`); +console.log(`[BIKEGEAR] Concurrency: listing=${LISTING_CONCURRENCY} detail=${DETAIL_CONCURRENCY} delay=${BROWSER_REQUEST_DELAY}ms rateLimitDelay=${BROWSER_429_DELAY}ms`); function getProxyUrl() { if (process.env.BIKEGEAR_PROXY_URL) { @@ -97,6 +104,7 @@ const STEALTH_SCRIPT = ` `; let _context = null; // launchPersistentContext = browser + context in one +let _browserFetchQueue = Promise.resolve(); async function getBrowserContext() { if (_context) return _context; @@ -152,38 +160,52 @@ async function recycleBrowserContext() { } } -async function fetchHtmlWithBrowser(url, attempt = 1) { +async function fetchHtmlWithBrowserQueued(url) { + const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url)); + _browserFetchQueue = run.catch(() => {}); + return run; +} + +async function fetchHtmlWithBrowser(url) { const context = await getBrowserContext(); - const page = await context.newPage(); - try { - const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 }); - const status = response?.status() ?? 0; - if (status === 429 || status >= 500) { - if (attempt <= 4) { - const wait = attempt * 3000; - console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`); - await sleep(wait); - return fetchHtmlWithBrowser(url, attempt + 1); + for (let attempt = 1; attempt <= 5; attempt += 1) { + const page = await context.newPage(); + try { + const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45000 }); + const status = response?.status() ?? 0; + + if (status === 429 || status >= 500) { + if (attempt < 5) { + const wait = status === 429 ? attempt * BROWSER_429_DELAY : attempt * 5000; + console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/5)`); + await page.close().catch(() => {}); + await sleep(wait); + continue; + } + throw new Error(`HTTP ${status} after retries`); } - throw new Error(`HTTP ${status} after retries`); - } - if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`); + if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`); - // Wait for Cloudflare JS challenge to resolve, then add polite delay - await page.waitForTimeout(1500); - const html = await page.content(); - if (BROWSER_REQUEST_DELAY > 0) await sleep(BROWSER_REQUEST_DELAY); - return html; - } catch (err) { - if (attempt <= 4 && (err.message?.includes("timeout") || err.message?.includes("net::"))) { - await sleep(attempt * 2000); - return fetchHtmlWithBrowser(url, attempt + 1); + // Wait for Cloudflare JS challenge to resolve, then add polite delay + await page.waitForTimeout(2500); + const html = await page.content(); + if (BROWSER_REQUEST_DELAY > 0) await sleep(BROWSER_REQUEST_DELAY); + return html; + } catch (err) { + if (attempt < 5 && (err.message?.includes("timeout") || err.message?.includes("net::"))) { + const wait = attempt * 5000; + console.log(`[BIKEGEAR] RETRY(browser) ${url} ${err.message} — waiting ${wait}ms (${attempt}/5)`); + await sleep(wait); + continue; + } + throw err; + } finally { + await page.close().catch(() => {}); } - throw err; - } finally { - await page.close().catch(() => {}); } + + throw new Error("Browser fetch failed after retries"); } // ── HTTP fetch (axios) ───────────────────────────────────────────────────── @@ -244,7 +266,7 @@ function sleep(ms) { function fetchHtml(url, attempt = 1) { return FETCH_MODE === "browser" - ? fetchHtmlWithBrowser(url, attempt) + ? fetchHtmlWithBrowserQueued(url) : fetchHtmlWithHttp(url, attempt); }