diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index dbb8381..879d76f 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -3,13 +3,23 @@ * 2-phase: listing pages → product URLs, then detail pages → full product data. * * Prices are in INR (Indian Rupees) as scraped from the site. + * + * Set BIKEGEAR_PROXY_URL (e.g. http://user:pass@host:port) to route through a + * residential proxy — required when running from a datacenter IP that bikegear.in + * Cloudflare blocks with 403. */ +const axios = require("axios"); const { BRANDS } = require("./brands"); const BASE_URL = "https://bikegear.in"; const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3); const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2); +const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null; + +if (PROXY_URL) { + console.log(`[BIKEGEAR] Using proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://@")}`); +} const FETCH_HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", @@ -31,27 +41,54 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** Build axios proxy config from a proxy URL string. */ +function buildProxyConfig(proxyUrl) { + const u = new URL(proxyUrl); + const cfg = { + host: u.hostname, + port: parseInt(u.port, 10), + protocol: u.protocol.replace(":", ""), + }; + if (u.username) { + cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) }; + } + return cfg; +} + async function fetchHtml(url, attempt = 1) { try { - const res = await fetch(url, { headers: FETCH_HEADERS }); - - if (res.status === 429 || res.status >= 500) { - if (attempt <= 4) { - const wait = attempt * 2000; - console.log(`[BIKEGEAR] RETRY ${url} HTTP ${res.status} — waiting ${wait}ms (${attempt}/4)`); - await sleep(wait); - return fetchHtml(url, attempt + 1); - } - throw new Error(`HTTP ${res.status} after retries`); + const axiosOpts = { + headers: FETCH_HEADERS, + timeout: 30000, + responseType: "text", + maxRedirects: 5, + }; + if (PROXY_URL) { + axiosOpts.proxy = buildProxyConfig(PROXY_URL); } - if (!res.ok) throw new Error(`HTTP ${res.status}`); - return res.text(); + const res = await axios.get(url, axiosOpts); + return res.data; } catch (err) { - if (attempt <= 4 && (err.code === "UND_ERR_CONNECT_TIMEOUT" || err.code === "ECONNRESET" || err.message?.includes("fetch failed"))) { + const status = err.response?.status; + + if ((status === 429 || status >= 500) && attempt <= 4) { + const wait = attempt * 2000; + console.log(`[BIKEGEAR] RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`); + await sleep(wait); + return fetchHtml(url, attempt + 1); + } + + if (status) throw new Error(`HTTP ${status}`); + + const isNetwork = err.code === "ECONNRESET" || err.code === "ECONNREFUSED" || + err.code === "ETIMEDOUT" || err.code === "ENOTFOUND" || + err.message?.includes("timeout") || err.message?.includes("connect"); + if (isNetwork && attempt <= 4) { await sleep(attempt * 2000); return fetchHtml(url, attempt + 1); } + throw err; } }