diff --git a/src/business-logic/import-pipeline/sources/bikegear/index.js b/src/business-logic/import-pipeline/sources/bikegear/index.js index a9f0daf..286a192 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/index.js +++ b/src/business-logic/import-pipeline/sources/bikegear/index.js @@ -1,7 +1,7 @@ const fs = require("node:fs/promises"); const path = require("node:path"); const { BRANDS } = require("./brands"); -const { scrapeBikeGear } = require("./scraper"); +const { scrapeBikeGear, closeBrowser } = require("./scraper"); const { convertBikeGearJsonToShopifyProducts } = require("./converter"); const sourceKey = "bikegear"; @@ -54,7 +54,12 @@ async function fetchWebsiteData({ paths: runPaths }) { } const now = new Date().toISOString(); - const rawProducts = await scrapeBikeGear(); + let rawProducts; + try { + rawProducts = await scrapeBikeGear(); + } finally { + await closeBrowser(); + } const successfulProducts = rawProducts.filter((p) => !p.scrapeError); const successCount = successfulProducts.length; diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index 879d76f..98a793e 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -4,23 +4,92 @@ * * Prices are in INR (Indian Rupees) as scraped from the site. * - * Set BIKEGEAR_PROXY_URL (e.g. http://user:pass@host:port) to route through a - * residential proxy — required when running from a datacenter IP that bikegear.in - * Cloudflare blocks with 403. + * Fetch modes (set in .env): + * BIKEGEAR_FETCH_MODE=browser — use Playwright Chromium (default; bypasses Cloudflare on VPS IPs) + * BIKEGEAR_FETCH_MODE=http — use axios (works only from residential/whitelisted IPs) + * BIKEGEAR_PROXY_URL=http://user:pass@host:port — proxy for http mode */ const axios = require("axios"); const { BRANDS } = require("./brands"); const BASE_URL = "https://bikegear.in"; -const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3); +const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 2); const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2); const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null; +// Default to browser mode — it bypasses Cloudflare without needing a proxy +const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; -if (PROXY_URL) { - console.log(`[BIKEGEAR] Using proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://@")}`); +console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://@")})` : ""}`); + +// ── Browser pool (Playwright) ────────────────────────────────────────────── + +let _browser = null; + +async function getBrowser() { + if (_browser) return _browser; + const { chromium } = require("playwright"); + _browser = await chromium.launch({ + headless: true, + args: [ + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-blink-features=AutomationControlled", + ], + }); + return _browser; } +async function closeBrowser() { + if (_browser) { + await _browser.close().catch(() => {}); + _browser = null; + } +} + +async function fetchHtmlWithBrowser(url, attempt = 1) { + const browser = await getBrowser(); + const context = await browser.newContext({ + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + locale: "en-US", + viewport: { width: 1280, height: 800 }, + extraHTTPHeaders: { + "Accept-Language": "en-US,en;q=0.9", + }, + }); + const page = await context.newPage(); + try { + const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 }); + const status = response?.status() ?? 0; + + if (status === 429 || status >= 500) { + if (attempt <= 4) { + const wait = attempt * 3000; + console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`); + await sleep(wait); + return fetchHtmlWithBrowser(url, attempt + 1); + } + throw new Error(`HTTP ${status} after retries`); + } + if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`); + + // Wait a moment for any JS-rendered content (Cloudflare challenge resolution) + await page.waitForTimeout(1500); + return await page.content(); + } catch (err) { + if (attempt <= 4 && (err.message?.includes("timeout") || err.message?.includes("net::"))) { + await sleep(attempt * 2000); + return fetchHtmlWithBrowser(url, attempt + 1); + } + throw err; + } finally { + await page.close().catch(() => {}); + await context.close().catch(() => {}); + } +} + +// ── HTTP fetch (axios) ───────────────────────────────────────────────────── + const FETCH_HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", @@ -37,62 +106,50 @@ const FETCH_HEADERS = { "Sec-CH-UA-Platform": '"Windows"', }; -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** Build axios proxy config from a proxy URL string. */ function buildProxyConfig(proxyUrl) { const u = new URL(proxyUrl); - const cfg = { - host: u.hostname, - port: parseInt(u.port, 10), - protocol: u.protocol.replace(":", ""), - }; - if (u.username) { - cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) }; - } + const cfg = { host: u.hostname, port: parseInt(u.port, 10), protocol: u.protocol.replace(":", "") }; + if (u.username) cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) }; return cfg; } -async function fetchHtml(url, attempt = 1) { +async function fetchHtmlWithHttp(url, attempt = 1) { try { - const axiosOpts = { - headers: FETCH_HEADERS, - timeout: 30000, - responseType: "text", - maxRedirects: 5, - }; - if (PROXY_URL) { - axiosOpts.proxy = buildProxyConfig(PROXY_URL); - } - - const res = await axios.get(url, axiosOpts); + const opts = { headers: FETCH_HEADERS, timeout: 30000, responseType: "text", maxRedirects: 5 }; + if (PROXY_URL) opts.proxy = buildProxyConfig(PROXY_URL); + const res = await axios.get(url, opts); return res.data; } catch (err) { const status = err.response?.status; - if ((status === 429 || status >= 500) && attempt <= 4) { const wait = attempt * 2000; console.log(`[BIKEGEAR] RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`); await sleep(wait); - return fetchHtml(url, attempt + 1); + return fetchHtmlWithHttp(url, attempt + 1); } - if (status) throw new Error(`HTTP ${status}`); - - const isNetwork = err.code === "ECONNRESET" || err.code === "ECONNREFUSED" || - err.code === "ETIMEDOUT" || err.code === "ENOTFOUND" || + const isNetwork = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"].includes(err.code) || err.message?.includes("timeout") || err.message?.includes("connect"); if (isNetwork && attempt <= 4) { await sleep(attempt * 2000); - return fetchHtml(url, attempt + 1); + return fetchHtmlWithHttp(url, attempt + 1); } - throw err; } } +// ── Unified fetch entry point ────────────────────────────────────────────── + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function fetchHtml(url, attempt = 1) { + return FETCH_MODE === "browser" + ? fetchHtmlWithBrowser(url, attempt) + : fetchHtmlWithHttp(url, attempt); +} + /** Run up to `limit` tasks from `items` at a time. */ async function runConcurrent(items, fn, limit) { const results = []; @@ -346,4 +403,4 @@ async function scrapeBikeGear() { return allProducts; } -module.exports = { scrapeBikeGear }; +module.exports = { scrapeBikeGear, closeBrowser };