fix(bikegear): use Playwright browser mode to bypass Cloudflare on VPS IPs

HTTP requests from datacenter IPs get 403 from bikegear.in Cloudflare WAF.
Adds BIKEGEAR_FETCH_MODE=browser (default) which runs a headless Chromium via
Playwright — already installed — and navigates pages like a real user, bypassing
the JS challenge without needing a paid proxy.

Set BIKEGEAR_FETCH_MODE=http to keep the old axios path (for local/residential IPs).
Browser instance is shared across the full scrape run and closed cleanly on finish.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/business-logic/import-pipeline/sources/bikegear/index.js

@@ -1,7 +1,7 @@
 const fs = require("node:fs/promises");
 const path = require("node:path");
 const { BRANDS } = require("./brands");
-const { scrapeBikeGear } = require("./scraper");
+const { scrapeBikeGear, closeBrowser } = require("./scraper");
 const { convertBikeGearJsonToShopifyProducts } = require("./converter");
 
 const sourceKey = "bikegear";
@@ -54,7 +54,12 @@ async function fetchWebsiteData({ paths: runPaths }) {
   }
 
   const now = new Date().toISOString();
-  const rawProducts = await scrapeBikeGear();
+  let rawProducts;
+  try {
+    rawProducts = await scrapeBikeGear();
+  } finally {
+    await closeBrowser();
+  }
 
   const successfulProducts = rawProducts.filter((p) => !p.scrapeError);
   const successCount = successfulProducts.length;

src/business-logic/import-pipeline/sources/bikegear/scraper.js

@@ -4,23 +4,92 @@
  *
  * Prices are in INR (Indian Rupees) as scraped from the site.
  *
- * Set BIKEGEAR_PROXY_URL (e.g. http://user:pass@host:port) to route through a
+ * Fetch modes (set in .env):
- * residential proxy — required when running from a datacenter IP that bikegear.in
+ *   BIKEGEAR_FETCH_MODE=browser  — use Playwright Chromium (default; bypasses Cloudflare on VPS IPs)
- * Cloudflare blocks with 403.
+ *   BIKEGEAR_FETCH_MODE=http     — use axios (works only from residential/whitelisted IPs)
+ *   BIKEGEAR_PROXY_URL=http://user:pass@host:port  — proxy for http mode
  */
 
 const axios = require("axios");
 const { BRANDS } = require("./brands");
 
 const BASE_URL = "https://bikegear.in";
-const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3);
+const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 2);
 const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2);
 const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
+// Default to browser mode — it bypasses Cloudflare without needing a proxy
+const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
 
-if (PROXY_URL) {
+console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://<redacted>@")})` : ""}`);
-  console.log(`[BIKEGEAR] Using proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://<redacted>@")}`);
+
+// ── Browser pool (Playwright) ──────────────────────────────────────────────
+
+let _browser = null;
+
+async function getBrowser() {
+  if (_browser) return _browser;
+  const { chromium } = require("playwright");
+  _browser = await chromium.launch({
+    headless: true,
+    args: [
+      "--no-sandbox",
+      "--disable-setuid-sandbox",
+      "--disable-blink-features=AutomationControlled",
+    ],
+  });
+  return _browser;
 }
 
+async function closeBrowser() {
+  if (_browser) {
+    await _browser.close().catch(() => {});
+    _browser = null;
+  }
+}
+
+async function fetchHtmlWithBrowser(url, attempt = 1) {
+  const browser = await getBrowser();
+  const context = await browser.newContext({
+    userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    locale: "en-US",
+    viewport: { width: 1280, height: 800 },
+    extraHTTPHeaders: {
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+  });
+  const page = await context.newPage();
+  try {
+    const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
+    const status = response?.status() ?? 0;
+
+    if (status === 429 || status >= 500) {
+      if (attempt <= 4) {
+        const wait = attempt * 3000;
+        console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
+        await sleep(wait);
+        return fetchHtmlWithBrowser(url, attempt + 1);
+      }
+      throw new Error(`HTTP ${status} after retries`);
+    }
+    if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`);
+
+    // Wait a moment for any JS-rendered content (Cloudflare challenge resolution)
+    await page.waitForTimeout(1500);
+    return await page.content();
+  } catch (err) {
+    if (attempt <= 4 && (err.message?.includes("timeout") || err.message?.includes("net::"))) {
+      await sleep(attempt * 2000);
+      return fetchHtmlWithBrowser(url, attempt + 1);
+    }
+    throw err;
+  } finally {
+    await page.close().catch(() => {});
+    await context.close().catch(() => {});
+  }
+}
+
+// ── HTTP fetch (axios) ─────────────────────────────────────────────────────
+
 const FETCH_HEADERS = {
   "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
   "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@@ -37,62 +106,50 @@ const FETCH_HEADERS = {
   "Sec-CH-UA-Platform": '"Windows"',
 };
 
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-/** Build axios proxy config from a proxy URL string. */
 function buildProxyConfig(proxyUrl) {
   const u = new URL(proxyUrl);
-  const cfg = {
+  const cfg = { host: u.hostname, port: parseInt(u.port, 10), protocol: u.protocol.replace(":", "") };
-    host: u.hostname,
+  if (u.username) cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) };
-    port: parseInt(u.port, 10),
-    protocol: u.protocol.replace(":", ""),
-  };
-  if (u.username) {
-    cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) };
-  }
   return cfg;
 }
 
-async function fetchHtml(url, attempt = 1) {
+async function fetchHtmlWithHttp(url, attempt = 1) {
   try {
-    const axiosOpts = {
+    const opts = { headers: FETCH_HEADERS, timeout: 30000, responseType: "text", maxRedirects: 5 };
-      headers: FETCH_HEADERS,
+    if (PROXY_URL) opts.proxy = buildProxyConfig(PROXY_URL);
-      timeout: 30000,
+    const res = await axios.get(url, opts);
-      responseType: "text",
-      maxRedirects: 5,
-    };
-    if (PROXY_URL) {
-      axiosOpts.proxy = buildProxyConfig(PROXY_URL);
-    }
-
-    const res = await axios.get(url, axiosOpts);
     return res.data;
   } catch (err) {
     const status = err.response?.status;
-
     if ((status === 429 || status >= 500) && attempt <= 4) {
       const wait = attempt * 2000;
       console.log(`[BIKEGEAR] RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
       await sleep(wait);
-      return fetchHtml(url, attempt + 1);
+      return fetchHtmlWithHttp(url, attempt + 1);
     }
-
     if (status) throw new Error(`HTTP ${status}`);
-
+    const isNetwork = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"].includes(err.code) ||
-    const isNetwork = err.code === "ECONNRESET" || err.code === "ECONNREFUSED" ||
-      err.code === "ETIMEDOUT" || err.code === "ENOTFOUND" ||
       err.message?.includes("timeout") || err.message?.includes("connect");
     if (isNetwork && attempt <= 4) {
       await sleep(attempt * 2000);
-      return fetchHtml(url, attempt + 1);
+      return fetchHtmlWithHttp(url, attempt + 1);
     }
-
     throw err;
   }
 }
 
+// ── Unified fetch entry point ──────────────────────────────────────────────
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function fetchHtml(url, attempt = 1) {
+  return FETCH_MODE === "browser"
+    ? fetchHtmlWithBrowser(url, attempt)
+    : fetchHtmlWithHttp(url, attempt);
+}
+
 /** Run up to `limit` tasks from `items` at a time. */
 async function runConcurrent(items, fn, limit) {
   const results = [];
@@ -346,4 +403,4 @@ async function scrapeBikeGear() {
   return allProducts;
 }
 
-module.exports = { scrapeBikeGear };
+module.exports = { scrapeBikeGear, closeBrowser };