admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): use Playwright browser mode to bypass Cloudflare on VPS IPs

Browse Source 
HTTP requests from datacenter IPs get 403 from bikegear.in Cloudflare WAF.
Adds BIKEGEAR_FETCH_MODE=browser (default) which runs a headless Chromium via
Playwright — already installed — and navigates pages like a real user, bypassing
the JS challenge without needing a paid proxy.

Set BIKEGEAR_FETCH_MODE=http to keep the old axios path (for local/residential IPs).
Browser instance is shared across the full scrape run and closed cleanly on finish.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-07-22 18:05:24 +05:30
parent 846e467b85
commit 0fbe75dd3d
2 changed files with 104 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/business-logic/import-pipeline/sources/bikegear/index.js
View File

@ -1,7 +1,7 @@
const fs = require("node:fs/promises");
const path = require("node:path");
const { BRANDS } = require("./brands");
const { scrapeBikeGear } = require("./scraper");
const { scrapeBikeGear, closeBrowser } = require("./scraper");
const { convertBikeGearJsonToShopifyProducts } = require("./converter");
const sourceKey = "bikegear";
@ -54,7 +54,12 @@ async function fetchWebsiteData({ paths: runPaths }) {
}
const now = new Date().toISOString();
const rawProducts = await scrapeBikeGear();
let rawProducts;
try {
rawProducts = await scrapeBikeGear();
} finally {
await closeBrowser();
}
const successfulProducts = rawProducts.filter((p) => !p.scrapeError);
const successCount = successfulProducts.length;

137
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -4,23 +4,92 @@
*
* Prices are in INR (Indian Rupees) as scraped from the site.
*
* Set BIKEGEAR_PROXY_URL (e.g. http://user:pass@host:port) to route through a
* residential proxy — required when running from a datacenter IP that bikegear.in
* Cloudflare blocks with 403.
* Fetch modes (set in .env):
* BIKEGEAR_FETCH_MODE=browser — use Playwright Chromium (default; bypasses Cloudflare on VPS IPs)
* BIKEGEAR_FETCH_MODE=http — use axios (works only from residential/whitelisted IPs)
* BIKEGEAR_PROXY_URL=http://user:pass@host:port — proxy for http mode
*/
const axios = require("axios");
const { BRANDS } = require("./brands");
const BASE_URL = "https://bikegear.in";
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3);
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 2);
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2);
const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
// Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
if (PROXY_URL) {
console.log(`[BIKEGEAR] Using proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://<redacted>@")}`);
console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://<redacted>@")})` : ""}`);
// ── Browser pool (Playwright) ──────────────────────────────────────────────
let _browser = null;
async function getBrowser() {
if (_browser) return _browser;
const { chromium } = require("playwright");
_browser = await chromium.launch({
headless: true,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-blink-features=AutomationControlled",
],
});
return _browser;
}
async function closeBrowser() {
if (_browser) {
await _browser.close().catch(() => {});
_browser = null;
}
}
async function fetchHtmlWithBrowser(url, attempt = 1) {
const browser = await getBrowser();
const context = await browser.newContext({
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
locale: "en-US",
viewport: { width: 1280, height: 800 },
extraHTTPHeaders: {
"Accept-Language": "en-US,en;q=0.9",
},
});
const page = await context.newPage();
try {
const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
const status = response?.status() ?? 0;
if (status === 429 || status >= 500) {
if (attempt <= 4) {
const wait = attempt * 3000;
console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
await sleep(wait);
return fetchHtmlWithBrowser(url, attempt + 1);
}
throw new Error(`HTTP ${status} after retries`);
}
if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`);
// Wait a moment for any JS-rendered content (Cloudflare challenge resolution)
await page.waitForTimeout(1500);
return await page.content();
} catch (err) {
if (attempt <= 4 && (err.message?.includes("timeout") || err.message?.includes("net::"))) {
await sleep(attempt * 2000);
return fetchHtmlWithBrowser(url, attempt + 1);
}
throw err;
} finally {
await page.close().catch(() => {});
await context.close().catch(() => {});
}
}
// ── HTTP fetch (axios) ─────────────────────────────────────────────────────
const FETCH_HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@ -37,62 +106,50 @@ const FETCH_HEADERS = {
"Sec-CH-UA-Platform": '"Windows"',
};
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Build axios proxy config from a proxy URL string. */
function buildProxyConfig(proxyUrl) {
const u = new URL(proxyUrl);
const cfg = {
host: u.hostname,
port: parseInt(u.port, 10),
protocol: u.protocol.replace(":", ""),
};
if (u.username) {
cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) };
}
const cfg = { host: u.hostname, port: parseInt(u.port, 10), protocol: u.protocol.replace(":", "") };
if (u.username) cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) };
return cfg;
}
async function fetchHtml(url, attempt = 1) {
async function fetchHtmlWithHttp(url, attempt = 1) {
try {
const axiosOpts = {
headers: FETCH_HEADERS,
timeout: 30000,
responseType: "text",
maxRedirects: 5,
};
if (PROXY_URL) {
axiosOpts.proxy = buildProxyConfig(PROXY_URL);
}
const res = await axios.get(url, axiosOpts);
const opts = { headers: FETCH_HEADERS, timeout: 30000, responseType: "text", maxRedirects: 5 };
if (PROXY_URL) opts.proxy = buildProxyConfig(PROXY_URL);
const res = await axios.get(url, opts);
return res.data;
} catch (err) {
const status = err.response?.status;
if ((status === 429 || status >= 500) && attempt <= 4) {
const wait = attempt * 2000;
console.log(`[BIKEGEAR] RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
await sleep(wait);
return fetchHtml(url, attempt + 1);
return fetchHtmlWithHttp(url, attempt + 1);
}
if (status) throw new Error(`HTTP ${status}`);
const isNetwork = err.code === "ECONNRESET" || err.code === "ECONNREFUSED" ||
err.code === "ETIMEDOUT" || err.code === "ENOTFOUND" ||
const isNetwork = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"].includes(err.code) ||
err.message?.includes("timeout") || err.message?.includes("connect");
if (isNetwork && attempt <= 4) {
await sleep(attempt * 2000);
return fetchHtml(url, attempt + 1);
return fetchHtmlWithHttp(url, attempt + 1);
}
throw err;
}
}
// ── Unified fetch entry point ──────────────────────────────────────────────
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function fetchHtml(url, attempt = 1) {
return FETCH_MODE === "browser"
? fetchHtmlWithBrowser(url, attempt)
: fetchHtmlWithHttp(url, attempt);
}
/** Run up to `limit` tasks from `items` at a time. */
async function runConcurrent(items, fn, limit) {
const results = [];
@ -346,4 +403,4 @@ async function scrapeBikeGear() {
return allProducts;
}
module.exports = { scrapeBikeGear };
module.exports = { scrapeBikeGear, closeBrowser };