admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): switch to persistent browser profile + stealth script to beat Cloudflare detection

Browse Source 
Previous approach (newContext per run) was still detected via navigator.plugins=0
and window.chrome=undefined. Two fixes:
1. launchPersistentContext writes a real browser profile to disk — cookies,
   cache and fingerprint survive restarts so Cloudflare trusts the same "user"
2. addInitScript patches plugins, chrome runtime, webdriver and languages on
   every page, hiding remaining headless Chromium signals

recycleBrowserContext now just closes stale pages (profile stays on disk).
closeBrowser just closes the browser process — profile already persisted.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-07-29 07:25:35 +05:30
parent e4b3292533
commit 76bd69a92f
1 changed files with 47 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -25,79 +25,85 @@ const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://<redacted>@")})` : ""}`);
// ── Browser pool (Playwright) ──────────────────────────────────────────────
// ── Browser pool (Playwright persistent context) ───────────────────────────
// Uses launchPersistentContext so the browser profile (cookies, cache, fingerprint)
// survives across runs on disk — Cloudflare sees the same "user" every time.
const path = require("node:path");
const fs = require("node:fs/promises");
const COOKIE_STATE_PATH = path.resolve("data/sources/bikegear/.browser-state.json");
// Profile dir persists between runs — Cloudflare builds trust over multiple visits
const BROWSER_PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile");
let _browser = null;
// Shared persistent context — all requests reuse the same cookies/session
let _context = null;
// Patches injected into every page to hide headless Chromium signals
const STEALTH_SCRIPT = `
(() => {
// Hide webdriver flag
Object.defineProperty(navigator, 'webdriver', { get: () => false });
// Fake plugins so navigator.plugins.length > 0
const fakePlugin = (name, file, mimes) => {
const p = Object.create(Plugin.prototype);
Object.defineProperty(p, 'name', { get: () => name });
Object.defineProperty(p, 'filename', { get: () => file });
Object.defineProperty(p, 'length', { get: () => mimes.length });
return p;
};
const plugins = [
fakePlugin('Chrome PDF Plugin', 'internal-pdf-viewer', ['application/x-google-chrome-pdf']),
fakePlugin('Chrome PDF Viewer', 'mhjfbmdgcfjbbpaeojofohoefgiehjai', ['application/pdf']),
fakePlugin('Native Client', 'internal-nacl-plugin', ['application/x-nacl']),
];
Object.defineProperty(navigator, 'plugins', { get: () => plugins });
// Fake chrome runtime so window.chrome is defined
if (!window.chrome) window.chrome = { runtime: {}, loadTimes: () => {}, csi: () => {}, app: {} };
// Consistent languages
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
})();
`;
let _context = null; // launchPersistentContext = browser + context in one
async function getBrowserContext() {
if (_context) return _context;
const { chromium } = require("playwright");
_browser = await chromium.launch({
await fs.mkdir(BROWSER_PROFILE_PATH, { recursive: true });
_context = await chromium.launchPersistentContext(BROWSER_PROFILE_PATH, {
headless: true,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-blink-features=AutomationControlled",
],
});
// Load saved cookies if they exist (skips Cloudflare challenge on warm runs)
let storageState;
try {
await fs.access(COOKIE_STATE_PATH);
storageState = COOKIE_STATE_PATH;
console.log("[BIKEGEAR] Loaded saved browser session (cookies).");
} catch {
// No saved state yet — will solve challenge fresh and save afterwards
}
_context = await _browser.newContext({
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
locale: "en-US",
viewport: { width: 1280, height: 800 },
extraHTTPHeaders: { "Accept-Language": "en-US,en;q=0.9" },
...(storageState ? { storageState } : {}),
});
// Apply stealth patches to every new page opened in this context
await _context.addInitScript(STEALTH_SCRIPT);
console.log("[BIKEGEAR] Browser started with persistent profile.");
return _context;
}
async function saveBrowserState() {
if (!_context) return;
try {
await fs.mkdir(path.dirname(COOKIE_STATE_PATH), { recursive: true });
await _context.storageState({ path: COOKIE_STATE_PATH });
console.log("[BIKEGEAR] Browser session (cookies) saved for next run.");
} catch (err) {
console.warn(`[BIKEGEAR] Could not save browser state: ${err.message}`);
}
}
async function closeBrowser() {
await saveBrowserState();
if (_browser) {
await _browser.close().catch(() => {});
_browser = null;
if (_context) {
await _context.close().catch(() => {});
_context = null;
console.log("[BIKEGEAR] Browser closed. Profile saved to disk.");
}
}
/** Recycle the browser context to free accumulated memory, keeping cookies. */
/** Close all open pages to free memory — profile stays alive on disk. */
async function recycleBrowserContext() {
if (!_context) return;
console.log("[BIKEGEAR] Recycling browser context to free memory...");
await saveBrowserState();
await _context.close().catch(() => {});
_context = null;
// _browser stays alive — only the context is replaced
console.log("[BIKEGEAR] Cleaning up browser pages to free memory...");
for (const page of _context.pages()) {
await page.close().catch(() => {});
}
}
async function fetchHtmlWithBrowser(url, attempt = 1) {