diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index cccb043..e16648e 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -25,79 +25,85 @@ const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://@")})` : ""}`); -// ── Browser pool (Playwright) ────────────────────────────────────────────── +// ── Browser pool (Playwright persistent context) ─────────────────────────── +// Uses launchPersistentContext so the browser profile (cookies, cache, fingerprint) +// survives across runs on disk — Cloudflare sees the same "user" every time. const path = require("node:path"); const fs = require("node:fs/promises"); -const COOKIE_STATE_PATH = path.resolve("data/sources/bikegear/.browser-state.json"); +// Profile dir persists between runs — Cloudflare builds trust over multiple visits +const BROWSER_PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile"); -let _browser = null; -// Shared persistent context — all requests reuse the same cookies/session -let _context = null; +// Patches injected into every page to hide headless Chromium signals +const STEALTH_SCRIPT = ` + (() => { + // Hide webdriver flag + Object.defineProperty(navigator, 'webdriver', { get: () => false }); + // Fake plugins so navigator.plugins.length > 0 + const fakePlugin = (name, file, mimes) => { + const p = Object.create(Plugin.prototype); + Object.defineProperty(p, 'name', { get: () => name }); + Object.defineProperty(p, 'filename', { get: () => file }); + Object.defineProperty(p, 'length', { get: () => mimes.length }); + return p; + }; + const plugins = [ + fakePlugin('Chrome PDF Plugin', 'internal-pdf-viewer', ['application/x-google-chrome-pdf']), + fakePlugin('Chrome PDF Viewer', 'mhjfbmdgcfjbbpaeojofohoefgiehjai', ['application/pdf']), + fakePlugin('Native Client', 'internal-nacl-plugin', ['application/x-nacl']), + ]; + Object.defineProperty(navigator, 'plugins', { get: () => plugins }); + // Fake chrome runtime so window.chrome is defined + if (!window.chrome) window.chrome = { runtime: {}, loadTimes: () => {}, csi: () => {}, app: {} }; + // Consistent languages + Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); + })(); +`; + +let _context = null; // launchPersistentContext = browser + context in one async function getBrowserContext() { if (_context) return _context; const { chromium } = require("playwright"); - _browser = await chromium.launch({ + await fs.mkdir(BROWSER_PROFILE_PATH, { recursive: true }); + + _context = await chromium.launchPersistentContext(BROWSER_PROFILE_PATH, { headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-blink-features=AutomationControlled", ], - }); - - // Load saved cookies if they exist (skips Cloudflare challenge on warm runs) - let storageState; - try { - await fs.access(COOKIE_STATE_PATH); - storageState = COOKIE_STATE_PATH; - console.log("[BIKEGEAR] Loaded saved browser session (cookies)."); - } catch { - // No saved state yet — will solve challenge fresh and save afterwards - } - - _context = await _browser.newContext({ userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", locale: "en-US", viewport: { width: 1280, height: 800 }, extraHTTPHeaders: { "Accept-Language": "en-US,en;q=0.9" }, - ...(storageState ? { storageState } : {}), }); + // Apply stealth patches to every new page opened in this context + await _context.addInitScript(STEALTH_SCRIPT); + + console.log("[BIKEGEAR] Browser started with persistent profile."); return _context; } -async function saveBrowserState() { - if (!_context) return; - try { - await fs.mkdir(path.dirname(COOKIE_STATE_PATH), { recursive: true }); - await _context.storageState({ path: COOKIE_STATE_PATH }); - console.log("[BIKEGEAR] Browser session (cookies) saved for next run."); - } catch (err) { - console.warn(`[BIKEGEAR] Could not save browser state: ${err.message}`); - } -} - async function closeBrowser() { - await saveBrowserState(); - if (_browser) { - await _browser.close().catch(() => {}); - _browser = null; + if (_context) { + await _context.close().catch(() => {}); _context = null; + console.log("[BIKEGEAR] Browser closed. Profile saved to disk."); } } -/** Recycle the browser context to free accumulated memory, keeping cookies. */ +/** Close all open pages to free memory — profile stays alive on disk. */ async function recycleBrowserContext() { if (!_context) return; - console.log("[BIKEGEAR] Recycling browser context to free memory..."); - await saveBrowserState(); - await _context.close().catch(() => {}); - _context = null; - // _browser stays alive — only the context is replaced + console.log("[BIKEGEAR] Cleaning up browser pages to free memory..."); + for (const page of _context.pages()) { + await page.close().catch(() => {}); + } } async function fetchHtmlWithBrowser(url, attempt = 1) {