From 46d018c56ee9507549e7c4ade04cd791e2a85445 Mon Sep 17 00:00:00 2001
From: MOHAN <mohan326856@gmail.com>
Date: Fri, 24 Jul 2026 16:27:00 +0530
Subject: [PATCH] fix(bikegear): recycle browser context every 20 brands to
 prevent memory OOM restart

Running Playwright Chromium for 2+ hours with one shared context accumulates
memory until PM2/OOM kills the process with no error. Fix: save cookies and
close/reopen the context every 20 brands so Chromium memory is freed mid-run.
Cookie persistence means the new context resumes with the cf_clearance session.
Override interval with BIKEGEAR_RECYCLE_EVERY env var.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../sources/bikegear/scraper.js               | 88 ++++++++++++++-----
 1 file changed, 68 insertions(+), 20 deletions(-)

diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js
index 9998ccb..5653437 100644
--- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js
+++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js
@@ -27,10 +27,18 @@ console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY
 
 // ── Browser pool (Playwright) ──────────────────────────────────────────────
 
-let _browser = null;
+const path = require("node:path");
+const fs = require("node:fs/promises");
+
+const COOKIE_STATE_PATH = path.resolve("data/sources/bikegear/.browser-state.json");
+
+let _browser = null;
+// Shared persistent context — all requests reuse the same cookies/session
+let _context = null;
+
+async function getBrowserContext() {
+  if (_context) return _context;
 
-async function getBrowser() {
-  if (_browser) return _browser;
   const { chromium } = require("playwright");
   _browser = await chromium.launch({
     headless: true,
@@ -40,26 +48,60 @@ async function getBrowser() {
       "--disable-blink-features=AutomationControlled",
     ],
   });
-  return _browser;
-}
 
-async function closeBrowser() {
-  if (_browser) {
-    await _browser.close().catch(() => {});
-    _browser = null;
+  // Load saved cookies if they exist (skips Cloudflare challenge on warm runs)
+  let storageState;
+  try {
+    await fs.access(COOKIE_STATE_PATH);
+    storageState = COOKIE_STATE_PATH;
+    console.log("[BIKEGEAR] Loaded saved browser session (cookies).");
+  } catch {
+    // No saved state yet — will solve challenge fresh and save afterwards
   }
-}
 
-async function fetchHtmlWithBrowser(url, attempt = 1) {
-  const browser = await getBrowser();
-  const context = await browser.newContext({
+  _context = await _browser.newContext({
     userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     locale: "en-US",
     viewport: { width: 1280, height: 800 },
-    extraHTTPHeaders: {
-      "Accept-Language": "en-US,en;q=0.9",
-    },
+    extraHTTPHeaders: { "Accept-Language": "en-US,en;q=0.9" },
+    ...(storageState ? { storageState } : {}),
   });
+
+  return _context;
+}
+
+async function saveBrowserState() {
+  if (!_context) return;
+  try {
+    await fs.mkdir(path.dirname(COOKIE_STATE_PATH), { recursive: true });
+    await _context.storageState({ path: COOKIE_STATE_PATH });
+    console.log("[BIKEGEAR] Browser session (cookies) saved for next run.");
+  } catch (err) {
+    console.warn(`[BIKEGEAR] Could not save browser state: ${err.message}`);
+  }
+}
+
+async function closeBrowser() {
+  await saveBrowserState();
+  if (_browser) {
+    await _browser.close().catch(() => {});
+    _browser = null;
+    _context = null;
+  }
+}
+
+/** Recycle the browser context to free accumulated memory, keeping cookies. */
+async function recycleBrowserContext() {
+  if (!_context) return;
+  console.log("[BIKEGEAR] Recycling browser context to free memory...");
+  await saveBrowserState();
+  await _context.close().catch(() => {});
+  _context = null;
+  // _browser stays alive — only the context is replaced
+}
+
+async function fetchHtmlWithBrowser(url, attempt = 1) {
+  const context = await getBrowserContext();
   const page = await context.newPage();
   try {
     const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
@@ -89,7 +131,6 @@ async function fetchHtmlWithBrowser(url, attempt = 1) {
     throw err;
   } finally {
     await page.close().catch(() => {});
-    await context.close().catch(() => {});
   }
 }
 
@@ -365,10 +406,13 @@ async function scrapeProductDetail(productUrl, brandName) {
   }
 }
 
+// Recycle the browser context every N brands to prevent memory accumulation
+const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 20);
+
 /** Main entry point: scrape all configured brands and return flat product array. */
 async function scrapeBikeGear() {
   const allProducts = [];
-  let totalBrands = BRANDS.length;
+  const totalBrands = BRANDS.length;
   let brandsDone = 0;
 
   for (const brand of BRANDS) {
@@ -401,8 +445,12 @@ async function scrapeBikeGear() {
     allProducts.push(...products);
     brandsDone++;
 
-    // Polite delay between brands
-    if (brandsDone < totalBrands) await sleep(1000);
+    // Recycle browser context every N brands to free memory
+    if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0 && brandsDone < totalBrands) {
+      await recycleBrowserContext();
+    } else if (brandsDone < totalBrands) {
+      await sleep(1000);
+    }
   }
 
   return allProducts;