fix(bikegear): click through listing pages for product details

scripts/testBikegearProxy.js

@@ -128,11 +128,17 @@ async function main() {
       console.log(`[BIKEGEAR-PROXY-TEST] warmStatus=${warmResponse?.status() || 0}`);
     }
 
-    const response = await page.goto(TEST_URL, {
+    let response;
+    if (TEST_URL !== WARM_URL && !TEST_URL.includes("?page=")) {
+      response = await clickLinkForUrl(page, TEST_URL);
+    }
+    if (!response) {
+      response = await page.goto(TEST_URL, {
         waitUntil: "domcontentloaded",
         timeout: 45000,
         ...(TEST_URL !== WARM_URL ? { referer: WARM_URL } : {}),
       });
+    }
     await page.waitForTimeout(8000);
 
     const status = response?.status() || 0;
@@ -160,6 +166,22 @@ async function main() {
   }
 }
 
+async function clickLinkForUrl(page, url) {
+  const navigationPromise = page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 45000 })
+    .catch(() => null);
+  const clicked = await page.evaluate((targetUrl) => {
+    const links = Array.from(document.querySelectorAll("a[href]"));
+    const link = links.find((a) => a.href === targetUrl);
+    if (!link) return false;
+    link.click();
+    return true;
+  }, url).catch(() => false);
+
+  if (!clicked) return null;
+  await page.waitForURL(url, { timeout: 45000 }).catch(() => {});
+  return navigationPromise;
+}
+
 main().catch((error) => {
   process.exitCode = 1;
   console.error(`[BIKEGEAR-PROXY-TEST] ERROR ${error.message}`);

src/business-logic/import-pipeline/sources/bikegear/scraper.js

@@ -125,6 +125,7 @@ const STEALTH_SCRIPT = `
 
 let _context = null; // launchPersistentContext = browser + context in one
 let _browserFetchQueue = Promise.resolve();
+const PRODUCT_REFERERS = new Map();
 
 async function getBrowserContext() {
   if (_context) return _context;
@@ -192,11 +193,32 @@ async function fetchHtmlWithBrowser(url, referer = null) {
   for (let attempt = 1; attempt <= 5; attempt += 1) {
     const page = await context.newPage();
     try {
-      const response = await page.goto(url, {
+      let response;
+      if (referer && referer !== url && !url.includes("?page=")) {
+        const warmResponse = await page.goto(referer, { waitUntil: "domcontentloaded", timeout: 45000 });
+        const warmStatus = warmResponse?.status() ?? 0;
+        if (warmStatus && warmStatus !== 200 && warmStatus !== 304) {
+          throw new Error(`Referer HTTP ${warmStatus}`);
+        }
+        await page.waitForTimeout(2500);
+
+        response = await clickLinkForUrl(page, url);
+        if (response) {
+          // Navigation happened through the listing page link.
+        } else {
+          response = await page.goto(url, {
+            waitUntil: "domcontentloaded",
+            timeout: 45000,
+            referer,
+          });
+        }
+      } else {
+        response = await page.goto(url, {
           waitUntil: "domcontentloaded",
           timeout: 45000,
           ...(referer ? { referer } : {}),
         });
+      }
       const status = response?.status() ?? 0;
 
       if (status === 429 || status >= 500) {
@@ -232,6 +254,22 @@ async function fetchHtmlWithBrowser(url, referer = null) {
   throw new Error("Browser fetch failed after retries");
 }
 
+async function clickLinkForUrl(page, url) {
+  const navigationPromise = page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 45000 })
+    .catch(() => null);
+  const clicked = await page.evaluate((targetUrl) => {
+    const links = Array.from(document.querySelectorAll("a[href]"));
+    const link = links.find((a) => a.href === targetUrl);
+    if (!link) return false;
+    link.click();
+    return true;
+  }, url).catch(() => false);
+
+  if (!clicked) return null;
+  await page.waitForURL(url, { timeout: 45000 }).catch(() => {});
+  return navigationPromise;
+}
+
 // ── HTTP fetch (axios) ─────────────────────────────────────────────────────
 
 const FETCH_HEADERS = {
@@ -331,6 +369,14 @@ function extractProductUrls(html) {
   return [...urls];
 }
 
+function rememberProductReferers(productUrls, refererUrl) {
+  for (const productUrl of productUrls) {
+    if (!PRODUCT_REFERERS.has(productUrl)) {
+      PRODUCT_REFERERS.set(productUrl, refererUrl);
+    }
+  }
+}
+
 /** Scrape all listing pages for a brand and collect product detail URLs. */
 async function scrapeBrandProductUrls(brand) {
   console.log(`[BIKEGEAR][${brand.name}] Fetching page 1...`);
@@ -344,6 +390,7 @@ async function scrapeBrandProductUrls(brand) {
 
   const maxPage = extractMaxPage(html);
   const urls = new Set(extractProductUrls(html));
+  rememberProductReferers(urls, brand.url);
 
   if (maxPage > 1) {
     console.log(`[BIKEGEAR][${brand.name}] Found ${maxPage} pages — fetching pages 2–${maxPage}...`);
@@ -353,7 +400,9 @@ async function scrapeBrandProductUrls(brand) {
       const pageUrl = `${brand.url}?page=${pageNum}`;
       try {
         const pageHtml = await fetchHtml(pageUrl, 1, brand.url);
-        for (const u of extractProductUrls(pageHtml)) urls.add(u);
+        const pageProductUrls = extractProductUrls(pageHtml);
+        rememberProductReferers(pageProductUrls, pageUrl);
+        for (const u of pageProductUrls) urls.add(u);
       } catch (err) {
         console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`);
       }
@@ -524,7 +573,7 @@ async function scrapeBikeGear() {
 
       let done = 0;
       const products = await runConcurrent(productUrls, async (url) => {
-        const product = await scrapeProductDetail(url, brand.name, brand.url);
+        const product = await scrapeProductDetail(url, brand.name, PRODUCT_REFERERS.get(url) || brand.url);
         done++;
         if (product.scrapeError) {
           console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);