From d89db25aee7ba8d294174ebd807be02b2e875f11 Mon Sep 17 00:00:00 2001 From: MOHAN Date: Sun, 2 Aug 2026 16:06:29 +0530 Subject: [PATCH] fix(bikegear): click through listing pages for product details --- scripts/testBikegearProxy.js | 32 ++++++++-- .../sources/bikegear/scraper.js | 63 ++++++++++++++++--- 2 files changed, 83 insertions(+), 12 deletions(-) diff --git a/scripts/testBikegearProxy.js b/scripts/testBikegearProxy.js index d1cf17d..d6784cd 100644 --- a/scripts/testBikegearProxy.js +++ b/scripts/testBikegearProxy.js @@ -128,11 +128,17 @@ async function main() { console.log(`[BIKEGEAR-PROXY-TEST] warmStatus=${warmResponse?.status() || 0}`); } - const response = await page.goto(TEST_URL, { - waitUntil: "domcontentloaded", - timeout: 45000, - ...(TEST_URL !== WARM_URL ? { referer: WARM_URL } : {}), - }); + let response; + if (TEST_URL !== WARM_URL && !TEST_URL.includes("?page=")) { + response = await clickLinkForUrl(page, TEST_URL); + } + if (!response) { + response = await page.goto(TEST_URL, { + waitUntil: "domcontentloaded", + timeout: 45000, + ...(TEST_URL !== WARM_URL ? { referer: WARM_URL } : {}), + }); + } await page.waitForTimeout(8000); const status = response?.status() || 0; @@ -160,6 +166,22 @@ async function main() { } } +async function clickLinkForUrl(page, url) { + const navigationPromise = page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 45000 }) + .catch(() => null); + const clicked = await page.evaluate((targetUrl) => { + const links = Array.from(document.querySelectorAll("a[href]")); + const link = links.find((a) => a.href === targetUrl); + if (!link) return false; + link.click(); + return true; + }, url).catch(() => false); + + if (!clicked) return null; + await page.waitForURL(url, { timeout: 45000 }).catch(() => {}); + return navigationPromise; +} + main().catch((error) => { process.exitCode = 1; console.error(`[BIKEGEAR-PROXY-TEST] ERROR ${error.message}`); diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index e8bfdec..56ee940 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -125,6 +125,7 @@ const STEALTH_SCRIPT = ` let _context = null; // launchPersistentContext = browser + context in one let _browserFetchQueue = Promise.resolve(); +const PRODUCT_REFERERS = new Map(); async function getBrowserContext() { if (_context) return _context; @@ -192,11 +193,32 @@ async function fetchHtmlWithBrowser(url, referer = null) { for (let attempt = 1; attempt <= 5; attempt += 1) { const page = await context.newPage(); try { - const response = await page.goto(url, { - waitUntil: "domcontentloaded", - timeout: 45000, - ...(referer ? { referer } : {}), - }); + let response; + if (referer && referer !== url && !url.includes("?page=")) { + const warmResponse = await page.goto(referer, { waitUntil: "domcontentloaded", timeout: 45000 }); + const warmStatus = warmResponse?.status() ?? 0; + if (warmStatus && warmStatus !== 200 && warmStatus !== 304) { + throw new Error(`Referer HTTP ${warmStatus}`); + } + await page.waitForTimeout(2500); + + response = await clickLinkForUrl(page, url); + if (response) { + // Navigation happened through the listing page link. + } else { + response = await page.goto(url, { + waitUntil: "domcontentloaded", + timeout: 45000, + referer, + }); + } + } else { + response = await page.goto(url, { + waitUntil: "domcontentloaded", + timeout: 45000, + ...(referer ? { referer } : {}), + }); + } const status = response?.status() ?? 0; if (status === 429 || status >= 500) { @@ -232,6 +254,22 @@ async function fetchHtmlWithBrowser(url, referer = null) { throw new Error("Browser fetch failed after retries"); } +async function clickLinkForUrl(page, url) { + const navigationPromise = page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 45000 }) + .catch(() => null); + const clicked = await page.evaluate((targetUrl) => { + const links = Array.from(document.querySelectorAll("a[href]")); + const link = links.find((a) => a.href === targetUrl); + if (!link) return false; + link.click(); + return true; + }, url).catch(() => false); + + if (!clicked) return null; + await page.waitForURL(url, { timeout: 45000 }).catch(() => {}); + return navigationPromise; +} + // ── HTTP fetch (axios) ───────────────────────────────────────────────────── const FETCH_HEADERS = { @@ -331,6 +369,14 @@ function extractProductUrls(html) { return [...urls]; } +function rememberProductReferers(productUrls, refererUrl) { + for (const productUrl of productUrls) { + if (!PRODUCT_REFERERS.has(productUrl)) { + PRODUCT_REFERERS.set(productUrl, refererUrl); + } + } +} + /** Scrape all listing pages for a brand and collect product detail URLs. */ async function scrapeBrandProductUrls(brand) { console.log(`[BIKEGEAR][${brand.name}] Fetching page 1...`); @@ -344,6 +390,7 @@ async function scrapeBrandProductUrls(brand) { const maxPage = extractMaxPage(html); const urls = new Set(extractProductUrls(html)); + rememberProductReferers(urls, brand.url); if (maxPage > 1) { console.log(`[BIKEGEAR][${brand.name}] Found ${maxPage} pages — fetching pages 2–${maxPage}...`); @@ -353,7 +400,9 @@ async function scrapeBrandProductUrls(brand) { const pageUrl = `${brand.url}?page=${pageNum}`; try { const pageHtml = await fetchHtml(pageUrl, 1, brand.url); - for (const u of extractProductUrls(pageHtml)) urls.add(u); + const pageProductUrls = extractProductUrls(pageHtml); + rememberProductReferers(pageProductUrls, pageUrl); + for (const u of pageProductUrls) urls.add(u); } catch (err) { console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`); } @@ -524,7 +573,7 @@ async function scrapeBikeGear() { let done = 0; const products = await runConcurrent(productUrls, async (url) => { - const product = await scrapeProductDetail(url, brand.name, brand.url); + const product = await scrapeProductDetail(url, brand.name, PRODUCT_REFERERS.get(url) || brand.url); done++; if (product.scrapeError) { console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);