diff --git a/scripts/testBikegearProxy.js b/scripts/testBikegearProxy.js index ae7785c..d1cf17d 100644 --- a/scripts/testBikegearProxy.js +++ b/scripts/testBikegearProxy.js @@ -6,6 +6,7 @@ const { chromium } = require("playwright"); const DEFAULT_URL = "https://bikegear.in/acerbis"; const TEST_URL = process.argv[2] || process.env.BIKEGEAR_PROXY_TEST_URL || DEFAULT_URL; +const WARM_URL = process.env.BIKEGEAR_PROXY_TEST_WARM_URL || DEFAULT_URL; const PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile-test"); const STEALTH_SCRIPT = ` @@ -100,6 +101,7 @@ async function main() { await fs.mkdir(PROFILE_PATH, { recursive: true }); console.log(`[BIKEGEAR-PROXY-TEST] url=${TEST_URL}`); + console.log(`[BIKEGEAR-PROXY-TEST] warmUrl=${TEST_URL === WARM_URL ? "-" : WARM_URL}`); console.log(`[BIKEGEAR-PROXY-TEST] proxy=${maskProxyUrl(proxyUrl)}`); const context = await chromium.launchPersistentContext(PROFILE_PATH, { @@ -119,7 +121,18 @@ async function main() { try { await context.addInitScript(STEALTH_SCRIPT); const page = await context.newPage(); - const response = await page.goto(TEST_URL, { waitUntil: "domcontentloaded", timeout: 45000 }); + + if (TEST_URL !== WARM_URL) { + const warmResponse = await page.goto(WARM_URL, { waitUntil: "domcontentloaded", timeout: 45000 }); + await page.waitForTimeout(8000); + console.log(`[BIKEGEAR-PROXY-TEST] warmStatus=${warmResponse?.status() || 0}`); + } + + const response = await page.goto(TEST_URL, { + waitUntil: "domcontentloaded", + timeout: 45000, + ...(TEST_URL !== WARM_URL ? { referer: WARM_URL } : {}), + }); await page.waitForTimeout(8000); const status = response?.status() || 0; diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index e4bf81d..e8bfdec 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -180,19 +180,23 @@ async function recycleBrowserContext() { } } -async function fetchHtmlWithBrowserQueued(url) { - const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url)); +async function fetchHtmlWithBrowserQueued(url, referer = null) { + const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url, referer)); _browserFetchQueue = run.catch(() => {}); return run; } -async function fetchHtmlWithBrowser(url) { +async function fetchHtmlWithBrowser(url, referer = null) { const context = await getBrowserContext(); for (let attempt = 1; attempt <= 5; attempt += 1) { const page = await context.newPage(); try { - const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45000 }); + const response = await page.goto(url, { + waitUntil: "domcontentloaded", + timeout: 45000, + ...(referer ? { referer } : {}), + }); const status = response?.status() ?? 0; if (status === 429 || status >= 500) { @@ -284,9 +288,9 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } -function fetchHtml(url, attempt = 1) { +function fetchHtml(url, attempt = 1, referer = null) { return FETCH_MODE === "browser" - ? fetchHtmlWithBrowserQueued(url) + ? fetchHtmlWithBrowserQueued(url, referer) : fetchHtmlWithHttp(url, attempt); } @@ -348,7 +352,7 @@ async function scrapeBrandProductUrls(brand) { await runConcurrent(pageNums, async (pageNum) => { const pageUrl = `${brand.url}?page=${pageNum}`; try { - const pageHtml = await fetchHtml(pageUrl); + const pageHtml = await fetchHtml(pageUrl, 1, brand.url); for (const u of extractProductUrls(pageHtml)) urls.add(u); } catch (err) { console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`); @@ -463,10 +467,10 @@ function parsePrice(str) { } /** Scrape a single product detail page and return structured data. */ -async function scrapeProductDetail(productUrl, brandName) { +async function scrapeProductDetail(productUrl, brandName, refererUrl = null) { let html; try { - html = await fetchHtml(productUrl); + html = await fetchHtml(productUrl, 1, refererUrl); } catch (err) { return { url: productUrl, scrapeError: err.message }; } @@ -520,7 +524,7 @@ async function scrapeBikeGear() { let done = 0; const products = await runConcurrent(productUrls, async (url) => { - const product = await scrapeProductDetail(url, brand.name); + const product = await scrapeProductDetail(url, brand.name, brand.url); done++; if (product.scrapeError) { console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);