diff --git a/scripts/testBikegearProxy.js b/scripts/testBikegearProxy.js index f93b764..88f5e78 100644 --- a/scripts/testBikegearProxy.js +++ b/scripts/testBikegearProxy.js @@ -7,6 +7,7 @@ const { chromium } = require("playwright"); const DEFAULT_URL = "https://bikegear.in/acerbis"; const TEST_URL = process.argv[2] || process.env.BIKEGEAR_PROXY_TEST_URL || DEFAULT_URL; const WARM_URL = process.env.BIKEGEAR_PROXY_TEST_WARM_URL || DEFAULT_URL; +const WARM_DELAY = Number(process.env.BIKEGEAR_WARM_DELAY || 8000); const PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile-test"); const STEALTH_SCRIPT = ` @@ -124,12 +125,12 @@ async function main() { if (TEST_URL !== WARM_URL) { const warmResponse = await page.goto(WARM_URL, { waitUntil: "domcontentloaded", timeout: 45000 }); - await page.waitForTimeout(8000); + await page.waitForTimeout(WARM_DELAY); console.log(`[BIKEGEAR-PROXY-TEST] warmStatus=${warmResponse?.status() || 0}`); } let response; - if (TEST_URL !== WARM_URL) { + if (TEST_URL !== WARM_URL && !TEST_URL.includes("?page=")) { const clickResult = await clickLinkForUrl(page, TEST_URL); response = clickResult.response; console.log(`[BIKEGEAR-PROXY-TEST] clickedLink=${clickResult.clicked}`); diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index 5dfa547..983fc18 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -32,12 +32,13 @@ const DETAIL_CONCURRENCY = IS_HTTP_MODE // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500); const BROWSER_429_DELAY = Number(process.env.BIKEGEAR_429_DELAY ?? 30000); +const BROWSER_WARM_DELAY = Number(process.env.BIKEGEAR_WARM_DELAY ?? 8000); const PROXY_URL = getProxyUrl(); // Default to browser mode — it bypasses Cloudflare without needing a proxy const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`); -console.log(`[BIKEGEAR] Concurrency: listing=${LISTING_CONCURRENCY} detail=${DETAIL_CONCURRENCY} delay=${BROWSER_REQUEST_DELAY}ms rateLimitDelay=${BROWSER_429_DELAY}ms`); +console.log(`[BIKEGEAR] Concurrency: listing=${LISTING_CONCURRENCY} detail=${DETAIL_CONCURRENCY} delay=${BROWSER_REQUEST_DELAY}ms rateLimitDelay=${BROWSER_429_DELAY}ms warmDelay=${BROWSER_WARM_DELAY}ms`); function getProxyUrl() { if (process.env.BIKEGEAR_PROXY_URL) { @@ -200,18 +201,26 @@ async function fetchHtmlWithBrowser(url, referer = null) { if (warmStatus && warmStatus !== 200 && warmStatus !== 304) { throw new Error(`Referer HTTP ${warmStatus}`); } - await page.waitForTimeout(2500); + await page.waitForTimeout(BROWSER_WARM_DELAY); - const clickResult = await clickLinkForUrl(page, url); - response = clickResult.response; - if (clickResult.clicked) { - // Navigation happened through the listing page link. - } else { + if (url.includes("?page=")) { response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45000, referer, }); + } else { + const clickResult = await clickLinkForUrl(page, url); + response = clickResult.response; + if (clickResult.clicked) { + // Navigation happened through the listing page link. + } else { + response = await page.goto(url, { + waitUntil: "domcontentloaded", + timeout: 45000, + referer, + }); + } } } else { response = await page.goto(url, {