admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): use warmed referrer navigation for pagination

Browse Source
This commit is contained in:
MOHAN 2026-08-02 16:19:01 +05:30
parent ed877e3a78
commit b6115cc637
2 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/testBikegearProxy.js
View File

@ -7,6 +7,7 @@ const { chromium } = require("playwright");
const DEFAULT_URL = "https://bikegear.in/acerbis"; const DEFAULT_URL = "https://bikegear.in/acerbis";
const TEST_URL = process.argv[2] || process.env.BIKEGEAR_PROXY_TEST_URL || DEFAULT_URL; const TEST_URL = process.argv[2] || process.env.BIKEGEAR_PROXY_TEST_URL || DEFAULT_URL;
const WARM_URL = process.env.BIKEGEAR_PROXY_TEST_WARM_URL || DEFAULT_URL; const WARM_URL = process.env.BIKEGEAR_PROXY_TEST_WARM_URL || DEFAULT_URL;
const WARM_DELAY = Number(process.env.BIKEGEAR_WARM_DELAY || 8000);
const PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile-test"); const PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile-test");
const STEALTH_SCRIPT = ` const STEALTH_SCRIPT = `
@ -124,12 +125,12 @@ async function main() {
if (TEST_URL !== WARM_URL) { if (TEST_URL !== WARM_URL) {
const warmResponse = await page.goto(WARM_URL, { waitUntil: "domcontentloaded", timeout: 45000 }); const warmResponse = await page.goto(WARM_URL, { waitUntil: "domcontentloaded", timeout: 45000 });
await page.waitForTimeout(8000); await page.waitForTimeout(WARM_DELAY);
console.log(`[BIKEGEAR-PROXY-TEST] warmStatus=${warmResponse?.status() || 0}`); console.log(`[BIKEGEAR-PROXY-TEST] warmStatus=${warmResponse?.status() || 0}`);
} }
let response; let response;
if (TEST_URL !== WARM_URL) { if (TEST_URL !== WARM_URL && !TEST_URL.includes("?page=")) {
const clickResult = await clickLinkForUrl(page, TEST_URL); const clickResult = await clickLinkForUrl(page, TEST_URL);
response = clickResult.response; response = clickResult.response;
console.log(`[BIKEGEAR-PROXY-TEST] clickedLink=${clickResult.clicked}`); console.log(`[BIKEGEAR-PROXY-TEST] clickedLink=${clickResult.clicked}`);

13
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -32,12 +32,13 @@ const DETAIL_CONCURRENCY = IS_HTTP_MODE
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500); const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
const BROWSER_429_DELAY = Number(process.env.BIKEGEAR_429_DELAY ?? 30000); const BROWSER_429_DELAY = Number(process.env.BIKEGEAR_429_DELAY ?? 30000);
const BROWSER_WARM_DELAY = Number(process.env.BIKEGEAR_WARM_DELAY ?? 8000);
const PROXY_URL = getProxyUrl(); const PROXY_URL = getProxyUrl();
// Default to browser mode — it bypasses Cloudflare without needing a proxy // Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`); console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`);
console.log(`[BIKEGEAR] Concurrency: listing=${LISTING_CONCURRENCY} detail=${DETAIL_CONCURRENCY} delay=${BROWSER_REQUEST_DELAY}ms rateLimitDelay=${BROWSER_429_DELAY}ms`); console.log(`[BIKEGEAR] Concurrency: listing=${LISTING_CONCURRENCY} detail=${DETAIL_CONCURRENCY} delay=${BROWSER_REQUEST_DELAY}ms rateLimitDelay=${BROWSER_429_DELAY}ms warmDelay=${BROWSER_WARM_DELAY}ms`);
function getProxyUrl() { function getProxyUrl() {
if (process.env.BIKEGEAR_PROXY_URL) { if (process.env.BIKEGEAR_PROXY_URL) {
@ -200,8 +201,15 @@ async function fetchHtmlWithBrowser(url, referer = null) {
if (warmStatus && warmStatus !== 200 && warmStatus !== 304) { if (warmStatus && warmStatus !== 200 && warmStatus !== 304) {
throw new Error(`Referer HTTP ${warmStatus}`); throw new Error(`Referer HTTP ${warmStatus}`);
} }
await page.waitForTimeout(2500); await page.waitForTimeout(BROWSER_WARM_DELAY);
if (url.includes("?page=")) {
response = await page.goto(url, {
waitUntil: "domcontentloaded",
timeout: 45000,
referer,
});
} else {
const clickResult = await clickLinkForUrl(page, url); const clickResult = await clickLinkForUrl(page, url);
response = clickResult.response; response = clickResult.response;
if (clickResult.clicked) { if (clickResult.clicked) {
@ -213,6 +221,7 @@ async function fetchHtmlWithBrowser(url, referer = null) {
referer, referer,
}); });
} }
}
} else { } else {
response = await page.goto(url, { response = await page.goto(url, {
waitUntil: "domcontentloaded", waitUntil: "domcontentloaded",