admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): preserve listing referrer for paged and detail requests

Browse Source
This commit is contained in:
MOHAN 2026-08-02 15:59:15 +05:30
parent 57b174bb3c
commit b460853806
2 changed files with 28 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scripts/testBikegearProxy.js
View File

@ -6,6 +6,7 @@ const { chromium } = require("playwright");
const DEFAULT_URL = "https://bikegear.in/acerbis"; const DEFAULT_URL = "https://bikegear.in/acerbis";
const TEST_URL = process.argv[2] || process.env.BIKEGEAR_PROXY_TEST_URL || DEFAULT_URL; const TEST_URL = process.argv[2] || process.env.BIKEGEAR_PROXY_TEST_URL || DEFAULT_URL;
const WARM_URL = process.env.BIKEGEAR_PROXY_TEST_WARM_URL || DEFAULT_URL;
const PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile-test"); const PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile-test");
const STEALTH_SCRIPT = ` const STEALTH_SCRIPT = `
@ -100,6 +101,7 @@ async function main() {
await fs.mkdir(PROFILE_PATH, { recursive: true }); await fs.mkdir(PROFILE_PATH, { recursive: true });
console.log(`[BIKEGEAR-PROXY-TEST] url=${TEST_URL}`); console.log(`[BIKEGEAR-PROXY-TEST] url=${TEST_URL}`);
console.log(`[BIKEGEAR-PROXY-TEST] warmUrl=${TEST_URL === WARM_URL ? "-" : WARM_URL}`);
console.log(`[BIKEGEAR-PROXY-TEST] proxy=${maskProxyUrl(proxyUrl)}`); console.log(`[BIKEGEAR-PROXY-TEST] proxy=${maskProxyUrl(proxyUrl)}`);
const context = await chromium.launchPersistentContext(PROFILE_PATH, { const context = await chromium.launchPersistentContext(PROFILE_PATH, {
@ -119,7 +121,18 @@ async function main() {
try { try {
await context.addInitScript(STEALTH_SCRIPT); await context.addInitScript(STEALTH_SCRIPT);
const page = await context.newPage(); const page = await context.newPage();
const response = await page.goto(TEST_URL, { waitUntil: "domcontentloaded", timeout: 45000 });
if (TEST_URL !== WARM_URL) {
const warmResponse = await page.goto(WARM_URL, { waitUntil: "domcontentloaded", timeout: 45000 });
await page.waitForTimeout(8000);
console.log(`[BIKEGEAR-PROXY-TEST] warmStatus=${warmResponse?.status() || 0}`);
}
const response = await page.goto(TEST_URL, {
waitUntil: "domcontentloaded",
timeout: 45000,
...(TEST_URL !== WARM_URL ? { referer: WARM_URL } : {}),
});
await page.waitForTimeout(8000); await page.waitForTimeout(8000);
const status = response?.status() || 0; const status = response?.status() || 0;

24
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -180,19 +180,23 @@ async function recycleBrowserContext() {
} }
} }
async function fetchHtmlWithBrowserQueued(url) { async function fetchHtmlWithBrowserQueued(url, referer = null) {
const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url)); const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url, referer));
_browserFetchQueue = run.catch(() => {}); _browserFetchQueue = run.catch(() => {});
return run; return run;
} }
async function fetchHtmlWithBrowser(url) { async function fetchHtmlWithBrowser(url, referer = null) {
const context = await getBrowserContext(); const context = await getBrowserContext();
for (let attempt = 1; attempt <= 5; attempt += 1) { for (let attempt = 1; attempt <= 5; attempt += 1) {
const page = await context.newPage(); const page = await context.newPage();
try { try {
const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45000 }); const response = await page.goto(url, {
waitUntil: "domcontentloaded",
timeout: 45000,
...(referer ? { referer } : {}),
});
const status = response?.status() ?? 0; const status = response?.status() ?? 0;
if (status === 429 || status >= 500) { if (status === 429 || status >= 500) {
@ -284,9 +288,9 @@ function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
function fetchHtml(url, attempt = 1) { function fetchHtml(url, attempt = 1, referer = null) {
return FETCH_MODE === "browser" return FETCH_MODE === "browser"
? fetchHtmlWithBrowserQueued(url) ? fetchHtmlWithBrowserQueued(url, referer)
: fetchHtmlWithHttp(url, attempt); : fetchHtmlWithHttp(url, attempt);
} }
@ -348,7 +352,7 @@ async function scrapeBrandProductUrls(brand) {
await runConcurrent(pageNums, async (pageNum) => { await runConcurrent(pageNums, async (pageNum) => {
const pageUrl = `${brand.url}?page=${pageNum}`; const pageUrl = `${brand.url}?page=${pageNum}`;
try { try {
const pageHtml = await fetchHtml(pageUrl); const pageHtml = await fetchHtml(pageUrl, 1, brand.url);
for (const u of extractProductUrls(pageHtml)) urls.add(u); for (const u of extractProductUrls(pageHtml)) urls.add(u);
} catch (err) { } catch (err) {
console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`); console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`);
@ -463,10 +467,10 @@ function parsePrice(str) {
} }
/** Scrape a single product detail page and return structured data. */ /** Scrape a single product detail page and return structured data. */
async function scrapeProductDetail(productUrl, brandName) { async function scrapeProductDetail(productUrl, brandName, refererUrl = null) {
let html; let html;
try { try {
html = await fetchHtml(productUrl); html = await fetchHtml(productUrl, 1, refererUrl);
} catch (err) { } catch (err) {
return { url: productUrl, scrapeError: err.message }; return { url: productUrl, scrapeError: err.message };
} }
@ -520,7 +524,7 @@ async function scrapeBikeGear() {
let done = 0; let done = 0;
const products = await runConcurrent(productUrls, async (url) => { const products = await runConcurrent(productUrls, async (url) => {
const product = await scrapeProductDetail(url, brand.name); const product = await scrapeProductDetail(url, brand.name, brand.url);
done++; done++;
if (product.scrapeError) { if (product.scrapeError) {
console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`); console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);