admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): preserve listing referrer for paged and detail requests

Browse Source
This commit is contained in:
MOHAN 2026-08-02 15:59:15 +05:30
parent 57b174bb3c
commit b460853806
2 changed files with 28 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scripts/testBikegearProxy.js
View File

@ -6,6 +6,7 @@ const { chromium } = require("playwright");
const DEFAULT_URL = "https://bikegear.in/acerbis";
const TEST_URL = process.argv[2] || process.env.BIKEGEAR_PROXY_TEST_URL || DEFAULT_URL;
const WARM_URL = process.env.BIKEGEAR_PROXY_TEST_WARM_URL || DEFAULT_URL;
const PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile-test");
const STEALTH_SCRIPT = `
@ -100,6 +101,7 @@ async function main() {
await fs.mkdir(PROFILE_PATH, { recursive: true });
console.log(`[BIKEGEAR-PROXY-TEST] url=${TEST_URL}`);
console.log(`[BIKEGEAR-PROXY-TEST] warmUrl=${TEST_URL === WARM_URL ? "-" : WARM_URL}`);
console.log(`[BIKEGEAR-PROXY-TEST] proxy=${maskProxyUrl(proxyUrl)}`);
const context = await chromium.launchPersistentContext(PROFILE_PATH, {
@ -119,7 +121,18 @@ async function main() {
try {
await context.addInitScript(STEALTH_SCRIPT);
const page = await context.newPage();
const response = await page.goto(TEST_URL, { waitUntil: "domcontentloaded", timeout: 45000 });
if (TEST_URL !== WARM_URL) {
const warmResponse = await page.goto(WARM_URL, { waitUntil: "domcontentloaded", timeout: 45000 });
await page.waitForTimeout(8000);
console.log(`[BIKEGEAR-PROXY-TEST] warmStatus=${warmResponse?.status() || 0}`);
}
const response = await page.goto(TEST_URL, {
waitUntil: "domcontentloaded",
timeout: 45000,
...(TEST_URL !== WARM_URL ? { referer: WARM_URL } : {}),
});
await page.waitForTimeout(8000);
const status = response?.status() || 0;

24
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -180,19 +180,23 @@ async function recycleBrowserContext() {
}
}
async function fetchHtmlWithBrowserQueued(url) {
const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url));
async function fetchHtmlWithBrowserQueued(url, referer = null) {
const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url, referer));
_browserFetchQueue = run.catch(() => {});
return run;
}
async function fetchHtmlWithBrowser(url) {
async function fetchHtmlWithBrowser(url, referer = null) {
const context = await getBrowserContext();
for (let attempt = 1; attempt <= 5; attempt += 1) {
const page = await context.newPage();
try {
const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45000 });
const response = await page.goto(url, {
waitUntil: "domcontentloaded",
timeout: 45000,
...(referer ? { referer } : {}),
});
const status = response?.status() ?? 0;
if (status === 429 || status >= 500) {
@ -284,9 +288,9 @@ function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function fetchHtml(url, attempt = 1) {
function fetchHtml(url, attempt = 1, referer = null) {
return FETCH_MODE === "browser"
? fetchHtmlWithBrowserQueued(url)
? fetchHtmlWithBrowserQueued(url, referer)
: fetchHtmlWithHttp(url, attempt);
}
@ -348,7 +352,7 @@ async function scrapeBrandProductUrls(brand) {
await runConcurrent(pageNums, async (pageNum) => {
const pageUrl = `${brand.url}?page=${pageNum}`;
try {
const pageHtml = await fetchHtml(pageUrl);
const pageHtml = await fetchHtml(pageUrl, 1, brand.url);
for (const u of extractProductUrls(pageHtml)) urls.add(u);
} catch (err) {
console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`);
@ -463,10 +467,10 @@ function parsePrice(str) {
}
/** Scrape a single product detail page and return structured data. */
async function scrapeProductDetail(productUrl, brandName) {
async function scrapeProductDetail(productUrl, brandName, refererUrl = null) {
let html;
try {
html = await fetchHtml(productUrl);
html = await fetchHtml(productUrl, 1, refererUrl);
} catch (err) {
return { url: productUrl, scrapeError: err.message };
}
@ -520,7 +524,7 @@ async function scrapeBikeGear() {
let done = 0;
const products = await runConcurrent(productUrls, async (url) => {
const product = await scrapeProductDetail(url, brand.name);
const product = await scrapeProductDetail(url, brand.name, brand.url);
done++;
if (product.scrapeError) {
console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);