533 lines
19 KiB
JavaScript
533 lines
19 KiB
JavaScript
/**
|
||
* BikeGear.in scraper (OpenCart / Journal3 theme).
|
||
* 2-phase: listing pages → product URLs, then detail pages → full product data.
|
||
*
|
||
* Prices are in INR (Indian Rupees) as scraped from the site.
|
||
*
|
||
* Fetch modes (set in .env):
|
||
* BIKEGEAR_FETCH_MODE=browser — use Playwright Chromium (default; bypasses Cloudflare on VPS IPs)
|
||
* BIKEGEAR_FETCH_MODE=http — use axios (works only from residential/whitelisted IPs)
|
||
* BIKEGEAR_PROXY_URL=http://user:pass@host:port
|
||
* Or Webshare-style parts:
|
||
* BIKEGEAR_PROXY_HOST=p.webshare.io
|
||
* BIKEGEAR_PROXY_PORT=80
|
||
* BIKEGEAR_PROXY_USERNAME=...
|
||
* BIKEGEAR_PROXY_PASSWORD=...
|
||
*/
|
||
|
||
const axios = require("axios");
|
||
const { BRANDS } = require("./brands");
|
||
|
||
const BASE_URL = "https://bikegear.in";
|
||
const IS_HTTP_MODE = process.env.BIKEGEAR_FETCH_MODE === "http";
|
||
// Browser mode must run sequentially to avoid Cloudflare 429s — 1 page at a time
|
||
const LISTING_CONCURRENCY = IS_HTTP_MODE
|
||
? Math.max(1, Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3))
|
||
: 1;
|
||
const DETAIL_CONCURRENCY = IS_HTTP_MODE
|
||
? Math.max(1, Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2))
|
||
: 1;
|
||
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
|
||
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
|
||
const BROWSER_429_DELAY = Number(process.env.BIKEGEAR_429_DELAY ?? 30000);
|
||
const PROXY_URL = getProxyUrl();
|
||
// Default to browser mode — it bypasses Cloudflare without needing a proxy
|
||
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
|
||
|
||
console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`);
|
||
console.log(`[BIKEGEAR] Concurrency: listing=${LISTING_CONCURRENCY} detail=${DETAIL_CONCURRENCY} delay=${BROWSER_REQUEST_DELAY}ms rateLimitDelay=${BROWSER_429_DELAY}ms`);
|
||
|
||
function getProxyUrl() {
|
||
if (process.env.BIKEGEAR_PROXY_URL) {
|
||
return process.env.BIKEGEAR_PROXY_URL;
|
||
}
|
||
|
||
const host = process.env.BIKEGEAR_PROXY_HOST;
|
||
const port = process.env.BIKEGEAR_PROXY_PORT;
|
||
if (!host || !port) {
|
||
return null;
|
||
}
|
||
|
||
const protocol = String(process.env.BIKEGEAR_PROXY_PROTOCOL || "http").replace(/:$/, "");
|
||
const username = process.env.BIKEGEAR_PROXY_USERNAME || "";
|
||
const password = process.env.BIKEGEAR_PROXY_PASSWORD || "";
|
||
const auth = username
|
||
? `${encodeURIComponent(username)}:${encodeURIComponent(password)}@`
|
||
: "";
|
||
|
||
return `${protocol}://${auth}${host}:${port}`;
|
||
}
|
||
|
||
function maskProxyUrl(proxyUrl) {
|
||
try {
|
||
const u = new URL(proxyUrl);
|
||
return `${u.protocol}//${u.username ? "<redacted>@" : ""}${u.host}`;
|
||
} catch {
|
||
return "<invalid proxy url>";
|
||
}
|
||
}
|
||
|
||
// ── Browser pool (Playwright persistent context) ───────────────────────────
|
||
// Uses launchPersistentContext so the browser profile (cookies, cache, fingerprint)
|
||
// survives across runs on disk — Cloudflare sees the same "user" every time.
|
||
|
||
const path = require("node:path");
|
||
const fs = require("node:fs/promises");
|
||
|
||
// Profile dir persists between runs — Cloudflare builds trust over multiple visits
|
||
const BROWSER_PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile");
|
||
|
||
// Patches injected into every page to hide headless Chromium signals
|
||
const STEALTH_SCRIPT = `
|
||
(() => {
|
||
// Hide webdriver flag
|
||
Object.defineProperty(navigator, 'webdriver', { get: () => false });
|
||
// Fake plugins so navigator.plugins.length > 0
|
||
const fakePlugin = (name, file, mimes) => {
|
||
const p = Object.create(Plugin.prototype);
|
||
Object.defineProperty(p, 'name', { get: () => name });
|
||
Object.defineProperty(p, 'filename', { get: () => file });
|
||
Object.defineProperty(p, 'length', { get: () => mimes.length });
|
||
return p;
|
||
};
|
||
const plugins = [
|
||
fakePlugin('Chrome PDF Plugin', 'internal-pdf-viewer', ['application/x-google-chrome-pdf']),
|
||
fakePlugin('Chrome PDF Viewer', 'mhjfbmdgcfjbbpaeojofohoefgiehjai', ['application/pdf']),
|
||
fakePlugin('Native Client', 'internal-nacl-plugin', ['application/x-nacl']),
|
||
];
|
||
Object.defineProperty(navigator, 'plugins', { get: () => plugins });
|
||
// Fake chrome runtime so window.chrome is defined
|
||
if (!window.chrome) window.chrome = { runtime: {}, loadTimes: () => {}, csi: () => {}, app: {} };
|
||
// Consistent languages
|
||
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
||
})();
|
||
`;
|
||
|
||
let _context = null; // launchPersistentContext = browser + context in one
|
||
let _browserFetchQueue = Promise.resolve();
|
||
|
||
async function getBrowserContext() {
|
||
if (_context) return _context;
|
||
|
||
const { chromium } = require("playwright");
|
||
await fs.mkdir(BROWSER_PROFILE_PATH, { recursive: true });
|
||
|
||
const proxyOpts = {};
|
||
if (PROXY_URL) {
|
||
const u = new URL(PROXY_URL);
|
||
proxyOpts.proxy = {
|
||
server: `${u.protocol}//${u.host}`,
|
||
...(u.username ? { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) } : {}),
|
||
};
|
||
console.log(`[BIKEGEAR] Browser using proxy: ${u.protocol}//${u.host}`);
|
||
}
|
||
|
||
_context = await chromium.launchPersistentContext(BROWSER_PROFILE_PATH, {
|
||
headless: true,
|
||
args: [
|
||
"--no-sandbox",
|
||
"--disable-setuid-sandbox",
|
||
"--disable-blink-features=AutomationControlled",
|
||
],
|
||
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||
locale: "en-US",
|
||
viewport: { width: 1280, height: 800 },
|
||
extraHTTPHeaders: { "Accept-Language": "en-US,en;q=0.9" },
|
||
...proxyOpts,
|
||
});
|
||
|
||
// Apply stealth patches to every new page opened in this context
|
||
await _context.addInitScript(STEALTH_SCRIPT);
|
||
|
||
console.log("[BIKEGEAR] Browser started with persistent profile.");
|
||
return _context;
|
||
}
|
||
|
||
async function closeBrowser() {
|
||
if (_context) {
|
||
await _context.close().catch(() => {});
|
||
_context = null;
|
||
console.log("[BIKEGEAR] Browser closed. Profile saved to disk.");
|
||
}
|
||
}
|
||
|
||
/** Close all open pages to free memory — profile stays alive on disk. */
|
||
async function recycleBrowserContext() {
|
||
if (!_context) return;
|
||
console.log("[BIKEGEAR] Cleaning up browser pages to free memory...");
|
||
for (const page of _context.pages()) {
|
||
await page.close().catch(() => {});
|
||
}
|
||
}
|
||
|
||
async function fetchHtmlWithBrowserQueued(url) {
|
||
const run = _browserFetchQueue.then(() => fetchHtmlWithBrowser(url));
|
||
_browserFetchQueue = run.catch(() => {});
|
||
return run;
|
||
}
|
||
|
||
async function fetchHtmlWithBrowser(url) {
|
||
const context = await getBrowserContext();
|
||
|
||
for (let attempt = 1; attempt <= 5; attempt += 1) {
|
||
const page = await context.newPage();
|
||
try {
|
||
const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45000 });
|
||
const status = response?.status() ?? 0;
|
||
|
||
if (status === 429 || status >= 500) {
|
||
if (attempt < 5) {
|
||
const wait = status === 429 ? attempt * BROWSER_429_DELAY : attempt * 5000;
|
||
console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/5)`);
|
||
await page.close().catch(() => {});
|
||
await sleep(wait);
|
||
continue;
|
||
}
|
||
throw new Error(`HTTP ${status} after retries`);
|
||
}
|
||
if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`);
|
||
|
||
// Wait for Cloudflare JS challenge to resolve, then add polite delay
|
||
await page.waitForTimeout(2500);
|
||
const html = await page.content();
|
||
if (BROWSER_REQUEST_DELAY > 0) await sleep(BROWSER_REQUEST_DELAY);
|
||
return html;
|
||
} catch (err) {
|
||
if (attempt < 5 && (err.message?.includes("timeout") || err.message?.includes("net::"))) {
|
||
const wait = attempt * 5000;
|
||
console.log(`[BIKEGEAR] RETRY(browser) ${url} ${err.message} — waiting ${wait}ms (${attempt}/5)`);
|
||
await sleep(wait);
|
||
continue;
|
||
}
|
||
throw err;
|
||
} finally {
|
||
await page.close().catch(() => {});
|
||
}
|
||
}
|
||
|
||
throw new Error("Browser fetch failed after retries");
|
||
}
|
||
|
||
// ── HTTP fetch (axios) ─────────────────────────────────────────────────────
|
||
|
||
const FETCH_HEADERS = {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||
"Accept-Language": "en-US,en;q=0.5",
|
||
"Cache-Control": "no-cache",
|
||
"Pragma": "no-cache",
|
||
"Upgrade-Insecure-Requests": "1",
|
||
"Sec-Fetch-Dest": "document",
|
||
"Sec-Fetch-Mode": "navigate",
|
||
"Sec-Fetch-Site": "none",
|
||
"Sec-Fetch-User": "?1",
|
||
"Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
|
||
"Sec-CH-UA-Mobile": "?0",
|
||
"Sec-CH-UA-Platform": '"Windows"',
|
||
};
|
||
|
||
function buildProxyConfig(proxyUrl) {
|
||
const u = new URL(proxyUrl);
|
||
const cfg = { host: u.hostname, port: parseInt(u.port, 10), protocol: u.protocol.replace(":", "") };
|
||
if (u.username) cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) };
|
||
return cfg;
|
||
}
|
||
|
||
async function fetchHtmlWithHttp(url, attempt = 1) {
|
||
try {
|
||
const opts = { headers: FETCH_HEADERS, timeout: 30000, responseType: "text", maxRedirects: 5 };
|
||
if (PROXY_URL) opts.proxy = buildProxyConfig(PROXY_URL);
|
||
const res = await axios.get(url, opts);
|
||
return res.data;
|
||
} catch (err) {
|
||
const status = err.response?.status;
|
||
if ((status === 429 || status >= 500) && attempt <= 4) {
|
||
const wait = attempt * 2000;
|
||
console.log(`[BIKEGEAR] RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
|
||
await sleep(wait);
|
||
return fetchHtmlWithHttp(url, attempt + 1);
|
||
}
|
||
if (status) throw new Error(`HTTP ${status}`);
|
||
const isNetwork = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"].includes(err.code) ||
|
||
err.message?.includes("timeout") || err.message?.includes("connect");
|
||
if (isNetwork && attempt <= 4) {
|
||
await sleep(attempt * 2000);
|
||
return fetchHtmlWithHttp(url, attempt + 1);
|
||
}
|
||
throw err;
|
||
}
|
||
}
|
||
|
||
// ── Unified fetch entry point ──────────────────────────────────────────────
|
||
|
||
function sleep(ms) {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
function fetchHtml(url, attempt = 1) {
|
||
return FETCH_MODE === "browser"
|
||
? fetchHtmlWithBrowserQueued(url)
|
||
: fetchHtmlWithHttp(url, attempt);
|
||
}
|
||
|
||
/** Run up to `limit` tasks from `items` at a time. */
|
||
async function runConcurrent(items, fn, limit) {
|
||
const results = [];
|
||
let idx = 0;
|
||
|
||
async function worker() {
|
||
while (idx < items.length) {
|
||
const i = idx++;
|
||
results[i] = await fn(items[i], i);
|
||
}
|
||
}
|
||
|
||
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
||
await Promise.all(workers);
|
||
return results;
|
||
}
|
||
|
||
/** Extract the max page number from a listing page's HTML. */
|
||
function extractMaxPage(html) {
|
||
const pageNums = [...html.matchAll(/\?page=(\d+)/g)].map((m) => parseInt(m[1], 10));
|
||
return pageNums.length ? Math.max(...pageNums) : 1;
|
||
}
|
||
|
||
/** Extract product detail page URLs from a brand listing page HTML. */
|
||
function extractProductUrls(html) {
|
||
const urls = new Set();
|
||
// Product links have class="product-img" — these are the card image anchors
|
||
for (const m of html.matchAll(/href="(https:\/\/bikegear\.in\/[^"]+)"\s+class="product-img/g)) {
|
||
const url = m[1];
|
||
// Exclude manufacturer/category/route pages — product slugs are plain paths
|
||
if (!url.includes("route=") && !url.includes("manufacturer") && !url.includes("category")) {
|
||
urls.add(url);
|
||
}
|
||
}
|
||
return [...urls];
|
||
}
|
||
|
||
/** Scrape all listing pages for a brand and collect product detail URLs. */
|
||
async function scrapeBrandProductUrls(brand) {
|
||
console.log(`[BIKEGEAR][${brand.name}] Fetching page 1...`);
|
||
let html;
|
||
try {
|
||
html = await fetchHtml(brand.url);
|
||
} catch (err) {
|
||
console.warn(`[BIKEGEAR][${brand.name}] Failed page 1: ${err.message}`);
|
||
return [];
|
||
}
|
||
|
||
const maxPage = extractMaxPage(html);
|
||
const urls = new Set(extractProductUrls(html));
|
||
|
||
if (maxPage > 1) {
|
||
console.log(`[BIKEGEAR][${brand.name}] Found ${maxPage} pages — fetching pages 2–${maxPage}...`);
|
||
const pageNums = Array.from({ length: maxPage - 1 }, (_, i) => i + 2);
|
||
|
||
await runConcurrent(pageNums, async (pageNum) => {
|
||
const pageUrl = `${brand.url}?page=${pageNum}`;
|
||
try {
|
||
const pageHtml = await fetchHtml(pageUrl);
|
||
for (const u of extractProductUrls(pageHtml)) urls.add(u);
|
||
} catch (err) {
|
||
console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`);
|
||
}
|
||
}, LISTING_CONCURRENCY);
|
||
}
|
||
|
||
const result = [...urls];
|
||
console.log(`[BIKEGEAR][${brand.name}] Collected ${result.length} product URLs across ${maxPage} page(s).`);
|
||
return result;
|
||
}
|
||
|
||
/** Parse the best product JSON-LD block from the page.
|
||
* Prefers the block with offers.price (detailed one), then any Product block.
|
||
* Out-of-stock products omit the sku field so we don't require it. */
|
||
function parseProductJsonLd(html) {
|
||
const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
|
||
const candidates = [];
|
||
for (const b of blocks) {
|
||
try {
|
||
const data = JSON.parse(b[1]);
|
||
if (data["@type"] === "Product") {
|
||
candidates.push(data);
|
||
}
|
||
} catch {
|
||
// malformed JSON-LD — skip
|
||
}
|
||
}
|
||
if (!candidates.length) return null;
|
||
// Prefer block that has price info (in-stock) or description (detailed one)
|
||
return candidates.find((d) => d.offers?.price || d.description) || candidates[0];
|
||
}
|
||
|
||
/** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */
|
||
function extractProductId(jsonLd) {
|
||
const imgUrl = jsonLd?.image || "";
|
||
const m = imgUrl.match(/\/catalog\/products\/(\d+)\//);
|
||
return m ? m[1] : null;
|
||
}
|
||
|
||
/**
|
||
* Extract all unique gallery images for the current product from the page HTML.
|
||
* Strategy 1: match data-src on the product-img anchor (the main image carousel).
|
||
* Strategy 2 (fallback): JSON-LD image field.
|
||
* Skips tiny thumbnails; upgrades 500x500 → 800x800 for consistency.
|
||
*/
|
||
function extractProductImages(html, productId, jsonLdImage) {
|
||
const seen = new Set();
|
||
const images = [];
|
||
|
||
// Strategy 1a: images in /catalog/products/{productId}/ (standard path)
|
||
if (productId) {
|
||
const re = new RegExp(
|
||
`data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`,
|
||
"g"
|
||
);
|
||
for (const m of html.matchAll(re)) {
|
||
const url = m[1];
|
||
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
|
||
seen.add(url);
|
||
images.push(url.replace("-500x500.", "-800x800."));
|
||
}
|
||
}
|
||
}
|
||
|
||
// Strategy 1b: images on the product-img anchor (catches non-standard paths like /catalog/KTM/...)
|
||
if (!images.length) {
|
||
const productImgBlock = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/);
|
||
if (productImgBlock) {
|
||
for (const m of productImgBlock[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) {
|
||
const url = m[1];
|
||
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
|
||
seen.add(url);
|
||
images.push(url.replace("-500x500.", "-800x800."));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Fallback: JSON-LD image
|
||
if (!images.length && jsonLdImage) {
|
||
images.push(jsonLdImage);
|
||
}
|
||
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* Extract the feature/description table from the product-extra-description tab-pane section.
|
||
* Specifically targets the `tab-pane` variant (not the popup variant).
|
||
* Returns an HTML string of the table, or empty string if not found.
|
||
*/
|
||
function extractDescriptionHtml(html) {
|
||
// Match the tab-pane description div — stop at the next product-extra section
|
||
const sectionMatch = html.match(
|
||
/<div[^>]*product-extra-description[^>]*tab-pane[^>]*>([\s\S]*?)(?=<div[^>]*product-extra-(?:attributes|reviews)|<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>)/
|
||
);
|
||
if (!sectionMatch) return "";
|
||
|
||
const section = sectionMatch[1];
|
||
const tableMatch = section.match(/<table[^>]*>([\s\S]*?)<\/table>/);
|
||
if (!tableMatch) return "";
|
||
|
||
return `<table border="1">${tableMatch[1]}</table>`;
|
||
}
|
||
|
||
/** Parse price from a string like "₹21,497.28" or just "21497.28" */
|
||
function parsePrice(str) {
|
||
if (!str) return 0;
|
||
const n = parseFloat(String(str).replace(/[^\d.]/g, ""));
|
||
return Number.isNaN(n) ? 0 : n;
|
||
}
|
||
|
||
/** Scrape a single product detail page and return structured data. */
|
||
async function scrapeProductDetail(productUrl, brandName) {
|
||
let html;
|
||
try {
|
||
html = await fetchHtml(productUrl);
|
||
} catch (err) {
|
||
return { url: productUrl, scrapeError: err.message };
|
||
}
|
||
|
||
try {
|
||
const jsonLd = parseProductJsonLd(html);
|
||
if (!jsonLd) {
|
||
return { url: productUrl, scrapeError: "No product JSON-LD found" };
|
||
}
|
||
|
||
const productId = extractProductId(jsonLd);
|
||
const images = extractProductImages(html, productId, jsonLd.image);
|
||
const descriptionHtml = extractDescriptionHtml(html);
|
||
const price = parsePrice(jsonLd.offers?.price);
|
||
const currency = jsonLd.offers?.priceCurrency || "INR";
|
||
|
||
return {
|
||
url: productUrl,
|
||
sku: jsonLd.sku || productUrl.split("/").pop(),
|
||
name: jsonLd.name || "",
|
||
brand: jsonLd.brand?.name || brandName,
|
||
price,
|
||
currency,
|
||
availability: jsonLd.offers?.availability?.replace("https://schema.org/", "") || "Unknown",
|
||
descriptionHtml,
|
||
images,
|
||
productId,
|
||
};
|
||
} catch (err) {
|
||
return { url: productUrl, scrapeError: `Parse error: ${err.message}` };
|
||
}
|
||
}
|
||
|
||
// Recycle the browser context every N brands to prevent memory accumulation
|
||
const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 10);
|
||
|
||
/** Main entry point: scrape all configured brands and return flat product array. */
|
||
async function scrapeBikeGear() {
|
||
const allProducts = [];
|
||
const totalBrands = BRANDS.length;
|
||
let brandsDone = 0;
|
||
|
||
for (const brand of BRANDS) {
|
||
const productUrls = await scrapeBrandProductUrls(brand);
|
||
brandsDone++;
|
||
|
||
if (!productUrls.length) {
|
||
console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`);
|
||
} else {
|
||
console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`);
|
||
|
||
let done = 0;
|
||
const products = await runConcurrent(productUrls, async (url) => {
|
||
const product = await scrapeProductDetail(url, brand.name);
|
||
done++;
|
||
if (product.scrapeError) {
|
||
console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);
|
||
} else {
|
||
console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`);
|
||
}
|
||
return product;
|
||
}, DETAIL_CONCURRENCY);
|
||
|
||
const ok = products.filter((p) => !p.scrapeError).length;
|
||
const fail = products.length - ok;
|
||
console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
|
||
allProducts.push(...products);
|
||
}
|
||
|
||
// Always delay/recycle between brands — even skipped ones — to avoid rapid-fire requests
|
||
if (brandsDone < totalBrands) {
|
||
if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0) {
|
||
await recycleBrowserContext();
|
||
} else {
|
||
await sleep(1000);
|
||
}
|
||
}
|
||
}
|
||
|
||
return allProducts;
|
||
}
|
||
|
||
module.exports = { scrapeBikeGear, closeBrowser };
|