MOHAN
ac0c37670b
Self-contained — all brands and scraper logic inlined, no project imports. Only needs: npm install axios, then node bikegear-scraper.js. Supports resume on Ctrl+C, saves bikegear_output.json in server-compatible format. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
501 lines
24 KiB
JavaScript
501 lines
24 KiB
JavaScript
/**
|
|
* BikeGear.in Standalone Scraper
|
|
* ================================
|
|
* Setup (one time):
|
|
* npm install axios
|
|
*
|
|
* Run:
|
|
* node bikegear-scraper.js
|
|
*
|
|
* Pause anytime with Ctrl+C — progress saves automatically.
|
|
* Re-run the same command to resume from where you stopped.
|
|
*
|
|
* Output: bikegear_output.json (send this file to the server)
|
|
*/
|
|
|
|
const axios = require("axios");
|
|
const path = require("node:path");
|
|
const fs = require("node:fs/promises");
|
|
|
|
// ─── Config ───────────────────────────────────────────────────────────────────
|
|
|
|
const LISTING_CONCURRENCY = 5; // how many listing pages to fetch at once
|
|
const DETAIL_CONCURRENCY = 5; // how many product pages to fetch at once
|
|
const PROGRESS_FILE = path.resolve(__dirname, "bikegear_progress.json");
|
|
const OUTPUT_FILE = path.resolve(__dirname, "bikegear_output.json");
|
|
|
|
// ─── Brand list ───────────────────────────────────────────────────────────────
|
|
|
|
const BRANDS = [
|
|
{ name: "Acerbis", url: "https://bikegear.in/acerbis" },
|
|
{ name: "Aimshop", url: "https://bikegear.in/aimshop" },
|
|
{ name: "Airoh Helmets", url: "https://bikegear.in/airoh-helmets" },
|
|
{ name: "Akrapovic Exhaust Systems", url: "https://bikegear.in/akrapovic-exhaust-systems" },
|
|
{ name: "Alpinestars", url: "https://bikegear.in/alpinestars" },
|
|
{ name: "Arai Helmet", url: "https://bikegear.in/arai-helmet-india" },
|
|
{ name: "Arai Helmet USA", url: "https://bikegear.in/arai-helmet-usa" },
|
|
{ name: "Arrow Exhausts", url: "https://bikegear.in/arrow-exhausts" },
|
|
{ name: "Austin Racing", url: "https://bikegear.in/austin-racing" },
|
|
{ name: "Bassani Exhaust", url: "https://bikegear.in/bassani-exhaust" },
|
|
{ name: "Beeline", url: "https://bikegear.in/beeline" },
|
|
{ name: "Bell", url: "https://bikegear.in/bell-helmet" },
|
|
{ name: "Bell Helmets", url: "https://bikegear.in/bell-helmets" },
|
|
{ name: "Biltwell Inc.", url: "https://bikegear.in/biltwell-inc" },
|
|
{ name: "Bitubo Suspension", url: "https://bikegear.in/bitubo-suspension" },
|
|
{ name: "Black Widow", url: "https://bikegear.in/black-widow" },
|
|
{ name: "British Customs", url: "https://bikegear.in/british-customs" },
|
|
{ name: "Brock's Performance", url: "https://bikegear.in/brocks-performance" },
|
|
{ name: "BT Moto", url: "https://bikegear.in/bt-moto" },
|
|
{ name: "Caberg", url: "https://bikegear.in/caberg-helmets" },
|
|
{ name: "Cardo Systems", url: "https://bikegear.in/cardo-systems" },
|
|
{ name: "Chigee", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=403" },
|
|
{ name: "Clover", url: "https://bikegear.in/clover-online" },
|
|
{ name: "CNC Racing", url: "https://bikegear.in/cnc-racing" },
|
|
{ name: "Cobra Exhaust", url: "https://bikegear.in/cobra-exhaust" },
|
|
{ name: "Competition Werkes", url: "https://bikegear.in/competition-werkes" },
|
|
{ name: "Daytona", url: "https://bikegear.in/daytona" },
|
|
{ name: "Dominator", url: "https://bikegear.in/dominator-exhaust-systems-in-india" },
|
|
{ name: "Ducabike", url: "https://bikegear.in/ducabike" },
|
|
{ name: "Dynojet Power Commander", url: "https://bikegear.in/dynojet-power-commander-research-inc" },
|
|
{ name: "Evotech Performance", url: "https://bikegear.in/evotech-performance-online-india" },
|
|
{ name: "Falco", url: "https://bikegear.in/falco" },
|
|
{ name: "Five", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=422" },
|
|
{ name: "Five Gloves", url: "https://bikegear.in/five-glove-india" },
|
|
{ name: "Forcefield", url: "https://bikegear.in/forcefield" },
|
|
{ name: "Forma Boots", url: "https://bikegear.in/forma-boots" },
|
|
{ name: "Forma Boots (2)", url: "https://bikegear.in/forma-boots-1-1" },
|
|
{ name: "Fox", url: "https://bikegear.in/fox-racing-gear" },
|
|
{ name: "Freedom Performance", url: "https://bikegear.in/freedom-performance-exhaust" },
|
|
{ name: "Full Six Carbon", url: "https://bikegear.in/full-six-carbon" },
|
|
{ name: "G.P.R. Exhaust System", url: "https://bikegear.in/gpr-exhaust-system" },
|
|
{ name: "Gaerne", url: "https://bikegear.in/gaerne" },
|
|
{ name: "GB Racing", url: "https://bikegear.in/gb-racing" },
|
|
{ name: "Gilles Tooling", url: "https://bikegear.in/gilles-tooling-india" },
|
|
{ name: "Graves Motorsports", url: "https://bikegear.in/graves-motorsports" },
|
|
{ name: "Held", url: "https://bikegear.in/held" },
|
|
{ name: "Hindle Exhaust", url: "https://bikegear.in/hindle-exhaust" },
|
|
{ name: "HJC Helmets", url: "https://bikegear.in/hjc-helmets" },
|
|
{ name: "Honda OEM", url: "https://bikegear.in/honda-oem" },
|
|
{ name: "HP Corse", url: "https://bikegear.in/hp-corse-exhaust" },
|
|
{ name: "Icon", url: "https://bikegear.in/icon-helmet" },
|
|
{ name: "Icon (2)", url: "https://bikegear.in/icon-helmet-1" },
|
|
{ name: "Ilmberger Carbon", url: "https://bikegear.in/ilmberger-carbon" },
|
|
{ name: "IXIL Exhaust", url: "https://bikegear.in/ixil-exhaust" },
|
|
{ name: "Ixon", url: "https://bikegear.in/ixon-motorcycle-gears" },
|
|
{ name: "Ixon (2)", url: "https://bikegear.in/ixon-motorcycle-gears-1" },
|
|
{ name: "IXS", url: "https://bikegear.in/ixs" },
|
|
{ name: "Jetprime", url: "https://bikegear.in/jet-prime" },
|
|
{ name: "Just-1", url: "https://bikegear.in/just-1" },
|
|
{ name: "Kabuto", url: "https://bikegear.in/kabuto" },
|
|
{ name: "Kineo Wheels", url: "https://bikegear.in/kineo-wheels" },
|
|
{ name: "KLIM", url: "https://bikegear.in/klim" },
|
|
{ name: "Knox", url: "https://bikegear.in/knox" },
|
|
{ name: "KTM", url: "https://bikegear.in/ktm" },
|
|
{ name: "Leatt", url: "https://bikegear.in/leatt-1" },
|
|
{ name: "Leo Vince", url: "https://bikegear.in/leo-vince-motorcycle-exhausts-system" },
|
|
{ name: "Leovince", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=426" },
|
|
{ name: "Lextek", url: "https://bikegear.in/lextek-motorcycle-products" },
|
|
{ name: "Livall", url: "https://bikegear.in/livall-helmet-india" },
|
|
{ name: "LS2 Helmets", url: "https://bikegear.in/ls2-helmet" },
|
|
{ name: "M4 Exhaust", url: "https://bikegear.in/m4-exhaust" },
|
|
{ name: "Macna", url: "https://bikegear.in/macna-brand" },
|
|
{ name: "Marchesini Wheels", url: "https://bikegear.in/marchesini-wheels" },
|
|
{ name: "Mivv Exhausts", url: "https://bikegear.in/mivv-exhausts" },
|
|
{ name: "Mosko Moto", url: "https://bikegear.in/mosko-moto" },
|
|
{ name: "Nexx Helmets", url: "https://bikegear.in/nexx-helmets" },
|
|
{ name: "Nolan", url: "https://bikegear.in/nolan-helmets" },
|
|
{ name: "Ohlins Racing", url: "https://bikegear.in/ohlins-racing" },
|
|
{ name: "Oxford", url: "https://bikegear.in/oxford" },
|
|
{ name: "OZ Motorbike", url: "https://bikegear.in/oz-motorbike" },
|
|
{ name: "Premier", url: "https://bikegear.in/premier-helmet" },
|
|
{ name: "QD Exhausts", url: "https://bikegear.in/qd-exhaust" },
|
|
{ name: "R&G Racing", url: "https://bikegear.in/r-and-g-racing" },
|
|
{ name: "Racefit Exhausts", url: "https://bikegear.in/racefit-exhausts" },
|
|
{ name: "Rapid Bike", url: "https://bikegear.in/rapid-bike" },
|
|
{ name: "Rekluse", url: "https://bikegear.in/rekluse-clutch-plates" },
|
|
{ name: "Remus Exhausts", url: "https://bikegear.in/remus-exhaust" },
|
|
{ name: "REV'IT", url: "https://bikegear.in/revit" },
|
|
{ name: "Rizoma", url: "https://bikegear.in/rizoma" },
|
|
{ name: "Rotobox Wheels", url: "https://bikegear.in/rotobox-wheels" },
|
|
{ name: "RPM Carbon", url: "https://bikegear.in/rpm-carbon" },
|
|
{ name: "Rukka Motorcycle Clothing", url: "https://bikegear.in/rukka-motorcycle-clothing" },
|
|
{ name: "Ruroc", url: "https://bikegear.in/ruroc-helmet" },
|
|
{ name: "Samcosport", url: "https://bikegear.in/samco-sport" },
|
|
{ name: "SC-Project", url: "https://bikegear.in/sc-project" },
|
|
{ name: "SC-Project (2)", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=415" },
|
|
{ name: "Schuberth Helmets", url: "https://bikegear.in/schuberth-helmets" },
|
|
{ name: "Scorpion Exhausts", url: "https://bikegear.in/scorpion-exhaust" },
|
|
{ name: "Scorpion EXO", url: "https://bikegear.in/scorpion-exo-motorcycle-gears" },
|
|
{ name: "Scorpion EXO (2)", url: "https://bikegear.in/scorpion-exo-motorcycle-gears-1" },
|
|
{ name: "Scorpion EXO USA", url: "https://bikegear.in/scorpion-helmet" },
|
|
{ name: "Sena", url: "https://bikegear.in/sena-motorcycle-intercom" },
|
|
{ name: "Shark", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=423" },
|
|
{ name: "Shark Helmets", url: "https://bikegear.in/shark-helmets" },
|
|
{ name: "Shark Helmets (2)", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=412" },
|
|
{ name: "Shima", url: "https://bikegear.in/shima" },
|
|
{ name: "Shoei Helmets", url: "https://bikegear.in/shoei-helmets" },
|
|
{ name: "Sidi", url: "https://bikegear.in/sidi-boots" },
|
|
{ name: "Simpson", url: "https://bikegear.in/simpson-helmets" },
|
|
{ name: "Spark Exhaust", url: "https://bikegear.in/spark-exhaust" },
|
|
{ name: "Spidi", url: "https://bikegear.in/spidi" },
|
|
{ name: "Stadler", url: "https://bikegear.in/stadler-india" },
|
|
{ name: "Suomy Helmets", url: "https://bikegear.in/suomy-helmets" },
|
|
{ name: "T-Rex Racing", url: "https://bikegear.in/t-rex-racing" },
|
|
{ name: "TCX Boots", url: "https://bikegear.in/tcx-motorcycle-boots" },
|
|
{ name: "TEC Bike Parts", url: "https://bikegear.in/tec-bike-parts" },
|
|
{ name: "Termignoni", url: "https://bikegear.in/termignoni-exhausts" },
|
|
{ name: "Toce Performance", url: "https://bikegear.in/toce-performance" },
|
|
{ name: "Touratech", url: "https://bikegear.in/touratech-accessories" },
|
|
{ name: "Tractive Suspension", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=424" },
|
|
{ name: "Two Brothers Racing", url: "https://bikegear.in/two-brothers-racing" },
|
|
{ name: "Vance & Hines", url: "https://bikegear.in/vance-and-hines" },
|
|
{ name: "Vandemon", url: "https://bikegear.in/vandemon-exhaust-and-performance-parts" },
|
|
{ name: "Voodoo", url: "https://bikegear.in/voodoo-exhaust" },
|
|
{ name: "X-Lite Helmets", url: "https://bikegear.in/x-lite-helmets" },
|
|
{ name: "XPD", url: "https://bikegear.in/xpd-boots" },
|
|
{ name: "Yoshimura", url: "https://bikegear.in/yoshimura" },
|
|
{ name: "Yoshimura Japan", url: "https://bikegear.in/yoshimura-japan" },
|
|
{ name: "YSS Suspension", url: "https://bikegear.in/yss-suspension" },
|
|
{ name: "Zard Exhausts", url: "https://bikegear.in/zard-exhausts" },
|
|
];
|
|
|
|
// ─── HTTP fetch (axios) ───────────────────────────────────────────────────────
|
|
|
|
const HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
"Cache-Control": "no-cache",
|
|
"Pragma": "no-cache",
|
|
"Upgrade-Insecure-Requests": "1",
|
|
"Sec-Fetch-Dest": "document",
|
|
"Sec-Fetch-Mode": "navigate",
|
|
"Sec-Fetch-Site": "none",
|
|
"Sec-Fetch-User": "?1",
|
|
"Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
|
|
"Sec-CH-UA-Mobile": "?0",
|
|
"Sec-CH-UA-Platform": '"Windows"',
|
|
};
|
|
|
|
function sleep(ms) {
|
|
return new Promise((r) => setTimeout(r, ms));
|
|
}
|
|
|
|
async function fetchHtml(url, attempt = 1) {
|
|
try {
|
|
const res = await axios.get(url, { headers: HEADERS, timeout: 30000, responseType: "text", maxRedirects: 5 });
|
|
return res.data;
|
|
} catch (err) {
|
|
const status = err.response?.status;
|
|
if ((status === 429 || status >= 500) && attempt <= 4) {
|
|
const wait = attempt * 2000;
|
|
console.log(` RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
|
|
await sleep(wait);
|
|
return fetchHtml(url, attempt + 1);
|
|
}
|
|
if (status) throw new Error(`HTTP ${status}`);
|
|
const isNetwork = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"].includes(err.code) ||
|
|
err.message?.includes("timeout") || err.message?.includes("connect");
|
|
if (isNetwork && attempt <= 4) {
|
|
await sleep(attempt * 2000);
|
|
return fetchHtml(url, attempt + 1);
|
|
}
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ─── Concurrent runner ────────────────────────────────────────────────────────
|
|
|
|
async function runConcurrent(items, fn, limit) {
|
|
const results = new Array(items.length);
|
|
let idx = 0;
|
|
async function worker() {
|
|
while (idx < items.length) {
|
|
const i = idx++;
|
|
results[i] = await fn(items[i], i);
|
|
}
|
|
}
|
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
return results;
|
|
}
|
|
|
|
// ─── Listing page helpers ─────────────────────────────────────────────────────
|
|
|
|
function extractMaxPage(html) {
|
|
const nums = [...html.matchAll(/\?page=(\d+)/g)].map((m) => parseInt(m[1], 10));
|
|
return nums.length ? Math.max(...nums) : 1;
|
|
}
|
|
|
|
function extractProductUrls(html) {
|
|
const urls = new Set();
|
|
for (const m of html.matchAll(/href="(https:\/\/bikegear\.in\/[^"]+)"\s+class="product-img/g)) {
|
|
const url = m[1];
|
|
if (!url.includes("route=") && !url.includes("manufacturer") && !url.includes("category")) {
|
|
urls.add(url);
|
|
}
|
|
}
|
|
return [...urls];
|
|
}
|
|
|
|
async function scrapeBrandProductUrls(brand) {
|
|
let html;
|
|
try {
|
|
html = await fetchHtml(brand.url);
|
|
} catch (err) {
|
|
console.warn(` [${brand.name}] Failed page 1: ${err.message}`);
|
|
return [];
|
|
}
|
|
|
|
const maxPage = extractMaxPage(html);
|
|
const urls = new Set(extractProductUrls(html));
|
|
|
|
if (maxPage > 1) {
|
|
const pageNums = Array.from({ length: maxPage - 1 }, (_, i) => i + 2);
|
|
await runConcurrent(pageNums, async (pageNum) => {
|
|
try {
|
|
const pageHtml = await fetchHtml(`${brand.url}?page=${pageNum}`);
|
|
for (const u of extractProductUrls(pageHtml)) urls.add(u);
|
|
} catch (err) {
|
|
console.warn(` [${brand.name}] Failed page ${pageNum}: ${err.message}`);
|
|
}
|
|
}, LISTING_CONCURRENCY);
|
|
}
|
|
|
|
const result = [...urls];
|
|
if (result.length) console.log(` [${brand.name}] Found ${result.length} products across ${maxPage} page(s).`);
|
|
return result;
|
|
}
|
|
|
|
// ─── Product detail helpers ───────────────────────────────────────────────────
|
|
|
|
function parseProductJsonLd(html) {
|
|
const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
|
|
const candidates = [];
|
|
for (const b of blocks) {
|
|
try {
|
|
const data = JSON.parse(b[1]);
|
|
if (data["@type"] === "Product") candidates.push(data);
|
|
} catch {}
|
|
}
|
|
if (!candidates.length) return null;
|
|
return candidates.find((d) => d.offers?.price || d.description) || candidates[0];
|
|
}
|
|
|
|
function extractProductId(jsonLd) {
|
|
const m = (jsonLd?.image || "").match(/\/catalog\/products\/(\d+)\//);
|
|
return m ? m[1] : null;
|
|
}
|
|
|
|
function extractProductImages(html, productId, jsonLdImage) {
|
|
const seen = new Set();
|
|
const images = [];
|
|
|
|
if (productId) {
|
|
const re = new RegExp(
|
|
`data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`, "g"
|
|
);
|
|
for (const m of html.matchAll(re)) {
|
|
const url = m[1];
|
|
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
|
|
seen.add(url);
|
|
images.push(url.replace("-500x500.", "-800x800."));
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!images.length) {
|
|
const block = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/);
|
|
if (block) {
|
|
for (const m of block[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) {
|
|
const url = m[1];
|
|
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
|
|
seen.add(url);
|
|
images.push(url.replace("-500x500.", "-800x800."));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!images.length && jsonLdImage) images.push(jsonLdImage);
|
|
return images;
|
|
}
|
|
|
|
function extractDescriptionHtml(html) {
|
|
const sec = html.match(
|
|
/<div[^>]*product-extra-description[^>]*tab-pane[^>]*>([\s\S]*?)(?=<div[^>]*product-extra-(?:attributes|reviews)|<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>)/
|
|
);
|
|
if (!sec) return "";
|
|
const tbl = sec[1].match(/<table[^>]*>([\s\S]*?)<\/table>/);
|
|
return tbl ? `<table border="1">${tbl[1]}</table>` : "";
|
|
}
|
|
|
|
function parsePrice(str) {
|
|
if (!str) return 0;
|
|
const n = parseFloat(String(str).replace(/[^\d.]/g, ""));
|
|
return Number.isNaN(n) ? 0 : n;
|
|
}
|
|
|
|
async function scrapeProductDetail(productUrl, brandName) {
|
|
let html;
|
|
try {
|
|
html = await fetchHtml(productUrl);
|
|
} catch (err) {
|
|
return { url: productUrl, scrapeError: err.message };
|
|
}
|
|
try {
|
|
const jsonLd = parseProductJsonLd(html);
|
|
if (!jsonLd) return { url: productUrl, scrapeError: "No product JSON-LD found" };
|
|
const productId = extractProductId(jsonLd);
|
|
const images = extractProductImages(html, productId, jsonLd.image);
|
|
const descriptionHtml = extractDescriptionHtml(html);
|
|
const price = parsePrice(jsonLd.offers?.price);
|
|
return {
|
|
url: productUrl,
|
|
sku: jsonLd.sku || productUrl.split("/").pop(),
|
|
name: jsonLd.name || "",
|
|
brand: jsonLd.brand?.name || brandName,
|
|
price,
|
|
currency: jsonLd.offers?.priceCurrency || "INR",
|
|
availability: jsonLd.offers?.availability?.replace("https://schema.org/", "") || "Unknown",
|
|
descriptionHtml,
|
|
images,
|
|
productId,
|
|
};
|
|
} catch (err) {
|
|
return { url: productUrl, scrapeError: `Parse error: ${err.message}` };
|
|
}
|
|
}
|
|
|
|
// ─── Output helpers ───────────────────────────────────────────────────────────
|
|
|
|
function wrapProduct(product) {
|
|
return {
|
|
productId: product.sku || product.url,
|
|
sourceKey: "bikegear",
|
|
brand: product.brand || "BikeGear",
|
|
bikeModel: "",
|
|
productSummary: {
|
|
id: product.sku,
|
|
name: product.name,
|
|
img: product.images || [],
|
|
cost: { mrp: product.price || 0 },
|
|
},
|
|
scraped: product,
|
|
};
|
|
}
|
|
|
|
async function writeOutput(products) {
|
|
const ok = products.filter((p) => !(p.scraped || p).scrapeError);
|
|
const fail = products.length - ok.length;
|
|
const payload = {
|
|
generatedAt: new Date().toISOString(),
|
|
sourceKey: "bikegear",
|
|
sourceLabel: "BikeGear India",
|
|
analysis: {
|
|
timestamp: new Date().toISOString(),
|
|
totalProducts: products.length,
|
|
totalProductsUnique: ok.length,
|
|
detailSuccess: ok.length,
|
|
detailFailed: fail,
|
|
},
|
|
products,
|
|
};
|
|
await fs.writeFile(OUTPUT_FILE, JSON.stringify(payload, null, 2), "utf8");
|
|
console.log(`\n Output saved → ${OUTPUT_FILE} (${ok.length} ok, ${fail} failed)`);
|
|
}
|
|
|
|
// ─── Progress ─────────────────────────────────────────────────────────────────
|
|
|
|
async function loadProgress() {
|
|
try {
|
|
return JSON.parse(await fs.readFile(PROGRESS_FILE, "utf8"));
|
|
} catch {
|
|
return { completedBrands: [], products: [] };
|
|
}
|
|
}
|
|
|
|
async function saveProgress(p) {
|
|
await fs.writeFile(PROGRESS_FILE, JSON.stringify(p, null, 2), "utf8");
|
|
}
|
|
|
|
// ─── Main ─────────────────────────────────────────────────────────────────────
|
|
|
|
async function main() {
|
|
console.log("BikeGear Scraper — starting");
|
|
console.log(`${BRANDS.length} brands total | listing concurrency: ${LISTING_CONCURRENCY} | detail concurrency: ${DETAIL_CONCURRENCY}\n`);
|
|
|
|
const progress = await loadProgress();
|
|
const done = new Set(progress.completedBrands);
|
|
const remaining = BRANDS.filter((b) => !done.has(b.name));
|
|
|
|
if (done.size > 0) {
|
|
console.log(`Resuming — ${done.size} brands already done, ${remaining.length} remaining.\n`);
|
|
}
|
|
|
|
let stopping = false;
|
|
|
|
async function gracefulExit() {
|
|
if (stopping) return;
|
|
stopping = true;
|
|
console.log("\nCtrl+C — saving progress...");
|
|
await saveProgress(progress);
|
|
await writeOutput(progress.products);
|
|
console.log("Stopped. Re-run to continue.");
|
|
process.exit(0);
|
|
}
|
|
|
|
process.on("SIGINT", gracefulExit);
|
|
process.on("SIGTERM", gracefulExit);
|
|
|
|
for (const brand of remaining) {
|
|
if (stopping) break;
|
|
|
|
process.stdout.write(`[${brand.name}] listing... `);
|
|
const productUrls = await scrapeBrandProductUrls(brand);
|
|
|
|
if (!productUrls.length) {
|
|
console.log("no products, skipping.");
|
|
progress.completedBrands.push(brand.name);
|
|
await saveProgress(progress);
|
|
continue;
|
|
}
|
|
|
|
console.log(`scraping ${productUrls.length} products...`);
|
|
let done2 = 0;
|
|
|
|
const rawProducts = await runConcurrent(productUrls, async (url) => {
|
|
if (stopping) return { url, scrapeError: "interrupted" };
|
|
const p = await scrapeProductDetail(url, brand.name);
|
|
done2++;
|
|
if (p.scrapeError) {
|
|
process.stdout.write(` ${done2}/${productUrls.length} ERR\n`);
|
|
} else {
|
|
process.stdout.write(` ${done2}/${productUrls.length} OK: ${p.name}\n`);
|
|
}
|
|
return p;
|
|
}, DETAIL_CONCURRENCY);
|
|
|
|
const okCount = rawProducts.filter((p) => !p.scrapeError).length;
|
|
console.log(`[${brand.name}] complete — ${okCount}/${productUrls.length} ok\n`);
|
|
|
|
progress.products.push(...rawProducts.map(wrapProduct));
|
|
progress.completedBrands.push(brand.name);
|
|
|
|
await saveProgress(progress);
|
|
await writeOutput(progress.products);
|
|
}
|
|
|
|
if (!stopping) {
|
|
console.log("\nAll brands done!");
|
|
await writeOutput(progress.products);
|
|
await fs.unlink(PROGRESS_FILE).catch(() => {});
|
|
console.log("Progress file removed (clean finish).");
|
|
console.log(`\nSend this file to the server:\n ${OUTPUT_FILE}\n`);
|
|
}
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("Fatal error:", err);
|
|
process.exit(1);
|
|
});
|