diff --git a/bikegear-local-scraper/bikegear-scraper.js b/bikegear-local-scraper/bikegear-scraper.js new file mode 100644 index 0000000..6231fcb --- /dev/null +++ b/bikegear-local-scraper/bikegear-scraper.js @@ -0,0 +1,500 @@ +/** + * BikeGear.in Standalone Scraper + * ================================ + * Setup (one time): + * npm install axios + * + * Run: + * node bikegear-scraper.js + * + * Pause anytime with Ctrl+C — progress saves automatically. + * Re-run the same command to resume from where you stopped. + * + * Output: bikegear_output.json (send this file to the server) + */ + +const axios = require("axios"); +const path = require("node:path"); +const fs = require("node:fs/promises"); + +// ─── Config ─────────────────────────────────────────────────────────────────── + +const LISTING_CONCURRENCY = 5; // how many listing pages to fetch at once +const DETAIL_CONCURRENCY = 5; // how many product pages to fetch at once +const PROGRESS_FILE = path.resolve(__dirname, "bikegear_progress.json"); +const OUTPUT_FILE = path.resolve(__dirname, "bikegear_output.json"); + +// ─── Brand list ─────────────────────────────────────────────────────────────── + +const BRANDS = [ + { name: "Acerbis", url: "https://bikegear.in/acerbis" }, + { name: "Aimshop", url: "https://bikegear.in/aimshop" }, + { name: "Airoh Helmets", url: "https://bikegear.in/airoh-helmets" }, + { name: "Akrapovic Exhaust Systems", url: "https://bikegear.in/akrapovic-exhaust-systems" }, + { name: "Alpinestars", url: "https://bikegear.in/alpinestars" }, + { name: "Arai Helmet", url: "https://bikegear.in/arai-helmet-india" }, + { name: "Arai Helmet USA", url: "https://bikegear.in/arai-helmet-usa" }, + { name: "Arrow Exhausts", url: "https://bikegear.in/arrow-exhausts" }, + { name: "Austin Racing", url: "https://bikegear.in/austin-racing" }, + { name: "Bassani Exhaust", url: "https://bikegear.in/bassani-exhaust" }, + { name: "Beeline", url: "https://bikegear.in/beeline" }, + { name: "Bell", url: "https://bikegear.in/bell-helmet" }, + { name: "Bell Helmets", url: "https://bikegear.in/bell-helmets" }, + { name: "Biltwell Inc.", url: "https://bikegear.in/biltwell-inc" }, + { name: "Bitubo Suspension", url: "https://bikegear.in/bitubo-suspension" }, + { name: "Black Widow", url: "https://bikegear.in/black-widow" }, + { name: "British Customs", url: "https://bikegear.in/british-customs" }, + { name: "Brock's Performance", url: "https://bikegear.in/brocks-performance" }, + { name: "BT Moto", url: "https://bikegear.in/bt-moto" }, + { name: "Caberg", url: "https://bikegear.in/caberg-helmets" }, + { name: "Cardo Systems", url: "https://bikegear.in/cardo-systems" }, + { name: "Chigee", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=403" }, + { name: "Clover", url: "https://bikegear.in/clover-online" }, + { name: "CNC Racing", url: "https://bikegear.in/cnc-racing" }, + { name: "Cobra Exhaust", url: "https://bikegear.in/cobra-exhaust" }, + { name: "Competition Werkes", url: "https://bikegear.in/competition-werkes" }, + { name: "Daytona", url: "https://bikegear.in/daytona" }, + { name: "Dominator", url: "https://bikegear.in/dominator-exhaust-systems-in-india" }, + { name: "Ducabike", url: "https://bikegear.in/ducabike" }, + { name: "Dynojet Power Commander", url: "https://bikegear.in/dynojet-power-commander-research-inc" }, + { name: "Evotech Performance", url: "https://bikegear.in/evotech-performance-online-india" }, + { name: "Falco", url: "https://bikegear.in/falco" }, + { name: "Five", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=422" }, + { name: "Five Gloves", url: "https://bikegear.in/five-glove-india" }, + { name: "Forcefield", url: "https://bikegear.in/forcefield" }, + { name: "Forma Boots", url: "https://bikegear.in/forma-boots" }, + { name: "Forma Boots (2)", url: "https://bikegear.in/forma-boots-1-1" }, + { name: "Fox", url: "https://bikegear.in/fox-racing-gear" }, + { name: "Freedom Performance", url: "https://bikegear.in/freedom-performance-exhaust" }, + { name: "Full Six Carbon", url: "https://bikegear.in/full-six-carbon" }, + { name: "G.P.R. Exhaust System", url: "https://bikegear.in/gpr-exhaust-system" }, + { name: "Gaerne", url: "https://bikegear.in/gaerne" }, + { name: "GB Racing", url: "https://bikegear.in/gb-racing" }, + { name: "Gilles Tooling", url: "https://bikegear.in/gilles-tooling-india" }, + { name: "Graves Motorsports", url: "https://bikegear.in/graves-motorsports" }, + { name: "Held", url: "https://bikegear.in/held" }, + { name: "Hindle Exhaust", url: "https://bikegear.in/hindle-exhaust" }, + { name: "HJC Helmets", url: "https://bikegear.in/hjc-helmets" }, + { name: "Honda OEM", url: "https://bikegear.in/honda-oem" }, + { name: "HP Corse", url: "https://bikegear.in/hp-corse-exhaust" }, + { name: "Icon", url: "https://bikegear.in/icon-helmet" }, + { name: "Icon (2)", url: "https://bikegear.in/icon-helmet-1" }, + { name: "Ilmberger Carbon", url: "https://bikegear.in/ilmberger-carbon" }, + { name: "IXIL Exhaust", url: "https://bikegear.in/ixil-exhaust" }, + { name: "Ixon", url: "https://bikegear.in/ixon-motorcycle-gears" }, + { name: "Ixon (2)", url: "https://bikegear.in/ixon-motorcycle-gears-1" }, + { name: "IXS", url: "https://bikegear.in/ixs" }, + { name: "Jetprime", url: "https://bikegear.in/jet-prime" }, + { name: "Just-1", url: "https://bikegear.in/just-1" }, + { name: "Kabuto", url: "https://bikegear.in/kabuto" }, + { name: "Kineo Wheels", url: "https://bikegear.in/kineo-wheels" }, + { name: "KLIM", url: "https://bikegear.in/klim" }, + { name: "Knox", url: "https://bikegear.in/knox" }, + { name: "KTM", url: "https://bikegear.in/ktm" }, + { name: "Leatt", url: "https://bikegear.in/leatt-1" }, + { name: "Leo Vince", url: "https://bikegear.in/leo-vince-motorcycle-exhausts-system" }, + { name: "Leovince", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=426" }, + { name: "Lextek", url: "https://bikegear.in/lextek-motorcycle-products" }, + { name: "Livall", url: "https://bikegear.in/livall-helmet-india" }, + { name: "LS2 Helmets", url: "https://bikegear.in/ls2-helmet" }, + { name: "M4 Exhaust", url: "https://bikegear.in/m4-exhaust" }, + { name: "Macna", url: "https://bikegear.in/macna-brand" }, + { name: "Marchesini Wheels", url: "https://bikegear.in/marchesini-wheels" }, + { name: "Mivv Exhausts", url: "https://bikegear.in/mivv-exhausts" }, + { name: "Mosko Moto", url: "https://bikegear.in/mosko-moto" }, + { name: "Nexx Helmets", url: "https://bikegear.in/nexx-helmets" }, + { name: "Nolan", url: "https://bikegear.in/nolan-helmets" }, + { name: "Ohlins Racing", url: "https://bikegear.in/ohlins-racing" }, + { name: "Oxford", url: "https://bikegear.in/oxford" }, + { name: "OZ Motorbike", url: "https://bikegear.in/oz-motorbike" }, + { name: "Premier", url: "https://bikegear.in/premier-helmet" }, + { name: "QD Exhausts", url: "https://bikegear.in/qd-exhaust" }, + { name: "R&G Racing", url: "https://bikegear.in/r-and-g-racing" }, + { name: "Racefit Exhausts", url: "https://bikegear.in/racefit-exhausts" }, + { name: "Rapid Bike", url: "https://bikegear.in/rapid-bike" }, + { name: "Rekluse", url: "https://bikegear.in/rekluse-clutch-plates" }, + { name: "Remus Exhausts", url: "https://bikegear.in/remus-exhaust" }, + { name: "REV'IT", url: "https://bikegear.in/revit" }, + { name: "Rizoma", url: "https://bikegear.in/rizoma" }, + { name: "Rotobox Wheels", url: "https://bikegear.in/rotobox-wheels" }, + { name: "RPM Carbon", url: "https://bikegear.in/rpm-carbon" }, + { name: "Rukka Motorcycle Clothing", url: "https://bikegear.in/rukka-motorcycle-clothing" }, + { name: "Ruroc", url: "https://bikegear.in/ruroc-helmet" }, + { name: "Samcosport", url: "https://bikegear.in/samco-sport" }, + { name: "SC-Project", url: "https://bikegear.in/sc-project" }, + { name: "SC-Project (2)", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=415" }, + { name: "Schuberth Helmets", url: "https://bikegear.in/schuberth-helmets" }, + { name: "Scorpion Exhausts", url: "https://bikegear.in/scorpion-exhaust" }, + { name: "Scorpion EXO", url: "https://bikegear.in/scorpion-exo-motorcycle-gears" }, + { name: "Scorpion EXO (2)", url: "https://bikegear.in/scorpion-exo-motorcycle-gears-1" }, + { name: "Scorpion EXO USA", url: "https://bikegear.in/scorpion-helmet" }, + { name: "Sena", url: "https://bikegear.in/sena-motorcycle-intercom" }, + { name: "Shark", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=423" }, + { name: "Shark Helmets", url: "https://bikegear.in/shark-helmets" }, + { name: "Shark Helmets (2)", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=412" }, + { name: "Shima", url: "https://bikegear.in/shima" }, + { name: "Shoei Helmets", url: "https://bikegear.in/shoei-helmets" }, + { name: "Sidi", url: "https://bikegear.in/sidi-boots" }, + { name: "Simpson", url: "https://bikegear.in/simpson-helmets" }, + { name: "Spark Exhaust", url: "https://bikegear.in/spark-exhaust" }, + { name: "Spidi", url: "https://bikegear.in/spidi" }, + { name: "Stadler", url: "https://bikegear.in/stadler-india" }, + { name: "Suomy Helmets", url: "https://bikegear.in/suomy-helmets" }, + { name: "T-Rex Racing", url: "https://bikegear.in/t-rex-racing" }, + { name: "TCX Boots", url: "https://bikegear.in/tcx-motorcycle-boots" }, + { name: "TEC Bike Parts", url: "https://bikegear.in/tec-bike-parts" }, + { name: "Termignoni", url: "https://bikegear.in/termignoni-exhausts" }, + { name: "Toce Performance", url: "https://bikegear.in/toce-performance" }, + { name: "Touratech", url: "https://bikegear.in/touratech-accessories" }, + { name: "Tractive Suspension", url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=424" }, + { name: "Two Brothers Racing", url: "https://bikegear.in/two-brothers-racing" }, + { name: "Vance & Hines", url: "https://bikegear.in/vance-and-hines" }, + { name: "Vandemon", url: "https://bikegear.in/vandemon-exhaust-and-performance-parts" }, + { name: "Voodoo", url: "https://bikegear.in/voodoo-exhaust" }, + { name: "X-Lite Helmets", url: "https://bikegear.in/x-lite-helmets" }, + { name: "XPD", url: "https://bikegear.in/xpd-boots" }, + { name: "Yoshimura", url: "https://bikegear.in/yoshimura" }, + { name: "Yoshimura Japan", url: "https://bikegear.in/yoshimura-japan" }, + { name: "YSS Suspension", url: "https://bikegear.in/yss-suspension" }, + { name: "Zard Exhausts", url: "https://bikegear.in/zard-exhausts" }, +]; + +// ─── HTTP fetch (axios) ─────────────────────────────────────────────────────── + +const HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', +}; + +function sleep(ms) { + return new Promise((r) => setTimeout(r, ms)); +} + +async function fetchHtml(url, attempt = 1) { + try { + const res = await axios.get(url, { headers: HEADERS, timeout: 30000, responseType: "text", maxRedirects: 5 }); + return res.data; + } catch (err) { + const status = err.response?.status; + if ((status === 429 || status >= 500) && attempt <= 4) { + const wait = attempt * 2000; + console.log(` RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`); + await sleep(wait); + return fetchHtml(url, attempt + 1); + } + if (status) throw new Error(`HTTP ${status}`); + const isNetwork = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"].includes(err.code) || + err.message?.includes("timeout") || err.message?.includes("connect"); + if (isNetwork && attempt <= 4) { + await sleep(attempt * 2000); + return fetchHtml(url, attempt + 1); + } + throw err; + } +} + +// ─── Concurrent runner ──────────────────────────────────────────────────────── + +async function runConcurrent(items, fn, limit) { + const results = new Array(items.length); + let idx = 0; + async function worker() { + while (idx < items.length) { + const i = idx++; + results[i] = await fn(items[i], i); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker())); + return results; +} + +// ─── Listing page helpers ───────────────────────────────────────────────────── + +function extractMaxPage(html) { + const nums = [...html.matchAll(/\?page=(\d+)/g)].map((m) => parseInt(m[1], 10)); + return nums.length ? Math.max(...nums) : 1; +} + +function extractProductUrls(html) { + const urls = new Set(); + for (const m of html.matchAll(/href="(https:\/\/bikegear\.in\/[^"]+)"\s+class="product-img/g)) { + const url = m[1]; + if (!url.includes("route=") && !url.includes("manufacturer") && !url.includes("category")) { + urls.add(url); + } + } + return [...urls]; +} + +async function scrapeBrandProductUrls(brand) { + let html; + try { + html = await fetchHtml(brand.url); + } catch (err) { + console.warn(` [${brand.name}] Failed page 1: ${err.message}`); + return []; + } + + const maxPage = extractMaxPage(html); + const urls = new Set(extractProductUrls(html)); + + if (maxPage > 1) { + const pageNums = Array.from({ length: maxPage - 1 }, (_, i) => i + 2); + await runConcurrent(pageNums, async (pageNum) => { + try { + const pageHtml = await fetchHtml(`${brand.url}?page=${pageNum}`); + for (const u of extractProductUrls(pageHtml)) urls.add(u); + } catch (err) { + console.warn(` [${brand.name}] Failed page ${pageNum}: ${err.message}`); + } + }, LISTING_CONCURRENCY); + } + + const result = [...urls]; + if (result.length) console.log(` [${brand.name}] Found ${result.length} products across ${maxPage} page(s).`); + return result; +} + +// ─── Product detail helpers ─────────────────────────────────────────────────── + +function parseProductJsonLd(html) { + const blocks = [...html.matchAll(/([\s\S]*?)<\/script>/g)]; + const candidates = []; + for (const b of blocks) { + try { + const data = JSON.parse(b[1]); + if (data["@type"] === "Product") candidates.push(data); + } catch {} + } + if (!candidates.length) return null; + return candidates.find((d) => d.offers?.price || d.description) || candidates[0]; +} + +function extractProductId(jsonLd) { + const m = (jsonLd?.image || "").match(/\/catalog\/products\/(\d+)\//); + return m ? m[1] : null; +} + +function extractProductImages(html, productId, jsonLdImage) { + const seen = new Set(); + const images = []; + + if (productId) { + const re = new RegExp( + `data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`, "g" + ); + for (const m of html.matchAll(re)) { + const url = m[1]; + if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) { + seen.add(url); + images.push(url.replace("-500x500.", "-800x800.")); + } + } + } + + if (!images.length) { + const block = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/); + if (block) { + for (const m of block[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) { + const url = m[1]; + if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) { + seen.add(url); + images.push(url.replace("-500x500.", "-800x800.")); + } + } + } + } + + if (!images.length && jsonLdImage) images.push(jsonLdImage); + return images; +} + +function extractDescriptionHtml(html) { + const sec = html.match( + /]*product-extra-description[^>]*tab-pane[^>]*>([\s\S]*?)(?=]*product-extra-(?:attributes|reviews)|<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>)/ + ); + if (!sec) return ""; + const tbl = sec[1].match(/]*>([\s\S]*?)<\/table>/); + return tbl ? `${tbl[1]}
` : ""; +} + +function parsePrice(str) { + if (!str) return 0; + const n = parseFloat(String(str).replace(/[^\d.]/g, "")); + return Number.isNaN(n) ? 0 : n; +} + +async function scrapeProductDetail(productUrl, brandName) { + let html; + try { + html = await fetchHtml(productUrl); + } catch (err) { + return { url: productUrl, scrapeError: err.message }; + } + try { + const jsonLd = parseProductJsonLd(html); + if (!jsonLd) return { url: productUrl, scrapeError: "No product JSON-LD found" }; + const productId = extractProductId(jsonLd); + const images = extractProductImages(html, productId, jsonLd.image); + const descriptionHtml = extractDescriptionHtml(html); + const price = parsePrice(jsonLd.offers?.price); + return { + url: productUrl, + sku: jsonLd.sku || productUrl.split("/").pop(), + name: jsonLd.name || "", + brand: jsonLd.brand?.name || brandName, + price, + currency: jsonLd.offers?.priceCurrency || "INR", + availability: jsonLd.offers?.availability?.replace("https://schema.org/", "") || "Unknown", + descriptionHtml, + images, + productId, + }; + } catch (err) { + return { url: productUrl, scrapeError: `Parse error: ${err.message}` }; + } +} + +// ─── Output helpers ─────────────────────────────────────────────────────────── + +function wrapProduct(product) { + return { + productId: product.sku || product.url, + sourceKey: "bikegear", + brand: product.brand || "BikeGear", + bikeModel: "", + productSummary: { + id: product.sku, + name: product.name, + img: product.images || [], + cost: { mrp: product.price || 0 }, + }, + scraped: product, + }; +} + +async function writeOutput(products) { + const ok = products.filter((p) => !(p.scraped || p).scrapeError); + const fail = products.length - ok.length; + const payload = { + generatedAt: new Date().toISOString(), + sourceKey: "bikegear", + sourceLabel: "BikeGear India", + analysis: { + timestamp: new Date().toISOString(), + totalProducts: products.length, + totalProductsUnique: ok.length, + detailSuccess: ok.length, + detailFailed: fail, + }, + products, + }; + await fs.writeFile(OUTPUT_FILE, JSON.stringify(payload, null, 2), "utf8"); + console.log(`\n Output saved → ${OUTPUT_FILE} (${ok.length} ok, ${fail} failed)`); +} + +// ─── Progress ───────────────────────────────────────────────────────────────── + +async function loadProgress() { + try { + return JSON.parse(await fs.readFile(PROGRESS_FILE, "utf8")); + } catch { + return { completedBrands: [], products: [] }; + } +} + +async function saveProgress(p) { + await fs.writeFile(PROGRESS_FILE, JSON.stringify(p, null, 2), "utf8"); +} + +// ─── Main ───────────────────────────────────────────────────────────────────── + +async function main() { + console.log("BikeGear Scraper — starting"); + console.log(`${BRANDS.length} brands total | listing concurrency: ${LISTING_CONCURRENCY} | detail concurrency: ${DETAIL_CONCURRENCY}\n`); + + const progress = await loadProgress(); + const done = new Set(progress.completedBrands); + const remaining = BRANDS.filter((b) => !done.has(b.name)); + + if (done.size > 0) { + console.log(`Resuming — ${done.size} brands already done, ${remaining.length} remaining.\n`); + } + + let stopping = false; + + async function gracefulExit() { + if (stopping) return; + stopping = true; + console.log("\nCtrl+C — saving progress..."); + await saveProgress(progress); + await writeOutput(progress.products); + console.log("Stopped. Re-run to continue."); + process.exit(0); + } + + process.on("SIGINT", gracefulExit); + process.on("SIGTERM", gracefulExit); + + for (const brand of remaining) { + if (stopping) break; + + process.stdout.write(`[${brand.name}] listing... `); + const productUrls = await scrapeBrandProductUrls(brand); + + if (!productUrls.length) { + console.log("no products, skipping."); + progress.completedBrands.push(brand.name); + await saveProgress(progress); + continue; + } + + console.log(`scraping ${productUrls.length} products...`); + let done2 = 0; + + const rawProducts = await runConcurrent(productUrls, async (url) => { + if (stopping) return { url, scrapeError: "interrupted" }; + const p = await scrapeProductDetail(url, brand.name); + done2++; + if (p.scrapeError) { + process.stdout.write(` ${done2}/${productUrls.length} ERR\n`); + } else { + process.stdout.write(` ${done2}/${productUrls.length} OK: ${p.name}\n`); + } + return p; + }, DETAIL_CONCURRENCY); + + const okCount = rawProducts.filter((p) => !p.scrapeError).length; + console.log(`[${brand.name}] complete — ${okCount}/${productUrls.length} ok\n`); + + progress.products.push(...rawProducts.map(wrapProduct)); + progress.completedBrands.push(brand.name); + + await saveProgress(progress); + await writeOutput(progress.products); + } + + if (!stopping) { + console.log("\nAll brands done!"); + await writeOutput(progress.products); + await fs.unlink(PROGRESS_FILE).catch(() => {}); + console.log("Progress file removed (clean finish)."); + console.log(`\nSend this file to the server:\n ${OUTPUT_FILE}\n`); + } +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +});