feat: add bikegear.in import source (Alpinestars as first brand)
Adds a new OpenCart-based source (bikegear.in) to the import pipeline. 2-phase scraper: listing pages collect product URLs, detail pages extract JSON-LD (name, price in INR, brand, SKU) + description table HTML + gallery images. - sources/bikegear/brands.js — brand list (Alpinestars active, 40+ commented) - sources/bikegear/scraper.js — HTML scraper with full Sec-Fetch browser headers - sources/bikegear/converter.js — Shopify-ready converter (record.scraped || record pattern) - sources/bikegear/index.js — 24h cache, zero-success guard, brands[] export - sources/index.js — register bikegear source Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
4b5b7cab9f
commit
c2e037310d
@ -0,0 +1,56 @@
|
|||||||
|
/**
|
||||||
|
* BikeGear.in brand list.
|
||||||
|
* Source: https://bikegear.in/index.php?route=product/manufacturer
|
||||||
|
* Each brand has a `name` and `url` (the brand collection page on bikegear.in).
|
||||||
|
* Add / remove entries here to control which brands are scraped.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const BRANDS = [
|
||||||
|
// added 2026-07-13
|
||||||
|
{ name: "Alpinestars", url: "https://bikegear.in/alpinestars" },
|
||||||
|
|
||||||
|
// Uncomment to add more brands:
|
||||||
|
// { name: "Acerbis", url: "https://bikegear.in/acerbis" },
|
||||||
|
// { name: "Airoh Helmets", url: "https://bikegear.in/airoh-helmets" },
|
||||||
|
// { name: "Akrapovic", url: "https://bikegear.in/akrapovic-exhaust-systems" },
|
||||||
|
// { name: "Arai Helmet", url: "https://bikegear.in/arai-helmet-india" },
|
||||||
|
// { name: "Arrow Exhausts", url: "https://bikegear.in/arrow-exhausts" },
|
||||||
|
// { name: "Austin Racing", url: "https://bikegear.in/austin-racing" },
|
||||||
|
// { name: "Bell", url: "https://bikegear.in/bell-helmets" },
|
||||||
|
// { name: "Caberg", url: "https://bikegear.in/caberg-helmets" },
|
||||||
|
// { name: "CNC Racing", url: "https://bikegear.in/cnc-racing" },
|
||||||
|
// { name: "Ducabike", url: "https://bikegear.in/ducabike" },
|
||||||
|
// { name: "Evotech Performance",url: "https://bikegear.in/evotech-performance-online-india" },
|
||||||
|
// { name: "Forma Boots", url: "https://bikegear.in/forma-boots" },
|
||||||
|
// { name: "Gaerne", url: "https://bikegear.in/gaerne" },
|
||||||
|
// { name: "GB Racing", url: "https://bikegear.in/gb-racing" },
|
||||||
|
// { name: "HJC Helmets", url: "https://bikegear.in/hjc-helmets" },
|
||||||
|
// { name: "HP Corse", url: "https://bikegear.in/hp-corse-exhaust" },
|
||||||
|
// { name: "IXS", url: "https://bikegear.in/ixs" },
|
||||||
|
// { name: "Kabuto", url: "https://bikegear.in/kabuto" },
|
||||||
|
// { name: "KLIM", url: "https://bikegear.in/klim" },
|
||||||
|
// { name: "Leo Vince", url: "https://bikegear.in/leo-vince-motorcycle-exhausts-system" },
|
||||||
|
// { name: "LS2 Helmets", url: "https://bikegear.in/ls2-helmet" },
|
||||||
|
// { name: "Mivv Exhausts", url: "https://bikegear.in/mivv-exhausts" },
|
||||||
|
// { name: "Nexx Helmets", url: "https://bikegear.in/nexx-helmets" },
|
||||||
|
// { name: "Nolan", url: "https://bikegear.in/nolan-helmets" },
|
||||||
|
// { name: "Ohlins Racing", url: "https://bikegear.in/ohlins-racing" },
|
||||||
|
// { name: "R&G Racing", url: "https://bikegear.in/r-and-g-racing" },
|
||||||
|
// { name: "REV'IT", url: "https://bikegear.in/revit" },
|
||||||
|
// { name: "Rizoma", url: "https://bikegear.in/rizoma" },
|
||||||
|
// { name: "Rukka", url: "https://bikegear.in/rukka-motorcycle-clothing" },
|
||||||
|
// { name: "SC-Project", url: "https://bikegear.in/sc-project" },
|
||||||
|
// { name: "Schuberth Helmets", url: "https://bikegear.in/schuberth-helmets" },
|
||||||
|
// { name: "Scorpion EXO", url: "https://bikegear.in/scorpion-exo-motorcycle-gears" },
|
||||||
|
// { name: "Sena", url: "https://bikegear.in/sena-motorcycle-intercom" },
|
||||||
|
// { name: "Shark Helmets", url: "https://bikegear.in/shark-helmets" },
|
||||||
|
// { name: "Shoei Helmets", url: "https://bikegear.in/shoei-helmets" },
|
||||||
|
// { name: "Sidi", url: "https://bikegear.in/sidi-boots" },
|
||||||
|
// { name: "Spidi", url: "https://bikegear.in/spidi" },
|
||||||
|
// { name: "TCX Boots", url: "https://bikegear.in/tcx-motorcycle-boots" },
|
||||||
|
// { name: "Termignoni", url: "https://bikegear.in/termignoni-exhausts" },
|
||||||
|
// { name: "Touratech", url: "https://bikegear.in/touratech-accessories" },
|
||||||
|
// { name: "Yoshimura", url: "https://bikegear.in/yoshimura" },
|
||||||
|
];
|
||||||
|
|
||||||
|
module.exports = { BRANDS };
|
||||||
@ -0,0 +1,91 @@
|
|||||||
|
const path = require("node:path");
|
||||||
|
|
||||||
|
function slugify(str) {
|
||||||
|
return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function getUploadedImageUrl(imgPath, uploadedImageMap) {
|
||||||
|
if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null;
|
||||||
|
const bySourcePath = uploadedImageMap.bySourcePath || {};
|
||||||
|
return bySourcePath[String(imgPath)]?.url || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getImageFileName(imagePath) {
|
||||||
|
try { return path.basename(new URL(imagePath).pathname); } catch { return path.basename(String(imagePath || "").split(/[?#]/)[0]); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function convertBikeGearRecordToShopifyReady(record, options = {}) {
|
||||||
|
const brand = record.brand || options.brand || "BikeGear";
|
||||||
|
const uploadedImageMap = options.uploadedImageMap || null;
|
||||||
|
|
||||||
|
const sku = record.sku || slugify(record.name || "unknown");
|
||||||
|
const productId = sku;
|
||||||
|
const title = record.name || "Untitled Product";
|
||||||
|
const price = Number(record.price ?? 0);
|
||||||
|
const currency = record.currency || "INR";
|
||||||
|
const descriptionHtml = record.descriptionHtml || "";
|
||||||
|
const imagePaths = Array.isArray(record.images) ? record.images : [];
|
||||||
|
|
||||||
|
const files = imagePaths
|
||||||
|
.map((imgPath) => ({
|
||||||
|
type: "Image",
|
||||||
|
url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath,
|
||||||
|
media_content: getImageFileName(imgPath),
|
||||||
|
source_path: imgPath,
|
||||||
|
}))
|
||||||
|
.filter((f) => f.url);
|
||||||
|
|
||||||
|
const quantity = 0;
|
||||||
|
const tags = [brand, sku, `currency:${currency}`, "bikegear-in"].filter(Boolean).map((t) => String(t).trim());
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: productId,
|
||||||
|
source: {
|
||||||
|
productId,
|
||||||
|
sourceKey: "bikegear",
|
||||||
|
url: record.url || null,
|
||||||
|
image_paths: imagePaths,
|
||||||
|
currency,
|
||||||
|
},
|
||||||
|
attributes: {
|
||||||
|
product_name: title,
|
||||||
|
brand,
|
||||||
|
category: "Motorcycle Gear",
|
||||||
|
subcategory: "",
|
||||||
|
part_number: sku,
|
||||||
|
mfr_part_number: sku,
|
||||||
|
price,
|
||||||
|
compare_price: null,
|
||||||
|
purchase_cost: null,
|
||||||
|
barcode: "",
|
||||||
|
price_group: null,
|
||||||
|
units_per_sku: null,
|
||||||
|
part_description: descriptionHtml,
|
||||||
|
descriptions: descriptionHtml ? [{ type: "Market Description", description: descriptionHtml }] : [],
|
||||||
|
files,
|
||||||
|
image_paths: imagePaths,
|
||||||
|
dimensions: [{ weight: 0 }],
|
||||||
|
total_quantity: quantity,
|
||||||
|
inventorydata: { inventory: { main: quantity } },
|
||||||
|
options: [],
|
||||||
|
variants: [{ sku, price, compare_price: null, quantity, optionValues: [] }],
|
||||||
|
fitmentTags: { make: [], model: [], year: [], drive: [], baseModel: [] },
|
||||||
|
tags,
|
||||||
|
handle: slugify(`${brand}-${title}-${productId}`),
|
||||||
|
source_url: record.url || null,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function convertBikeGearJsonToShopifyProducts(input, options = {}) {
|
||||||
|
const records = Array.isArray(input?.products) ? input.products : [];
|
||||||
|
return records.map((record) => {
|
||||||
|
const data = record.scraped || record;
|
||||||
|
return convertBikeGearRecordToShopifyReady(data, {
|
||||||
|
...options,
|
||||||
|
brand: data.brand || record.brand || options.brand,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { convertBikeGearRecordToShopifyReady, convertBikeGearJsonToShopifyProducts };
|
||||||
135
src/business-logic/import-pipeline/sources/bikegear/index.js
Normal file
135
src/business-logic/import-pipeline/sources/bikegear/index.js
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
const fs = require("node:fs/promises");
|
||||||
|
const path = require("node:path");
|
||||||
|
const { BRANDS } = require("./brands");
|
||||||
|
const { scrapeBikeGear } = require("./scraper");
|
||||||
|
const { convertBikeGearJsonToShopifyProducts } = require("./converter");
|
||||||
|
|
||||||
|
const sourceKey = "bikegear";
|
||||||
|
const label = "BikeGear India";
|
||||||
|
const dataDir = path.join("data", "sources", sourceKey);
|
||||||
|
|
||||||
|
function paths() {
|
||||||
|
return {
|
||||||
|
aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
|
||||||
|
historyJson: path.join(dataDir, "01_products_run_history.json"),
|
||||||
|
downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"),
|
||||||
|
watermarkState: path.join(dataDir, "03_watermark_state.json"),
|
||||||
|
imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"),
|
||||||
|
uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"),
|
||||||
|
shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"),
|
||||||
|
logsDir: path.join(dataDir, "99_run_logs"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchWebsiteData({ paths: runPaths }) {
|
||||||
|
// Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.
|
||||||
|
const cacheMaxHours = Number(process.env.BIKEGEAR_CACHE_HOURS ?? 24);
|
||||||
|
const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson);
|
||||||
|
|
||||||
|
if (cacheMaxHours > 0) {
|
||||||
|
try {
|
||||||
|
const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
|
||||||
|
const products = existing?.products;
|
||||||
|
const successCount = Array.isArray(products)
|
||||||
|
? products.filter((p) => !(p.scraped || p).scrapeError).length
|
||||||
|
: 0;
|
||||||
|
if (existing?.generatedAt && Array.isArray(products) && successCount > 0) {
|
||||||
|
const ageHours = (Date.now() - new Date(existing.generatedAt).getTime()) / (1000 * 60 * 60);
|
||||||
|
if (ageHours < cacheMaxHours) {
|
||||||
|
console.log(`[BIKEGEAR] Using cached data (${ageHours.toFixed(1)}h old) — ${successCount} ok products. Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.`);
|
||||||
|
return {
|
||||||
|
analysis: existing.analysis || {},
|
||||||
|
outputJsonPath: absAggregatedPath,
|
||||||
|
historyPath: path.resolve(process.cwd(), runPaths.historyJson),
|
||||||
|
cached: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
console.log(`[BIKEGEAR] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`);
|
||||||
|
} else if (Array.isArray(products) && products.length > 0 && successCount === 0) {
|
||||||
|
console.log(`[BIKEGEAR] Cached data has 0 successful products — ignoring cache and re-scraping.`);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// no cache yet
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
const rawProducts = await scrapeBikeGear();
|
||||||
|
|
||||||
|
const successfulProducts = rawProducts.filter((p) => !p.scrapeError);
|
||||||
|
const successCount = successfulProducts.length;
|
||||||
|
const failCount = rawProducts.length - successCount;
|
||||||
|
|
||||||
|
if (successCount === 0) {
|
||||||
|
throw new Error(
|
||||||
|
`[BIKEGEAR] All ${rawProducts.length} product scrapes failed. First error: ${rawProducts[0]?.scrapeError || "unknown"}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const allProducts = successfulProducts.map((product) => ({
|
||||||
|
productId: product.sku || product.url,
|
||||||
|
sourceKey,
|
||||||
|
brand: product.brand || "BikeGear",
|
||||||
|
bikeModel: "",
|
||||||
|
productSummary: {
|
||||||
|
id: product.sku,
|
||||||
|
name: product.name,
|
||||||
|
img: product.images || [],
|
||||||
|
cost: { mrp: product.price || 0 },
|
||||||
|
},
|
||||||
|
scraped: product,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const analysis = {
|
||||||
|
timestamp: now,
|
||||||
|
totalProducts: rawProducts.length,
|
||||||
|
totalProductsUnique: allProducts.length,
|
||||||
|
detailSuccess: successCount,
|
||||||
|
detailFailed: failCount,
|
||||||
|
};
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
generatedAt: now,
|
||||||
|
sourceKey,
|
||||||
|
sourceLabel: label,
|
||||||
|
analysis,
|
||||||
|
products: allProducts,
|
||||||
|
};
|
||||||
|
|
||||||
|
await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true });
|
||||||
|
await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8");
|
||||||
|
|
||||||
|
let history = [];
|
||||||
|
const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson);
|
||||||
|
try {
|
||||||
|
history = JSON.parse(await fs.readFile(absHistoryPath, "utf8"));
|
||||||
|
if (!Array.isArray(history)) history = [];
|
||||||
|
} catch {
|
||||||
|
history = [];
|
||||||
|
}
|
||||||
|
history.push(analysis);
|
||||||
|
await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8");
|
||||||
|
|
||||||
|
console.log(`[BIKEGEAR] Saved ${allProducts.length} products (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`);
|
||||||
|
|
||||||
|
return { analysis, outputJsonPath: absAggregatedPath, historyPath: absHistoryPath };
|
||||||
|
}
|
||||||
|
|
||||||
|
function convertToShopifyProducts(input, options = {}) {
|
||||||
|
return convertBikeGearJsonToShopifyProducts(input, {
|
||||||
|
brand: options.brand || label,
|
||||||
|
uploadedImageMap: options.uploadedImageMap,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
sourceKey,
|
||||||
|
label,
|
||||||
|
brands: BRANDS.map((b) => b.name),
|
||||||
|
defaultBrand: "BikeGear India",
|
||||||
|
defaultImageBaseUrl: "",
|
||||||
|
envImageBaseUrl: "BIKEGEAR_IMAGE_BASE_URL",
|
||||||
|
paths,
|
||||||
|
fetchWebsiteData,
|
||||||
|
convertToShopifyProducts,
|
||||||
|
};
|
||||||
294
src/business-logic/import-pipeline/sources/bikegear/scraper.js
Normal file
294
src/business-logic/import-pipeline/sources/bikegear/scraper.js
Normal file
@ -0,0 +1,294 @@
|
|||||||
|
/**
|
||||||
|
* BikeGear.in scraper (OpenCart / Journal3 theme).
|
||||||
|
* 2-phase: listing pages → product URLs, then detail pages → full product data.
|
||||||
|
*
|
||||||
|
* Prices are in INR (Indian Rupees) as scraped from the site.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const { BRANDS } = require("./brands");
|
||||||
|
|
||||||
|
const BASE_URL = "https://bikegear.in";
|
||||||
|
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3);
|
||||||
|
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2);
|
||||||
|
|
||||||
|
const FETCH_HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||||
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
|
"Cache-Control": "no-cache",
|
||||||
|
"Pragma": "no-cache",
|
||||||
|
"Upgrade-Insecure-Requests": "1",
|
||||||
|
"Sec-Fetch-Dest": "document",
|
||||||
|
"Sec-Fetch-Mode": "navigate",
|
||||||
|
"Sec-Fetch-Site": "none",
|
||||||
|
"Sec-Fetch-User": "?1",
|
||||||
|
"Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
|
||||||
|
"Sec-CH-UA-Mobile": "?0",
|
||||||
|
"Sec-CH-UA-Platform": '"Windows"',
|
||||||
|
};
|
||||||
|
|
||||||
|
function sleep(ms) {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchHtml(url, attempt = 1) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(url, { headers: FETCH_HEADERS });
|
||||||
|
|
||||||
|
if (res.status === 429 || res.status >= 500) {
|
||||||
|
if (attempt <= 4) {
|
||||||
|
const wait = attempt * 2000;
|
||||||
|
console.log(`[BIKEGEAR] RETRY ${url} HTTP ${res.status} — waiting ${wait}ms (${attempt}/4)`);
|
||||||
|
await sleep(wait);
|
||||||
|
return fetchHtml(url, attempt + 1);
|
||||||
|
}
|
||||||
|
throw new Error(`HTTP ${res.status} after retries`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||||
|
return res.text();
|
||||||
|
} catch (err) {
|
||||||
|
if (attempt <= 4 && (err.code === "UND_ERR_CONNECT_TIMEOUT" || err.code === "ECONNRESET" || err.message?.includes("fetch failed"))) {
|
||||||
|
await sleep(attempt * 2000);
|
||||||
|
return fetchHtml(url, attempt + 1);
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Run up to `limit` tasks from `items` at a time. */
|
||||||
|
async function runConcurrent(items, fn, limit) {
|
||||||
|
const results = [];
|
||||||
|
let idx = 0;
|
||||||
|
|
||||||
|
async function worker() {
|
||||||
|
while (idx < items.length) {
|
||||||
|
const i = idx++;
|
||||||
|
results[i] = await fn(items[i], i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
||||||
|
await Promise.all(workers);
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Extract the max page number from a listing page's HTML. */
|
||||||
|
function extractMaxPage(html) {
|
||||||
|
const pageNums = [...html.matchAll(/\?page=(\d+)/g)].map((m) => parseInt(m[1], 10));
|
||||||
|
return pageNums.length ? Math.max(...pageNums) : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Extract product detail page URLs from a brand listing page HTML. */
|
||||||
|
function extractProductUrls(html) {
|
||||||
|
const urls = new Set();
|
||||||
|
// Product links have class="product-img" — these are the card image anchors
|
||||||
|
for (const m of html.matchAll(/href="(https:\/\/bikegear\.in\/[^"]+)"\s+class="product-img/g)) {
|
||||||
|
const url = m[1];
|
||||||
|
// Exclude manufacturer/category/route pages — product slugs are plain paths
|
||||||
|
if (!url.includes("route=") && !url.includes("manufacturer") && !url.includes("category")) {
|
||||||
|
urls.add(url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [...urls];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scrape all listing pages for a brand and collect product detail URLs. */
|
||||||
|
async function scrapeBrandProductUrls(brand) {
|
||||||
|
console.log(`[BIKEGEAR][${brand.name}] Fetching page 1...`);
|
||||||
|
let html;
|
||||||
|
try {
|
||||||
|
html = await fetchHtml(brand.url);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[BIKEGEAR][${brand.name}] Failed page 1: ${err.message}`);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxPage = extractMaxPage(html);
|
||||||
|
const urls = new Set(extractProductUrls(html));
|
||||||
|
|
||||||
|
if (maxPage > 1) {
|
||||||
|
console.log(`[BIKEGEAR][${brand.name}] Found ${maxPage} pages — fetching pages 2–${maxPage}...`);
|
||||||
|
const pageNums = Array.from({ length: maxPage - 1 }, (_, i) => i + 2);
|
||||||
|
|
||||||
|
await runConcurrent(pageNums, async (pageNum) => {
|
||||||
|
const pageUrl = `${brand.url}?page=${pageNum}`;
|
||||||
|
try {
|
||||||
|
const pageHtml = await fetchHtml(pageUrl);
|
||||||
|
for (const u of extractProductUrls(pageHtml)) urls.add(u);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`);
|
||||||
|
}
|
||||||
|
}, LISTING_CONCURRENCY);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = [...urls];
|
||||||
|
console.log(`[BIKEGEAR][${brand.name}] Collected ${result.length} product URLs across ${maxPage} page(s).`);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parse the product JSON-LD block that contains sku/offers fields. */
|
||||||
|
function parseProductJsonLd(html) {
|
||||||
|
const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
|
||||||
|
for (const b of blocks) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(b[1]);
|
||||||
|
// Use the block that has sku field (the more detailed one)
|
||||||
|
if (data["@type"] === "Product" && data.sku) {
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// malformed JSON-LD — skip
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */
|
||||||
|
function extractProductId(jsonLd) {
|
||||||
|
const imgUrl = jsonLd?.image || "";
|
||||||
|
const m = imgUrl.match(/\/catalog\/products\/(\d+)\//);
|
||||||
|
return m ? m[1] : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract all unique 800x800 gallery images for the current product from the page HTML.
|
||||||
|
* Falls back to 900x900 or 500x500 if 800x800 not found.
|
||||||
|
*/
|
||||||
|
function extractProductImages(html, productId, jsonLdImage) {
|
||||||
|
const seen = new Set();
|
||||||
|
const images = [];
|
||||||
|
|
||||||
|
if (productId) {
|
||||||
|
// Match data-src attributes belonging to this product's images
|
||||||
|
const re = new RegExp(
|
||||||
|
`data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`,
|
||||||
|
"g"
|
||||||
|
);
|
||||||
|
for (const m of html.matchAll(re)) {
|
||||||
|
const url = m[1];
|
||||||
|
// Prefer large sizes, skip tiny thumbnails
|
||||||
|
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
|
||||||
|
seen.add(url);
|
||||||
|
// Upgrade 500x500 → 800x800
|
||||||
|
images.push(url.replace("-500x500.", "-800x800."));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: use the JSON-LD image if we found nothing
|
||||||
|
if (!images.length && jsonLdImage) {
|
||||||
|
images.push(jsonLdImage);
|
||||||
|
}
|
||||||
|
|
||||||
|
return images;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract the feature/description table from the product-extra-description tab-pane section.
|
||||||
|
* Specifically targets the `tab-pane` variant (not the popup variant).
|
||||||
|
* Returns an HTML string of the table, or empty string if not found.
|
||||||
|
*/
|
||||||
|
function extractDescriptionHtml(html) {
|
||||||
|
// Match the tab-pane description div — stop at the next product-extra section
|
||||||
|
const sectionMatch = html.match(
|
||||||
|
/<div[^>]*product-extra-description[^>]*tab-pane[^>]*>([\s\S]*?)(?=<div[^>]*product-extra-(?:attributes|reviews)|<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>)/
|
||||||
|
);
|
||||||
|
if (!sectionMatch) return "";
|
||||||
|
|
||||||
|
const section = sectionMatch[1];
|
||||||
|
const tableMatch = section.match(/<table[^>]*>([\s\S]*?)<\/table>/);
|
||||||
|
if (!tableMatch) return "";
|
||||||
|
|
||||||
|
return `<table border="1">${tableMatch[1]}</table>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parse price from a string like "₹21,497.28" or just "21497.28" */
|
||||||
|
function parsePrice(str) {
|
||||||
|
if (!str) return 0;
|
||||||
|
const n = parseFloat(String(str).replace(/[^\d.]/g, ""));
|
||||||
|
return Number.isNaN(n) ? 0 : n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scrape a single product detail page and return structured data. */
|
||||||
|
async function scrapeProductDetail(productUrl, brandName) {
|
||||||
|
let html;
|
||||||
|
try {
|
||||||
|
html = await fetchHtml(productUrl);
|
||||||
|
} catch (err) {
|
||||||
|
return { url: productUrl, scrapeError: err.message };
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const jsonLd = parseProductJsonLd(html);
|
||||||
|
if (!jsonLd) {
|
||||||
|
return { url: productUrl, scrapeError: "No product JSON-LD found" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const productId = extractProductId(jsonLd);
|
||||||
|
const images = extractProductImages(html, productId, jsonLd.image);
|
||||||
|
const descriptionHtml = extractDescriptionHtml(html);
|
||||||
|
const price = parsePrice(jsonLd.offers?.price);
|
||||||
|
const currency = jsonLd.offers?.priceCurrency || "INR";
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: productUrl,
|
||||||
|
sku: jsonLd.sku || productUrl.split("/").pop(),
|
||||||
|
name: jsonLd.name || "",
|
||||||
|
brand: jsonLd.brand?.name || brandName,
|
||||||
|
price,
|
||||||
|
currency,
|
||||||
|
availability: jsonLd.offers?.availability?.replace("https://schema.org/", "") || "Unknown",
|
||||||
|
descriptionHtml,
|
||||||
|
images,
|
||||||
|
productId,
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return { url: productUrl, scrapeError: `Parse error: ${err.message}` };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Main entry point: scrape all configured brands and return flat product array. */
|
||||||
|
async function scrapeBikeGear() {
|
||||||
|
const allProducts = [];
|
||||||
|
let totalBrands = BRANDS.length;
|
||||||
|
let brandsDone = 0;
|
||||||
|
|
||||||
|
for (const brand of BRANDS) {
|
||||||
|
const productUrls = await scrapeBrandProductUrls(brand);
|
||||||
|
|
||||||
|
if (!productUrls.length) {
|
||||||
|
console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`);
|
||||||
|
brandsDone++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`);
|
||||||
|
|
||||||
|
let done = 0;
|
||||||
|
const products = await runConcurrent(productUrls, async (url, i) => {
|
||||||
|
const product = await scrapeProductDetail(url, brand.name);
|
||||||
|
done++;
|
||||||
|
if (product.scrapeError) {
|
||||||
|
console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);
|
||||||
|
} else {
|
||||||
|
console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`);
|
||||||
|
}
|
||||||
|
return product;
|
||||||
|
}, DETAIL_CONCURRENCY);
|
||||||
|
|
||||||
|
const ok = products.filter((p) => !p.scrapeError).length;
|
||||||
|
const fail = products.length - ok;
|
||||||
|
console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
|
||||||
|
|
||||||
|
allProducts.push(...products);
|
||||||
|
brandsDone++;
|
||||||
|
|
||||||
|
// Polite delay between brands
|
||||||
|
if (brandsDone < totalBrands) await sleep(1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
return allProducts;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { scrapeBikeGear };
|
||||||
@ -4,6 +4,7 @@ const motousher = require("./motousher");
|
|||||||
const dirtstreet = require("./dirtstreet");
|
const dirtstreet = require("./dirtstreet");
|
||||||
const retrorides = require("./retrorides");
|
const retrorides = require("./retrorides");
|
||||||
const yuasa = require("./yuasa");
|
const yuasa = require("./yuasa");
|
||||||
|
const bikegear = require("./bikegear");
|
||||||
|
|
||||||
const sources = {
|
const sources = {
|
||||||
[kyt.sourceKey]: kyt,
|
[kyt.sourceKey]: kyt,
|
||||||
@ -12,6 +13,7 @@ const sources = {
|
|||||||
[dirtstreet.sourceKey]: dirtstreet,
|
[dirtstreet.sourceKey]: dirtstreet,
|
||||||
[retrorides.sourceKey]: retrorides,
|
[retrorides.sourceKey]: retrorides,
|
||||||
[yuasa.sourceKey]: yuasa,
|
[yuasa.sourceKey]: yuasa,
|
||||||
|
[bikegear.sourceKey]: bikegear,
|
||||||
};
|
};
|
||||||
|
|
||||||
function normalizeSourceKey(sourceKey) {
|
function normalizeSourceKey(sourceKey) {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user