diff --git a/src/business-logic/import-pipeline/sources/bikegear/brands.js b/src/business-logic/import-pipeline/sources/bikegear/brands.js new file mode 100644 index 0000000..592fe57 --- /dev/null +++ b/src/business-logic/import-pipeline/sources/bikegear/brands.js @@ -0,0 +1,56 @@ +/** + * BikeGear.in brand list. + * Source: https://bikegear.in/index.php?route=product/manufacturer + * Each brand has a `name` and `url` (the brand collection page on bikegear.in). + * Add / remove entries here to control which brands are scraped. + */ + +const BRANDS = [ + // added 2026-07-13 + { name: "Alpinestars", url: "https://bikegear.in/alpinestars" }, + + // Uncomment to add more brands: + // { name: "Acerbis", url: "https://bikegear.in/acerbis" }, + // { name: "Airoh Helmets", url: "https://bikegear.in/airoh-helmets" }, + // { name: "Akrapovic", url: "https://bikegear.in/akrapovic-exhaust-systems" }, + // { name: "Arai Helmet", url: "https://bikegear.in/arai-helmet-india" }, + // { name: "Arrow Exhausts", url: "https://bikegear.in/arrow-exhausts" }, + // { name: "Austin Racing", url: "https://bikegear.in/austin-racing" }, + // { name: "Bell", url: "https://bikegear.in/bell-helmets" }, + // { name: "Caberg", url: "https://bikegear.in/caberg-helmets" }, + // { name: "CNC Racing", url: "https://bikegear.in/cnc-racing" }, + // { name: "Ducabike", url: "https://bikegear.in/ducabike" }, + // { name: "Evotech Performance",url: "https://bikegear.in/evotech-performance-online-india" }, + // { name: "Forma Boots", url: "https://bikegear.in/forma-boots" }, + // { name: "Gaerne", url: "https://bikegear.in/gaerne" }, + // { name: "GB Racing", url: "https://bikegear.in/gb-racing" }, + // { name: "HJC Helmets", url: "https://bikegear.in/hjc-helmets" }, + // { name: "HP Corse", url: "https://bikegear.in/hp-corse-exhaust" }, + // { name: "IXS", url: "https://bikegear.in/ixs" }, + // { name: "Kabuto", url: "https://bikegear.in/kabuto" }, + // { name: "KLIM", url: "https://bikegear.in/klim" }, + // { name: "Leo Vince", url: "https://bikegear.in/leo-vince-motorcycle-exhausts-system" }, + // { name: "LS2 Helmets", url: "https://bikegear.in/ls2-helmet" }, + // { name: "Mivv Exhausts", url: "https://bikegear.in/mivv-exhausts" }, + // { name: "Nexx Helmets", url: "https://bikegear.in/nexx-helmets" }, + // { name: "Nolan", url: "https://bikegear.in/nolan-helmets" }, + // { name: "Ohlins Racing", url: "https://bikegear.in/ohlins-racing" }, + // { name: "R&G Racing", url: "https://bikegear.in/r-and-g-racing" }, + // { name: "REV'IT", url: "https://bikegear.in/revit" }, + // { name: "Rizoma", url: "https://bikegear.in/rizoma" }, + // { name: "Rukka", url: "https://bikegear.in/rukka-motorcycle-clothing" }, + // { name: "SC-Project", url: "https://bikegear.in/sc-project" }, + // { name: "Schuberth Helmets", url: "https://bikegear.in/schuberth-helmets" }, + // { name: "Scorpion EXO", url: "https://bikegear.in/scorpion-exo-motorcycle-gears" }, + // { name: "Sena", url: "https://bikegear.in/sena-motorcycle-intercom" }, + // { name: "Shark Helmets", url: "https://bikegear.in/shark-helmets" }, + // { name: "Shoei Helmets", url: "https://bikegear.in/shoei-helmets" }, + // { name: "Sidi", url: "https://bikegear.in/sidi-boots" }, + // { name: "Spidi", url: "https://bikegear.in/spidi" }, + // { name: "TCX Boots", url: "https://bikegear.in/tcx-motorcycle-boots" }, + // { name: "Termignoni", url: "https://bikegear.in/termignoni-exhausts" }, + // { name: "Touratech", url: "https://bikegear.in/touratech-accessories" }, + // { name: "Yoshimura", url: "https://bikegear.in/yoshimura" }, +]; + +module.exports = { BRANDS }; diff --git a/src/business-logic/import-pipeline/sources/bikegear/converter.js b/src/business-logic/import-pipeline/sources/bikegear/converter.js new file mode 100644 index 0000000..c95ce07 --- /dev/null +++ b/src/business-logic/import-pipeline/sources/bikegear/converter.js @@ -0,0 +1,91 @@ +const path = require("node:path"); + +function slugify(str) { + return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, ""); +} + +function getUploadedImageUrl(imgPath, uploadedImageMap) { + if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null; + const bySourcePath = uploadedImageMap.bySourcePath || {}; + return bySourcePath[String(imgPath)]?.url || null; +} + +function getImageFileName(imagePath) { + try { return path.basename(new URL(imagePath).pathname); } catch { return path.basename(String(imagePath || "").split(/[?#]/)[0]); } +} + +function convertBikeGearRecordToShopifyReady(record, options = {}) { + const brand = record.brand || options.brand || "BikeGear"; + const uploadedImageMap = options.uploadedImageMap || null; + + const sku = record.sku || slugify(record.name || "unknown"); + const productId = sku; + const title = record.name || "Untitled Product"; + const price = Number(record.price ?? 0); + const currency = record.currency || "INR"; + const descriptionHtml = record.descriptionHtml || ""; + const imagePaths = Array.isArray(record.images) ? record.images : []; + + const files = imagePaths + .map((imgPath) => ({ + type: "Image", + url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath, + media_content: getImageFileName(imgPath), + source_path: imgPath, + })) + .filter((f) => f.url); + + const quantity = 0; + const tags = [brand, sku, `currency:${currency}`, "bikegear-in"].filter(Boolean).map((t) => String(t).trim()); + + return { + id: productId, + source: { + productId, + sourceKey: "bikegear", + url: record.url || null, + image_paths: imagePaths, + currency, + }, + attributes: { + product_name: title, + brand, + category: "Motorcycle Gear", + subcategory: "", + part_number: sku, + mfr_part_number: sku, + price, + compare_price: null, + purchase_cost: null, + barcode: "", + price_group: null, + units_per_sku: null, + part_description: descriptionHtml, + descriptions: descriptionHtml ? [{ type: "Market Description", description: descriptionHtml }] : [], + files, + image_paths: imagePaths, + dimensions: [{ weight: 0 }], + total_quantity: quantity, + inventorydata: { inventory: { main: quantity } }, + options: [], + variants: [{ sku, price, compare_price: null, quantity, optionValues: [] }], + fitmentTags: { make: [], model: [], year: [], drive: [], baseModel: [] }, + tags, + handle: slugify(`${brand}-${title}-${productId}`), + source_url: record.url || null, + }, + }; +} + +function convertBikeGearJsonToShopifyProducts(input, options = {}) { + const records = Array.isArray(input?.products) ? input.products : []; + return records.map((record) => { + const data = record.scraped || record; + return convertBikeGearRecordToShopifyReady(data, { + ...options, + brand: data.brand || record.brand || options.brand, + }); + }); +} + +module.exports = { convertBikeGearRecordToShopifyReady, convertBikeGearJsonToShopifyProducts }; diff --git a/src/business-logic/import-pipeline/sources/bikegear/index.js b/src/business-logic/import-pipeline/sources/bikegear/index.js new file mode 100644 index 0000000..a9f0daf --- /dev/null +++ b/src/business-logic/import-pipeline/sources/bikegear/index.js @@ -0,0 +1,135 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { BRANDS } = require("./brands"); +const { scrapeBikeGear } = require("./scraper"); +const { convertBikeGearJsonToShopifyProducts } = require("./converter"); + +const sourceKey = "bikegear"; +const label = "BikeGear India"; +const dataDir = path.join("data", "sources", sourceKey); + +function paths() { + return { + aggregatedJson: path.join(dataDir, "01_products_aggregated.json"), + historyJson: path.join(dataDir, "01_products_run_history.json"), + downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"), + watermarkState: path.join(dataDir, "03_watermark_state.json"), + imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"), + uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"), + shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"), + logsDir: path.join(dataDir, "99_run_logs"), + }; +} + +async function fetchWebsiteData({ paths: runPaths }) { + // Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape. + const cacheMaxHours = Number(process.env.BIKEGEAR_CACHE_HOURS ?? 24); + const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson); + + if (cacheMaxHours > 0) { + try { + const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); + const products = existing?.products; + const successCount = Array.isArray(products) + ? products.filter((p) => !(p.scraped || p).scrapeError).length + : 0; + if (existing?.generatedAt && Array.isArray(products) && successCount > 0) { + const ageHours = (Date.now() - new Date(existing.generatedAt).getTime()) / (1000 * 60 * 60); + if (ageHours < cacheMaxHours) { + console.log(`[BIKEGEAR] Using cached data (${ageHours.toFixed(1)}h old) — ${successCount} ok products. Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.`); + return { + analysis: existing.analysis || {}, + outputJsonPath: absAggregatedPath, + historyPath: path.resolve(process.cwd(), runPaths.historyJson), + cached: true, + }; + } + console.log(`[BIKEGEAR] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`); + } else if (Array.isArray(products) && products.length > 0 && successCount === 0) { + console.log(`[BIKEGEAR] Cached data has 0 successful products — ignoring cache and re-scraping.`); + } + } catch { + // no cache yet + } + } + + const now = new Date().toISOString(); + const rawProducts = await scrapeBikeGear(); + + const successfulProducts = rawProducts.filter((p) => !p.scrapeError); + const successCount = successfulProducts.length; + const failCount = rawProducts.length - successCount; + + if (successCount === 0) { + throw new Error( + `[BIKEGEAR] All ${rawProducts.length} product scrapes failed. First error: ${rawProducts[0]?.scrapeError || "unknown"}` + ); + } + + const allProducts = successfulProducts.map((product) => ({ + productId: product.sku || product.url, + sourceKey, + brand: product.brand || "BikeGear", + bikeModel: "", + productSummary: { + id: product.sku, + name: product.name, + img: product.images || [], + cost: { mrp: product.price || 0 }, + }, + scraped: product, + })); + + const analysis = { + timestamp: now, + totalProducts: rawProducts.length, + totalProductsUnique: allProducts.length, + detailSuccess: successCount, + detailFailed: failCount, + }; + + const payload = { + generatedAt: now, + sourceKey, + sourceLabel: label, + analysis, + products: allProducts, + }; + + await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true }); + await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8"); + + let history = []; + const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson); + try { + history = JSON.parse(await fs.readFile(absHistoryPath, "utf8")); + if (!Array.isArray(history)) history = []; + } catch { + history = []; + } + history.push(analysis); + await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8"); + + console.log(`[BIKEGEAR] Saved ${allProducts.length} products (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`); + + return { analysis, outputJsonPath: absAggregatedPath, historyPath: absHistoryPath }; +} + +function convertToShopifyProducts(input, options = {}) { + return convertBikeGearJsonToShopifyProducts(input, { + brand: options.brand || label, + uploadedImageMap: options.uploadedImageMap, + }); +} + +module.exports = { + sourceKey, + label, + brands: BRANDS.map((b) => b.name), + defaultBrand: "BikeGear India", + defaultImageBaseUrl: "", + envImageBaseUrl: "BIKEGEAR_IMAGE_BASE_URL", + paths, + fetchWebsiteData, + convertToShopifyProducts, +}; diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js new file mode 100644 index 0000000..a70579d --- /dev/null +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -0,0 +1,294 @@ +/** + * BikeGear.in scraper (OpenCart / Journal3 theme). + * 2-phase: listing pages → product URLs, then detail pages → full product data. + * + * Prices are in INR (Indian Rupees) as scraped from the site. + */ + +const { BRANDS } = require("./brands"); + +const BASE_URL = "https://bikegear.in"; +const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? 3); +const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? 2); + +const FETCH_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', +}; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function fetchHtml(url, attempt = 1) { + try { + const res = await fetch(url, { headers: FETCH_HEADERS }); + + if (res.status === 429 || res.status >= 500) { + if (attempt <= 4) { + const wait = attempt * 2000; + console.log(`[BIKEGEAR] RETRY ${url} HTTP ${res.status} — waiting ${wait}ms (${attempt}/4)`); + await sleep(wait); + return fetchHtml(url, attempt + 1); + } + throw new Error(`HTTP ${res.status} after retries`); + } + + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return res.text(); + } catch (err) { + if (attempt <= 4 && (err.code === "UND_ERR_CONNECT_TIMEOUT" || err.code === "ECONNRESET" || err.message?.includes("fetch failed"))) { + await sleep(attempt * 2000); + return fetchHtml(url, attempt + 1); + } + throw err; + } +} + +/** Run up to `limit` tasks from `items` at a time. */ +async function runConcurrent(items, fn, limit) { + const results = []; + let idx = 0; + + async function worker() { + while (idx < items.length) { + const i = idx++; + results[i] = await fn(items[i], i); + } + } + + const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker()); + await Promise.all(workers); + return results; +} + +/** Extract the max page number from a listing page's HTML. */ +function extractMaxPage(html) { + const pageNums = [...html.matchAll(/\?page=(\d+)/g)].map((m) => parseInt(m[1], 10)); + return pageNums.length ? Math.max(...pageNums) : 1; +} + +/** Extract product detail page URLs from a brand listing page HTML. */ +function extractProductUrls(html) { + const urls = new Set(); + // Product links have class="product-img" — these are the card image anchors + for (const m of html.matchAll(/href="(https:\/\/bikegear\.in\/[^"]+)"\s+class="product-img/g)) { + const url = m[1]; + // Exclude manufacturer/category/route pages — product slugs are plain paths + if (!url.includes("route=") && !url.includes("manufacturer") && !url.includes("category")) { + urls.add(url); + } + } + return [...urls]; +} + +/** Scrape all listing pages for a brand and collect product detail URLs. */ +async function scrapeBrandProductUrls(brand) { + console.log(`[BIKEGEAR][${brand.name}] Fetching page 1...`); + let html; + try { + html = await fetchHtml(brand.url); + } catch (err) { + console.warn(`[BIKEGEAR][${brand.name}] Failed page 1: ${err.message}`); + return []; + } + + const maxPage = extractMaxPage(html); + const urls = new Set(extractProductUrls(html)); + + if (maxPage > 1) { + console.log(`[BIKEGEAR][${brand.name}] Found ${maxPage} pages — fetching pages 2–${maxPage}...`); + const pageNums = Array.from({ length: maxPage - 1 }, (_, i) => i + 2); + + await runConcurrent(pageNums, async (pageNum) => { + const pageUrl = `${brand.url}?page=${pageNum}`; + try { + const pageHtml = await fetchHtml(pageUrl); + for (const u of extractProductUrls(pageHtml)) urls.add(u); + } catch (err) { + console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`); + } + }, LISTING_CONCURRENCY); + } + + const result = [...urls]; + console.log(`[BIKEGEAR][${brand.name}] Collected ${result.length} product URLs across ${maxPage} page(s).`); + return result; +} + +/** Parse the product JSON-LD block that contains sku/offers fields. */ +function parseProductJsonLd(html) { + const blocks = [...html.matchAll(/([\s\S]*?)<\/script>/g)]; + for (const b of blocks) { + try { + const data = JSON.parse(b[1]); + // Use the block that has sku field (the more detailed one) + if (data["@type"] === "Product" && data.sku) { + return data; + } + } catch { + // malformed JSON-LD — skip + } + } + return null; +} + +/** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */ +function extractProductId(jsonLd) { + const imgUrl = jsonLd?.image || ""; + const m = imgUrl.match(/\/catalog\/products\/(\d+)\//); + return m ? m[1] : null; +} + +/** + * Extract all unique 800x800 gallery images for the current product from the page HTML. + * Falls back to 900x900 or 500x500 if 800x800 not found. + */ +function extractProductImages(html, productId, jsonLdImage) { + const seen = new Set(); + const images = []; + + if (productId) { + // Match data-src attributes belonging to this product's images + const re = new RegExp( + `data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`, + "g" + ); + for (const m of html.matchAll(re)) { + const url = m[1]; + // Prefer large sizes, skip tiny thumbnails + if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) { + seen.add(url); + // Upgrade 500x500 → 800x800 + images.push(url.replace("-500x500.", "-800x800.")); + } + } + } + + // Fallback: use the JSON-LD image if we found nothing + if (!images.length && jsonLdImage) { + images.push(jsonLdImage); + } + + return images; +} + +/** + * Extract the feature/description table from the product-extra-description tab-pane section. + * Specifically targets the `tab-pane` variant (not the popup variant). + * Returns an HTML string of the table, or empty string if not found. + */ +function extractDescriptionHtml(html) { + // Match the tab-pane description div — stop at the next product-extra section + const sectionMatch = html.match( + /]*product-extra-description[^>]*tab-pane[^>]*>([\s\S]*?)(?=]*product-extra-(?:attributes|reviews)|<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>)/ + ); + if (!sectionMatch) return ""; + + const section = sectionMatch[1]; + const tableMatch = section.match(/]*>([\s\S]*?)<\/table>/); + if (!tableMatch) return ""; + + return `${tableMatch[1]}
`; +} + +/** Parse price from a string like "₹21,497.28" or just "21497.28" */ +function parsePrice(str) { + if (!str) return 0; + const n = parseFloat(String(str).replace(/[^\d.]/g, "")); + return Number.isNaN(n) ? 0 : n; +} + +/** Scrape a single product detail page and return structured data. */ +async function scrapeProductDetail(productUrl, brandName) { + let html; + try { + html = await fetchHtml(productUrl); + } catch (err) { + return { url: productUrl, scrapeError: err.message }; + } + + try { + const jsonLd = parseProductJsonLd(html); + if (!jsonLd) { + return { url: productUrl, scrapeError: "No product JSON-LD found" }; + } + + const productId = extractProductId(jsonLd); + const images = extractProductImages(html, productId, jsonLd.image); + const descriptionHtml = extractDescriptionHtml(html); + const price = parsePrice(jsonLd.offers?.price); + const currency = jsonLd.offers?.priceCurrency || "INR"; + + return { + url: productUrl, + sku: jsonLd.sku || productUrl.split("/").pop(), + name: jsonLd.name || "", + brand: jsonLd.brand?.name || brandName, + price, + currency, + availability: jsonLd.offers?.availability?.replace("https://schema.org/", "") || "Unknown", + descriptionHtml, + images, + productId, + }; + } catch (err) { + return { url: productUrl, scrapeError: `Parse error: ${err.message}` }; + } +} + +/** Main entry point: scrape all configured brands and return flat product array. */ +async function scrapeBikeGear() { + const allProducts = []; + let totalBrands = BRANDS.length; + let brandsDone = 0; + + for (const brand of BRANDS) { + const productUrls = await scrapeBrandProductUrls(brand); + + if (!productUrls.length) { + console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`); + brandsDone++; + continue; + } + + console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`); + + let done = 0; + const products = await runConcurrent(productUrls, async (url, i) => { + const product = await scrapeProductDetail(url, brand.name); + done++; + if (product.scrapeError) { + console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`); + } else { + console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`); + } + return product; + }, DETAIL_CONCURRENCY); + + const ok = products.filter((p) => !p.scrapeError).length; + const fail = products.length - ok; + console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`); + + allProducts.push(...products); + brandsDone++; + + // Polite delay between brands + if (brandsDone < totalBrands) await sleep(1000); + } + + return allProducts; +} + +module.exports = { scrapeBikeGear }; diff --git a/src/business-logic/import-pipeline/sources/index.js b/src/business-logic/import-pipeline/sources/index.js index cfe0cb7..817097a 100644 --- a/src/business-logic/import-pipeline/sources/index.js +++ b/src/business-logic/import-pipeline/sources/index.js @@ -4,6 +4,7 @@ const motousher = require("./motousher"); const dirtstreet = require("./dirtstreet"); const retrorides = require("./retrorides"); const yuasa = require("./yuasa"); +const bikegear = require("./bikegear"); const sources = { [kyt.sourceKey]: kyt, @@ -12,6 +13,7 @@ const sources = { [dirtstreet.sourceKey]: dirtstreet, [retrorides.sourceKey]: retrorides, [yuasa.sourceKey]: yuasa, + [bikegear.sourceKey]: bikegear, }; function normalizeSourceKey(sourceKey) {