From 95e100c9b46aa743f60a39568062db1b9aa59316 Mon Sep 17 00:00:00 2001 From: MOHAN Date: Sat, 27 Jun 2026 18:57:09 +0530 Subject: [PATCH] feat: add Yuasa Batteries as new import pipeline source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scraper.js: fetches product detail pages from yuasabatteries.com for a configurable battery ID list; auto-discovers and downloads the current MSRP xlsx from the Yuasa MSRP page to build a battery-type → price map using the xlsx package; extracts specs, images, and vehicle fitment from HTML tables - converter.js: converts scraped battery data to Shopify-ready format (title, SKU, price from MSRP, description from specs, images) - index.js: standard pipeline integration with 24h cache (YUASA_CACHE_HOURS=0 to bypass); battery ID list configurable via YUASA_BATTERY_IDS env var (comma-separated) - sources/index.js: registers yuasa as the 6th import source Co-Authored-By: Claude Sonnet 4.6 --- .../import-pipeline/sources/index.js | 2 + .../sources/yuasa/converter.js | 127 ++++++++++++ .../import-pipeline/sources/yuasa/index.js | 130 ++++++++++++ .../import-pipeline/sources/yuasa/scraper.js | 196 ++++++++++++++++++ 4 files changed, 455 insertions(+) create mode 100644 src/business-logic/import-pipeline/sources/yuasa/converter.js create mode 100644 src/business-logic/import-pipeline/sources/yuasa/index.js create mode 100644 src/business-logic/import-pipeline/sources/yuasa/scraper.js diff --git a/src/business-logic/import-pipeline/sources/index.js b/src/business-logic/import-pipeline/sources/index.js index 6654c25..54b676f 100644 --- a/src/business-logic/import-pipeline/sources/index.js +++ b/src/business-logic/import-pipeline/sources/index.js @@ -3,6 +3,7 @@ const brocksPerformance = require("./brocks-performance"); const motousher = require("./motousher"); const dirtstreet = require("./dirtstreet"); const retrorides = require("./retrorides"); +const yuasa = require("./yuasa"); const sources = { [kyt.sourceKey]: kyt, @@ -10,6 +11,7 @@ const sources = { [motousher.sourceKey]: motousher, [dirtstreet.sourceKey]: dirtstreet, [retrorides.sourceKey]: retrorides, + [yuasa.sourceKey]: yuasa, }; function normalizeSourceKey(sourceKey) { diff --git a/src/business-logic/import-pipeline/sources/yuasa/converter.js b/src/business-logic/import-pipeline/sources/yuasa/converter.js new file mode 100644 index 0000000..3f9c64f --- /dev/null +++ b/src/business-logic/import-pipeline/sources/yuasa/converter.js @@ -0,0 +1,127 @@ +const path = require("node:path"); + +function slugify(str) { + return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/, ""); +} + +function getUploadedImageUrl(imgPath, uploadedImageMap) { + if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null; + const bySourcePath = uploadedImageMap.bySourcePath || {}; + return bySourcePath[String(imgPath)]?.url || null; +} + +function getImageFileName(imagePath) { + try { + return path.basename(new URL(imagePath).pathname); + } catch { + return path.basename(String(imagePath || "").split(/[?#]/)[0]); + } +} + +function convertYuasaRecordToShopifyReady(record, options = {}) { + const uploadedImageMap = options.uploadedImageMap || null; + + const batteryId = record.id || ""; + const details = record.details || {}; + const batteryType = details["Battery Type"] || batteryId; + const batteryFamily = details["Battery Family"] || "Powersports Battery"; + const voltage = details["Voltage"] || ""; + const capacity = details["Capacity (10-HR)"] || ""; + const cca = details["C.C.A"] || ""; + const weight = details["Weight"] || ""; + const country = details["Country"] || ""; + const dimensions = details["Dimensions"] || ""; + const metricDimensions = details["Metric Dimensions"] || ""; + + const title = "Yuasa " + batteryType + " Battery"; + const sku = batteryType.toUpperCase(); + const handle = slugify("yuasa-" + batteryType); + const price = typeof record.msrp === "number" ? record.msrp : 0; + const quantity = 10; + + const imagePaths = (record.images || []).map(function (img) { + return typeof img === "string" ? img : img.src; + }).filter(Boolean); + + const files = imagePaths + .map(function (imgPath) { + return { + type: "Image", + url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath, + media_content: getImageFileName(imgPath), + source_path: imgPath, + }; + }) + .filter(function (f) { return f.url; }); + + const descParts = []; + if (batteryFamily) descParts.push("Battery Family: " + batteryFamily); + if (voltage) descParts.push("Voltage: " + voltage + "V"); + if (capacity) descParts.push("Capacity (10-HR): " + capacity + " AH"); + if (cca) descParts.push("Cold Cranking Amps: " + cca); + if (dimensions) descParts.push("Dimensions: " + dimensions); + if (metricDimensions) descParts.push("Metric Dimensions: " + metricDimensions); + if (weight) descParts.push("Weight: " + weight); + if (country) descParts.push("Country of Origin: " + country); + const descriptionHtml = descParts.map(function (p) { return "

" + p + "

"; }).join(""); + + const rawTags = [ + "Yuasa", + batteryType, + batteryFamily, + "Powersports Battery", + voltage ? voltage + "V" : null, + country ? "Made in " + country : null, + sku, + ]; + const tags = [...new Set(rawTags.filter(Boolean).map(function (t) { return String(t).trim(); }))]; + + const productId = handle; + + return { + id: productId, + source: { + productId, + sourceKey: "yuasa", + url: record.url || null, + image_paths: imagePaths, + }, + attributes: { + product_name: title, + brand: "Yuasa", + category: batteryFamily, + subcategory: "Powersports Battery", + part_number: sku, + mfr_part_number: sku, + price, + compare_price: null, + purchase_cost: null, + barcode: "", + price_group: null, + units_per_sku: null, + part_description: descriptionHtml, + descriptions: descriptionHtml ? [{ type: "Market Description", description: descriptionHtml }] : [], + files, + image_paths: imagePaths, + dimensions: [{ weight: 0 }], + total_quantity: quantity, + inventorydata: { inventory: { main: quantity } }, + options: [], + variants: [{ sku, price, compare_price: null, quantity, optionValues: [] }], + fitmentTags: { make: [], model: [], year: [], drive: [], baseModel: [] }, + tags, + handle, + source_url: record.url || null, + }, + }; +} + +function convertYuasaJsonToShopifyProducts(input, options = {}) { + const records = Array.isArray(input?.products) ? input.products : []; + return records.map(function (record) { + const data = record.scraped || record; + return convertYuasaRecordToShopifyReady(data, options); + }); +} + +module.exports = { convertYuasaRecordToShopifyReady, convertYuasaJsonToShopifyProducts }; diff --git a/src/business-logic/import-pipeline/sources/yuasa/index.js b/src/business-logic/import-pipeline/sources/yuasa/index.js new file mode 100644 index 0000000..c7bfdde --- /dev/null +++ b/src/business-logic/import-pipeline/sources/yuasa/index.js @@ -0,0 +1,130 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { scrapeYuasaBatteries } = require("./scraper"); +const { convertYuasaJsonToShopifyProducts } = require("./converter"); + +const sourceKey = "yuasa"; +const label = "Yuasa Batteries"; +const dataDir = path.join("data", "sources", sourceKey); + +function paths() { + return { + aggregatedJson: path.join(dataDir, "01_products_aggregated.json"), + historyJson: path.join(dataDir, "01_products_run_history.json"), + downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"), + watermarkState: path.join(dataDir, "03_watermark_state.json"), + imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"), + uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"), + shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"), + logsDir: path.join(dataDir, "99_run_logs"), + }; +} + +async function fetchWebsiteData({ paths: runPaths }) { + // Cache for YUASA_CACHE_HOURS (default 24h). Set YUASA_CACHE_HOURS=0 to force re-scrape. + const cacheMaxHours = Number(process.env.YUASA_CACHE_HOURS ?? 24); + const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson); + + if (cacheMaxHours > 0) { + try { + const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); + if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) { + const ageMs = Date.now() - new Date(existing.generatedAt).getTime(); + const ageHours = ageMs / (1000 * 60 * 60); + if (ageHours < cacheMaxHours) { + console.log( + `[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${existing.products.length} batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.` + ); + return { + analysis: existing.analysis || {}, + outputJsonPath: absAggregatedPath, + historyPath: path.resolve(process.cwd(), runPaths.historyJson), + cached: true, + }; + } + console.log(`[YUASA] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`); + } + } catch { + // no cache yet + } + } + + const now = new Date().toISOString(); + const rawProducts = await scrapeYuasaBatteries(); + + const successCount = rawProducts.filter((p) => !p.scrapeError).length; + const failCount = rawProducts.filter((p) => p.scrapeError).length; + + // Normalize into pipeline-compatible aggregated format + const allProducts = rawProducts.map(function (product) { + const imageSrcs = (product.images || []).map(function (img) { + return typeof img === "string" ? img : img.src; + }).filter(Boolean); + return { + productId: product.id, + sourceKey, + brand: "Yuasa", + bikeModel: "", + productSummary: { + id: product.id, + name: "Yuasa " + (product.details["Battery Type"] || product.id) + " Battery", + img: imageSrcs, + cost: { mrp: product.msrp || 0 }, + }, + scraped: product, + }; + }); + + const analysis = { + timestamp: now, + totalProducts: rawProducts.length, + totalProductsUnique: allProducts.length, + detailSuccess: successCount, + detailFailed: failCount, + }; + + const payload = { + generatedAt: now, + sourceKey, + sourceLabel: label, + analysis, + products: allProducts, + }; + + await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true }); + await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8"); + + let history = []; + const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson); + try { + const raw = await fs.readFile(absHistoryPath, "utf8"); + history = JSON.parse(raw); + if (!Array.isArray(history)) history = []; + } catch { + history = []; + } + history.push(analysis); + await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8"); + + console.log(`[YUASA] Saved ${allProducts.length} batteries (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`); + + return { analysis, outputJsonPath: absAggregatedPath, historyPath: absHistoryPath }; +} + +function convertToShopifyProducts(input, options = {}) { + return convertYuasaJsonToShopifyProducts(input, { + brand: options.brand || label, + uploadedImageMap: options.uploadedImageMap, + }); +} + +module.exports = { + sourceKey, + label, + defaultBrand: "Yuasa", + defaultImageBaseUrl: "", + envImageBaseUrl: "YUASA_IMAGE_BASE_URL", + paths, + fetchWebsiteData, + convertToShopifyProducts, +}; diff --git a/src/business-logic/import-pipeline/sources/yuasa/scraper.js b/src/business-logic/import-pipeline/sources/yuasa/scraper.js new file mode 100644 index 0000000..28e65f8 --- /dev/null +++ b/src/business-logic/import-pipeline/sources/yuasa/scraper.js @@ -0,0 +1,196 @@ +const XLSX = require("xlsx"); + +const BASE_URL = "https://www.yuasabatteries.com/battery/"; +const MSRP_PAGE_URL = "https://www.yuasabatteries.com/about-us/msrp/"; + +const FETCH_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "identity", + "Connection": "keep-alive", + "Cache-Control": "no-cache", +}; + +// Default battery ID list -- override with YUASA_BATTERY_IDS env var (comma-separated) +const DEFAULT_BATTERY_IDS = [ + "YTZ7S", "YTZ10S", "YTZ14S", + "YT7B-BS", "YT12A-BS", "YT12B-BS", + "YTX9-BS", "YTX12-BS", "YTX14-BS", "YTX14L-BS", "YTX16-BS", + "YTX20L-BS", "YTX20HL-BS", +]; + +function sleep(ms) { + return new Promise(function (resolve) { setTimeout(resolve, ms); }); +} + +async function fetchText(url) { + for (let attempt = 1; attempt <= 4; attempt++) { + try { + const response = await fetch(url, { headers: FETCH_HEADERS }); + if (!response.ok) throw new Error("HTTP " + response.status + " " + response.statusText); + return await response.text(); + } catch (err) { + if (attempt === 4) throw err; + await sleep(attempt * 2000); + } + } +} + +async function fetchBuffer(url) { + for (let attempt = 1; attempt <= 4; attempt++) { + try { + const response = await fetch(url, { headers: FETCH_HEADERS }); + if (!response.ok) throw new Error("HTTP " + response.status + " " + response.statusText); + return Buffer.from(await response.arrayBuffer()); + } catch (err) { + if (attempt === 4) throw err; + await sleep(attempt * 2000); + } + } +} + +async function fetchMsrpPriceMap() { + // Scrape MSRP page to find the current xlsx URL (filename changes with pricing dates) + const html = await fetchText(MSRP_PAGE_URL); + const xlsxMatch = html.match(/href=["']([^"']+\.xlsx[^"']*?)["']/i); + if (!xlsxMatch) throw new Error("Could not find MSRP xlsx link on " + MSRP_PAGE_URL); + const xlsxUrl = xlsxMatch[1]; + console.log("[YUASA] Downloading MSRP pricing: " + xlsxUrl); + const buf = await fetchBuffer(xlsxUrl); + const wb = XLSX.read(buf, { type: "buffer" }); + // US-CANADA sheet: col B (index 1) = Battery Type, col F (index 5) = MSRP + const ws = wb.Sheets["US-CANADA"]; + if (!ws) throw new Error("US-CANADA sheet not found in MSRP xlsx"); + const rows = XLSX.utils.sheet_to_json(ws, { header: 1, defval: null }); + const priceMap = {}; + for (const row of rows) { + const btype = row[1] ? String(row[1]).trim() : null; + const msrp = row[5]; + if (btype && typeof msrp === "number" && msrp > 0) { + priceMap[btype.toUpperCase()] = msrp; + } + } + console.log("[YUASA] Loaded MSRP prices for " + Object.keys(priceMap).length + " battery models"); + return priceMap; +} + +function extractImagesFromHtml(html) { + const imgs = []; + const seen = new Set(); + for (const m of html.matchAll(/]+>/gi)) { + const tag = m[0]; + const srcMatch = tag.match(/src=["']([^"']+)["']/i); + if (!srcMatch) continue; + const src = srcMatch[1]; + if (src.indexOf("/uploads/") === -1) continue; + if (seen.has(src)) continue; + seen.add(src); + const altMatch = tag.match(/alt=["']([^"']*)["']/i); + imgs.push({ src: src, alt: altMatch ? altMatch[1] : "" }); + } + return imgs; +} + +function extractTablesFromHtml(html) { + const tables = []; + for (const tm of html.matchAll(/]*>([\s\S]*?)<\/table>/gi)) { + const rows = []; + for (const rm of tm[1].matchAll(/]*>([\s\S]*?)<\/tr>/gi)) { + const cells = Array.from( + rm[1].matchAll(/]*>([\s\S]*?)<\/t[dh]>/gi) + ).map(function (cm) { + return cm[1].replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim(); + }); + if (cells.length > 0) rows.push(cells); + } + if (rows.length > 0) tables.push(rows); + } + return tables; +} + +async function scrapeBatteryPage(batteryId, priceMap) { + priceMap = priceMap || {}; + const url = BASE_URL + batteryId.toLowerCase(); + let html; + try { + html = await fetchText(url); + } catch (err) { + return { id: batteryId, url: url, scrapeError: err.message, images: [], details: {}, vehicles: [], msrp: null }; + } + + const images = extractImagesFromHtml(html); + const tables = extractTablesFromHtml(html); + + // First table = battery spec key/value pairs + const details = {}; + if (tables[0]) { + for (const row of tables[0]) { + if (row.length >= 2) { + const key = row[0].replace(/:$/, "").trim(); + details[key] = row[1].trim(); + } + } + } + + // Second table = vehicle fitment + const vehicles = []; + if (tables[1] && tables[1].length > 1) { + const headers = tables[1][0]; + for (const row of tables[1].slice(1)) { + const vehicle = {}; + headers.forEach(function (h, i) { vehicle[h] = row[i] || ""; }); + vehicles.push(vehicle); + } + } + + // Price lookup: use battery type from scraped details, fallback to the given ID + const btypeKey = (details["Battery Type"] || batteryId).trim().toUpperCase(); + const msrp = priceMap[btypeKey] || priceMap[batteryId.toUpperCase()] || null; + + return { id: batteryId, url: url, images: images, details: details, vehicles: vehicles, msrp: msrp, scrapeError: null }; +} + +async function scrapeYuasaBatteries(options) { + options = options || {}; + + // Determine which battery IDs to scrape + let batteryIds = DEFAULT_BATTERY_IDS; + const envIds = process.env.YUASA_BATTERY_IDS; + if (envIds && envIds.trim()) { + batteryIds = envIds.split(",").map(function (s) { return s.trim(); }).filter(Boolean); + } + if (options.batteryIds && Array.isArray(options.batteryIds)) { + batteryIds = options.batteryIds; + } + + // Fetch MSRP pricing (auto-discovers current xlsx URL from MSRP page each run) + let priceMap = {}; + try { + priceMap = await fetchMsrpPriceMap(); + } catch (err) { + console.log("[YUASA] WARNING: Could not load MSRP pricing: " + err.message + " -- prices will be 0"); + } + + console.log("[YUASA] Scraping " + batteryIds.length + " batteries from yuasabatteries.com..."); + const products = []; + + for (let i = 0; i < batteryIds.length; i++) { + const id = batteryIds[i]; + try { + const product = await scrapeBatteryPage(id, priceMap); + products.push(product); + const status = product.scrapeError ? "ERR: " + product.scrapeError : "ok"; + const priceStr = product.msrp ? " USD " + product.msrp : " (no price)"; + console.log("[YUASA] " + (i + 1) + "/" + batteryIds.length + " " + id + " -- " + status + priceStr); + } catch (err) { + console.log("[YUASA] FAIL " + id + ": " + err.message); + products.push({ id: id, url: BASE_URL + id.toLowerCase(), scrapeError: err.message, images: [], details: {}, vehicles: [], msrp: null }); + } + if (i < batteryIds.length - 1) await sleep(1000); + } + + return products; +} + +module.exports = { scrapeYuasaBatteries, scrapeBatteryPage, fetchMsrpPriceMap, DEFAULT_BATTERY_IDS };