feat: add Yuasa Batteries as new import pipeline source

- scraper.js: fetches product detail pages from yuasabatteries.com for a configurable battery ID list; auto-discovers and downloads the current MSRP xlsx from the Yuasa MSRP page to build a battery-type → price map using the xlsx package; extracts specs, images, and vehicle fitment from HTML tables
- converter.js: converts scraped battery data to Shopify-ready format (title, SKU, price from MSRP, description from specs, images)
- index.js: standard pipeline integration with 24h cache (YUASA_CACHE_HOURS=0 to bypass); battery ID list configurable via YUASA_BATTERY_IDS env var (comma-separated)
- sources/index.js: registers yuasa as the 6th import source

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/business-logic/import-pipeline/sources/index.js

@@ -3,6 +3,7 @@ const brocksPerformance = require("./brocks-performance");
 const motousher = require("./motousher");
 const dirtstreet = require("./dirtstreet");
 const retrorides = require("./retrorides");
+const yuasa = require("./yuasa");
 
 const sources = {
   [kyt.sourceKey]: kyt,
@@ -10,6 +11,7 @@ const sources = {
   [motousher.sourceKey]: motousher,
   [dirtstreet.sourceKey]: dirtstreet,
   [retrorides.sourceKey]: retrorides,
+  [yuasa.sourceKey]: yuasa,
 };
 
 function normalizeSourceKey(sourceKey) {

src/business-logic/import-pipeline/sources/yuasa/converter.js

@@ -0,0 +1,127 @@
+const path = require("node:path");
+
+function slugify(str) {
+  return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/, "");
+}
+
+function getUploadedImageUrl(imgPath, uploadedImageMap) {
+  if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null;
+  const bySourcePath = uploadedImageMap.bySourcePath || {};
+  return bySourcePath[String(imgPath)]?.url || null;
+}
+
+function getImageFileName(imagePath) {
+  try {
+    return path.basename(new URL(imagePath).pathname);
+  } catch {
+    return path.basename(String(imagePath || "").split(/[?#]/)[0]);
+  }
+}
+
+function convertYuasaRecordToShopifyReady(record, options = {}) {
+  const uploadedImageMap = options.uploadedImageMap || null;
+
+  const batteryId = record.id || "";
+  const details = record.details || {};
+  const batteryType = details["Battery Type"] || batteryId;
+  const batteryFamily = details["Battery Family"] || "Powersports Battery";
+  const voltage = details["Voltage"] || "";
+  const capacity = details["Capacity (10-HR)"] || "";
+  const cca = details["C.C.A"] || "";
+  const weight = details["Weight"] || "";
+  const country = details["Country"] || "";
+  const dimensions = details["Dimensions"] || "";
+  const metricDimensions = details["Metric Dimensions"] || "";
+
+  const title = "Yuasa " + batteryType + " Battery";
+  const sku = batteryType.toUpperCase();
+  const handle = slugify("yuasa-" + batteryType);
+  const price = typeof record.msrp === "number" ? record.msrp : 0;
+  const quantity = 10;
+
+  const imagePaths = (record.images || []).map(function (img) {
+    return typeof img === "string" ? img : img.src;
+  }).filter(Boolean);
+
+  const files = imagePaths
+    .map(function (imgPath) {
+      return {
+        type: "Image",
+        url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath,
+        media_content: getImageFileName(imgPath),
+        source_path: imgPath,
+      };
+    })
+    .filter(function (f) { return f.url; });
+
+  const descParts = [];
+  if (batteryFamily) descParts.push("Battery Family: " + batteryFamily);
+  if (voltage) descParts.push("Voltage: " + voltage + "V");
+  if (capacity) descParts.push("Capacity (10-HR): " + capacity + " AH");
+  if (cca) descParts.push("Cold Cranking Amps: " + cca);
+  if (dimensions) descParts.push("Dimensions: " + dimensions);
+  if (metricDimensions) descParts.push("Metric Dimensions: " + metricDimensions);
+  if (weight) descParts.push("Weight: " + weight);
+  if (country) descParts.push("Country of Origin: " + country);
+  const descriptionHtml = descParts.map(function (p) { return "<p>" + p + "</p>"; }).join("");
+
+  const rawTags = [
+    "Yuasa",
+    batteryType,
+    batteryFamily,
+    "Powersports Battery",
+    voltage ? voltage + "V" : null,
+    country ? "Made in " + country : null,
+    sku,
+  ];
+  const tags = [...new Set(rawTags.filter(Boolean).map(function (t) { return String(t).trim(); }))];
+
+  const productId = handle;
+
+  return {
+    id: productId,
+    source: {
+      productId,
+      sourceKey: "yuasa",
+      url: record.url || null,
+      image_paths: imagePaths,
+    },
+    attributes: {
+      product_name: title,
+      brand: "Yuasa",
+      category: batteryFamily,
+      subcategory: "Powersports Battery",
+      part_number: sku,
+      mfr_part_number: sku,
+      price,
+      compare_price: null,
+      purchase_cost: null,
+      barcode: "",
+      price_group: null,
+      units_per_sku: null,
+      part_description: descriptionHtml,
+      descriptions: descriptionHtml ? [{ type: "Market Description", description: descriptionHtml }] : [],
+      files,
+      image_paths: imagePaths,
+      dimensions: [{ weight: 0 }],
+      total_quantity: quantity,
+      inventorydata: { inventory: { main: quantity } },
+      options: [],
+      variants: [{ sku, price, compare_price: null, quantity, optionValues: [] }],
+      fitmentTags: { make: [], model: [], year: [], drive: [], baseModel: [] },
+      tags,
+      handle,
+      source_url: record.url || null,
+    },
+  };
+}
+
+function convertYuasaJsonToShopifyProducts(input, options = {}) {
+  const records = Array.isArray(input?.products) ? input.products : [];
+  return records.map(function (record) {
+    const data = record.scraped || record;
+    return convertYuasaRecordToShopifyReady(data, options);
+  });
+}
+
+module.exports = { convertYuasaRecordToShopifyReady, convertYuasaJsonToShopifyProducts };

src/business-logic/import-pipeline/sources/yuasa/index.js

@@ -0,0 +1,130 @@
+const fs = require("node:fs/promises");
+const path = require("node:path");
+const { scrapeYuasaBatteries } = require("./scraper");
+const { convertYuasaJsonToShopifyProducts } = require("./converter");
+
+const sourceKey = "yuasa";
+const label = "Yuasa Batteries";
+const dataDir = path.join("data", "sources", sourceKey);
+
+function paths() {
+  return {
+    aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
+    historyJson: path.join(dataDir, "01_products_run_history.json"),
+    downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"),
+    watermarkState: path.join(dataDir, "03_watermark_state.json"),
+    imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"),
+    uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"),
+    shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"),
+    logsDir: path.join(dataDir, "99_run_logs"),
+  };
+}
+
+async function fetchWebsiteData({ paths: runPaths }) {
+  // Cache for YUASA_CACHE_HOURS (default 24h). Set YUASA_CACHE_HOURS=0 to force re-scrape.
+  const cacheMaxHours = Number(process.env.YUASA_CACHE_HOURS ?? 24);
+  const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson);
+
+  if (cacheMaxHours > 0) {
+    try {
+      const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
+      if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) {
+        const ageMs = Date.now() - new Date(existing.generatedAt).getTime();
+        const ageHours = ageMs / (1000 * 60 * 60);
+        if (ageHours < cacheMaxHours) {
+          console.log(
+            `[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${existing.products.length} batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.`
+          );
+          return {
+            analysis: existing.analysis || {},
+            outputJsonPath: absAggregatedPath,
+            historyPath: path.resolve(process.cwd(), runPaths.historyJson),
+            cached: true,
+          };
+        }
+        console.log(`[YUASA] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`);
+      }
+    } catch {
+      // no cache yet
+    }
+  }
+
+  const now = new Date().toISOString();
+  const rawProducts = await scrapeYuasaBatteries();
+
+  const successCount = rawProducts.filter((p) => !p.scrapeError).length;
+  const failCount = rawProducts.filter((p) => p.scrapeError).length;
+
+  // Normalize into pipeline-compatible aggregated format
+  const allProducts = rawProducts.map(function (product) {
+    const imageSrcs = (product.images || []).map(function (img) {
+      return typeof img === "string" ? img : img.src;
+    }).filter(Boolean);
+    return {
+      productId: product.id,
+      sourceKey,
+      brand: "Yuasa",
+      bikeModel: "",
+      productSummary: {
+        id: product.id,
+        name: "Yuasa " + (product.details["Battery Type"] || product.id) + " Battery",
+        img: imageSrcs,
+        cost: { mrp: product.msrp || 0 },
+      },
+      scraped: product,
+    };
+  });
+
+  const analysis = {
+    timestamp: now,
+    totalProducts: rawProducts.length,
+    totalProductsUnique: allProducts.length,
+    detailSuccess: successCount,
+    detailFailed: failCount,
+  };
+
+  const payload = {
+    generatedAt: now,
+    sourceKey,
+    sourceLabel: label,
+    analysis,
+    products: allProducts,
+  };
+
+  await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true });
+  await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8");
+
+  let history = [];
+  const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson);
+  try {
+    const raw = await fs.readFile(absHistoryPath, "utf8");
+    history = JSON.parse(raw);
+    if (!Array.isArray(history)) history = [];
+  } catch {
+    history = [];
+  }
+  history.push(analysis);
+  await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8");
+
+  console.log(`[YUASA] Saved ${allProducts.length} batteries (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`);
+
+  return { analysis, outputJsonPath: absAggregatedPath, historyPath: absHistoryPath };
+}
+
+function convertToShopifyProducts(input, options = {}) {
+  return convertYuasaJsonToShopifyProducts(input, {
+    brand: options.brand || label,
+    uploadedImageMap: options.uploadedImageMap,
+  });
+}
+
+module.exports = {
+  sourceKey,
+  label,
+  defaultBrand: "Yuasa",
+  defaultImageBaseUrl: "",
+  envImageBaseUrl: "YUASA_IMAGE_BASE_URL",
+  paths,
+  fetchWebsiteData,
+  convertToShopifyProducts,
+};

src/business-logic/import-pipeline/sources/yuasa/scraper.js

@@ -0,0 +1,196 @@
+const XLSX = require("xlsx");
+
+const BASE_URL = "https://www.yuasabatteries.com/battery/";
+const MSRP_PAGE_URL = "https://www.yuasabatteries.com/about-us/msrp/";
+
+const FETCH_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.9",
+  "Accept-Encoding": "identity",
+  "Connection": "keep-alive",
+  "Cache-Control": "no-cache",
+};
+
+// Default battery ID list -- override with YUASA_BATTERY_IDS env var (comma-separated)
+const DEFAULT_BATTERY_IDS = [
+  "YTZ7S", "YTZ10S", "YTZ14S",
+  "YT7B-BS", "YT12A-BS", "YT12B-BS",
+  "YTX9-BS", "YTX12-BS", "YTX14-BS", "YTX14L-BS", "YTX16-BS",
+  "YTX20L-BS", "YTX20HL-BS",
+];
+
+function sleep(ms) {
+  return new Promise(function (resolve) { setTimeout(resolve, ms); });
+}
+
+async function fetchText(url) {
+  for (let attempt = 1; attempt <= 4; attempt++) {
+    try {
+      const response = await fetch(url, { headers: FETCH_HEADERS });
+      if (!response.ok) throw new Error("HTTP " + response.status + " " + response.statusText);
+      return await response.text();
+    } catch (err) {
+      if (attempt === 4) throw err;
+      await sleep(attempt * 2000);
+    }
+  }
+}
+
+async function fetchBuffer(url) {
+  for (let attempt = 1; attempt <= 4; attempt++) {
+    try {
+      const response = await fetch(url, { headers: FETCH_HEADERS });
+      if (!response.ok) throw new Error("HTTP " + response.status + " " + response.statusText);
+      return Buffer.from(await response.arrayBuffer());
+    } catch (err) {
+      if (attempt === 4) throw err;
+      await sleep(attempt * 2000);
+    }
+  }
+}
+
+async function fetchMsrpPriceMap() {
+  // Scrape MSRP page to find the current xlsx URL (filename changes with pricing dates)
+  const html = await fetchText(MSRP_PAGE_URL);
+  const xlsxMatch = html.match(/href=["']([^"']+\.xlsx[^"']*?)["']/i);
+  if (!xlsxMatch) throw new Error("Could not find MSRP xlsx link on " + MSRP_PAGE_URL);
+  const xlsxUrl = xlsxMatch[1];
+  console.log("[YUASA] Downloading MSRP pricing: " + xlsxUrl);
+  const buf = await fetchBuffer(xlsxUrl);
+  const wb = XLSX.read(buf, { type: "buffer" });
+  // US-CANADA sheet: col B (index 1) = Battery Type, col F (index 5) = MSRP
+  const ws = wb.Sheets["US-CANADA"];
+  if (!ws) throw new Error("US-CANADA sheet not found in MSRP xlsx");
+  const rows = XLSX.utils.sheet_to_json(ws, { header: 1, defval: null });
+  const priceMap = {};
+  for (const row of rows) {
+    const btype = row[1] ? String(row[1]).trim() : null;
+    const msrp = row[5];
+    if (btype && typeof msrp === "number" && msrp > 0) {
+      priceMap[btype.toUpperCase()] = msrp;
+    }
+  }
+  console.log("[YUASA] Loaded MSRP prices for " + Object.keys(priceMap).length + " battery models");
+  return priceMap;
+}
+
+function extractImagesFromHtml(html) {
+  const imgs = [];
+  const seen = new Set();
+  for (const m of html.matchAll(/<img[^>]+>/gi)) {
+    const tag = m[0];
+    const srcMatch = tag.match(/src=["']([^"']+)["']/i);
+    if (!srcMatch) continue;
+    const src = srcMatch[1];
+    if (src.indexOf("/uploads/") === -1) continue;
+    if (seen.has(src)) continue;
+    seen.add(src);
+    const altMatch = tag.match(/alt=["']([^"']*)["']/i);
+    imgs.push({ src: src, alt: altMatch ? altMatch[1] : "" });
+  }
+  return imgs;
+}
+
+function extractTablesFromHtml(html) {
+  const tables = [];
+  for (const tm of html.matchAll(/<table[^>]*>([\s\S]*?)<\/table>/gi)) {
+    const rows = [];
+    for (const rm of tm[1].matchAll(/<tr[^>]*>([\s\S]*?)<\/tr>/gi)) {
+      const cells = Array.from(
+        rm[1].matchAll(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi)
+      ).map(function (cm) {
+        return cm[1].replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+      });
+      if (cells.length > 0) rows.push(cells);
+    }
+    if (rows.length > 0) tables.push(rows);
+  }
+  return tables;
+}
+
+async function scrapeBatteryPage(batteryId, priceMap) {
+  priceMap = priceMap || {};
+  const url = BASE_URL + batteryId.toLowerCase();
+  let html;
+  try {
+    html = await fetchText(url);
+  } catch (err) {
+    return { id: batteryId, url: url, scrapeError: err.message, images: [], details: {}, vehicles: [], msrp: null };
+  }
+
+  const images = extractImagesFromHtml(html);
+  const tables = extractTablesFromHtml(html);
+
+  // First table = battery spec key/value pairs
+  const details = {};
+  if (tables[0]) {
+    for (const row of tables[0]) {
+      if (row.length >= 2) {
+        const key = row[0].replace(/:$/, "").trim();
+        details[key] = row[1].trim();
+      }
+    }
+  }
+
+  // Second table = vehicle fitment
+  const vehicles = [];
+  if (tables[1] && tables[1].length > 1) {
+    const headers = tables[1][0];
+    for (const row of tables[1].slice(1)) {
+      const vehicle = {};
+      headers.forEach(function (h, i) { vehicle[h] = row[i] || ""; });
+      vehicles.push(vehicle);
+    }
+  }
+
+  // Price lookup: use battery type from scraped details, fallback to the given ID
+  const btypeKey = (details["Battery Type"] || batteryId).trim().toUpperCase();
+  const msrp = priceMap[btypeKey] || priceMap[batteryId.toUpperCase()] || null;
+
+  return { id: batteryId, url: url, images: images, details: details, vehicles: vehicles, msrp: msrp, scrapeError: null };
+}
+
+async function scrapeYuasaBatteries(options) {
+  options = options || {};
+
+  // Determine which battery IDs to scrape
+  let batteryIds = DEFAULT_BATTERY_IDS;
+  const envIds = process.env.YUASA_BATTERY_IDS;
+  if (envIds && envIds.trim()) {
+    batteryIds = envIds.split(",").map(function (s) { return s.trim(); }).filter(Boolean);
+  }
+  if (options.batteryIds && Array.isArray(options.batteryIds)) {
+    batteryIds = options.batteryIds;
+  }
+
+  // Fetch MSRP pricing (auto-discovers current xlsx URL from MSRP page each run)
+  let priceMap = {};
+  try {
+    priceMap = await fetchMsrpPriceMap();
+  } catch (err) {
+    console.log("[YUASA] WARNING: Could not load MSRP pricing: " + err.message + " -- prices will be 0");
+  }
+
+  console.log("[YUASA] Scraping " + batteryIds.length + " batteries from yuasabatteries.com...");
+  const products = [];
+
+  for (let i = 0; i < batteryIds.length; i++) {
+    const id = batteryIds[i];
+    try {
+      const product = await scrapeBatteryPage(id, priceMap);
+      products.push(product);
+      const status = product.scrapeError ? "ERR: " + product.scrapeError : "ok";
+      const priceStr = product.msrp ? " USD " + product.msrp : " (no price)";
+      console.log("[YUASA] " + (i + 1) + "/" + batteryIds.length + " " + id + " -- " + status + priceStr);
+    } catch (err) {
+      console.log("[YUASA] FAIL " + id + ": " + err.message);
+      products.push({ id: id, url: BASE_URL + id.toLowerCase(), scrapeError: err.message, images: [], details: {}, vehicles: [], msrp: null });
+    }
+    if (i < batteryIds.length - 1) await sleep(1000);
+  }
+
+  return products;
+}
+
+module.exports = { scrapeYuasaBatteries, scrapeBatteryPage, fetchMsrpPriceMap, DEFAULT_BATTERY_IDS };