" + p + "
"; }).join(""); + + const rawTags = [ + "Yuasa", + batteryType, + batteryFamily, + "Powersports Battery", + voltage ? voltage + "V" : null, + country ? "Made in " + country : null, + sku, + ]; + const tags = [...new Set(rawTags.filter(Boolean).map(function (t) { return String(t).trim(); }))]; + + const productId = handle; + + return { + id: productId, + source: { + productId, + sourceKey: "yuasa", + url: record.url || null, + image_paths: imagePaths, + }, + attributes: { + product_name: title, + brand: "Yuasa", + category: batteryFamily, + subcategory: "Powersports Battery", + part_number: sku, + mfr_part_number: sku, + price, + compare_price: null, + purchase_cost: null, + barcode: "", + price_group: null, + units_per_sku: null, + part_description: descriptionHtml, + descriptions: descriptionHtml ? [{ type: "Market Description", description: descriptionHtml }] : [], + files, + image_paths: imagePaths, + dimensions: [{ weight: 0 }], + total_quantity: quantity, + inventorydata: { inventory: { main: quantity } }, + options: [], + variants: [{ sku, price, compare_price: null, quantity, optionValues: [] }], + fitmentTags: { make: [], model: [], year: [], drive: [], baseModel: [] }, + tags, + handle, + source_url: record.url || null, + }, + }; +} + +function convertYuasaJsonToShopifyProducts(input, options = {}) { + const records = Array.isArray(input?.products) ? input.products : []; + return records.map(function (record) { + const data = record.scraped || record; + return convertYuasaRecordToShopifyReady(data, options); + }); +} + +module.exports = { convertYuasaRecordToShopifyReady, convertYuasaJsonToShopifyProducts }; diff --git a/src/business-logic/import-pipeline/sources/yuasa/index.js b/src/business-logic/import-pipeline/sources/yuasa/index.js new file mode 100644 index 0000000..c7bfdde --- /dev/null +++ b/src/business-logic/import-pipeline/sources/yuasa/index.js @@ -0,0 +1,130 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { scrapeYuasaBatteries } = require("./scraper"); +const { convertYuasaJsonToShopifyProducts } = require("./converter"); + +const sourceKey = "yuasa"; +const label = "Yuasa Batteries"; +const dataDir = path.join("data", "sources", sourceKey); + +function paths() { + return { + aggregatedJson: path.join(dataDir, "01_products_aggregated.json"), + historyJson: path.join(dataDir, "01_products_run_history.json"), + downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"), + watermarkState: path.join(dataDir, "03_watermark_state.json"), + imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"), + uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"), + shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"), + logsDir: path.join(dataDir, "99_run_logs"), + }; +} + +async function fetchWebsiteData({ paths: runPaths }) { + // Cache for YUASA_CACHE_HOURS (default 24h). Set YUASA_CACHE_HOURS=0 to force re-scrape. + const cacheMaxHours = Number(process.env.YUASA_CACHE_HOURS ?? 24); + const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson); + + if (cacheMaxHours > 0) { + try { + const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); + if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) { + const ageMs = Date.now() - new Date(existing.generatedAt).getTime(); + const ageHours = ageMs / (1000 * 60 * 60); + if (ageHours < cacheMaxHours) { + console.log( + `[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${existing.products.length} batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.` + ); + return { + analysis: existing.analysis || {}, + outputJsonPath: absAggregatedPath, + historyPath: path.resolve(process.cwd(), runPaths.historyJson), + cached: true, + }; + } + console.log(`[YUASA] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`); + } + } catch { + // no cache yet + } + } + + const now = new Date().toISOString(); + const rawProducts = await scrapeYuasaBatteries(); + + const successCount = rawProducts.filter((p) => !p.scrapeError).length; + const failCount = rawProducts.filter((p) => p.scrapeError).length; + + // Normalize into pipeline-compatible aggregated format + const allProducts = rawProducts.map(function (product) { + const imageSrcs = (product.images || []).map(function (img) { + return typeof img === "string" ? img : img.src; + }).filter(Boolean); + return { + productId: product.id, + sourceKey, + brand: "Yuasa", + bikeModel: "", + productSummary: { + id: product.id, + name: "Yuasa " + (product.details["Battery Type"] || product.id) + " Battery", + img: imageSrcs, + cost: { mrp: product.msrp || 0 }, + }, + scraped: product, + }; + }); + + const analysis = { + timestamp: now, + totalProducts: rawProducts.length, + totalProductsUnique: allProducts.length, + detailSuccess: successCount, + detailFailed: failCount, + }; + + const payload = { + generatedAt: now, + sourceKey, + sourceLabel: label, + analysis, + products: allProducts, + }; + + await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true }); + await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8"); + + let history = []; + const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson); + try { + const raw = await fs.readFile(absHistoryPath, "utf8"); + history = JSON.parse(raw); + if (!Array.isArray(history)) history = []; + } catch { + history = []; + } + history.push(analysis); + await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8"); + + console.log(`[YUASA] Saved ${allProducts.length} batteries (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`); + + return { analysis, outputJsonPath: absAggregatedPath, historyPath: absHistoryPath }; +} + +function convertToShopifyProducts(input, options = {}) { + return convertYuasaJsonToShopifyProducts(input, { + brand: options.brand || label, + uploadedImageMap: options.uploadedImageMap, + }); +} + +module.exports = { + sourceKey, + label, + defaultBrand: "Yuasa", + defaultImageBaseUrl: "", + envImageBaseUrl: "YUASA_IMAGE_BASE_URL", + paths, + fetchWebsiteData, + convertToShopifyProducts, +}; diff --git a/src/business-logic/import-pipeline/sources/yuasa/scraper.js b/src/business-logic/import-pipeline/sources/yuasa/scraper.js new file mode 100644 index 0000000..28e65f8 --- /dev/null +++ b/src/business-logic/import-pipeline/sources/yuasa/scraper.js @@ -0,0 +1,196 @@ +const XLSX = require("xlsx"); + +const BASE_URL = "https://www.yuasabatteries.com/battery/"; +const MSRP_PAGE_URL = "https://www.yuasabatteries.com/about-us/msrp/"; + +const FETCH_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "identity", + "Connection": "keep-alive", + "Cache-Control": "no-cache", +}; + +// Default battery ID list -- override with YUASA_BATTERY_IDS env var (comma-separated) +const DEFAULT_BATTERY_IDS = [ + "YTZ7S", "YTZ10S", "YTZ14S", + "YT7B-BS", "YT12A-BS", "YT12B-BS", + "YTX9-BS", "YTX12-BS", "YTX14-BS", "YTX14L-BS", "YTX16-BS", + "YTX20L-BS", "YTX20HL-BS", +]; + +function sleep(ms) { + return new Promise(function (resolve) { setTimeout(resolve, ms); }); +} + +async function fetchText(url) { + for (let attempt = 1; attempt <= 4; attempt++) { + try { + const response = await fetch(url, { headers: FETCH_HEADERS }); + if (!response.ok) throw new Error("HTTP " + response.status + " " + response.statusText); + return await response.text(); + } catch (err) { + if (attempt === 4) throw err; + await sleep(attempt * 2000); + } + } +} + +async function fetchBuffer(url) { + for (let attempt = 1; attempt <= 4; attempt++) { + try { + const response = await fetch(url, { headers: FETCH_HEADERS }); + if (!response.ok) throw new Error("HTTP " + response.status + " " + response.statusText); + return Buffer.from(await response.arrayBuffer()); + } catch (err) { + if (attempt === 4) throw err; + await sleep(attempt * 2000); + } + } +} + +async function fetchMsrpPriceMap() { + // Scrape MSRP page to find the current xlsx URL (filename changes with pricing dates) + const html = await fetchText(MSRP_PAGE_URL); + const xlsxMatch = html.match(/href=["']([^"']+\.xlsx[^"']*?)["']/i); + if (!xlsxMatch) throw new Error("Could not find MSRP xlsx link on " + MSRP_PAGE_URL); + const xlsxUrl = xlsxMatch[1]; + console.log("[YUASA] Downloading MSRP pricing: " + xlsxUrl); + const buf = await fetchBuffer(xlsxUrl); + const wb = XLSX.read(buf, { type: "buffer" }); + // US-CANADA sheet: col B (index 1) = Battery Type, col F (index 5) = MSRP + const ws = wb.Sheets["US-CANADA"]; + if (!ws) throw new Error("US-CANADA sheet not found in MSRP xlsx"); + const rows = XLSX.utils.sheet_to_json(ws, { header: 1, defval: null }); + const priceMap = {}; + for (const row of rows) { + const btype = row[1] ? String(row[1]).trim() : null; + const msrp = row[5]; + if (btype && typeof msrp === "number" && msrp > 0) { + priceMap[btype.toUpperCase()] = msrp; + } + } + console.log("[YUASA] Loaded MSRP prices for " + Object.keys(priceMap).length + " battery models"); + return priceMap; +} + +function extractImagesFromHtml(html) { + const imgs = []; + const seen = new Set(); + for (const m of html.matchAll(/