diff --git a/src/business-logic/import-pipeline/sources/yuasa/index.js b/src/business-logic/import-pipeline/sources/yuasa/index.js index c7bfdde..c63472c 100644 --- a/src/business-logic/import-pipeline/sources/yuasa/index.js +++ b/src/business-logic/import-pipeline/sources/yuasa/index.js @@ -28,12 +28,17 @@ async function fetchWebsiteData({ paths: runPaths }) { if (cacheMaxHours > 0) { try { const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); - if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) { + const products = existing?.products; + const successCount = Array.isArray(products) + ? products.filter((p) => !(p.scraped || p).scrapeError).length + : 0; + // Only use cache if it has at least one successful product (not an all-failed run) + if (existing?.generatedAt && Array.isArray(products) && successCount > 0) { const ageMs = Date.now() - new Date(existing.generatedAt).getTime(); const ageHours = ageMs / (1000 * 60 * 60); if (ageHours < cacheMaxHours) { console.log( - `[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${existing.products.length} batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.` + `[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${successCount} ok batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.` ); return { analysis: existing.analysis || {}, @@ -43,6 +48,8 @@ async function fetchWebsiteData({ paths: runPaths }) { }; } console.log(`[YUASA] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`); + } else if (Array.isArray(products) && products.length > 0 && successCount === 0) { + console.log(`[YUASA] Cached data has 0 successful products — ignoring cache and re-scraping.`); } } catch { // no cache yet @@ -52,11 +59,18 @@ async function fetchWebsiteData({ paths: runPaths }) { const now = new Date().toISOString(); const rawProducts = await scrapeYuasaBatteries(); - const successCount = rawProducts.filter((p) => !p.scrapeError).length; - const failCount = rawProducts.filter((p) => p.scrapeError).length; + const successfulProducts = rawProducts.filter((p) => !p.scrapeError); + const successCount = successfulProducts.length; + const failCount = rawProducts.length - successCount; - // Normalize into pipeline-compatible aggregated format - const allProducts = rawProducts.map(function (product) { + if (successCount === 0) { + throw new Error( + `[YUASA] All ${rawProducts.length} battery scrapes failed. First error: ${rawProducts[0]?.scrapeError || "unknown"}` + ); + } + + // Only include successfully scraped products in the pipeline + const allProducts = successfulProducts.map(function (product) { const imageSrcs = (product.images || []).map(function (img) { return typeof img === "string" ? img : img.src; }).filter(Boolean);