From 9c70fcb86c3fd174d5fa0c0da74104f55e5c84e4 Mon Sep 17 00:00:00 2001
From: MOHAN <mohan326856@gmail.com>
Date: Sat, 27 Jun 2026 19:08:20 +0530
Subject: [PATCH] fix: don't cache or process all-failed Yuasa scrape results

- Cache is now skipped if every product in the cached file has scrapeError
  (prevents a 403-failed run from poisoning the next 24h of imports)
- If all scrapes fail, throw an error so the pipeline stops at step 1
  instead of silently creating 13 empty Shopify products with no images/price
- Only successfully scraped batteries are included in allProducts passed
  to the converter/download/upload/upsert stages

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../import-pipeline/sources/yuasa/index.js    | 26 ++++++++++++++-----
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/business-logic/import-pipeline/sources/yuasa/index.js b/src/business-logic/import-pipeline/sources/yuasa/index.js
index c7bfdde..c63472c 100644
--- a/src/business-logic/import-pipeline/sources/yuasa/index.js
+++ b/src/business-logic/import-pipeline/sources/yuasa/index.js
@@ -28,12 +28,17 @@ async function fetchWebsiteData({ paths: runPaths }) {
   if (cacheMaxHours > 0) {
     try {
       const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
-      if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) {
+      const products = existing?.products;
+      const successCount = Array.isArray(products)
+        ? products.filter((p) => !(p.scraped || p).scrapeError).length
+        : 0;
+      // Only use cache if it has at least one successful product (not an all-failed run)
+      if (existing?.generatedAt && Array.isArray(products) && successCount > 0) {
         const ageMs = Date.now() - new Date(existing.generatedAt).getTime();
         const ageHours = ageMs / (1000 * 60 * 60);
         if (ageHours < cacheMaxHours) {
           console.log(
-            `[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${existing.products.length} batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.`
+            `[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${successCount} ok batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.`
           );
           return {
             analysis: existing.analysis || {},
@@ -43,6 +48,8 @@ async function fetchWebsiteData({ paths: runPaths }) {
           };
         }
         console.log(`[YUASA] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`);
+      } else if (Array.isArray(products) && products.length > 0 && successCount === 0) {
+        console.log(`[YUASA] Cached data has 0 successful products — ignoring cache and re-scraping.`);
       }
     } catch {
       // no cache yet
@@ -52,11 +59,18 @@ async function fetchWebsiteData({ paths: runPaths }) {
   const now = new Date().toISOString();
   const rawProducts = await scrapeYuasaBatteries();
 
-  const successCount = rawProducts.filter((p) => !p.scrapeError).length;
-  const failCount = rawProducts.filter((p) => p.scrapeError).length;
+  const successfulProducts = rawProducts.filter((p) => !p.scrapeError);
+  const successCount = successfulProducts.length;
+  const failCount = rawProducts.length - successCount;
 
-  // Normalize into pipeline-compatible aggregated format
-  const allProducts = rawProducts.map(function (product) {
+  if (successCount === 0) {
+    throw new Error(
+      `[YUASA] All ${rawProducts.length} battery scrapes failed. First error: ${rawProducts[0]?.scrapeError || "unknown"}`
+    );
+  }
+
+  // Only include successfully scraped products in the pipeline
+  const allProducts = successfulProducts.map(function (product) {
     const imageSrcs = (product.images || []).map(function (img) {
       return typeof img === "string" ? img : img.src;
     }).filter(Boolean);