From 6ba4dc7d83bcf161cab75d1f8f38ead263f5501c Mon Sep 17 00:00:00 2001
From: MOHAN <mohan326856@gmail.com>
Date: Sat, 8 Aug 2026 21:54:35 +0530
Subject: [PATCH] feat(bikegear): retry failed products up to 3 times after all
 brands complete

After the main scrape loop finishes, collects all products that had scrapeError
and retries them sequentially up to 3 times with a 2s pause between passes.
Updates the output in-place so recovered products count as successes.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 bikegear-local-scraper/bikegear-scraper.js | 37 +++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/bikegear-local-scraper/bikegear-scraper.js b/bikegear-local-scraper/bikegear-scraper.js
index 6231fcb..482ef78 100644
--- a/bikegear-local-scraper/bikegear-scraper.js
+++ b/bikegear-local-scraper/bikegear-scraper.js
@@ -486,7 +486,42 @@ async function main() {
   }
 
   if (!stopping) {
-    console.log("\nAll brands done!");
+    // ── Retry pass: re-attempt all failed products up to 3 times ──────────────
+    const MAX_RETRIES = 3;
+    let failedIndexes = progress.products
+      .map((p, i) => (p.scraped?.scrapeError ? i : -1))
+      .filter((i) => i !== -1);
+
+    if (failedIndexes.length > 0) {
+      console.log(`\nRetry pass — ${failedIndexes.length} failed products, up to ${MAX_RETRIES} attempts each...\n`);
+
+      for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+        if (!failedIndexes.length) break;
+        console.log(`  Retry attempt ${attempt}/${MAX_RETRIES} — ${failedIndexes.length} products...`);
+        await sleep(2000);
+
+        const stillFailed = [];
+        for (const idx of failedIndexes) {
+          const wrapped = progress.products[idx];
+          const url = wrapped.scraped?.url || wrapped.productId;
+          const brandName = wrapped.brand;
+          const result = await scrapeProductDetail(url, brandName);
+          if (result.scrapeError) {
+            console.log(`    ERR [${attempt}/${MAX_RETRIES}] ${url}: ${result.scrapeError}`);
+            stillFailed.push(idx);
+          } else {
+            console.log(`    OK  ${result.name}`);
+            progress.products[idx] = wrapProduct(result);
+          }
+        }
+        failedIndexes = stillFailed;
+      }
+
+      const remaining = failedIndexes.length;
+      const fixed = progress.products.filter((p) => !p.scraped?.scrapeError).length;
+      console.log(`\nRetry done — ${fixed} ok total, ${remaining} still failed after ${MAX_RETRIES} attempts.`);
+    }
+
     await writeOutput(progress.products);
     await fs.unlink(PROGRESS_FILE).catch(() => {});
     console.log("Progress file removed (clean finish).");