diff --git a/bikegear-local-scraper/bikegear-scraper.js b/bikegear-local-scraper/bikegear-scraper.js index 6231fcb..482ef78 100644 --- a/bikegear-local-scraper/bikegear-scraper.js +++ b/bikegear-local-scraper/bikegear-scraper.js @@ -486,7 +486,42 @@ async function main() { } if (!stopping) { - console.log("\nAll brands done!"); + // ── Retry pass: re-attempt all failed products up to 3 times ────────────── + const MAX_RETRIES = 3; + let failedIndexes = progress.products + .map((p, i) => (p.scraped?.scrapeError ? i : -1)) + .filter((i) => i !== -1); + + if (failedIndexes.length > 0) { + console.log(`\nRetry pass — ${failedIndexes.length} failed products, up to ${MAX_RETRIES} attempts each...\n`); + + for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { + if (!failedIndexes.length) break; + console.log(` Retry attempt ${attempt}/${MAX_RETRIES} — ${failedIndexes.length} products...`); + await sleep(2000); + + const stillFailed = []; + for (const idx of failedIndexes) { + const wrapped = progress.products[idx]; + const url = wrapped.scraped?.url || wrapped.productId; + const brandName = wrapped.brand; + const result = await scrapeProductDetail(url, brandName); + if (result.scrapeError) { + console.log(` ERR [${attempt}/${MAX_RETRIES}] ${url}: ${result.scrapeError}`); + stillFailed.push(idx); + } else { + console.log(` OK ${result.name}`); + progress.products[idx] = wrapProduct(result); + } + } + failedIndexes = stillFailed; + } + + const remaining = failedIndexes.length; + const fixed = progress.products.filter((p) => !p.scraped?.scrapeError).length; + console.log(`\nRetry done — ${fixed} ok total, ${remaining} still failed after ${MAX_RETRIES} attempts.`); + } + await writeOutput(progress.products); await fs.unlink(PROGRESS_FILE).catch(() => {}); console.log("Progress file removed (clean finish).");