admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(bikegear): retry failed products up to 3 times after all brands complete

Browse Source 
After the main scrape loop finishes, collects all products that had scrapeError
and retries them sequentially up to 3 times with a 2s pause between passes.
Updates the output in-place so recovered products count as successes.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-08-08 21:54:35 +05:30
parent ead5d75309
commit 6ba4dc7d83
1 changed files with 36 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
bikegear-local-scraper/bikegear-scraper.js
View File

@ -486,7 +486,42 @@ async function main() {
}
if (!stopping) {
console.log("\nAll brands done!");
// ── Retry pass: re-attempt all failed products up to 3 times ──────────────
const MAX_RETRIES = 3;
let failedIndexes = progress.products
.map((p, i) => (p.scraped?.scrapeError ? i : -1))
.filter((i) => i !== -1);
if (failedIndexes.length > 0) {
console.log(`\nRetry pass — ${failedIndexes.length} failed products, up to ${MAX_RETRIES} attempts each...\n`);
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
if (!failedIndexes.length) break;
console.log(` Retry attempt ${attempt}/${MAX_RETRIES} — ${failedIndexes.length} products...`);
await sleep(2000);
const stillFailed = [];
for (const idx of failedIndexes) {
const wrapped = progress.products[idx];
const url = wrapped.scraped?.url || wrapped.productId;
const brandName = wrapped.brand;
const result = await scrapeProductDetail(url, brandName);
if (result.scrapeError) {
console.log(` ERR [${attempt}/${MAX_RETRIES}] ${url}: ${result.scrapeError}`);
stillFailed.push(idx);
} else {
console.log(` OK ${result.name}`);
progress.products[idx] = wrapProduct(result);
}
}
failedIndexes = stillFailed;
}
const remaining = failedIndexes.length;
const fixed = progress.products.filter((p) => !p.scraped?.scrapeError).length;
console.log(`\nRetry done — ${fixed} ok total, ${remaining} still failed after ${MAX_RETRIES} attempts.`);
}
await writeOutput(progress.products);
await fs.unlink(PROGRESS_FILE).catch(() => {});
console.log("Progress file removed (clean finish).");