feat(bikegear): retry failed products up to 3 times after all brands complete
After the main scrape loop finishes, collects all products that had scrapeError and retries them sequentially up to 3 times with a 2s pause between passes. Updates the output in-place so recovered products count as successes. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
ead5d75309
commit
6ba4dc7d83
@ -486,7 +486,42 @@ async function main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!stopping) {
|
if (!stopping) {
|
||||||
console.log("\nAll brands done!");
|
// ── Retry pass: re-attempt all failed products up to 3 times ──────────────
|
||||||
|
const MAX_RETRIES = 3;
|
||||||
|
let failedIndexes = progress.products
|
||||||
|
.map((p, i) => (p.scraped?.scrapeError ? i : -1))
|
||||||
|
.filter((i) => i !== -1);
|
||||||
|
|
||||||
|
if (failedIndexes.length > 0) {
|
||||||
|
console.log(`\nRetry pass — ${failedIndexes.length} failed products, up to ${MAX_RETRIES} attempts each...\n`);
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||||
|
if (!failedIndexes.length) break;
|
||||||
|
console.log(` Retry attempt ${attempt}/${MAX_RETRIES} — ${failedIndexes.length} products...`);
|
||||||
|
await sleep(2000);
|
||||||
|
|
||||||
|
const stillFailed = [];
|
||||||
|
for (const idx of failedIndexes) {
|
||||||
|
const wrapped = progress.products[idx];
|
||||||
|
const url = wrapped.scraped?.url || wrapped.productId;
|
||||||
|
const brandName = wrapped.brand;
|
||||||
|
const result = await scrapeProductDetail(url, brandName);
|
||||||
|
if (result.scrapeError) {
|
||||||
|
console.log(` ERR [${attempt}/${MAX_RETRIES}] ${url}: ${result.scrapeError}`);
|
||||||
|
stillFailed.push(idx);
|
||||||
|
} else {
|
||||||
|
console.log(` OK ${result.name}`);
|
||||||
|
progress.products[idx] = wrapProduct(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
failedIndexes = stillFailed;
|
||||||
|
}
|
||||||
|
|
||||||
|
const remaining = failedIndexes.length;
|
||||||
|
const fixed = progress.products.filter((p) => !p.scraped?.scrapeError).length;
|
||||||
|
console.log(`\nRetry done — ${fixed} ok total, ${remaining} still failed after ${MAX_RETRIES} attempts.`);
|
||||||
|
}
|
||||||
|
|
||||||
await writeOutput(progress.products);
|
await writeOutput(progress.products);
|
||||||
await fs.unlink(PROGRESS_FILE).catch(() => {});
|
await fs.unlink(PROGRESS_FILE).catch(() => {});
|
||||||
console.log("Progress file removed (clean finish).");
|
console.log("Progress file removed (clean finish).");
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user