fix: don't cache or process all-failed Yuasa scrape results
- Cache is now skipped if every product in the cached file has scrapeError (prevents a 403-failed run from poisoning the next 24h of imports) - If all scrapes fail, throw an error so the pipeline stops at step 1 instead of silently creating 13 empty Shopify products with no images/price - Only successfully scraped batteries are included in allProducts passed to the converter/download/upload/upsert stages Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
54b2ad381c
commit
9c70fcb86c
@ -28,12 +28,17 @@ async function fetchWebsiteData({ paths: runPaths }) {
|
|||||||
if (cacheMaxHours > 0) {
|
if (cacheMaxHours > 0) {
|
||||||
try {
|
try {
|
||||||
const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
|
const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
|
||||||
if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) {
|
const products = existing?.products;
|
||||||
|
const successCount = Array.isArray(products)
|
||||||
|
? products.filter((p) => !(p.scraped || p).scrapeError).length
|
||||||
|
: 0;
|
||||||
|
// Only use cache if it has at least one successful product (not an all-failed run)
|
||||||
|
if (existing?.generatedAt && Array.isArray(products) && successCount > 0) {
|
||||||
const ageMs = Date.now() - new Date(existing.generatedAt).getTime();
|
const ageMs = Date.now() - new Date(existing.generatedAt).getTime();
|
||||||
const ageHours = ageMs / (1000 * 60 * 60);
|
const ageHours = ageMs / (1000 * 60 * 60);
|
||||||
if (ageHours < cacheMaxHours) {
|
if (ageHours < cacheMaxHours) {
|
||||||
console.log(
|
console.log(
|
||||||
`[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${existing.products.length} batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.`
|
`[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${successCount} ok batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.`
|
||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
analysis: existing.analysis || {},
|
analysis: existing.analysis || {},
|
||||||
@ -43,6 +48,8 @@ async function fetchWebsiteData({ paths: runPaths }) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
console.log(`[YUASA] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`);
|
console.log(`[YUASA] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`);
|
||||||
|
} else if (Array.isArray(products) && products.length > 0 && successCount === 0) {
|
||||||
|
console.log(`[YUASA] Cached data has 0 successful products — ignoring cache and re-scraping.`);
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// no cache yet
|
// no cache yet
|
||||||
@ -52,11 +59,18 @@ async function fetchWebsiteData({ paths: runPaths }) {
|
|||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const rawProducts = await scrapeYuasaBatteries();
|
const rawProducts = await scrapeYuasaBatteries();
|
||||||
|
|
||||||
const successCount = rawProducts.filter((p) => !p.scrapeError).length;
|
const successfulProducts = rawProducts.filter((p) => !p.scrapeError);
|
||||||
const failCount = rawProducts.filter((p) => p.scrapeError).length;
|
const successCount = successfulProducts.length;
|
||||||
|
const failCount = rawProducts.length - successCount;
|
||||||
|
|
||||||
// Normalize into pipeline-compatible aggregated format
|
if (successCount === 0) {
|
||||||
const allProducts = rawProducts.map(function (product) {
|
throw new Error(
|
||||||
|
`[YUASA] All ${rawProducts.length} battery scrapes failed. First error: ${rawProducts[0]?.scrapeError || "unknown"}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only include successfully scraped products in the pipeline
|
||||||
|
const allProducts = successfulProducts.map(function (product) {
|
||||||
const imageSrcs = (product.images || []).map(function (img) {
|
const imageSrcs = (product.images || []).map(function (img) {
|
||||||
return typeof img === "string" ? img : img.src;
|
return typeof img === "string" ? img : img.src;
|
||||||
}).filter(Boolean);
|
}).filter(Boolean);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user