admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: don't cache or process all-failed Yuasa scrape results

Browse Source 
- Cache is now skipped if every product in the cached file has scrapeError
  (prevents a 403-failed run from poisoning the next 24h of imports)
- If all scrapes fail, throw an error so the pipeline stops at step 1
  instead of silently creating 13 empty Shopify products with no images/price
- Only successfully scraped batteries are included in allProducts passed
  to the converter/download/upload/upsert stages

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-06-27 19:08:20 +05:30
parent 54b2ad381c
commit 9c70fcb86c
1 changed files with 20 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/business-logic/import-pipeline/sources/yuasa/index.js
View File

@ -28,12 +28,17 @@ async function fetchWebsiteData({ paths: runPaths }) {
if (cacheMaxHours > 0) {
try {
const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) {
const products = existing?.products;
const successCount = Array.isArray(products)
? products.filter((p) => !(p.scraped || p).scrapeError).length
: 0;
// Only use cache if it has at least one successful product (not an all-failed run)
if (existing?.generatedAt && Array.isArray(products) && successCount > 0) {
const ageMs = Date.now() - new Date(existing.generatedAt).getTime();
const ageHours = ageMs / (1000 * 60 * 60);
if (ageHours < cacheMaxHours) {
console.log(
`[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${existing.products.length} batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.`
`[YUASA] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) -- ${successCount} ok batteries. Set YUASA_CACHE_HOURS=0 to force re-scrape.`
);
return {
analysis: existing.analysis || {},
@ -43,6 +48,8 @@ async function fetchWebsiteData({ paths: runPaths }) {
};
}
console.log(`[YUASA] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`);
} else if (Array.isArray(products) && products.length > 0 && successCount === 0) {
console.log(`[YUASA] Cached data has 0 successful products — ignoring cache and re-scraping.`);
}
} catch {
// no cache yet
@ -52,11 +59,18 @@ async function fetchWebsiteData({ paths: runPaths }) {
const now = new Date().toISOString();
const rawProducts = await scrapeYuasaBatteries();
const successCount = rawProducts.filter((p) => !p.scrapeError).length;
const failCount = rawProducts.filter((p) => p.scrapeError).length;
const successfulProducts = rawProducts.filter((p) => !p.scrapeError);
const successCount = successfulProducts.length;
const failCount = rawProducts.length - successCount;
// Normalize into pipeline-compatible aggregated format
const allProducts = rawProducts.map(function (product) {
if (successCount === 0) {
throw new Error(
`[YUASA] All ${rawProducts.length} battery scrapes failed. First error: ${rawProducts[0]?.scrapeError || "unknown"}`
);
}
// Only include successfully scraped products in the pipeline
const allProducts = successfulProducts.map(function (product) {
const imageSrcs = (product.images || []).map(function (img) {
return typeof img === "string" ? img : img.src;
}).filter(Boolean);