const fs = require("node:fs/promises"); const path = require("node:path"); const { BRANDS } = require("./brands"); const { scrapeBikeGear, closeBrowser } = require("./scraper"); const { convertBikeGearJsonToShopifyProducts } = require("./converter"); const sourceKey = "bikegear"; const label = "BikeGear India"; const dataDir = path.join("data", "sources", sourceKey); function paths() { return { aggregatedJson: path.join(dataDir, "01_products_aggregated.json"), historyJson: path.join(dataDir, "01_products_run_history.json"), downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"), watermarkState: path.join(dataDir, "03_watermark_state.json"), imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"), uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"), shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"), logsDir: path.join(dataDir, "99_run_logs"), }; } async function fetchWebsiteData({ paths: runPaths }) { // Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape. const cacheMaxHours = Number(process.env.BIKEGEAR_CACHE_HOURS ?? 24); const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson); if (cacheMaxHours > 0) { try { const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); const products = existing?.products; const successCount = Array.isArray(products) ? products.filter((p) => !(p.scraped || p).scrapeError).length : 0; if (existing?.generatedAt && Array.isArray(products) && successCount > 0) { const ageHours = (Date.now() - new Date(existing.generatedAt).getTime()) / (1000 * 60 * 60); if (ageHours < cacheMaxHours) { console.log(`[BIKEGEAR] Using cached data (${ageHours.toFixed(1)}h old) — ${successCount} ok products. Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.`); return { analysis: existing.analysis || {}, outputJsonPath: absAggregatedPath, historyPath: path.resolve(process.cwd(), runPaths.historyJson), cached: true, }; } console.log(`[BIKEGEAR] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`); } else if (Array.isArray(products) && products.length > 0 && successCount === 0) { console.log(`[BIKEGEAR] Cached data has 0 successful products — ignoring cache and re-scraping.`); } } catch { // no cache yet } } const now = new Date().toISOString(); let rawProducts; try { rawProducts = await scrapeBikeGear(); } finally { await closeBrowser(); } const successfulProducts = rawProducts.filter((p) => !p.scrapeError); const successCount = successfulProducts.length; const failCount = rawProducts.length - successCount; if (successCount === 0) { throw new Error( `[BIKEGEAR] All ${rawProducts.length} product scrapes failed. First error: ${rawProducts[0]?.scrapeError || "unknown"}` ); } const allProducts = successfulProducts.map((product) => ({ productId: product.sku || product.url, sourceKey, brand: product.brand || "BikeGear", bikeModel: "", productSummary: { id: product.sku, name: product.name, img: product.images || [], cost: { mrp: product.price || 0 }, }, scraped: product, })); const analysis = { timestamp: now, totalProducts: rawProducts.length, totalProductsUnique: allProducts.length, detailSuccess: successCount, detailFailed: failCount, }; const payload = { generatedAt: now, sourceKey, sourceLabel: label, analysis, products: allProducts, }; await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true }); await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8"); let history = []; const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson); try { history = JSON.parse(await fs.readFile(absHistoryPath, "utf8")); if (!Array.isArray(history)) history = []; } catch { history = []; } history.push(analysis); await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8"); console.log(`[BIKEGEAR] Saved ${allProducts.length} products (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`); return { analysis, outputJsonPath: absAggregatedPath, historyPath: absHistoryPath }; } function convertToShopifyProducts(input, options = {}) { return convertBikeGearJsonToShopifyProducts(input, { brand: options.brand || label, uploadedImageMap: options.uploadedImageMap, }); } module.exports = { sourceKey, label, brands: BRANDS.map((b) => b.name), defaultBrand: "BikeGear India", defaultImageBaseUrl: "", envImageBaseUrl: "BIKEGEAR_IMAGE_BASE_URL", paths, fetchWebsiteData, convertToShopifyProducts, };