const fs = require("node:fs/promises"); const path = require("node:path"); const { BRANDS } = require("./brands"); const { scrapeMotousherBrand } = require("./scraper"); const { convertMotousherJsonToShopifyProducts } = require("./converter"); const sourceKey = "motousher"; const label = "Motousher"; const dataDir = path.join("data", "sources", sourceKey); function paths() { return { aggregatedJson: path.join(dataDir, "01_products_aggregated.json"), historyJson: path.join(dataDir, "01_products_run_history.json"), downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"), watermarkState: path.join(dataDir, "03_watermark_state.json"), imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"), uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"), shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"), logsDir: path.join(dataDir, "99_run_logs"), }; } async function fetchWebsiteData({ paths: runPaths }) { // Use cached aggregated JSON if it exists and is fresh (default 24h). // Set MOTOUSHER_CACHE_HOURS=0 to always force a full re-scrape. const cacheMaxHours = Number(process.env.MOTOUSHER_CACHE_HOURS ?? 24); const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson); if (cacheMaxHours > 0) { try { const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) { const ageMs = Date.now() - new Date(existing.generatedAt).getTime(); const ageHours = ageMs / (1000 * 60 * 60); if (ageHours < cacheMaxHours) { console.log( `[MOTOUSHER] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) — ${existing.products.length} products. Set MOTOUSHER_CACHE_HOURS=0 to force re-scrape.` ); return { analysis: existing.analysis || {}, outputJsonPath: absAggregatedPath, historyPath: path.resolve(process.cwd(), runPaths.historyJson), cached: true, }; } console.log(`[MOTOUSHER] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping all brands...`); } } catch { // no cache file yet, proceed with scraping } } const brandsToScrape = process.env.MOTOUSHER_BRANDS ? BRANDS.filter((b) => process.env.MOTOUSHER_BRANDS.split(",").map((s) => s.trim()).includes(b.slug)) : BRANDS; const allProducts = []; const runSummary = []; const now = new Date().toISOString(); for (const brand of brandsToScrape) { console.log(`[MOTOUSHER] Scraping brand: ${brand.name} (${brand.collectionUrl})`); const brandStartedAt = new Date().toISOString(); try { const products = await scrapeMotousherBrand(brand); // Normalize into pipeline-compatible aggregated format for (const product of products) { allProducts.push({ productId: product.handle || product.url, sourceKey, brand: product.brand, brandSlug: product.brandSlug, // productSummary.img is read by shared downloadImages + uploadImages utilities productSummary: { id: product.handle, name: product.title, img: product.images || [], cost: { mrp: product.price || 0 }, }, scraped: product, }); } runSummary.push({ brand: brand.name, slug: brand.slug, startedAt: brandStartedAt, completedAt: new Date().toISOString(), totalProducts: products.length, success: true, }); console.log(`[MOTOUSHER] ${brand.name}: ${products.length} products fetched`); } catch (err) { console.log(`[MOTOUSHER] ${brand.name} failed: ${err.message}`); runSummary.push({ brand: brand.name, slug: brand.slug, startedAt: brandStartedAt, completedAt: new Date().toISOString(), totalProducts: 0, success: false, error: err.message, }); } } const analysis = { timestamp: now, totalBrands: brandsToScrape.length, totalProductsUnique: allProducts.length, detailSuccess: allProducts.filter((p) => !p.scraped?.scrapeError).length, detailFailed: allProducts.filter((p) => p.scraped?.scrapeError).length, runSummary, }; const payload = { generatedAt: now, sourceKey, sourceLabel: label, analysis, products: allProducts, }; await fs.mkdir(path.dirname(path.resolve(process.cwd(), runPaths.aggregatedJson)), { recursive: true }); await fs.writeFile(path.resolve(process.cwd(), runPaths.aggregatedJson), JSON.stringify(payload, null, 2), "utf8"); // Append to history let history = []; try { const raw = await fs.readFile(path.resolve(process.cwd(), runPaths.historyJson), "utf8"); history = JSON.parse(raw); if (!Array.isArray(history)) history = []; } catch { history = []; } history.push(analysis); await fs.writeFile(path.resolve(process.cwd(), runPaths.historyJson), JSON.stringify(history, null, 2), "utf8"); console.log(`[MOTOUSHER] Saved ${allProducts.length} products to ${runPaths.aggregatedJson}`); return { analysis, outputJsonPath: path.resolve(process.cwd(), runPaths.aggregatedJson), historyPath: path.resolve(process.cwd(), runPaths.historyJson), }; } function convertToShopifyProducts(input, options = {}) { return convertMotousherJsonToShopifyProducts(input, { brand: options.brand || label, uploadedImageMap: options.uploadedImageMap, }); } module.exports = { sourceKey, label, brands: BRANDS.map((b) => b.name), defaultBrand: "Motousher", defaultImageBaseUrl: "", envImageBaseUrl: "MOTOUSHER_IMAGE_BASE_URL", paths, fetchWebsiteData, convertToShopifyProducts, };