Race-Nation-Shopify-Backend/src/business-logic/import-pipeline/sources/motousher/index.js

const fs = require("node:fs/promises");
const path = require("node:path");
const { BRANDS } = require("./brands");
const { scrapeMotousherBrand } = require("./scraper");
const { convertMotousherJsonToShopifyProducts } = require("./converter");

const sourceKey = "motousher";
const label = "Motousher";
const dataDir = path.join("data", "sources", sourceKey);

function paths() {
  return {
    aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
    historyJson: path.join(dataDir, "01_products_run_history.json"),
    downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"),
    watermarkState: path.join(dataDir, "03_watermark_state.json"),
    imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"),
    uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"),
    shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"),
    logsDir: path.join(dataDir, "99_run_logs"),
  };
}

async function fetchWebsiteData({ paths: runPaths }) {
  // Use cached aggregated JSON if it exists and is fresh (default 24h).
  // Set MOTOUSHER_CACHE_HOURS=0 to always force a full re-scrape.
  const cacheMaxHours = Number(process.env.MOTOUSHER_CACHE_HOURS ?? 24);
  const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson);

  if (cacheMaxHours > 0) {
    try {
      const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
      if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) {
        const ageMs = Date.now() - new Date(existing.generatedAt).getTime();
        const ageHours = ageMs / (1000 * 60 * 60);
        if (ageHours < cacheMaxHours) {
          console.log(
            `[MOTOUSHER] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) — ${existing.products.length} products. Set MOTOUSHER_CACHE_HOURS=0 to force re-scrape.`
          );
          return {
            analysis: existing.analysis || {},
            outputJsonPath: absAggregatedPath,
            historyPath: path.resolve(process.cwd(), runPaths.historyJson),
            cached: true,
          };
        }
        console.log(`[MOTOUSHER] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping all brands...`);
      }
    } catch {
      // no cache file yet, proceed with scraping
    }
  }

  const brandsToScrape = process.env.MOTOUSHER_BRANDS
    ? BRANDS.filter((b) => process.env.MOTOUSHER_BRANDS.split(",").map((s) => s.trim()).includes(b.slug))
    : BRANDS;

  const allProducts = [];
  const runSummary = [];
  const now = new Date().toISOString();

  for (const brand of brandsToScrape) {
    console.log(`[MOTOUSHER] Scraping brand: ${brand.name} (${brand.collectionUrl})`);
    const brandStartedAt = new Date().toISOString();

    try {
      const products = await scrapeMotousherBrand(brand);

      // Normalize into pipeline-compatible aggregated format
      for (const product of products) {
        allProducts.push({
          productId: product.handle || product.url,
          sourceKey,
          brand: product.brand,
          brandSlug: product.brandSlug,
          // productSummary.img is read by shared downloadImages + uploadImages utilities
          productSummary: {
            id: product.handle,
            name: product.title,
            img: product.images || [],
            cost: { mrp: product.price || 0 },
          },
          scraped: product,
        });
      }

      runSummary.push({
        brand: brand.name,
        slug: brand.slug,
        startedAt: brandStartedAt,
        completedAt: new Date().toISOString(),
        totalProducts: products.length,
        success: true,
      });

      console.log(`[MOTOUSHER] ${brand.name}: ${products.length} products fetched`);
    } catch (err) {
      console.log(`[MOTOUSHER] ${brand.name} failed: ${err.message}`);
      runSummary.push({
        brand: brand.name,
        slug: brand.slug,
        startedAt: brandStartedAt,
        completedAt: new Date().toISOString(),
        totalProducts: 0,
        success: false,
        error: err.message,
      });
    }
  }

  const analysis = {
    timestamp: now,
    totalBrands: brandsToScrape.length,
    totalProductsUnique: allProducts.length,
    detailSuccess: allProducts.filter((p) => !p.scraped?.scrapeError).length,
    detailFailed: allProducts.filter((p) => p.scraped?.scrapeError).length,
    runSummary,
  };

  const payload = {
    generatedAt: now,
    sourceKey,
    sourceLabel: label,
    analysis,
    products: allProducts,
  };

  await fs.mkdir(path.dirname(path.resolve(process.cwd(), runPaths.aggregatedJson)), { recursive: true });
  await fs.writeFile(path.resolve(process.cwd(), runPaths.aggregatedJson), JSON.stringify(payload, null, 2), "utf8");

  // Append to history
  let history = [];
  try {
    const raw = await fs.readFile(path.resolve(process.cwd(), runPaths.historyJson), "utf8");
    history = JSON.parse(raw);
    if (!Array.isArray(history)) history = [];
  } catch { history = []; }
  history.push(analysis);
  await fs.writeFile(path.resolve(process.cwd(), runPaths.historyJson), JSON.stringify(history, null, 2), "utf8");

  console.log(`[MOTOUSHER] Saved ${allProducts.length} products to ${runPaths.aggregatedJson}`);

  return {
    analysis,
    outputJsonPath: path.resolve(process.cwd(), runPaths.aggregatedJson),
    historyPath: path.resolve(process.cwd(), runPaths.historyJson),
  };
}

function convertToShopifyProducts(input, options = {}) {
  return convertMotousherJsonToShopifyProducts(input, {
    brand: options.brand || label,
    uploadedImageMap: options.uploadedImageMap,
  });
}

module.exports = {
  sourceKey,
  label,
  brands: BRANDS.map((b) => b.name),
  defaultBrand: "Motousher",
  defaultImageBaseUrl: "",
  envImageBaseUrl: "MOTOUSHER_IMAGE_BASE_URL",
  paths,
  fetchWebsiteData,
  convertToShopifyProducts,
};