Add Retro Rides source — sitemap scraper + converter + pipeline integration

- New source: retrorides (retrorides.co.in, 1168 products)
- Scraper reads all product URLs from WordPress sitemap in one call
- Extracts title/SKU/price/stock from JSON-LD, gallery images from data-large_image
- Bike brand/model from breadcrumb (case-insensitive BIKE_MAKES normalisation)
- Converts Jetpack CDN image URLs to direct retrorides.co.in URLs
- 24h cache with RETRORIDES_CACHE_HOURS env var, concurrency=3 default
- Converter follows same record.scraped pattern as motousher/dirtstreet
- Registered in sources/index.js alongside kyt, brocks-performance, motousher, dirtstreet

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/business-logic/import-pipeline/sources/index.js

@@ -2,12 +2,14 @@ const kyt = require("./kyt");
 const brocksPerformance = require("./brocks-performance");
 const motousher = require("./motousher");
 const dirtstreet = require("./dirtstreet");
+const retrorides = require("./retrorides");
 
 const sources = {
   [kyt.sourceKey]: kyt,
   [brocksPerformance.sourceKey]: brocksPerformance,
   [motousher.sourceKey]: motousher,
   [dirtstreet.sourceKey]: dirtstreet,
+  [retrorides.sourceKey]: retrorides,
 };
 
 function normalizeSourceKey(sourceKey) {

src/business-logic/import-pipeline/sources/retrorides/converter.js

@@ -0,0 +1,131 @@
+const path = require("node:path");
+
+function slugify(str) {
+  return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+
+function getUploadedImageUrl(imgPath, uploadedImageMap) {
+  if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null;
+  const bySourcePath = uploadedImageMap.bySourcePath || {};
+  return bySourcePath[String(imgPath)]?.url || null;
+}
+
+function getImageFileName(imagePath) {
+  try {
+    return path.basename(new URL(imagePath).pathname);
+  } catch {
+    return path.basename(String(imagePath || "").split(/[?#]/)[0]);
+  }
+}
+
+function convertRetroRidesRecordToShopifyReady(record, options = {}) {
+  const uploadedImageMap = options.uploadedImageMap || null;
+
+  const title = record.title || "Untitled Product";
+  const sku = record.sku || slugify(title);
+  const handle = record.handle || slugify(`${sku}-${title}`);
+  const price = Number(record.price ?? 0);
+  const quantity = Number(record.quantity ?? 0);
+  const descriptionHtml = record.description || "";
+
+  // Vendor: bike brand that the product is made for (BMW, Ducati, etc.)
+  // Falls back to store name if no bike brand found
+  const vendor = record.bikeBrand || options.brand || "Retro Rides";
+
+  // Product type derived from bike model (e.g., "S1000rr 2023+")
+  const productType = record.bikeModel
+    ? `${record.bikeBrand || ""} ${record.bikeModel}`.trim()
+    : "Motorcycle Accessories";
+
+  const imagePaths = Array.isArray(record.images) ? record.images : [];
+
+  const files = imagePaths
+    .map((imgPath) => ({
+      type: "Image",
+      url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath,
+      media_content: getImageFileName(imgPath),
+      source_path: imgPath,
+    }))
+    .filter((f) => f.url);
+
+  const variants = [
+    {
+      sku,
+      price,
+      compare_price: null,
+      quantity,
+      optionValues: [],
+    },
+  ];
+
+  // Tags: bike brand, bike model, bike categories, WooCommerce tags, sku
+  const rawTags = [
+    vendor,
+    record.bikeModel,
+    ...(Array.isArray(record.bikeCategories) ? record.bikeCategories : []),
+    ...(Array.isArray(record.tags) ? record.tags : []),
+    sku,
+  ];
+  const tags = [...new Set(rawTags.filter(Boolean).map((t) => String(t).trim()))];
+
+  const productId = handle || slugify(`retrorides-${sku}`);
+
+  return {
+    id: productId,
+    source: {
+      productId,
+      sourceKey: "retrorides",
+      url: record.url || null,
+      image_paths: imagePaths,
+    },
+    attributes: {
+      product_name: title,
+      brand: vendor,
+      category: productType,
+      subcategory: record.bikeModel || "",
+      part_number: sku,
+      mfr_part_number: sku,
+      price,
+      compare_price: null,
+      purchase_cost: null,
+      barcode: "",
+      price_group: null,
+      units_per_sku: null,
+      part_description: descriptionHtml,
+      descriptions: descriptionHtml
+        ? [{ type: "Market Description", description: descriptionHtml }]
+        : [],
+      files,
+      image_paths: imagePaths,
+      dimensions: [{ weight: 0 }],
+      total_quantity: quantity,
+      inventorydata: { inventory: { main: quantity } },
+      options: [],
+      variants,
+      fitmentTags: {
+        make: record.bikeBrand ? [record.bikeBrand] : [],
+        model: record.bikeModel ? [record.bikeModel] : [],
+        year: [],
+        drive: [],
+        baseModel: [],
+      },
+      tags,
+      handle: slugify(`${vendor}-${title}-${productId}`),
+      source_url: record.url || null,
+    },
+  };
+}
+
+function convertRetroRidesJsonToShopifyProducts(input, options = {}) {
+  const records = Array.isArray(input?.products) ? input.products : [];
+  return records.map((record) => {
+    // Aggregated JSON wraps scraped data inside record.scraped
+    const data = record.scraped || record;
+    return convertRetroRidesRecordToShopifyReady(data, {
+      ...options,
+      brand: data.bikeBrand || record.brand || options.brand,
+    });
+  });
+}
+
+module.exports = { convertRetroRidesRecordToShopifyReady, convertRetroRidesJsonToShopifyProducts };

src/business-logic/import-pipeline/sources/retrorides/index.js

@@ -0,0 +1,135 @@
+const fs = require("node:fs/promises");
+const path = require("node:path");
+const { scrapeRetroRidesProducts } = require("./scraper");
+const { convertRetroRidesJsonToShopifyProducts } = require("./converter");
+
+const sourceKey = "retrorides";
+const label = "Retro Rides";
+const dataDir = path.join("data", "sources", sourceKey);
+
+function paths() {
+  return {
+    aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
+    historyJson: path.join(dataDir, "01_products_run_history.json"),
+    downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"),
+    watermarkState: path.join(dataDir, "03_watermark_state.json"),
+    imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"),
+    uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"),
+    shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"),
+    logsDir: path.join(dataDir, "99_run_logs"),
+  };
+}
+
+async function fetchWebsiteData({ paths: runPaths }) {
+  // Use cached aggregated JSON if it exists and is fresh (default 24h).
+  // Set RETRORIDES_CACHE_HOURS=0 to always force a full re-scrape.
+  const cacheMaxHours = Number(process.env.RETRORIDES_CACHE_HOURS ?? 24);
+  const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson);
+
+  if (cacheMaxHours > 0) {
+    try {
+      const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
+      if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) {
+        const ageMs = Date.now() - new Date(existing.generatedAt).getTime();
+        const ageHours = ageMs / (1000 * 60 * 60);
+        if (ageHours < cacheMaxHours) {
+          console.log(
+            `[RETRORIDES] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) — ${existing.products.length} products. Set RETRORIDES_CACHE_HOURS=0 to force re-scrape.`
+          );
+          return {
+            analysis: existing.analysis || {},
+            outputJsonPath: absAggregatedPath,
+            historyPath: path.resolve(process.cwd(), runPaths.historyJson),
+            cached: true,
+          };
+        }
+        console.log(`[RETRORIDES] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping all products...`);
+      }
+    } catch {
+      // no cache file yet, proceed with scraping
+    }
+  }
+
+  const concurrency = Number(process.env.RETRORIDES_CONCURRENCY ?? 3);
+  const now = new Date().toISOString();
+
+  console.log(`[RETRORIDES] Scraping all products from retrorides.co.in (concurrency=${concurrency})...`);
+  const rawProducts = await scrapeRetroRidesProducts({ concurrency });
+
+  const successCount = rawProducts.filter((p) => !p.scrapeError).length;
+  const failCount = rawProducts.filter((p) => p.scrapeError).length;
+
+  // Normalize into pipeline-compatible aggregated format.
+  // productSummary.img is the field that shared download/upload utilities read.
+  const allProducts = rawProducts.map((product) => ({
+    productId: product.handle || product.url,
+    sourceKey,
+    brand: product.bikeBrand || "Retro Rides",
+    bikeModel: product.bikeModel || "",
+    productSummary: {
+      id: product.handle,
+      name: product.title,
+      img: product.images || [],
+      cost: { mrp: product.price || 0 },
+    },
+    scraped: product,
+  }));
+
+  const analysis = {
+    timestamp: now,
+    totalProducts: rawProducts.length,
+    totalProductsUnique: allProducts.length,
+    detailSuccess: successCount,
+    detailFailed: failCount,
+  };
+
+  const payload = {
+    generatedAt: now,
+    sourceKey,
+    sourceLabel: label,
+    analysis,
+    products: allProducts,
+  };
+
+  await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true });
+  await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8");
+
+  // Append to history
+  let history = [];
+  const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson);
+  try {
+    const raw = await fs.readFile(absHistoryPath, "utf8");
+    history = JSON.parse(raw);
+    if (!Array.isArray(history)) history = [];
+  } catch {
+    history = [];
+  }
+  history.push(analysis);
+  await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8");
+
+  console.log(`[RETRORIDES] Saved ${allProducts.length} products (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`);
+
+  return {
+    analysis,
+    outputJsonPath: absAggregatedPath,
+    historyPath: absHistoryPath,
+  };
+}
+
+function convertToShopifyProducts(input, options = {}) {
+  return convertRetroRidesJsonToShopifyProducts(input, {
+    brand: options.brand || label,
+    uploadedImageMap: options.uploadedImageMap,
+  });
+}
+
+module.exports = {
+  sourceKey,
+  label,
+  defaultBrand: "Retro Rides",
+  defaultImageBaseUrl: "",
+  envImageBaseUrl: "RETRORIDES_IMAGE_BASE_URL",
+  paths,
+  fetchWebsiteData,
+  convertToShopifyProducts,
+};

src/business-logic/import-pipeline/sources/retrorides/scraper.js

@@ -0,0 +1,207 @@
+const FETCH_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.9",
+  "Accept-Encoding": "identity",
+  "Connection": "keep-alive",
+  "Cache-Control": "no-cache",
+};
+
+const SITEMAP_URL = "https://retrorides.co.in/wp-sitemap-posts-product-1.xml";
+
+const BIKE_MAKES_LOWER = new Set(["bmw", "ducati", "aprilia", "honda", "kawasaki", "triumph", "suzuki", "yamaha", "ktm", "husqvarna"]);
+// Canonical display names for each bike make
+const BIKE_MAKE_CANONICAL = { bmw: "BMW", ducati: "Ducati", aprilia: "Aprilia", honda: "Honda", kawasaki: "Kawasaki", triumph: "Triumph", suzuki: "Suzuki", yamaha: "Yamaha", ktm: "KTM", husqvarna: "Husqvarna" };
+function normalizeBikeMake(name) {
+  const key = String(name || "").toLowerCase().trim();
+  return BIKE_MAKES_LOWER.has(key) ? BIKE_MAKE_CANONICAL[key] : "";
+}
+
+function sleep(ms) {
+  return new Promise(function(resolve) { setTimeout(resolve, ms); });
+}
+
+async function fetchText(url, maxAttempts) {
+  maxAttempts = maxAttempts || 4;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const response = await fetch(url, { headers: FETCH_HEADERS });
+      if (response.status === 429) {
+        const waitMs = attempt * 4000;
+        console.log("[RETRY] " + url + " HTTP 429, retrying in " + waitMs + "ms...");
+        await sleep(waitMs);
+        continue;
+      }
+      if (!response.ok) {
+        throw new Error("HTTP " + response.status + " " + response.statusText);
+      }
+      return await response.text();
+    } catch (err) {
+      if (attempt === maxAttempts) throw err;
+      await sleep(attempt * 1500);
+    }
+  }
+  throw new Error("fetchText: exhausted retries");
+}
+
+function decodeHtmlEntities(str) {
+  const ENT = {
+    "&amp;": "&", "&#038;": "&", "&lt;": "<", "&gt;": ">",
+    "&quot;": '"', "&#034;": '"', "&#039;": "'",
+    "&#8217;": "'", "&#8216;": "'", "&#8220;": '"', "&#8221;": '"',
+    "&nbsp;": " ", "&rsquo;": "'", "&lsquo;": "'", "&rdquo;": '"', "&ldquo;": '"',
+  };
+  let s = String(str || "");
+  for (let i = 0; i < 2; i++) {
+    s = s.replace(/&[#a-z0-9]+;/gi, function(e) { return ENT[e] !== undefined ? ENT[e] : e; });
+  }
+  return s.trim();
+}
+
+function stripHtml(html) {
+  return String(html || "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+}
+
+function slugify(str) {
+  return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+
+function extractJsonLd(html) {
+  const blocks = Array.from(html.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi));
+  for (const block of blocks) {
+    try {
+      const parsed = JSON.parse(block[1]);
+      const graph = Array.isArray(parsed && parsed["@graph"]) ? parsed["@graph"] : [parsed];
+      const product = graph.find(function(n) { return n && n["@type"] === "Product"; });
+      const breadcrumb = graph.find(function(n) { return n && n["@type"] === "BreadcrumbList"; });
+      if (product) return { product: product, breadcrumb: breadcrumb };
+    } catch (e) {}
+  }
+  return {};
+}
+
+function extractGalleryImages(html) {
+  const matches = Array.from(html.matchAll(/data-large_image=["']([^"']+)["']/g)).map(function(m) { return m[1]; });
+  const seen = new Set();
+  return matches
+    .map(function(raw) {
+      const decoded = decodeHtmlEntities(raw);
+      const cdnMatch = decoded.match(/^https?:\/\/i\d+\.wp\.com\/(.+?)(?:\?.*)?$/);
+      const url = cdnMatch ? "https://" + cdnMatch[1] : decoded.split("?")[0];
+      return url;
+    })
+    .filter(function(url) {
+      if (!url || url.indexOf("wp-content/uploads") === -1) return false;
+      if (seen.has(url)) return false;
+      seen.add(url);
+      return true;
+    });
+}
+
+function extractDescription(html) {
+  const panelMatch = html.match(/id=["']tab-description["'][^>]*>([\s\S]*?)<\/div>\s*(?:<\/div>|\s*<div\s+class=["']woocommerce-Tabs)/i);
+  if (panelMatch) {
+    const raw = panelMatch[1].replace(/<h2[^>]*>Description<\/h2>/gi, "").trim();
+    return decodeHtmlEntities(stripHtml(raw)).replace(/\s{2,}/g, " ").trim();
+  }
+  const shortMatch = html.match(/short-description["'][^>]*>([\s\S]*?)<\/div>/i);
+  if (shortMatch) {
+    const raw = decodeHtmlEntities(stripHtml(shortMatch[1])).trim();
+    if (raw && raw !== " ") return raw;
+  }
+  return "";
+}
+
+async function scrapeProductPage(url) {
+  const html = await fetchText(url);
+  const jsonLd = extractJsonLd(html);
+  const jsonLdProduct = jsonLd.product;
+  const breadcrumb = jsonLd.breadcrumb;
+
+  if (!jsonLdProduct) {
+    return { url: url, scrapeError: "No JSON-LD Product found on page" };
+  }
+
+  const title = decodeHtmlEntities(jsonLdProduct.name || "");
+  const sku = String(jsonLdProduct.sku || "").trim();
+
+  const offer = Array.isArray(jsonLdProduct.offers) ? jsonLdProduct.offers[0] : jsonLdProduct.offers;
+  const priceSpec = offer && offer.priceSpecification && offer.priceSpecification[0];
+  const price = Number((priceSpec && priceSpec.price) || (offer && offer.price) || 0);
+  const currency = (priceSpec && priceSpec.priceCurrency) || (offer && offer.priceCurrency) || "INR";
+  const availability = String((offer && offer.availability) || "");
+  const inStock = /InStock|BackOrder/i.test(availability);
+  const quantity = inStock ? 10 : 0;
+
+  const crumbs = ((breadcrumb && breadcrumb.itemListElement) || []).map(function(item) {
+    return decodeHtmlEntities((item && item.item && item.item.name) || (item && item.name) || "");
+  });
+
+  const rawBrand = crumbs[1] || "";
+  const rawModel = crumbs[2] || "";
+  const bikeBrand = normalizeBikeMake(rawBrand);
+  const bikeModel = bikeBrand ? rawModel : "";
+
+  const galleryImages = extractGalleryImages(html);
+  const jsonLdImage = jsonLdProduct.image ? String(jsonLdProduct.image) : "";
+  const images = galleryImages.length > 0 ? galleryImages : (jsonLdImage ? [jsonLdImage] : []);
+
+  const tags = Array.from(html.matchAll(/rel=["']tag["'][^>]*>([^<]+)<\/a>/g))
+    .map(function(m) { return m[1].trim(); })
+    .filter(function(t, i, arr) { return t && arr.indexOf(t) === i; });
+
+  const handleMatch = url.match(/\/product\/([^\/]+)\/?$/);
+  const handle = handleMatch ? handleMatch[1] : slugify(sku || title);
+  const description = extractDescription(html);
+
+  return {
+    url: url, handle: handle, title: title, sku: sku,
+    price: price, currency: currency, quantity: quantity,
+    availability: availability, inStock: inStock, images: images,
+    description: description, bikeBrand: bikeBrand, bikeModel: bikeModel,
+    bikeCategories: crumbs.slice(1, -1).filter(Boolean),
+    tags: tags, brand: bikeBrand || "Retro Rides", scrapeError: null,
+  };
+}
+
+async function getAllProductUrls() {
+  const xml = await fetchText(SITEMAP_URL);
+  return Array.from(xml.matchAll(/<loc>([^<]+)<\/loc>/g))
+    .map(function(m) { return m[1].trim(); })
+    .filter(function(u) { return u.indexOf("/product/") !== -1; });
+}
+
+async function mapWithConcurrency(items, concurrency, fn) {
+  const results = new Array(items.length);
+  let nextIndex = 0;
+  async function worker() {
+    while (true) {
+      const i = nextIndex++;
+      if (i >= items.length) return;
+      results[i] = await fn(items[i], i);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
+  return results;
+}
+
+async function scrapeRetroRidesProducts(options) {
+  const concurrency = (options && options.concurrency) || 3;
+  const urls = await getAllProductUrls();
+  console.log("[RETRORIDES] Found " + urls.length + " product URLs from sitemap");
+  const products = await mapWithConcurrency(urls, concurrency, async function(url, index) {
+    try {
+      const product = await scrapeProductPage(url);
+      if ((index + 1) % 25 === 0 || index === urls.length - 1) {
+        console.log("[RETRORIDES] " + (index + 1) + "/" + urls.length + " scraped");
+      }
+      return product;
+    } catch (err) {
+      console.log("[RETRORIDES] FAIL " + url + ": " + err.message);
+      return { url: url, scrapeError: err.message };
+    }
+  });
+  return products;
+}
+
+module.exports = { scrapeRetroRidesProducts: scrapeRetroRidesProducts, getAllProductUrls: getAllProductUrls, scrapeProductPage: scrapeProductPage };