diff --git a/src/business-logic/import-pipeline/sources/index.js b/src/business-logic/import-pipeline/sources/index.js index b78f398..6654c25 100644 --- a/src/business-logic/import-pipeline/sources/index.js +++ b/src/business-logic/import-pipeline/sources/index.js @@ -2,12 +2,14 @@ const kyt = require("./kyt"); const brocksPerformance = require("./brocks-performance"); const motousher = require("./motousher"); const dirtstreet = require("./dirtstreet"); +const retrorides = require("./retrorides"); const sources = { [kyt.sourceKey]: kyt, [brocksPerformance.sourceKey]: brocksPerformance, [motousher.sourceKey]: motousher, [dirtstreet.sourceKey]: dirtstreet, + [retrorides.sourceKey]: retrorides, }; function normalizeSourceKey(sourceKey) { diff --git a/src/business-logic/import-pipeline/sources/retrorides/converter.js b/src/business-logic/import-pipeline/sources/retrorides/converter.js new file mode 100644 index 0000000..5b906d1 --- /dev/null +++ b/src/business-logic/import-pipeline/sources/retrorides/converter.js @@ -0,0 +1,131 @@ +const path = require("node:path"); + +function slugify(str) { + return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, ""); +} + +function getUploadedImageUrl(imgPath, uploadedImageMap) { + if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null; + const bySourcePath = uploadedImageMap.bySourcePath || {}; + return bySourcePath[String(imgPath)]?.url || null; +} + +function getImageFileName(imagePath) { + try { + return path.basename(new URL(imagePath).pathname); + } catch { + return path.basename(String(imagePath || "").split(/[?#]/)[0]); + } +} + +function convertRetroRidesRecordToShopifyReady(record, options = {}) { + const uploadedImageMap = options.uploadedImageMap || null; + + const title = record.title || "Untitled Product"; + const sku = record.sku || slugify(title); + const handle = record.handle || slugify(`${sku}-${title}`); + const price = Number(record.price ?? 0); + const quantity = Number(record.quantity ?? 0); + const descriptionHtml = record.description || ""; + + // Vendor: bike brand that the product is made for (BMW, Ducati, etc.) + // Falls back to store name if no bike brand found + const vendor = record.bikeBrand || options.brand || "Retro Rides"; + + // Product type derived from bike model (e.g., "S1000rr 2023+") + const productType = record.bikeModel + ? `${record.bikeBrand || ""} ${record.bikeModel}`.trim() + : "Motorcycle Accessories"; + + const imagePaths = Array.isArray(record.images) ? record.images : []; + + const files = imagePaths + .map((imgPath) => ({ + type: "Image", + url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath, + media_content: getImageFileName(imgPath), + source_path: imgPath, + })) + .filter((f) => f.url); + + const variants = [ + { + sku, + price, + compare_price: null, + quantity, + optionValues: [], + }, + ]; + + // Tags: bike brand, bike model, bike categories, WooCommerce tags, sku + const rawTags = [ + vendor, + record.bikeModel, + ...(Array.isArray(record.bikeCategories) ? record.bikeCategories : []), + ...(Array.isArray(record.tags) ? record.tags : []), + sku, + ]; + const tags = [...new Set(rawTags.filter(Boolean).map((t) => String(t).trim()))]; + + const productId = handle || slugify(`retrorides-${sku}`); + + return { + id: productId, + source: { + productId, + sourceKey: "retrorides", + url: record.url || null, + image_paths: imagePaths, + }, + attributes: { + product_name: title, + brand: vendor, + category: productType, + subcategory: record.bikeModel || "", + part_number: sku, + mfr_part_number: sku, + price, + compare_price: null, + purchase_cost: null, + barcode: "", + price_group: null, + units_per_sku: null, + part_description: descriptionHtml, + descriptions: descriptionHtml + ? [{ type: "Market Description", description: descriptionHtml }] + : [], + files, + image_paths: imagePaths, + dimensions: [{ weight: 0 }], + total_quantity: quantity, + inventorydata: { inventory: { main: quantity } }, + options: [], + variants, + fitmentTags: { + make: record.bikeBrand ? [record.bikeBrand] : [], + model: record.bikeModel ? [record.bikeModel] : [], + year: [], + drive: [], + baseModel: [], + }, + tags, + handle: slugify(`${vendor}-${title}-${productId}`), + source_url: record.url || null, + }, + }; +} + +function convertRetroRidesJsonToShopifyProducts(input, options = {}) { + const records = Array.isArray(input?.products) ? input.products : []; + return records.map((record) => { + // Aggregated JSON wraps scraped data inside record.scraped + const data = record.scraped || record; + return convertRetroRidesRecordToShopifyReady(data, { + ...options, + brand: data.bikeBrand || record.brand || options.brand, + }); + }); +} + +module.exports = { convertRetroRidesRecordToShopifyReady, convertRetroRidesJsonToShopifyProducts }; diff --git a/src/business-logic/import-pipeline/sources/retrorides/index.js b/src/business-logic/import-pipeline/sources/retrorides/index.js new file mode 100644 index 0000000..f9b6622 --- /dev/null +++ b/src/business-logic/import-pipeline/sources/retrorides/index.js @@ -0,0 +1,135 @@ +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { scrapeRetroRidesProducts } = require("./scraper"); +const { convertRetroRidesJsonToShopifyProducts } = require("./converter"); + +const sourceKey = "retrorides"; +const label = "Retro Rides"; +const dataDir = path.join("data", "sources", sourceKey); + +function paths() { + return { + aggregatedJson: path.join(dataDir, "01_products_aggregated.json"), + historyJson: path.join(dataDir, "01_products_run_history.json"), + downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"), + watermarkState: path.join(dataDir, "03_watermark_state.json"), + imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"), + uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"), + shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"), + logsDir: path.join(dataDir, "99_run_logs"), + }; +} + +async function fetchWebsiteData({ paths: runPaths }) { + // Use cached aggregated JSON if it exists and is fresh (default 24h). + // Set RETRORIDES_CACHE_HOURS=0 to always force a full re-scrape. + const cacheMaxHours = Number(process.env.RETRORIDES_CACHE_HOURS ?? 24); + const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson); + + if (cacheMaxHours > 0) { + try { + const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); + if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) { + const ageMs = Date.now() - new Date(existing.generatedAt).getTime(); + const ageHours = ageMs / (1000 * 60 * 60); + if (ageHours < cacheMaxHours) { + console.log( + `[RETRORIDES] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) — ${existing.products.length} products. Set RETRORIDES_CACHE_HOURS=0 to force re-scrape.` + ); + return { + analysis: existing.analysis || {}, + outputJsonPath: absAggregatedPath, + historyPath: path.resolve(process.cwd(), runPaths.historyJson), + cached: true, + }; + } + console.log(`[RETRORIDES] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping all products...`); + } + } catch { + // no cache file yet, proceed with scraping + } + } + + const concurrency = Number(process.env.RETRORIDES_CONCURRENCY ?? 3); + const now = new Date().toISOString(); + + console.log(`[RETRORIDES] Scraping all products from retrorides.co.in (concurrency=${concurrency})...`); + const rawProducts = await scrapeRetroRidesProducts({ concurrency }); + + const successCount = rawProducts.filter((p) => !p.scrapeError).length; + const failCount = rawProducts.filter((p) => p.scrapeError).length; + + // Normalize into pipeline-compatible aggregated format. + // productSummary.img is the field that shared download/upload utilities read. + const allProducts = rawProducts.map((product) => ({ + productId: product.handle || product.url, + sourceKey, + brand: product.bikeBrand || "Retro Rides", + bikeModel: product.bikeModel || "", + productSummary: { + id: product.handle, + name: product.title, + img: product.images || [], + cost: { mrp: product.price || 0 }, + }, + scraped: product, + })); + + const analysis = { + timestamp: now, + totalProducts: rawProducts.length, + totalProductsUnique: allProducts.length, + detailSuccess: successCount, + detailFailed: failCount, + }; + + const payload = { + generatedAt: now, + sourceKey, + sourceLabel: label, + analysis, + products: allProducts, + }; + + await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true }); + await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8"); + + // Append to history + let history = []; + const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson); + try { + const raw = await fs.readFile(absHistoryPath, "utf8"); + history = JSON.parse(raw); + if (!Array.isArray(history)) history = []; + } catch { + history = []; + } + history.push(analysis); + await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8"); + + console.log(`[RETRORIDES] Saved ${allProducts.length} products (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`); + + return { + analysis, + outputJsonPath: absAggregatedPath, + historyPath: absHistoryPath, + }; +} + +function convertToShopifyProducts(input, options = {}) { + return convertRetroRidesJsonToShopifyProducts(input, { + brand: options.brand || label, + uploadedImageMap: options.uploadedImageMap, + }); +} + +module.exports = { + sourceKey, + label, + defaultBrand: "Retro Rides", + defaultImageBaseUrl: "", + envImageBaseUrl: "RETRORIDES_IMAGE_BASE_URL", + paths, + fetchWebsiteData, + convertToShopifyProducts, +}; diff --git a/src/business-logic/import-pipeline/sources/retrorides/scraper.js b/src/business-logic/import-pipeline/sources/retrorides/scraper.js new file mode 100644 index 0000000..f13dda4 --- /dev/null +++ b/src/business-logic/import-pipeline/sources/retrorides/scraper.js @@ -0,0 +1,207 @@ +const FETCH_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "identity", + "Connection": "keep-alive", + "Cache-Control": "no-cache", +}; + +const SITEMAP_URL = "https://retrorides.co.in/wp-sitemap-posts-product-1.xml"; + +const BIKE_MAKES_LOWER = new Set(["bmw", "ducati", "aprilia", "honda", "kawasaki", "triumph", "suzuki", "yamaha", "ktm", "husqvarna"]); +// Canonical display names for each bike make +const BIKE_MAKE_CANONICAL = { bmw: "BMW", ducati: "Ducati", aprilia: "Aprilia", honda: "Honda", kawasaki: "Kawasaki", triumph: "Triumph", suzuki: "Suzuki", yamaha: "Yamaha", ktm: "KTM", husqvarna: "Husqvarna" }; +function normalizeBikeMake(name) { + const key = String(name || "").toLowerCase().trim(); + return BIKE_MAKES_LOWER.has(key) ? BIKE_MAKE_CANONICAL[key] : ""; +} + +function sleep(ms) { + return new Promise(function(resolve) { setTimeout(resolve, ms); }); +} + +async function fetchText(url, maxAttempts) { + maxAttempts = maxAttempts || 4; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const response = await fetch(url, { headers: FETCH_HEADERS }); + if (response.status === 429) { + const waitMs = attempt * 4000; + console.log("[RETRY] " + url + " HTTP 429, retrying in " + waitMs + "ms..."); + await sleep(waitMs); + continue; + } + if (!response.ok) { + throw new Error("HTTP " + response.status + " " + response.statusText); + } + return await response.text(); + } catch (err) { + if (attempt === maxAttempts) throw err; + await sleep(attempt * 1500); + } + } + throw new Error("fetchText: exhausted retries"); +} + +function decodeHtmlEntities(str) { + const ENT = { + "&": "&", "&": "&", "<": "<", ">": ">", + """: '"', """: '"', "'": "'", + "’": "'", "‘": "'", "“": '"', "”": '"', + " ": " ", "’": "'", "‘": "'", "”": '"', "“": '"', + }; + let s = String(str || ""); + for (let i = 0; i < 2; i++) { + s = s.replace(/&[#a-z0-9]+;/gi, function(e) { return ENT[e] !== undefined ? ENT[e] : e; }); + } + return s.trim(); +} + +function stripHtml(html) { + return String(html || "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim(); +} + +function slugify(str) { + return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, ""); +} + +function extractJsonLd(html) { + const blocks = Array.from(html.matchAll(/]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)); + for (const block of blocks) { + try { + const parsed = JSON.parse(block[1]); + const graph = Array.isArray(parsed && parsed["@graph"]) ? parsed["@graph"] : [parsed]; + const product = graph.find(function(n) { return n && n["@type"] === "Product"; }); + const breadcrumb = graph.find(function(n) { return n && n["@type"] === "BreadcrumbList"; }); + if (product) return { product: product, breadcrumb: breadcrumb }; + } catch (e) {} + } + return {}; +} + +function extractGalleryImages(html) { + const matches = Array.from(html.matchAll(/data-large_image=["']([^"']+)["']/g)).map(function(m) { return m[1]; }); + const seen = new Set(); + return matches + .map(function(raw) { + const decoded = decodeHtmlEntities(raw); + const cdnMatch = decoded.match(/^https?:\/\/i\d+\.wp\.com\/(.+?)(?:\?.*)?$/); + const url = cdnMatch ? "https://" + cdnMatch[1] : decoded.split("?")[0]; + return url; + }) + .filter(function(url) { + if (!url || url.indexOf("wp-content/uploads") === -1) return false; + if (seen.has(url)) return false; + seen.add(url); + return true; + }); +} + +function extractDescription(html) { + const panelMatch = html.match(/id=["']tab-description["'][^>]*>([\s\S]*?)<\/div>\s*(?:<\/div>|\s*]*>Description<\/h2>/gi, "").trim(); + return decodeHtmlEntities(stripHtml(raw)).replace(/\s{2,}/g, " ").trim(); + } + const shortMatch = html.match(/short-description["'][^>]*>([\s\S]*?)<\/div>/i); + if (shortMatch) { + const raw = decodeHtmlEntities(stripHtml(shortMatch[1])).trim(); + if (raw && raw !== " ") return raw; + } + return ""; +} + +async function scrapeProductPage(url) { + const html = await fetchText(url); + const jsonLd = extractJsonLd(html); + const jsonLdProduct = jsonLd.product; + const breadcrumb = jsonLd.breadcrumb; + + if (!jsonLdProduct) { + return { url: url, scrapeError: "No JSON-LD Product found on page" }; + } + + const title = decodeHtmlEntities(jsonLdProduct.name || ""); + const sku = String(jsonLdProduct.sku || "").trim(); + + const offer = Array.isArray(jsonLdProduct.offers) ? jsonLdProduct.offers[0] : jsonLdProduct.offers; + const priceSpec = offer && offer.priceSpecification && offer.priceSpecification[0]; + const price = Number((priceSpec && priceSpec.price) || (offer && offer.price) || 0); + const currency = (priceSpec && priceSpec.priceCurrency) || (offer && offer.priceCurrency) || "INR"; + const availability = String((offer && offer.availability) || ""); + const inStock = /InStock|BackOrder/i.test(availability); + const quantity = inStock ? 10 : 0; + + const crumbs = ((breadcrumb && breadcrumb.itemListElement) || []).map(function(item) { + return decodeHtmlEntities((item && item.item && item.item.name) || (item && item.name) || ""); + }); + + const rawBrand = crumbs[1] || ""; + const rawModel = crumbs[2] || ""; + const bikeBrand = normalizeBikeMake(rawBrand); + const bikeModel = bikeBrand ? rawModel : ""; + + const galleryImages = extractGalleryImages(html); + const jsonLdImage = jsonLdProduct.image ? String(jsonLdProduct.image) : ""; + const images = galleryImages.length > 0 ? galleryImages : (jsonLdImage ? [jsonLdImage] : []); + + const tags = Array.from(html.matchAll(/rel=["']tag["'][^>]*>([^<]+)<\/a>/g)) + .map(function(m) { return m[1].trim(); }) + .filter(function(t, i, arr) { return t && arr.indexOf(t) === i; }); + + const handleMatch = url.match(/\/product\/([^\/]+)\/?$/); + const handle = handleMatch ? handleMatch[1] : slugify(sku || title); + const description = extractDescription(html); + + return { + url: url, handle: handle, title: title, sku: sku, + price: price, currency: currency, quantity: quantity, + availability: availability, inStock: inStock, images: images, + description: description, bikeBrand: bikeBrand, bikeModel: bikeModel, + bikeCategories: crumbs.slice(1, -1).filter(Boolean), + tags: tags, brand: bikeBrand || "Retro Rides", scrapeError: null, + }; +} + +async function getAllProductUrls() { + const xml = await fetchText(SITEMAP_URL); + return Array.from(xml.matchAll(/([^<]+)<\/loc>/g)) + .map(function(m) { return m[1].trim(); }) + .filter(function(u) { return u.indexOf("/product/") !== -1; }); +} + +async function mapWithConcurrency(items, concurrency, fn) { + const results = new Array(items.length); + let nextIndex = 0; + async function worker() { + while (true) { + const i = nextIndex++; + if (i >= items.length) return; + results[i] = await fn(items[i], i); + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker)); + return results; +} + +async function scrapeRetroRidesProducts(options) { + const concurrency = (options && options.concurrency) || 3; + const urls = await getAllProductUrls(); + console.log("[RETRORIDES] Found " + urls.length + " product URLs from sitemap"); + const products = await mapWithConcurrency(urls, concurrency, async function(url, index) { + try { + const product = await scrapeProductPage(url); + if ((index + 1) % 25 === 0 || index === urls.length - 1) { + console.log("[RETRORIDES] " + (index + 1) + "/" + urls.length + " scraped"); + } + return product; + } catch (err) { + console.log("[RETRORIDES] FAIL " + url + ": " + err.message); + return { url: url, scrapeError: err.message }; + } + }); + return products; +} + +module.exports = { scrapeRetroRidesProducts: scrapeRetroRidesProducts, getAllProductUrls: getAllProductUrls, scrapeProductPage: scrapeProductPage }; \ No newline at end of file