admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: admin:c0132ab0aae5f2d363de87966eaea19c7921d704
Branches Tags
admin:main
..
compare: admin:1d254a9009976070b0af130fe8c6820ca111d6c8
Branches Tags
admin:main

No commits in common. "c0132ab0aae5f2d363de87966eaea19c7921d704" and "1d254a9009976070b0af130fe8c6820ca111d6c8" have entirely different histories.
c0132ab0aa ... 1d254a9009

18 changed files with 0 additions and 2292 deletions
Show Stats Expand all files Collapse all files

14
src/business-logic/import-pipeline/sources/dirtstreet/brands.js
View File

@ -1,14 +0,0 @@
/**
* Dirtstreet.in brand list — production config.
* Source: https://dirtstreet.in/shop-by-brands/
*/
const BRANDS = [
{ name: "SC Project", slug: "scproject", brandUrl: "https://dirtstreet.in/brand/scproject/" },
{ name: "Evotech Performance", slug: "evotechperformance", brandUrl: "https://dirtstreet.in/brand/evotechperformance/" },
{ name: "DNA Air Filters", slug: "dnaairfilters", brandUrl: "https://dirtstreet.in/brand/dnaairfilters/" },
{ name: "WRS", slug: "wrs", brandUrl: "https://dirtstreet.in/brand/wrs/" },
{ name: "Zero Gravity Racing", slug: "zerogravityracing", brandUrl: "https://dirtstreet.in/brand/zerogravityracing/" },
];
module.exports = { BRANDS };

143
src/business-logic/import-pipeline/sources/dirtstreet/converter.js
View File

@ -1,143 +0,0 @@
const path = require("node:path");
function slugify(str) {
return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
}
function getUploadedImageUrl(imgPath, uploadedImageMap) {
if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null;
const bySourcePath = uploadedImageMap.bySourcePath || {};
return bySourcePath[String(imgPath)]?.url || null;
}
function getImageFileName(imagePath) {
try { return path.basename(new URL(imagePath).pathname); } catch { return path.basename(String(imagePath || "").split(/[?#]/)[0]); }
}
function availabilityToStockStatus(availability) {
if (!availability) return null;
const url = String(availability).toLowerCase();
if (url.includes("instock")) return "instock";
if (url.includes("outofstock")) return "outofstock";
if (url.includes("backorder") || url.includes("preorder")) return "onbackorder";
return null;
}
function buildDescriptionHtml(record) {
const parts = [];
if (record.descriptionHtml) {
parts.push(record.descriptionHtml);
} else if (record.description) {
parts.push(`<p>${record.description.replace(/\n/g, "<br/>")}</p>`);
}
if (record.shortDescription && !parts.join("").includes(record.shortDescription)) {
parts.unshift(`<p><strong>${record.shortDescription}</strong></p>`);
}
if (Array.isArray(record.attributes) && record.attributes.length) {
const rows = record.attributes
.map((attr) => `<tr><th>${attr.name}</th><td>${attr.value}</td></tr>`)
.join("");
parts.push(`<table><tbody>${rows}</tbody></table>`);
}
return parts.join("\n");
}
function convertDirtstreetRecordToShopifyReady(record, options = {}) {
const brand = options.brand || record.brand || "Dirtstreet";
const uploadedImageMap = options.uploadedImageMap || null;
const title = record.title || "Untitled Product";
const sku = record.sku || slugify(title);
const productId = sku || slugify(`${brand}-${title}`);
const price = Number(record.price ?? 0);
const compareAtPrice = record.compareAtPrice ? Number(record.compareAtPrice) : null;
const imagePaths = Array.isArray(record.images) ? record.images : [];
// Stock: prefer schema availability, fall back to stockStatus
const stockStatus = availabilityToStockStatus(record.availability) || record.stockStatus || null;
const quantity = stockStatus === "instock" ? 1 : 0;
const descriptionHtml = buildDescriptionHtml(record);
const files = imagePaths
.map((imgPath) => ({
type: "Image",
url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath,
media_content: getImageFileName(imgPath),
source_path: imgPath,
}))
.filter((f) => f.url);
// Categories become subcategory / collections
const categories = Array.isArray(record.categories) ? record.categories : [];
const [primaryCategory, ...subCategories] = categories;
const category = primaryCategory || "Motorcycle Parts";
const subcategory = subCategories.join(", ");
// Variants — dirtstreet products are typically simple (single variant)
const variants = [{ sku, price, compare_price: compareAtPrice, quantity, optionValues: [] }];
const tags = [
brand,
category,
...categories,
sku,
record.stockStatus || "",
...(Array.isArray(record.tags) ? record.tags : []),
].filter(Boolean).map((t) => String(t).trim());
return {
id: productId,
source: {
productId,
sourceKey: "dirtstreet",
url: record.url || null,
image_paths: imagePaths,
},
attributes: {
product_name: title,
brand,
category,
subcategory,
part_number: sku,
mfr_part_number: record.mpn || sku,
price,
compare_price: compareAtPrice,
purchase_cost: null,
barcode: record.gtin || record.barcode || "",
price_group: null,
units_per_sku: null,
part_description: descriptionHtml,
descriptions: descriptionHtml ? [{ type: "Market Description", description: descriptionHtml }] : [],
files,
image_paths: imagePaths,
dimensions: [{ weight: 0 }],
total_quantity: quantity,
inventorydata: { inventory: { main: quantity } },
options: [],
variants,
fitmentTags: { make: [], model: [], year: [], drive: [], baseModel: [] },
tags,
handle: slugify(`${brand}-${title}-${productId}`),
source_url: record.url || null,
},
};
}
function convertDirtstreetJsonToShopifyProducts(input, options = {}) {
const records = Array.isArray(input?.products) ? input.products : [];
return records.map((record) => {
// Each record in aggregated JSON has raw scraped data in record.scraped
const scraped = record.scraped || record;
return convertDirtstreetRecordToShopifyReady(scraped, {
brand: options.brand || scraped.brand || record.brand,
uploadedImageMap: options.uploadedImageMap,
});
});
}
module.exports = { convertDirtstreetRecordToShopifyReady, convertDirtstreetJsonToShopifyProducts };

135
src/business-logic/import-pipeline/sources/dirtstreet/index.js
View File

@ -1,135 +0,0 @@
const fs = require("node:fs/promises");
const path = require("node:path");
const { BRANDS } = require("./brands");
const { scrapeDirtstreetBrand } = require("./scraper");
const { convertDirtstreetJsonToShopifyProducts } = require("./converter");
const sourceKey = "dirtstreet";
const label = "Dirtstreet";
const dataDir = path.join("data", "sources", sourceKey);
function paths() {
return {
aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
historyJson: path.join(dataDir, "01_products_run_history.json"),
downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"),
watermarkState: path.join(dataDir, "03_watermark_state.json"),
imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"),
uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"),
shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"),
logsDir: path.join(dataDir, "99_run_logs"),
};
}
async function fetchWebsiteData({ paths: runPaths }) {
const brandsToScrape = process.env.DIRTSTREET_BRANDS
? BRANDS.filter((b) => process.env.DIRTSTREET_BRANDS.split(",").map((s) => s.trim()).includes(b.slug))
: BRANDS;
const allProducts = [];
const runSummary = [];
const now = new Date().toISOString();
for (const brand of brandsToScrape) {
console.log(`[DIRTSTREET] Scraping brand: ${brand.name} (${brand.brandUrl})`);
const brandStartedAt = new Date().toISOString();
try {
const products = await scrapeDirtstreetBrand(brand);
for (const product of products) {
allProducts.push({
productId: product.sku || product.url,
sourceKey,
brand: product.brand,
brandSlug: product.brandSlug,
// productSummary.img is read by shared downloadImages + uploadImages utilities
productSummary: {
id: product.sku || product.url,
name: product.title,
img: product.images || [],
cost: { mrp: product.price || 0 },
},
scraped: product,
});
}
runSummary.push({
brand: brand.name,
slug: brand.slug,
startedAt: brandStartedAt,
completedAt: new Date().toISOString(),
totalProducts: products.length,
success: true,
});
console.log(`[DIRTSTREET] ${brand.name}: ${products.length} products scraped`);
} catch (err) {
console.log(`[DIRTSTREET] ${brand.name} failed: ${err.message}`);
runSummary.push({
brand: brand.name,
slug: brand.slug,
startedAt: brandStartedAt,
completedAt: new Date().toISOString(),
totalProducts: 0,
success: false,
error: err.message,
});
}
}
const analysis = {
timestamp: now,
totalBrands: brandsToScrape.length,
totalProductsUnique: allProducts.length,
detailSuccess: allProducts.filter((p) => !p.scraped?.scrapeError).length,
detailFailed: allProducts.filter((p) => p.scraped?.scrapeError).length,
runSummary,
};
const payload = {
generatedAt: now,
sourceKey,
sourceLabel: label,
analysis,
products: allProducts,
};
await fs.mkdir(path.dirname(path.resolve(process.cwd(), runPaths.aggregatedJson)), { recursive: true });
await fs.writeFile(path.resolve(process.cwd(), runPaths.aggregatedJson), JSON.stringify(payload, null, 2), "utf8");
let history = [];
try {
const raw = await fs.readFile(path.resolve(process.cwd(), runPaths.historyJson), "utf8");
history = JSON.parse(raw);
if (!Array.isArray(history)) history = [];
} catch { history = []; }
history.push(analysis);
await fs.writeFile(path.resolve(process.cwd(), runPaths.historyJson), JSON.stringify(history, null, 2), "utf8");
console.log(`[DIRTSTREET] Saved ${allProducts.length} products to ${runPaths.aggregatedJson}`);
return {
analysis,
outputJsonPath: path.resolve(process.cwd(), runPaths.aggregatedJson),
historyPath: path.resolve(process.cwd(), runPaths.historyJson),
};
}
function convertToShopifyProducts(input, options = {}) {
return convertDirtstreetJsonToShopifyProducts(input, {
brand: options.brand || label,
uploadedImageMap: options.uploadedImageMap,
});
}
module.exports = {
sourceKey,
label,
defaultBrand: "Dirtstreet",
defaultImageBaseUrl: "",
envImageBaseUrl: "DIRTSTREET_IMAGE_BASE_URL",
paths,
fetchWebsiteData,
convertToShopifyProducts,
};

423
src/business-logic/import-pipeline/sources/dirtstreet/scraper.js
View File

@ -1,423 +0,0 @@
/**
* Dirtstreet.in scraper - plain fetch + JSON-LD extraction (no browser).
*
* Dirtstreet is a WooCommerce store. Product pages contain JSON-LD schema.org
* markup. We fetch HTML directly and extract all fields via regex.
*
* 1. Brand pages paginated at /brand/{slug}/page/N/
* 2. Product detail via HTML + JSON-LD
*/
const BASE_URL = "https://dirtstreet.in";
const CONCURRENCY = 3;
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function fetchHtml(url, attempt = 1) {
try {
const res = await fetch(url, {
headers: {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"accept-language": "en-US,en;q=0.9",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36",
"cache-control": "no-cache",
},
});
if (res.status === 429 || res.status >= 500) {
if (attempt <= 4) {
const wait = attempt * 2000;
console.log(`[RETRY] ${url} -> HTTP ${res.status}, retrying in ${wait}ms (attempt ${attempt}/4)`);
await sleep(wait);
return fetchHtml(url, attempt + 1);
}
throw new Error(`HTTP ${res.status} after retries`);
}
if (res.status === 404) return null;
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return res.text();
} catch (err) {
if (attempt <= 3 && (err.code === "UND_ERR_CONNECT_TIMEOUT" || err.code === "ECONNRESET")) {
await sleep(attempt * 2000);
return fetchHtml(url, attempt + 1);
}
throw err;
}
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function decodeEntities(str) {
return String(str || "")
.replace(/&amp;/g, "&")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&times;/g, "x")
.replace(/&nbsp;/g, " ")
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));
}
function cleanText(value) {
return decodeEntities(String(value || ""))
.replace(/<[^>]+>/g, " ")
.replace(/\s+/g, " ")
.trim();
}
function toNumber(value) {
if (!value) return null;
const match = String(value).match(/([\d,]+\.?\d*)/);
if (!match) return null;
const n = Number.parseFloat(match[1].replace(/,/g, ""));
return Number.isNaN(n) ? null : n;
}
function extractPriceFromHtml(html) {
// data-price attribute (WooCommerce sets this for JS cart)
const dp = html.match(/data-price=["']([\d.]+)["']/);
if (dp) { const n = Number.parseFloat(dp[1]); if (n > 0) return n; }
// Pattern on dirtstreet: &#8377;</span>198,000.00</bdi>
// Collect ALL such values, filter out 0, take the first valid one (the product price)
const bdiPattern = /(?:&#8377;|₹)<\/span>([\d,]+(?:\.\d+)?)<\/bdi>/gi;
let bdiMatch;
while ((bdiMatch = bdiPattern.exec(html)) !== null) {
const n = toNumber(bdiMatch[1]);
if (n > 0) return n;
}
// <ins> sale price block
const ins = html.match(/<ins>[\s\S]{0,300}?(?:&#8377;|₹)<\/span>([\d,]+(?:\.\d+)?)<\/bdi>[\s\S]{0,300}?<\/ins>/i);
if (ins) { const n = toNumber(ins[1]); if (n > 0) return n; }
// Fallback: &#8377; number anywhere
const entity = html.match(/&#8377;[^>]*>([\d,]+(?:\.\d+)?)/);
if (entity) { const n = toNumber(entity[1]); if (n > 0) return n; }
return null;
}
function availabilityToStockStatus(availability) {
if (!availability) return null;
const url = String(availability).toLowerCase();
if (url.includes("instock")) return "instock";
if (url.includes("outofstock")) return "outofstock";
if (url.includes("backorder") || url.includes("preorder")) return "onbackorder";
return null;
}
// ---------------------------------------------------------------------------
// JSON-LD extraction
// ---------------------------------------------------------------------------
function extractJsonLd(html) {
const results = [];
const regex = /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
let match;
while ((match = regex.exec(html)) !== null) {
try {
results.push(JSON.parse(match[1].trim()));
} catch {
// skip
}
}
return results;
}
// ---------------------------------------------------------------------------
// Collection page: extract product URLs
// ---------------------------------------------------------------------------
function extractProductUrls(html) {
const urls = new Set();
const regex = /href=["']((?:https?:\/\/dirtstreet\.in)?\/product\/[^"'#?]+)["']/gi;
let match;
while ((match = regex.exec(html)) !== null) {
let url = match[1];
if (url.startsWith("/")) url = `${BASE_URL}${url}`;
if (url.includes("/product-category/") || url.includes("/product-tag/")) continue;
// normalize trailing slash
url = url.replace(/\/?$/, "/");
urls.add(url);
}
return Array.from(urls);
}
function hasNextPage(html, currentPage, slug) {
const pattern = new RegExp(
`href=["'][^"']*/brand/${slug}/page/${currentPage + 1}/?["']`,
"i"
);
return pattern.test(html);
}
async function fetchBrandProductUrls(slug, brandUrl) {
const allUrls = new Set();
let page = 1;
while (true) {
const url = page === 1 ? brandUrl : `${brandUrl}page/${page}/`;
console.log(`[DIRTSTREET] Fetching brand page ${page}: ${url}`);
const html = await fetchHtml(url);
if (!html) break;
const urls = extractProductUrls(html);
const before = allUrls.size;
urls.forEach((u) => allUrls.add(u));
console.log(`[DIRTSTREET] Page ${page} -> ${urls.length} product URLs (total: ${allUrls.size})`);
if (!hasNextPage(html, page, slug) || allUrls.size === before) break;
page++;
await sleep(300);
}
return Array.from(allUrls);
}
// ---------------------------------------------------------------------------
// Extra WooCommerce fields from HTML
// ---------------------------------------------------------------------------
function extractExtraFields(html) {
const extra = {
sku: null,
categories: [],
tags: [],
shortDescription: null,
stockStatus: null,
attributes: [],
variations: [],
};
// SKU - strip "SKU:" label prefix
const skuMatch = html.match(/<span[^>]+class="[^"]*\bsku\b[^"]*"[^>]*>([\s\S]*?)<\/span>/i);
if (skuMatch) {
extra.sku = cleanText(skuMatch[1])
.replace(/^SKU\s*:\s*/i, "")
.trim();
}
// Stock status — read from the main product wrapper class only, not whole page
// The first <div/article class="product ... instock/outofstock ..."> is the current product
const productClassMatch = html.match(/class="[^"]*\bproduct\b[^"]*\b(instock|outofstock|on-backorder)\b[^"]*"/i);
if (productClassMatch) {
const cls = productClassMatch[1].toLowerCase();
if (cls === "instock") extra.stockStatus = "instock";
else if (cls === "outofstock") extra.stockStatus = "outofstock";
else extra.stockStatus = "onbackorder";
}
// Categories
const catMatch = html.match(/<span[^>]+class="[^"]*posted_in[^"]*"[^>]*>([\s\S]*?)<\/span>/i);
if (catMatch) {
const catRegex = /<a[^>]*>([\s\S]*?)<\/a>/gi;
let m;
while ((m = catRegex.exec(catMatch[1])) !== null) {
const cat = cleanText(m[1]);
if (cat) extra.categories.push(cat);
}
}
// Tags
const tagMatch = html.match(/<span[^>]+class="[^"]*tagged_as[^"]*"[^>]*>([\s\S]*?)<\/span>/i);
if (tagMatch) {
const tagRegex = /<a[^>]*>([\s\S]*?)<\/a>/gi;
let m;
while ((m = tagRegex.exec(tagMatch[1])) !== null) {
const tag = cleanText(m[1]);
if (tag) extra.tags.push(tag);
}
}
// Short description
const sdMatch = html.match(
/<div[^>]+class="[^"]*woocommerce-product-details__short-description[^"]*"[^>]*>([\s\S]*?)<\/div>/i
);
if (sdMatch) extra.shortDescription = cleanText(sdMatch[1]);
// Attributes table
const attrMatch = html.match(
/<table[^>]+class="[^"]*woocommerce-product-attributes[^"]*"[^>]*>([\s\S]*?)<\/table>/i
);
if (attrMatch) {
const rows = attrMatch[1].match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi) || [];
for (const row of rows) {
const th = row.match(/<th[^>]*>([\s\S]*?)<\/th>/i);
const td = row.match(/<td[^>]*>([\s\S]*?)<\/td>/i);
if (th && td) {
const key = cleanText(th[1]);
const value = cleanText(td[1]);
if (key && value) extra.attributes.push({ name: key, value });
}
}
}
// WooCommerce variations JSON in data attribute
const formMatch = html.match(/data-product_variations=["']([\[{][\s\S]*?)["']\s*>/);
if (formMatch) {
try {
extra.variations = JSON.parse(
formMatch[1].replace(/&quot;/g, '"').replace(/&#34;/g, '"')
);
} catch { /* skip */ }
}
return extra;
}
// ---------------------------------------------------------------------------
// Scrape one product detail page
// ---------------------------------------------------------------------------
async function scrapeProductDetail(productUrl, brandName, brandSlug) {
const html = await fetchHtml(productUrl);
if (!html) {
return { recordType: "product", source: "dirtstreet", brand: brandName, brandSlug, url: productUrl, scrapeError: "404" };
}
const jsonLdBlocks = extractJsonLd(html);
// Try direct Product type first, then search every @graph block
let schema = jsonLdBlocks.find((b) => b["@type"] === "Product");
if (!schema) {
for (const block of jsonLdBlocks) {
if (Array.isArray(block["@graph"])) {
const found = block["@graph"].find((g) => g["@type"] === "Product");
if (found) { schema = found; break; }
}
}
}
const extra = extractExtraFields(html);
const offer = Array.isArray(schema?.offers) ? schema.offers[0] : (schema?.offers || {});
// Price: JSON-LD offers.priceSpecification[0].price OR offers.price, then HTML fallback
const priceSpec = Array.isArray(offer?.priceSpecification)
? offer.priceSpecification[0]?.price
: null;
let price = toNumber(priceSpec ?? offer?.price ?? offer?.lowPrice);
if (!price || price === 0) price = extractPriceFromHtml(html);
if (price === 0) price = null;
// Stock: derive from JSON-LD availability (most reliable), fall back to HTML class
const stockFromSchema = availabilityToStockStatus(offer?.availability);
const stockStatus = stockFromSchema || extra.stockStatus || null;
const currency = offer?.priceCurrency || "INR";
const images = Array.isArray(schema?.image)
? schema.image.map((img) => (typeof img === "string" ? img : img?.url || img?.contentUrl)).filter(Boolean)
: schema?.image
? [typeof schema.image === "string" ? schema.image : (schema.image?.url || schema.image?.contentUrl)]
: [];
return {
recordType: "product",
source: "dirtstreet",
brand: schema?.brand?.name || brandName,
brandSlug,
url: productUrl,
title: cleanText(schema?.name || ""),
sku: extra.sku || cleanText(schema?.sku || ""),
mpn: cleanText(schema?.mpn || ""),
gtin: cleanText(schema?.gtin || schema?.gtin13 || ""),
price,
currency,
priceRaw: price != null ? String(price) : null,
compareAtPrice: toNumber(offer?.highPrice) || null,
availability: offer?.availability || null,
stockStatus,
image: images[0] || null,
images,
description: cleanText(schema?.description || ""),
shortDescription: extra.shortDescription,
categories: extra.categories,
tags: extra.tags,
attributes: extra.attributes,
variations: extra.variations,
aggregateRating: schema?.aggregateRating
? { ratingValue: schema.aggregateRating.ratingValue, reviewCount: schema.aggregateRating.reviewCount }
: null,
scrapeError: null,
};
}
// ---------------------------------------------------------------------------
// Concurrency helper
// ---------------------------------------------------------------------------
async function mapWithConcurrency(items, concurrency, worker) {
const results = new Array(items.length);
let index = 0;
async function run() {
while (true) {
const i = index++;
if (i >= items.length) return;
results[i] = await worker(items[i], i);
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run));
return results;
}
// ---------------------------------------------------------------------------
// Main export: scrape one brand
// ---------------------------------------------------------------------------
async function scrapeDirtstreetBrand({ name, slug, brandUrl }, options = {}) {
const limit = options.limit ? Number(options.limit) : null;
const onProgress = typeof options.onProgress === "function" ? options.onProgress : null;
const concurrency = options.concurrency || CONCURRENCY;
let productUrls = await fetchBrandProductUrls(slug, brandUrl);
console.log(`[DIRTSTREET:${name}] Found ${productUrls.length} products`);
if (limit && productUrls.length > limit) {
console.log(`[DIRTSTREET:${name}] Limit applied: fetching first ${limit} of ${productUrls.length}`);
productUrls = productUrls.slice(0, limit);
}
let done = 0;
const products = await mapWithConcurrency(productUrls, concurrency, async (url, i) => {
const product = await scrapeProductDetail(url, name, slug);
done++;
if (done % 10 === 0 || done === productUrls.length) {
console.log(`[DIRTSTREET:${name}] ${done}/${productUrls.length} products scraped`);
}
if (onProgress) onProgress({ done, total: productUrls.length, product: product.title || url });
return product;
});
return products;
}
// ---------------------------------------------------------------------------
// Standalone test
// ---------------------------------------------------------------------------
if (require.main === module) {
const { BRANDS } = require("./brands");
const brand = BRANDS[0];
console.log(`Testing scraper for: ${brand.name}`);
scrapeDirtstreetBrand(brand, { limit: 3 })
.then((products) => {
console.log(`\nScraped ${products.length} products`);
console.log("Sample product:");
console.log(JSON.stringify(products[0], null, 2));
})
.catch((err) => {
console.error("Scrape failed:", err.message);
process.exitCode = 1;
});
}
module.exports = { scrapeDirtstreetBrand, fetchBrandProductUrls, scrapeProductDetail };

4
src/business-logic/import-pipeline/sources/index.js
View File

@ -1,13 +1,9 @@
const kyt = require("./kyt"); const kyt = require("./kyt");
const brocksPerformance = require("./brocks-performance"); const brocksPerformance = require("./brocks-performance");
const motousher = require("./motousher");
const dirtstreet = require("./dirtstreet");
const sources = { const sources = {
[kyt.sourceKey]: kyt, [kyt.sourceKey]: kyt,
[brocksPerformance.sourceKey]: brocksPerformance, [brocksPerformance.sourceKey]: brocksPerformance,
[motousher.sourceKey]: motousher,
[dirtstreet.sourceKey]: dirtstreet,
}; };
function normalizeSourceKey(sourceKey) { function normalizeSourceKey(sourceKey) {

21
src/business-logic/import-pipeline/sources/motousher/brands.js
View File

@ -1,21 +0,0 @@
/**
* Motousher.com brand list — production config.
* Source: https://www.motousher.com/pages/partner-brands
*/
const BRANDS = [
{ name: "All Balls Racing", slug: "all-balls-racing", collectionUrl: "https://www.motousher.com/collections/all-balls-racing" },
{ name: "DID Chains", slug: "did-chains", collectionUrl: "https://www.motousher.com/collections/did-chains" },
{ name: "EBC Brakes", slug: "ebc-brakes", collectionUrl: "https://www.motousher.com/collections/ebc-brakes" },
{ name: "Esjot Sprockets", slug: "esjot-sprockets", collectionUrl: "https://www.motousher.com/collections/esjot-sprockets" },
{ name: "Evans Coolant", slug: "evans-coolant", collectionUrl: "https://www.motousher.com/collections/evans-coolant" },
{ name: "Grip Puppies", slug: "grip-puppies", collectionUrl: "https://www.motousher.com/collections/grip-puppies" },
{ name: "HiFlo Filters", slug: "hi-flo", collectionUrl: "https://www.motousher.com/collections/hi-flo" },
{ name: "JT Sprockets", slug: "jt-sprockets", collectionUrl: "https://www.motousher.com/collections/jt-sprockets" },
{ name: "Maxima Racing Oils", slug: "maxima-racing-oils", collectionUrl: "https://www.motousher.com/collections/maxima-racing-oils" },
{ name: "Putoline", slug: "putoline", collectionUrl: "https://www.motousher.com/collections/putoline" },
{ name: "Ram Mount", slug: "ram-mount", collectionUrl: "https://www.motousher.com/collections/ram-mount" },
{ name: "Wunderlich", slug: "wunderlich", collectionUrl: "https://www.motousher.com/collections/wunderlich" },
];
module.exports = { BRANDS };

113
src/business-logic/import-pipeline/sources/motousher/converter.js
View File

@ -1,113 +0,0 @@
const path = require("node:path");
function slugify(str) {
return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
}
function getUploadedImageUrl(imgPath, uploadedImageMap) {
if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null;
const bySourcePath = uploadedImageMap.bySourcePath || {};
return bySourcePath[String(imgPath)]?.url || null;
}
function getImageFileName(imagePath) {
try { return path.basename(new URL(imagePath).pathname); } catch { return path.basename(String(imagePath || "").split(/[?#]/)[0]); }
}
function convertMotousherRecordToShopifyReady(record, options = {}) {
const brand = options.brand || record.brand || "Motousher";
const uploadedImageMap = options.uploadedImageMap || null;
const productId = record.handle || slugify(`${record.title || ""}-${record.sku || ""}`);
const title = record.title || "Untitled Product";
const sku = record.sku || productId;
const price = Number(record.price ?? 0);
const compareAtPrice = record.compareAtPrice ? Number(record.compareAtPrice) : null;
const quantity = record.variants?.reduce((sum, v) => sum + (v.quantity || 0), 0) || 0;
const descriptionHtml = record.descriptionHtml || "";
const imagePaths = Array.isArray(record.images) ? record.images : [];
const files = imagePaths
.map((imgPath) => ({
type: "Image",
url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath,
media_content: getImageFileName(imgPath),
source_path: imgPath,
}))
.filter((f) => f.url);
// Options: filter out "Title"/"Default Title" Shopify placeholder
const sourceOptions = Array.isArray(record.options)
? record.options.filter((opt) => opt.name !== "Title" || !opt.values?.every((v) => v === "Default Title"))
: [];
// Variants
let variants;
if (Array.isArray(record.variants) && record.variants.length) {
variants = record.variants.map((v, index) => ({
sku: v.sku || (index === 0 ? sku : `${sku}-${index + 1}`),
price: Number(v.price ?? price),
compare_price: v.compare_price != null ? Number(v.compare_price) : compareAtPrice,
quantity: Number(v.quantity ?? 0),
optionValues: Array.isArray(v.optionValues) ? v.optionValues : [],
}));
} else {
variants = [{ sku, price, compare_price: compareAtPrice, quantity, optionValues: [] }];
}
const category = record.productType || "Motorcycle Parts";
const tags = [
brand,
category,
record.brandSlug,
sku,
...(Array.isArray(record.tags) ? record.tags : []),
].filter(Boolean).map((t) => String(t).trim());
return {
id: productId,
source: {
productId,
sourceKey: "motousher",
url: record.url || null,
image_paths: imagePaths,
},
attributes: {
product_name: title,
brand,
category,
subcategory: record.productType || "",
part_number: sku,
mfr_part_number: sku,
price,
compare_price: compareAtPrice,
purchase_cost: null,
barcode: record.barcode || "",
price_group: null,
units_per_sku: null,
part_description: descriptionHtml,
descriptions: descriptionHtml ? [{ type: "Market Description", description: descriptionHtml }] : [],
files,
image_paths: imagePaths,
dimensions: [{ weight: record.weight ? record.weight / 1000 : 0 }],
total_quantity: quantity,
inventorydata: { inventory: { main: quantity } },
options: sourceOptions.map((opt) => ({
name: opt.name,
values: Array.isArray(opt.values) ? opt.values : [],
})),
variants,
fitmentTags: { make: [], model: [], year: [], drive: [], baseModel: [] },
tags,
handle: slugify(`${brand}-${title}-${productId}`),
source_url: record.url || null,
},
};
}
function convertMotousherJsonToShopifyProducts(input, options = {}) {
const records = Array.isArray(input?.products) ? input.products : [];
return records.map((record) => convertMotousherRecordToShopifyReady(record, options));
}
module.exports = { convertMotousherRecordToShopifyReady, convertMotousherJsonToShopifyProducts };

137
src/business-logic/import-pipeline/sources/motousher/index.js
View File

@ -1,137 +0,0 @@
const fs = require("node:fs/promises");
const path = require("node:path");
const { BRANDS } = require("./brands");
const { scrapeMotousherBrand } = require("./scraper");
const { convertMotousherJsonToShopifyProducts } = require("./converter");
const sourceKey = "motousher";
const label = "Motousher";
const dataDir = path.join("data", "sources", sourceKey);
function paths() {
return {
aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
historyJson: path.join(dataDir, "01_products_run_history.json"),
downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"),
watermarkState: path.join(dataDir, "03_watermark_state.json"),
imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"),
uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"),
shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"),
logsDir: path.join(dataDir, "99_run_logs"),
};
}
async function fetchWebsiteData({ paths: runPaths }) {
const brandsToScrape = process.env.MOTOUSHER_BRANDS
? BRANDS.filter((b) => process.env.MOTOUSHER_BRANDS.split(",").map((s) => s.trim()).includes(b.slug))
: BRANDS;
const allProducts = [];
const runSummary = [];
const now = new Date().toISOString();
for (const brand of brandsToScrape) {
console.log(`[MOTOUSHER] Scraping brand: ${brand.name} (${brand.collectionUrl})`);
const brandStartedAt = new Date().toISOString();
try {
const products = await scrapeMotousherBrand(brand);
// Normalize into pipeline-compatible aggregated format
for (const product of products) {
allProducts.push({
productId: product.handle || product.url,
sourceKey,
brand: product.brand,
brandSlug: product.brandSlug,
// productSummary.img is read by shared downloadImages + uploadImages utilities
productSummary: {
id: product.handle,
name: product.title,
img: product.images || [],
cost: { mrp: product.price || 0 },
},
scraped: product,
});
}
runSummary.push({
brand: brand.name,
slug: brand.slug,
startedAt: brandStartedAt,
completedAt: new Date().toISOString(),
totalProducts: products.length,
success: true,
});
console.log(`[MOTOUSHER] ${brand.name}: ${products.length} products fetched`);
} catch (err) {
console.log(`[MOTOUSHER] ${brand.name} failed: ${err.message}`);
runSummary.push({
brand: brand.name,
slug: brand.slug,
startedAt: brandStartedAt,
completedAt: new Date().toISOString(),
totalProducts: 0,
success: false,
error: err.message,
});
}
}
const analysis = {
timestamp: now,
totalBrands: brandsToScrape.length,
totalProductsUnique: allProducts.length,
detailSuccess: allProducts.filter((p) => !p.scraped?.scrapeError).length,
detailFailed: allProducts.filter((p) => p.scraped?.scrapeError).length,
runSummary,
};
const payload = {
generatedAt: now,
sourceKey,
sourceLabel: label,
analysis,
products: allProducts,
};
await fs.mkdir(path.dirname(path.resolve(process.cwd(), runPaths.aggregatedJson)), { recursive: true });
await fs.writeFile(path.resolve(process.cwd(), runPaths.aggregatedJson), JSON.stringify(payload, null, 2), "utf8");
// Append to history
let history = [];
try {
const raw = await fs.readFile(path.resolve(process.cwd(), runPaths.historyJson), "utf8");
history = JSON.parse(raw);
if (!Array.isArray(history)) history = [];
} catch { history = []; }
history.push(analysis);
await fs.writeFile(path.resolve(process.cwd(), runPaths.historyJson), JSON.stringify(history, null, 2), "utf8");
console.log(`[MOTOUSHER] Saved ${allProducts.length} products to ${runPaths.aggregatedJson}`);
return {
analysis,
outputJsonPath: path.resolve(process.cwd(), runPaths.aggregatedJson),
historyPath: path.resolve(process.cwd(), runPaths.historyJson),
};
}
function convertToShopifyProducts(input, options = {}) {
return convertMotousherJsonToShopifyProducts(input, {
brand: options.brand || label,
uploadedImageMap: options.uploadedImageMap,
});
}
module.exports = {
sourceKey,
label,
defaultBrand: "Motousher",
defaultImageBaseUrl: "",
envImageBaseUrl: "MOTOUSHER_IMAGE_BASE_URL",
paths,
fetchWebsiteData,
convertToShopifyProducts,
};

185
src/business-logic/import-pipeline/sources/motousher/scraper.js
View File

@ -1,185 +0,0 @@
/**
* Motousher.com scraper — Shopify JSON API.
* Uses /collections/{slug}/products.json (paginated) + /products/{handle}.json
*/
const BASE_URL = "https://www.motousher.com";
const CONCURRENCY = 3;
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function fetchJson(url, attempt = 1) {
try {
const res = await fetch(url, {
headers: {
accept: "application/json",
"accept-language": "en-US,en;q=0.9",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36",
},
});
if (res.status === 429 || res.status >= 500) {
if (attempt <= 4) {
const wait = attempt * 1500;
console.log(`[RETRY] ${url} HTTP ${res.status}, retrying in ${wait}ms (${attempt}/4)`);
await sleep(wait);
return fetchJson(url, attempt + 1);
}
throw new Error(`HTTP ${res.status} after retries`);
}
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return res.json();
} catch (err) {
if (attempt <= 4 && (err.code === "UND_ERR_CONNECT_TIMEOUT" || err.code === "ECONNRESET")) {
await sleep(attempt * 1500);
return fetchJson(url, attempt + 1);
}
throw err;
}
}
async function fetchCollectionHandles(collectionSlug) {
const handles = [];
let page = 1;
while (true) {
const url = `${BASE_URL}/collections/${collectionSlug}/products.json?limit=250&page=${page}`;
const data = await fetchJson(url);
const products = Array.isArray(data?.products) ? data.products : [];
if (!products.length) break;
for (const p of products) {
if (p.handle) handles.push(p.handle);
}
if (products.length < 250) break;
page++;
}
return handles;
}
function toNumber(value) {
if (!value) return null;
const match = String(value).match(/([\d,]+\.?\d*)/);
if (!match) return null;
const n = Number.parseFloat(match[1].replace(/,/g, ""));
return Number.isNaN(n) ? null : n;
}
function bestImageUrl(src) {
if (!src) return null;
return src.replace(/(_\d+x\d*|_\d+x)(\.\w+)(\?|$)/, "$2$3");
}
function normalizeShopifyProduct(raw, brandName, brandSlug, collectionUrl) {
const images = (raw.images || []).map((img) => bestImageUrl(img.src)).filter(Boolean);
const firstVariant = raw.variants?.[0] || {};
const price = toNumber(firstVariant.price);
const compareAtPrice = toNumber(firstVariant.compare_at_price) || null;
const quantity = raw.variants?.reduce((sum, v) => sum + (v.inventory_quantity || 0), 0) || 0;
const options = (raw.options || [])
.filter((opt) => opt.name !== "Title" || (opt.values || []).join("") !== "Default Title")
.map((opt) => ({
name: opt.name,
values: opt.values || [],
}));
const hasRealOptions = options.length > 0;
const variants = (raw.variants || []).map((v, index) => ({
sku: v.sku || firstVariant.sku || raw.handle || "",
price: toNumber(v.price) ?? price,
compare_price: toNumber(v.compare_at_price) || null,
quantity: v.inventory_quantity || 0,
optionValues: hasRealOptions
? [v.option1, v.option2, v.option3]
.filter(Boolean)
.filter((val) => val !== "Default Title")
.map((val, i) => ({
name: options[i]?.name || `Option${i + 1}`,
value: val,
}))
: [],
}));
return {
recordType: "product",
source: "motousher",
brand: brandName,
brandSlug,
collectionUrl,
handle: raw.handle || null,
title: raw.title || null,
url: `${BASE_URL}/products/${raw.handle}`,
productType: raw.product_type || null,
vendor: raw.vendor || brandName,
tags: Array.isArray(raw.tags) ? raw.tags : typeof raw.tags === "string" ? raw.tags.split(",").map((t) => t.trim()).filter(Boolean) : [],
price,
compareAtPrice,
sku: firstVariant.sku || null,
barcode: firstVariant.barcode || null,
weight: firstVariant.grams || null,
available: raw.variants?.some((v) => v.available) ?? null,
image: images[0] || null,
images,
descriptionHtml: raw.body_html || null,
options,
variants,
publishedAt: raw.published_at || null,
scrapeError: null,
};
}
async function fetchProductDetail(handle, brandName, brandSlug, collectionUrl) {
const url = `${BASE_URL}/products/${handle}.json`;
try {
const data = await fetchJson(url);
if (!data?.product) throw new Error("No product in response");
return normalizeShopifyProduct(data.product, brandName, brandSlug, collectionUrl);
} catch (err) {
return { recordType: "product", source: "motousher", brand: brandName, brandSlug, collectionUrl, handle, url: `${BASE_URL}/products/${handle}`, scrapeError: err.message };
}
}
async function mapWithConcurrency(items, concurrency, worker) {
const results = new Array(items.length);
let index = 0;
async function run() {
while (true) {
const i = index++;
if (i >= items.length) return;
results[i] = await worker(items[i], i);
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run));
return results;
}
async function scrapeMotousherBrand({ name, slug, collectionUrl }, options = {}) {
const limit = options.limit ? Number(options.limit) : null;
const onProgress = typeof options.onProgress === "function" ? options.onProgress : null;
const concurrency = options.concurrency || CONCURRENCY;
let handles = await fetchCollectionHandles(slug);
if (limit && handles.length > limit) {
handles = handles.slice(0, limit);
}
let done = 0;
const products = await mapWithConcurrency(handles, concurrency, async (handle, i) => {
const product = await fetchProductDetail(handle, name, slug, collectionUrl);
done++;
if (done % 20 === 0 || done === handles.length) {
console.log(`[MOTOUSHER:${name}] ${done}/${handles.length} fetched`);
}
if (onProgress) onProgress({ done, total: handles.length, product: product.title || handle });
return product;
});
return products;
}
module.exports = { scrapeMotousherBrand, fetchCollectionHandles, fetchProductDetail };

1
src/business-logic/import-pipeline/test_source/.gitkeep
View File

@ -1 +0,0 @@

26
src/business-logic/import-pipeline/test_source/README.md
View File

@ -1,26 +0,0 @@
# Test Source Workspace
This folder is for experimental supplier scraping work.
Use one folder per supplier website or test target. Keep experiments here until
the scraper, converter, and data shape are confirmed safe to move into the active
`sources/` registry.
Suggested folder shape:
```txt
test_source/
supplier-key/
links.txt
scraper.js
sample-output.json
notes.md
```
Rules for this area:
- Do not register test sources in `../sources/index.js` until the scraper is ready.
- Store only brand/filter/listing URLs in `links.txt`; avoid committing secrets.
- Keep generated images, large raw exports, and run logs out of git.
- Document what each URL represents in the supplier folder notes.

35
src/business-logic/import-pipeline/test_source/dirtstreet/brands.js
View File

@ -1,35 +0,0 @@
/**
* Dirtstreet brand list.
*
* Brand index page: https://dirtstreet.in/shop-by-brands/
*/
const BRANDS = [
{
name: "SC Project",
slug: "scproject",
brandUrl: "https://dirtstreet.in/brand/scproject/",
},
{
name: "Evotech Performance",
slug: "evotechperformance",
brandUrl: "https://dirtstreet.in/brand/evotechperformance/",
},
{
name: "DNA Air Filters",
slug: "dnaairfilters",
brandUrl: "https://dirtstreet.in/brand/dnaairfilters/",
},
{
name: "WRS",
slug: "wrs",
brandUrl: "https://dirtstreet.in/brand/wrs/",
},
{
name: "Zero Gravity Racing",
slug: "zerogravityracing",
brandUrl: "https://dirtstreet.in/brand/zerogravityracing/",
},
];
module.exports = { BRANDS };

138
src/business-logic/import-pipeline/test_source/dirtstreet/index.js
View File

@ -1,138 +0,0 @@
/**
* Dirtstreet scraper runner.
*
* Usage:
* node index.js → scrape all brands in brands.js
* node index.js --brand scproject → scrape one brand only
* node index.js --limit 5 → first 5 products per brand (quick test)
*/
const fs = require("node:fs/promises");
const path = require("node:path");
const { BRANDS } = require("./brands");
const { scrapeDirtstreetBrand } = require("./scraper");
const OUTPUT_DIR = path.resolve(__dirname, "..", "..", "..", "..", "..", "data", "sources", "test_source", "dirtstreet");
function parseArgs(argv = process.argv.slice(2)) {
const out = { brandSlug: null, limit: null };
for (let i = 0; i < argv.length; i++) {
if ((argv[i] === "--brand" || argv[i] === "-b") && argv[i + 1]) {
out.brandSlug = argv[i + 1].toLowerCase().trim();
}
if ((argv[i] === "--limit" || argv[i] === "-n") && argv[i + 1]) {
const n = Number.parseInt(argv[i + 1], 10);
if (Number.isFinite(n) && n > 0) out.limit = n;
}
}
return out;
}
async function run() {
const { brandSlug, limit } = parseArgs();
await fs.mkdir(OUTPUT_DIR, { recursive: true });
const brandsToRun = brandSlug
? BRANDS.filter((b) => b.slug === brandSlug)
: BRANDS;
if (!brandsToRun.length) {
console.error(`No brand found matching slug: "${brandSlug}"`);
console.error(`Available: ${BRANDS.map((b) => b.slug).join(", ")}`);
process.exitCode = 1;
return;
}
const combinedProducts = [];
const runSummary = [];
const startedAt = new Date().toISOString();
for (const brand of brandsToRun) {
console.log(`\n${"=".repeat(60)}`);
console.log(`[DIRTSTREET] Scraping brand: ${brand.name}`);
console.log(`[DIRTSTREET] Brand URL: ${brand.brandUrl}`);
console.log(`${"=".repeat(60)}`);
const brandStartedAt = new Date().toISOString();
let products = [];
let error = null;
try {
products = await scrapeDirtstreetBrand(brand, { limit });
const brandFile = path.join(OUTPUT_DIR, `${brand.slug}.json`);
const brandPayload = {
generatedAt: new Date().toISOString(),
source: "dirtstreet",
brand: brand.name,
brandSlug: brand.slug,
brandUrl: brand.brandUrl,
totalProducts: products.length,
products,
};
await fs.writeFile(brandFile, JSON.stringify(brandPayload, null, 2), "utf8");
console.log(`[DIRTSTREET] Saved ${products.length} products → ${brandFile}`);
combinedProducts.push(...products);
runSummary.push({
brand: brand.name,
slug: brand.slug,
brandUrl: brand.brandUrl,
startedAt: brandStartedAt,
completedAt: new Date().toISOString(),
totalProducts: products.length,
success: true,
});
} catch (err) {
console.error(`[DIRTSTREET] Failed to scrape ${brand.name}: ${err.message}`);
error = err.message;
runSummary.push({
brand: brand.name,
slug: brand.slug,
brandUrl: brand.brandUrl,
startedAt: brandStartedAt,
completedAt: new Date().toISOString(),
totalProducts: 0,
success: false,
error,
});
}
}
// Save combined JSON
const combinedFile = path.join(OUTPUT_DIR, "combined.json");
const combinedPayload = {
generatedAt: new Date().toISOString(),
startedAt,
source: "dirtstreet",
siteUrl: "https://dirtstreet.in",
brandsScraped: runSummary.length,
totalProducts: combinedProducts.length,
runSummary,
products: combinedProducts,
};
await fs.writeFile(combinedFile, JSON.stringify(combinedPayload, null, 2), "utf8");
console.log(`\n${"=".repeat(60)}`);
console.log("[DIRTSTREET] RUN COMPLETE");
console.log(`${"=".repeat(60)}`);
console.log(`Brands scraped : ${runSummary.length}`);
console.log(`Total products : ${combinedProducts.length}`);
console.log(`Combined file : ${combinedFile}`);
console.log("\nBrand summary:");
for (const s of runSummary) {
const status = s.success ? "OK" : "FAILED";
console.log(` [${status}] ${s.brand} → ${s.totalProducts} products`);
}
return combinedPayload;
}
run().catch((err) => {
console.error("Runner failed:", err.message);
process.exitCode = 1;
});
module.exports = { run };

423
src/business-logic/import-pipeline/test_source/dirtstreet/scraper.js
View File

@ -1,423 +0,0 @@
/**
* Dirtstreet.in scraper - plain fetch + JSON-LD extraction (no browser).
*
* Dirtstreet is a WooCommerce store. Product pages contain JSON-LD schema.org
* markup. We fetch HTML directly and extract all fields via regex.
*
* 1. Brand pages paginated at /brand/{slug}/page/N/
* 2. Product detail via HTML + JSON-LD
*/
const BASE_URL = "https://dirtstreet.in";
const CONCURRENCY = 3;
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function fetchHtml(url, attempt = 1) {
try {
const res = await fetch(url, {
headers: {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"accept-language": "en-US,en;q=0.9",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36",
"cache-control": "no-cache",
},
});
if (res.status === 429 || res.status >= 500) {
if (attempt <= 4) {
const wait = attempt * 2000;
console.log(`[RETRY] ${url} -> HTTP ${res.status}, retrying in ${wait}ms (attempt ${attempt}/4)`);
await sleep(wait);
return fetchHtml(url, attempt + 1);
}
throw new Error(`HTTP ${res.status} after retries`);
}
if (res.status === 404) return null;
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return res.text();
} catch (err) {
if (attempt <= 3 && (err.code === "UND_ERR_CONNECT_TIMEOUT" || err.code === "ECONNRESET")) {
await sleep(attempt * 2000);
return fetchHtml(url, attempt + 1);
}
throw err;
}
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function decodeEntities(str) {
return String(str || "")
.replace(/&amp;/g, "&")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&times;/g, "x")
.replace(/&nbsp;/g, " ")
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)));
}
function cleanText(value) {
return decodeEntities(String(value || ""))
.replace(/<[^>]+>/g, " ")
.replace(/\s+/g, " ")
.trim();
}
function toNumber(value) {
if (!value) return null;
const match = String(value).match(/([\d,]+\.?\d*)/);
if (!match) return null;
const n = Number.parseFloat(match[1].replace(/,/g, ""));
return Number.isNaN(n) ? null : n;
}
function extractPriceFromHtml(html) {
// data-price attribute (WooCommerce sets this for JS cart)
const dp = html.match(/data-price=["']([\d.]+)["']/);
if (dp) { const n = Number.parseFloat(dp[1]); if (n > 0) return n; }
// Pattern on dirtstreet: &#8377;</span>198,000.00</bdi>
// Collect ALL such values, filter out 0, take the first valid one (the product price)
const bdiPattern = /(?:&#8377;|₹)<\/span>([\d,]+(?:\.\d+)?)<\/bdi>/gi;
let bdiMatch;
while ((bdiMatch = bdiPattern.exec(html)) !== null) {
const n = toNumber(bdiMatch[1]);
if (n > 0) return n;
}
// <ins> sale price block
const ins = html.match(/<ins>[\s\S]{0,300}?(?:&#8377;|₹)<\/span>([\d,]+(?:\.\d+)?)<\/bdi>[\s\S]{0,300}?<\/ins>/i);
if (ins) { const n = toNumber(ins[1]); if (n > 0) return n; }
// Fallback: &#8377; number anywhere
const entity = html.match(/&#8377;[^>]*>([\d,]+(?:\.\d+)?)/);
if (entity) { const n = toNumber(entity[1]); if (n > 0) return n; }
return null;
}
function availabilityToStockStatus(availability) {
if (!availability) return null;
const url = String(availability).toLowerCase();
if (url.includes("instock")) return "instock";
if (url.includes("outofstock")) return "outofstock";
if (url.includes("backorder") || url.includes("preorder")) return "onbackorder";
return null;
}
// ---------------------------------------------------------------------------
// JSON-LD extraction
// ---------------------------------------------------------------------------
function extractJsonLd(html) {
const results = [];
const regex = /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
let match;
while ((match = regex.exec(html)) !== null) {
try {
results.push(JSON.parse(match[1].trim()));
} catch {
// skip
}
}
return results;
}
// ---------------------------------------------------------------------------
// Collection page: extract product URLs
// ---------------------------------------------------------------------------
function extractProductUrls(html) {
const urls = new Set();
const regex = /href=["']((?:https?:\/\/dirtstreet\.in)?\/product\/[^"'#?]+)["']/gi;
let match;
while ((match = regex.exec(html)) !== null) {
let url = match[1];
if (url.startsWith("/")) url = `${BASE_URL}${url}`;
if (url.includes("/product-category/") || url.includes("/product-tag/")) continue;
// normalize trailing slash
url = url.replace(/\/?$/, "/");
urls.add(url);
}
return Array.from(urls);
}
function hasNextPage(html, currentPage, slug) {
const pattern = new RegExp(
`href=["'][^"']*/brand/${slug}/page/${currentPage + 1}/?["']`,
"i"
);
return pattern.test(html);
}
async function fetchBrandProductUrls(slug, brandUrl) {
const allUrls = new Set();
let page = 1;
while (true) {
const url = page === 1 ? brandUrl : `${brandUrl}page/${page}/`;
console.log(`[DIRTSTREET] Fetching brand page ${page}: ${url}`);
const html = await fetchHtml(url);
if (!html) break;
const urls = extractProductUrls(html);
const before = allUrls.size;
urls.forEach((u) => allUrls.add(u));
console.log(`[DIRTSTREET] Page ${page} -> ${urls.length} product URLs (total: ${allUrls.size})`);
if (!hasNextPage(html, page, slug) || allUrls.size === before) break;
page++;
await sleep(300);
}
return Array.from(allUrls);
}
// ---------------------------------------------------------------------------
// Extra WooCommerce fields from HTML
// ---------------------------------------------------------------------------
function extractExtraFields(html) {
const extra = {
sku: null,
categories: [],
tags: [],
shortDescription: null,
stockStatus: null,
attributes: [],
variations: [],
};
// SKU - strip "SKU:" label prefix
const skuMatch = html.match(/<span[^>]+class="[^"]*\bsku\b[^"]*"[^>]*>([\s\S]*?)<\/span>/i);
if (skuMatch) {
extra.sku = cleanText(skuMatch[1])
.replace(/^SKU\s*:\s*/i, "")
.trim();
}
// Stock status — read from the main product wrapper class only, not whole page
// The first <div/article class="product ... instock/outofstock ..."> is the current product
const productClassMatch = html.match(/class="[^"]*\bproduct\b[^"]*\b(instock|outofstock|on-backorder)\b[^"]*"/i);
if (productClassMatch) {
const cls = productClassMatch[1].toLowerCase();
if (cls === "instock") extra.stockStatus = "instock";
else if (cls === "outofstock") extra.stockStatus = "outofstock";
else extra.stockStatus = "onbackorder";
}
// Categories
const catMatch = html.match(/<span[^>]+class="[^"]*posted_in[^"]*"[^>]*>([\s\S]*?)<\/span>/i);
if (catMatch) {
const catRegex = /<a[^>]*>([\s\S]*?)<\/a>/gi;
let m;
while ((m = catRegex.exec(catMatch[1])) !== null) {
const cat = cleanText(m[1]);
if (cat) extra.categories.push(cat);
}
}
// Tags
const tagMatch = html.match(/<span[^>]+class="[^"]*tagged_as[^"]*"[^>]*>([\s\S]*?)<\/span>/i);
if (tagMatch) {
const tagRegex = /<a[^>]*>([\s\S]*?)<\/a>/gi;
let m;
while ((m = tagRegex.exec(tagMatch[1])) !== null) {
const tag = cleanText(m[1]);
if (tag) extra.tags.push(tag);
}
}
// Short description
const sdMatch = html.match(
/<div[^>]+class="[^"]*woocommerce-product-details__short-description[^"]*"[^>]*>([\s\S]*?)<\/div>/i
);
if (sdMatch) extra.shortDescription = cleanText(sdMatch[1]);
// Attributes table
const attrMatch = html.match(
/<table[^>]+class="[^"]*woocommerce-product-attributes[^"]*"[^>]*>([\s\S]*?)<\/table>/i
);
if (attrMatch) {
const rows = attrMatch[1].match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi) || [];
for (const row of rows) {
const th = row.match(/<th[^>]*>([\s\S]*?)<\/th>/i);
const td = row.match(/<td[^>]*>([\s\S]*?)<\/td>/i);
if (th && td) {
const key = cleanText(th[1]);
const value = cleanText(td[1]);
if (key && value) extra.attributes.push({ name: key, value });
}
}
}
// WooCommerce variations JSON in data attribute
const formMatch = html.match(/data-product_variations=["']([\[{][\s\S]*?)["']\s*>/);
if (formMatch) {
try {
extra.variations = JSON.parse(
formMatch[1].replace(/&quot;/g, '"').replace(/&#34;/g, '"')
);
} catch { /* skip */ }
}
return extra;
}
// ---------------------------------------------------------------------------
// Scrape one product detail page
// ---------------------------------------------------------------------------
async function scrapeProductDetail(productUrl, brandName, brandSlug) {
const html = await fetchHtml(productUrl);
if (!html) {
return { recordType: "product", source: "dirtstreet", brand: brandName, brandSlug, url: productUrl, scrapeError: "404" };
}
const jsonLdBlocks = extractJsonLd(html);
// Try direct Product type first, then search every @graph block
let schema = jsonLdBlocks.find((b) => b["@type"] === "Product");
if (!schema) {
for (const block of jsonLdBlocks) {
if (Array.isArray(block["@graph"])) {
const found = block["@graph"].find((g) => g["@type"] === "Product");
if (found) { schema = found; break; }
}
}
}
const extra = extractExtraFields(html);
const offer = Array.isArray(schema?.offers) ? schema.offers[0] : (schema?.offers || {});
// Price: JSON-LD offers.priceSpecification[0].price OR offers.price, then HTML fallback
const priceSpec = Array.isArray(offer?.priceSpecification)
? offer.priceSpecification[0]?.price
: null;
let price = toNumber(priceSpec ?? offer?.price ?? offer?.lowPrice);
if (!price || price === 0) price = extractPriceFromHtml(html);
if (price === 0) price = null;
// Stock: derive from JSON-LD availability (most reliable), fall back to HTML class
const stockFromSchema = availabilityToStockStatus(offer?.availability);
const stockStatus = stockFromSchema || extra.stockStatus || null;
const currency = offer?.priceCurrency || "INR";
const images = Array.isArray(schema?.image)
? schema.image.map((img) => (typeof img === "string" ? img : img?.url || img?.contentUrl)).filter(Boolean)
: schema?.image
? [typeof schema.image === "string" ? schema.image : (schema.image?.url || schema.image?.contentUrl)]
: [];
return {
recordType: "product",
source: "dirtstreet",
brand: schema?.brand?.name || brandName,
brandSlug,
url: productUrl,
title: cleanText(schema?.name || ""),
sku: extra.sku || cleanText(schema?.sku || ""),
mpn: cleanText(schema?.mpn || ""),
gtin: cleanText(schema?.gtin || schema?.gtin13 || ""),
price,
currency,
priceRaw: price != null ? String(price) : null,
compareAtPrice: toNumber(offer?.highPrice) || null,
availability: offer?.availability || null,
stockStatus,
image: images[0] || null,
images,
description: cleanText(schema?.description || ""),
shortDescription: extra.shortDescription,
categories: extra.categories,
tags: extra.tags,
attributes: extra.attributes,
variations: extra.variations,
aggregateRating: schema?.aggregateRating
? { ratingValue: schema.aggregateRating.ratingValue, reviewCount: schema.aggregateRating.reviewCount }
: null,
scrapeError: null,
};
}
// ---------------------------------------------------------------------------
// Concurrency helper
// ---------------------------------------------------------------------------
async function mapWithConcurrency(items, concurrency, worker) {
const results = new Array(items.length);
let index = 0;
async function run() {
while (true) {
const i = index++;
if (i >= items.length) return;
results[i] = await worker(items[i], i);
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run));
return results;
}
// ---------------------------------------------------------------------------
// Main export: scrape one brand
// ---------------------------------------------------------------------------
async function scrapeDirtstreetBrand({ name, slug, brandUrl }, options = {}) {
const limit = options.limit ? Number(options.limit) : null;
const onProgress = typeof options.onProgress === "function" ? options.onProgress : null;
const concurrency = options.concurrency || CONCURRENCY;
let productUrls = await fetchBrandProductUrls(slug, brandUrl);
console.log(`[DIRTSTREET:${name}] Found ${productUrls.length} products`);
if (limit && productUrls.length > limit) {
console.log(`[DIRTSTREET:${name}] Limit applied: fetching first ${limit} of ${productUrls.length}`);
productUrls = productUrls.slice(0, limit);
}
let done = 0;
const products = await mapWithConcurrency(productUrls, concurrency, async (url, i) => {
const product = await scrapeProductDetail(url, name, slug);
done++;
if (done % 10 === 0 || done === productUrls.length) {
console.log(`[DIRTSTREET:${name}] ${done}/${productUrls.length} products scraped`);
}
if (onProgress) onProgress({ done, total: productUrls.length, product: product.title || url });
return product;
});
return products;
}
// ---------------------------------------------------------------------------
// Standalone test
// ---------------------------------------------------------------------------
if (require.main === module) {
const { BRANDS } = require("./brands");
const brand = BRANDS[0];
console.log(`Testing scraper for: ${brand.name}`);
scrapeDirtstreetBrand(brand, { limit: 3 })
.then((products) => {
console.log(`\nScraped ${products.length} products`);
console.log("Sample product:");
console.log(JSON.stringify(products[0], null, 2));
})
.catch((err) => {
console.error("Scrape failed:", err.message);
process.exitCode = 1;
});
}
module.exports = { scrapeDirtstreetBrand, fetchBrandProductUrls, scrapeProductDetail };

73
src/business-logic/import-pipeline/test_source/motousher/brands.js
View File

@ -1,73 +0,0 @@
/**
* Motousher brand list.
*
* Each entry maps a brand name to its collection page URL on motousher.com.
* Add more brands here — the scraper loops this list automatically.
*
* Brand index page: https://www.motousher.com/pages/partner-brands
*/
const BRANDS = [
{
name: "All Balls Racing",
slug: "all-balls-racing",
collectionUrl: "https://www.motousher.com/collections/all-balls-racing",
},
{
name: "DID Chains",
slug: "did-chains",
collectionUrl: "https://www.motousher.com/collections/did-chains",
},
{
name: "EBC Brakes",
slug: "ebc-brakes",
collectionUrl: "https://www.motousher.com/collections/ebc-brakes",
},
{
name: "Esjot Sprockets",
slug: "esjot-sprockets",
collectionUrl: "https://www.motousher.com/collections/esjot-sprockets",
},
{
name: "Evans Coolant",
slug: "evans-coolant",
collectionUrl: "https://www.motousher.com/collections/evans-coolant",
},
{
name: "Grip Puppies",
slug: "grip-puppies",
collectionUrl: "https://www.motousher.com/collections/grip-puppies",
},
{
name: "HiFlo Filters",
slug: "hi-flo",
collectionUrl: "https://www.motousher.com/collections/hi-flo",
},
{
name: "JT Sprockets",
slug: "jt-sprockets",
collectionUrl: "https://www.motousher.com/collections/jt-sprockets",
},
{
name: "Maxima Racing Oils",
slug: "maxima-racing-oils",
collectionUrl: "https://www.motousher.com/collections/maxima-racing-oils",
},
{
name: "Putoline",
slug: "putoline",
collectionUrl: "https://www.motousher.com/collections/putoline",
},
{
name: "Ram Mount",
slug: "ram-mount",
collectionUrl: "https://www.motousher.com/collections/ram-mount",
},
{
name: "Wunderlich",
slug: "wunderlich",
collectionUrl: "https://www.motousher.com/collections/wunderlich",
},
];
module.exports = { BRANDS };

158
src/business-logic/import-pipeline/test_source/motousher/index.js
View File

@ -1,158 +0,0 @@
/**
* Motousher scraper runner.
*
* Loops through BRANDS in brands.js, scrapes each brand's collection page
* and all product detail pages, saves per-brand JSON files, then combines
* everything into a single combined.json.
*
* Usage:
* node index.js → scrape all brands in brands.js
* node index.js --brand all-balls-racing → scrape one brand only
* node index.js --limit 5 → only first 5 products per brand (for quick tests)
*/
const fs = require("node:fs/promises");
const path = require("node:path");
const { BRANDS } = require("./brands");
const { scrapeMotousherBrand } = require("./scraper");
const OUTPUT_DIR = path.resolve(__dirname, "..", "..", "..", "..", "..", "data", "sources", "test_source", "motousher");
function slugify(str) {
return String(str || "")
.trim()
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "");
}
function parseArgs(argv = process.argv.slice(2)) {
const out = { brandSlug: null, limit: null };
for (let i = 0; i < argv.length; i++) {
if ((argv[i] === "--brand" || argv[i] === "-b") && argv[i + 1]) {
out.brandSlug = argv[i + 1].toLowerCase().trim();
}
if ((argv[i] === "--limit" || argv[i] === "-n") && argv[i + 1]) {
const n = Number.parseInt(argv[i + 1], 10);
if (Number.isFinite(n) && n > 0) out.limit = n;
}
}
return out;
}
async function run() {
const { brandSlug, limit } = parseArgs();
await fs.mkdir(OUTPUT_DIR, { recursive: true });
const brandsToRun = brandSlug
? BRANDS.filter((b) => b.slug === brandSlug)
: BRANDS;
if (!brandsToRun.length) {
console.error(`No brand found matching slug: "${brandSlug}"`);
console.error(`Available: ${BRANDS.map((b) => b.slug).join(", ")}`);
process.exitCode = 1;
return;
}
const combinedProducts = [];
const runSummary = [];
const startedAt = new Date().toISOString();
for (const brand of brandsToRun) {
console.log(`\n${"=".repeat(60)}`);
console.log(`[MOTOUSHER] Scraping brand: ${brand.name}`);
console.log(`[MOTOUSHER] Collection URL: ${brand.collectionUrl}`);
console.log(`${"=".repeat(60)}`);
const brandStartedAt = new Date().toISOString();
let products = [];
let error = null;
try {
products = await scrapeMotousherBrand(brand, {
limit,
onProgress({ done, total, product }) {
// Inline progress already logged by scraper
},
});
// Save per-brand JSON
const brandFile = path.join(OUTPUT_DIR, `${brand.slug}.json`);
const brandPayload = {
generatedAt: new Date().toISOString(),
source: "motousher",
brand: brand.name,
brandSlug: brand.slug,
collectionUrl: brand.collectionUrl,
totalProducts: products.length,
products,
};
await fs.writeFile(brandFile, JSON.stringify(brandPayload, null, 2), "utf8");
console.log(`[MOTOUSHER] Saved ${products.length} products → ${brandFile}`);
combinedProducts.push(...products);
runSummary.push({
brand: brand.name,
slug: brand.slug,
collectionUrl: brand.collectionUrl,
startedAt: brandStartedAt,
completedAt: new Date().toISOString(),
totalProducts: products.length,
success: true,
});
} catch (err) {
console.error(`[MOTOUSHER] Failed to scrape ${brand.name}: ${err.message}`);
error = err.message;
runSummary.push({
brand: brand.name,
slug: brand.slug,
collectionUrl: brand.collectionUrl,
startedAt: brandStartedAt,
completedAt: new Date().toISOString(),
totalProducts: 0,
success: false,
error,
});
}
}
// Save combined JSON (all brands merged)
const combinedFile = path.join(OUTPUT_DIR, "combined.json");
const combinedPayload = {
generatedAt: new Date().toISOString(),
startedAt,
source: "motousher",
siteUrl: "https://www.motousher.com",
brandsScraped: runSummary.length,
totalProducts: combinedProducts.length,
runSummary,
products: combinedProducts,
};
await fs.writeFile(combinedFile, JSON.stringify(combinedPayload, null, 2), "utf8");
console.log(`\n${"=".repeat(60)}`);
console.log("[MOTOUSHER] RUN COMPLETE");
console.log(`${"=".repeat(60)}`);
console.log(`Brands scraped : ${runSummary.length}`);
console.log(`Total products : ${combinedProducts.length}`);
console.log(`Combined file : ${combinedFile}`);
console.log("\nBrand summary:");
for (const s of runSummary) {
const status = s.success ? "OK" : "FAILED";
console.log(` [${status}] ${s.brand} → ${s.totalProducts} products`);
}
return combinedPayload;
}
run().catch((err) => {
console.error("Runner failed:", err.message);
process.exitCode = 1;
});
module.exports = { run };

9
src/business-logic/import-pipeline/test_source/motousher/links.txt
View File

@ -1,9 +0,0 @@
# Motousher Partner Brand Links
# Source: https://www.motousher.com/pages/partner-brands
# Format: Brand Name | Collection URL
All Balls Racing | https://www.motousher.com/collections/all-balls-racing
# Add more brands below as needed, e.g.:
# Acerbis | https://www.motousher.com/collections/acerbis
# Twin Air | https://www.motousher.com/collections/twin-air

254
src/business-logic/import-pipeline/test_source/motousher/scraper.js
View File

@ -1,254 +0,0 @@
/**
* Motousher.com scraper — Shopify JSON API (fast, no browser needed).
*
* Motousher is a Shopify store. Every collection and product has a public
* JSON endpoint, so we use plain fetch instead of Playwright:
*
* Collection listing: /collections/{handle}/products.json?limit=250&page=N
* Product detail: /products/{handle}.json
*
* This is 10-20x faster than Playwright. Playwright is only used as a
* fallback if the JSON API returns nothing.
*/
const BASE_URL = "https://www.motousher.com";
const CONCURRENCY = 3; // parallel product detail fetches — motousher rate-limits above ~5
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function fetchJson(url, attempt = 1) {
try {
const res = await fetch(url, {
headers: {
"accept": "application/json",
"accept-language": "en-US,en;q=0.9",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36",
},
});
if (res.status === 429 || res.status >= 500) {
if (attempt <= 4) {
const wait = attempt * 1500;
console.log(`[RETRY] ${url} → HTTP ${res.status}, retrying in ${wait}ms (attempt ${attempt}/4)`);
await sleep(wait);
return fetchJson(url, attempt + 1);
}
throw new Error(`HTTP ${res.status} after retries`);
}
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return res.json();
} catch (err) {
if (attempt <= 4 && err.code === "UND_ERR_CONNECT_TIMEOUT") {
await sleep(attempt * 1500);
return fetchJson(url, attempt + 1);
}
throw err;
}
}
// ---------------------------------------------------------------------------
// Collection: fetch ALL product handles via paginated JSON API
// ---------------------------------------------------------------------------
async function fetchCollectionProductHandles(collectionHandle) {
const handles = [];
let page = 1;
while (true) {
const url = `${BASE_URL}/collections/${collectionHandle}/products.json?limit=250&page=${page}`;
console.log(`[MOTOUSHER] Fetching collection page ${page}: ${url}`);
const data = await fetchJson(url);
const products = Array.isArray(data?.products) ? data.products : [];
if (!products.length) break;
for (const p of products) {
if (p.handle) handles.push(p.handle);
}
console.log(`[MOTOUSHER] Page ${page} → ${products.length} products (total so far: ${handles.length})`);
if (products.length < 250) break; // last page
page++;
}
return handles;
}
// ---------------------------------------------------------------------------
// Product detail: fetch via /products/{handle}.json
// ---------------------------------------------------------------------------
function toNumber(value) {
if (!value) return null;
const match = String(value).match(/([\d,]+\.?\d*)/);
if (!match) return null;
const n = Number.parseFloat(match[1].replace(/,/g, ""));
return Number.isNaN(n) ? null : n;
}
function bestImageUrl(src) {
if (!src) return null;
// Remove Shopify width constraints to get the original full-size image
return src.replace(/(_\d+x\d*|_\d+x)(\.\w+)(\?|$)/, "$2$3");
}
function normalizeShopifyProduct(raw, brandName, brandSlug, collectionUrl) {
const images = (raw.images || []).map((img) => bestImageUrl(img.src)).filter(Boolean);
const options = (raw.options || []).map((opt) => ({
label: opt.name,
values: (opt.values || []).map((v) => ({ label: v, value: v })),
}));
const variants = (raw.variants || []).map((v) => ({
id: v.id,
title: v.title,
sku: v.sku || null,
price: toNumber(v.price),
compareAtPrice: toNumber(v.compare_at_price),
available: v.available,
options: [v.option1, v.option2, v.option3].filter(Boolean),
weight: v.grams || null,
barcode: v.barcode || null,
}));
const firstVariant = variants[0] || {};
return {
recordType: "product",
source: "motousher",
brand: brandName,
brandSlug,
collectionUrl,
title: raw.title || null,
url: `${BASE_URL}/products/${raw.handle}`,
handle: raw.handle || null,
productType: raw.product_type || null,
vendor: raw.vendor || brandName,
tags: Array.isArray(raw.tags) ? raw.tags : [],
priceRaw: firstVariant.price != null ? String(firstVariant.price) : null,
price: firstVariant.price ?? null,
compareAtPriceRaw: firstVariant.compareAtPrice != null ? String(firstVariant.compareAtPrice) : null,
compareAtPrice: firstVariant.compareAtPrice ?? null,
sku: firstVariant.sku || null,
barcode: firstVariant.barcode || null,
available: raw.variants?.some((v) => v.available) ?? null,
image: images[0] || null,
images,
descriptionHtml: raw.body_html || null,
description: raw.body_html
? raw.body_html.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim()
: null,
options,
variants,
publishedAt: raw.published_at || null,
createdAt: raw.created_at || null,
updatedAt: raw.updated_at || null,
scrapeError: null,
};
}
async function fetchProductDetail(handle, brandName, brandSlug, collectionUrl) {
const url = `${BASE_URL}/products/${handle}.json`;
try {
const data = await fetchJson(url);
if (!data?.product) throw new Error("No product in response");
return normalizeShopifyProduct(data.product, brandName, brandSlug, collectionUrl);
} catch (err) {
return {
recordType: "product",
source: "motousher",
brand: brandName,
brandSlug,
collectionUrl,
handle,
url: `${BASE_URL}/products/${handle}`,
scrapeError: err.message,
};
}
}
// ---------------------------------------------------------------------------
// Concurrency helper
// ---------------------------------------------------------------------------
async function mapWithConcurrency(items, concurrency, worker) {
const results = new Array(items.length);
let index = 0;
async function run() {
while (true) {
const i = index++;
if (i >= items.length) return;
results[i] = await worker(items[i], i);
}
}
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run));
return results;
}
// ---------------------------------------------------------------------------
// Main export: scrape one brand
// ---------------------------------------------------------------------------
async function scrapeMotousherBrand({ name, slug, collectionUrl }, options = {}) {
const limit = options.limit ? Number(options.limit) : null;
const onProgress = typeof options.onProgress === "function" ? options.onProgress : null;
const concurrency = options.concurrency || CONCURRENCY;
// Step 1: get all product handles from collection pages
let handles = await fetchCollectionProductHandles(slug);
console.log(`[MOTOUSHER:${name}] Found ${handles.length} products in collection`);
if (limit && handles.length > limit) {
console.log(`[MOTOUSHER:${name}] Limit applied: fetching first ${limit} of ${handles.length}`);
handles = handles.slice(0, limit);
}
// Step 2: fetch all product details in parallel
let done = 0;
const products = await mapWithConcurrency(handles, concurrency, async (handle, i) => {
const product = await fetchProductDetail(handle, name, slug, collectionUrl);
done++;
if (done % 20 === 0 || done === handles.length) {
console.log(`[MOTOUSHER:${name}] ${done}/${handles.length} products fetched`);
}
if (onProgress) onProgress({ done, total: handles.length, product: product.title || handle });
return product;
});
return products;
}
// ---------------------------------------------------------------------------
// Standalone test
// ---------------------------------------------------------------------------
if (require.main === module) {
const { BRANDS } = require("./brands");
const brand = BRANDS[0];
console.log(`Testing scraper for: ${brand.name}`);
scrapeMotousherBrand(brand, { limit: 5 })
.then((products) => {
console.log(`\nScraped ${products.length} products`);
console.log("Sample product:");
console.log(JSON.stringify(products[0], null, 2));
})
.catch((err) => {
console.error("Scrape failed:", err.message);
process.exitCode = 1;
});
}
module.exports = {
scrapeMotousherBrand,
fetchCollectionProductHandles,
fetchProductDetail,
};