Add Retro Rides source — sitemap scraper + converter + pipeline integration
- New source: retrorides (retrorides.co.in, 1168 products) - Scraper reads all product URLs from WordPress sitemap in one call - Extracts title/SKU/price/stock from JSON-LD, gallery images from data-large_image - Bike brand/model from breadcrumb (case-insensitive BIKE_MAKES normalisation) - Converts Jetpack CDN image URLs to direct retrorides.co.in URLs - 24h cache with RETRORIDES_CACHE_HOURS env var, concurrency=3 default - Converter follows same record.scraped pattern as motousher/dirtstreet - Registered in sources/index.js alongside kyt, brocks-performance, motousher, dirtstreet Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
08f21d9bc9
commit
4b2288187c
@ -2,12 +2,14 @@ const kyt = require("./kyt");
|
||||
const brocksPerformance = require("./brocks-performance");
|
||||
const motousher = require("./motousher");
|
||||
const dirtstreet = require("./dirtstreet");
|
||||
const retrorides = require("./retrorides");
|
||||
|
||||
const sources = {
|
||||
[kyt.sourceKey]: kyt,
|
||||
[brocksPerformance.sourceKey]: brocksPerformance,
|
||||
[motousher.sourceKey]: motousher,
|
||||
[dirtstreet.sourceKey]: dirtstreet,
|
||||
[retrorides.sourceKey]: retrorides,
|
||||
};
|
||||
|
||||
function normalizeSourceKey(sourceKey) {
|
||||
|
||||
@ -0,0 +1,131 @@
|
||||
const path = require("node:path");
|
||||
|
||||
function slugify(str) {
|
||||
return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
}
|
||||
|
||||
function getUploadedImageUrl(imgPath, uploadedImageMap) {
|
||||
if (!uploadedImageMap || typeof uploadedImageMap !== "object") return null;
|
||||
const bySourcePath = uploadedImageMap.bySourcePath || {};
|
||||
return bySourcePath[String(imgPath)]?.url || null;
|
||||
}
|
||||
|
||||
function getImageFileName(imagePath) {
|
||||
try {
|
||||
return path.basename(new URL(imagePath).pathname);
|
||||
} catch {
|
||||
return path.basename(String(imagePath || "").split(/[?#]/)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
function convertRetroRidesRecordToShopifyReady(record, options = {}) {
|
||||
const uploadedImageMap = options.uploadedImageMap || null;
|
||||
|
||||
const title = record.title || "Untitled Product";
|
||||
const sku = record.sku || slugify(title);
|
||||
const handle = record.handle || slugify(`${sku}-${title}`);
|
||||
const price = Number(record.price ?? 0);
|
||||
const quantity = Number(record.quantity ?? 0);
|
||||
const descriptionHtml = record.description || "";
|
||||
|
||||
// Vendor: bike brand that the product is made for (BMW, Ducati, etc.)
|
||||
// Falls back to store name if no bike brand found
|
||||
const vendor = record.bikeBrand || options.brand || "Retro Rides";
|
||||
|
||||
// Product type derived from bike model (e.g., "S1000rr 2023+")
|
||||
const productType = record.bikeModel
|
||||
? `${record.bikeBrand || ""} ${record.bikeModel}`.trim()
|
||||
: "Motorcycle Accessories";
|
||||
|
||||
const imagePaths = Array.isArray(record.images) ? record.images : [];
|
||||
|
||||
const files = imagePaths
|
||||
.map((imgPath) => ({
|
||||
type: "Image",
|
||||
url: getUploadedImageUrl(imgPath, uploadedImageMap) || imgPath,
|
||||
media_content: getImageFileName(imgPath),
|
||||
source_path: imgPath,
|
||||
}))
|
||||
.filter((f) => f.url);
|
||||
|
||||
const variants = [
|
||||
{
|
||||
sku,
|
||||
price,
|
||||
compare_price: null,
|
||||
quantity,
|
||||
optionValues: [],
|
||||
},
|
||||
];
|
||||
|
||||
// Tags: bike brand, bike model, bike categories, WooCommerce tags, sku
|
||||
const rawTags = [
|
||||
vendor,
|
||||
record.bikeModel,
|
||||
...(Array.isArray(record.bikeCategories) ? record.bikeCategories : []),
|
||||
...(Array.isArray(record.tags) ? record.tags : []),
|
||||
sku,
|
||||
];
|
||||
const tags = [...new Set(rawTags.filter(Boolean).map((t) => String(t).trim()))];
|
||||
|
||||
const productId = handle || slugify(`retrorides-${sku}`);
|
||||
|
||||
return {
|
||||
id: productId,
|
||||
source: {
|
||||
productId,
|
||||
sourceKey: "retrorides",
|
||||
url: record.url || null,
|
||||
image_paths: imagePaths,
|
||||
},
|
||||
attributes: {
|
||||
product_name: title,
|
||||
brand: vendor,
|
||||
category: productType,
|
||||
subcategory: record.bikeModel || "",
|
||||
part_number: sku,
|
||||
mfr_part_number: sku,
|
||||
price,
|
||||
compare_price: null,
|
||||
purchase_cost: null,
|
||||
barcode: "",
|
||||
price_group: null,
|
||||
units_per_sku: null,
|
||||
part_description: descriptionHtml,
|
||||
descriptions: descriptionHtml
|
||||
? [{ type: "Market Description", description: descriptionHtml }]
|
||||
: [],
|
||||
files,
|
||||
image_paths: imagePaths,
|
||||
dimensions: [{ weight: 0 }],
|
||||
total_quantity: quantity,
|
||||
inventorydata: { inventory: { main: quantity } },
|
||||
options: [],
|
||||
variants,
|
||||
fitmentTags: {
|
||||
make: record.bikeBrand ? [record.bikeBrand] : [],
|
||||
model: record.bikeModel ? [record.bikeModel] : [],
|
||||
year: [],
|
||||
drive: [],
|
||||
baseModel: [],
|
||||
},
|
||||
tags,
|
||||
handle: slugify(`${vendor}-${title}-${productId}`),
|
||||
source_url: record.url || null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function convertRetroRidesJsonToShopifyProducts(input, options = {}) {
|
||||
const records = Array.isArray(input?.products) ? input.products : [];
|
||||
return records.map((record) => {
|
||||
// Aggregated JSON wraps scraped data inside record.scraped
|
||||
const data = record.scraped || record;
|
||||
return convertRetroRidesRecordToShopifyReady(data, {
|
||||
...options,
|
||||
brand: data.bikeBrand || record.brand || options.brand,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { convertRetroRidesRecordToShopifyReady, convertRetroRidesJsonToShopifyProducts };
|
||||
135
src/business-logic/import-pipeline/sources/retrorides/index.js
Normal file
135
src/business-logic/import-pipeline/sources/retrorides/index.js
Normal file
@ -0,0 +1,135 @@
|
||||
const fs = require("node:fs/promises");
|
||||
const path = require("node:path");
|
||||
const { scrapeRetroRidesProducts } = require("./scraper");
|
||||
const { convertRetroRidesJsonToShopifyProducts } = require("./converter");
|
||||
|
||||
const sourceKey = "retrorides";
|
||||
const label = "Retro Rides";
|
||||
const dataDir = path.join("data", "sources", sourceKey);
|
||||
|
||||
function paths() {
|
||||
return {
|
||||
aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
|
||||
historyJson: path.join(dataDir, "01_products_run_history.json"),
|
||||
downloadedImagesDir: path.join(dataDir, "02_downloaded_product_images"),
|
||||
watermarkState: path.join(dataDir, "03_watermark_state.json"),
|
||||
imageUploadState: path.join(dataDir, "04_shopify_image_upload_state.json"),
|
||||
uploadedMapJson: path.join(dataDir, "04_shopify_uploaded_images_map.json"),
|
||||
shopifyReadyJson: path.join(dataDir, "05_shopify_products_ready.json"),
|
||||
logsDir: path.join(dataDir, "99_run_logs"),
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchWebsiteData({ paths: runPaths }) {
|
||||
// Use cached aggregated JSON if it exists and is fresh (default 24h).
|
||||
// Set RETRORIDES_CACHE_HOURS=0 to always force a full re-scrape.
|
||||
const cacheMaxHours = Number(process.env.RETRORIDES_CACHE_HOURS ?? 24);
|
||||
const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson);
|
||||
|
||||
if (cacheMaxHours > 0) {
|
||||
try {
|
||||
const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
|
||||
if (existing?.generatedAt && Array.isArray(existing?.products) && existing.products.length > 0) {
|
||||
const ageMs = Date.now() - new Date(existing.generatedAt).getTime();
|
||||
const ageHours = ageMs / (1000 * 60 * 60);
|
||||
if (ageHours < cacheMaxHours) {
|
||||
console.log(
|
||||
`[RETRORIDES] Using cached data (${ageHours.toFixed(1)}h old, limit ${cacheMaxHours}h) — ${existing.products.length} products. Set RETRORIDES_CACHE_HOURS=0 to force re-scrape.`
|
||||
);
|
||||
return {
|
||||
analysis: existing.analysis || {},
|
||||
outputJsonPath: absAggregatedPath,
|
||||
historyPath: path.resolve(process.cwd(), runPaths.historyJson),
|
||||
cached: true,
|
||||
};
|
||||
}
|
||||
console.log(`[RETRORIDES] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping all products...`);
|
||||
}
|
||||
} catch {
|
||||
// no cache file yet, proceed with scraping
|
||||
}
|
||||
}
|
||||
|
||||
const concurrency = Number(process.env.RETRORIDES_CONCURRENCY ?? 3);
|
||||
const now = new Date().toISOString();
|
||||
|
||||
console.log(`[RETRORIDES] Scraping all products from retrorides.co.in (concurrency=${concurrency})...`);
|
||||
const rawProducts = await scrapeRetroRidesProducts({ concurrency });
|
||||
|
||||
const successCount = rawProducts.filter((p) => !p.scrapeError).length;
|
||||
const failCount = rawProducts.filter((p) => p.scrapeError).length;
|
||||
|
||||
// Normalize into pipeline-compatible aggregated format.
|
||||
// productSummary.img is the field that shared download/upload utilities read.
|
||||
const allProducts = rawProducts.map((product) => ({
|
||||
productId: product.handle || product.url,
|
||||
sourceKey,
|
||||
brand: product.bikeBrand || "Retro Rides",
|
||||
bikeModel: product.bikeModel || "",
|
||||
productSummary: {
|
||||
id: product.handle,
|
||||
name: product.title,
|
||||
img: product.images || [],
|
||||
cost: { mrp: product.price || 0 },
|
||||
},
|
||||
scraped: product,
|
||||
}));
|
||||
|
||||
const analysis = {
|
||||
timestamp: now,
|
||||
totalProducts: rawProducts.length,
|
||||
totalProductsUnique: allProducts.length,
|
||||
detailSuccess: successCount,
|
||||
detailFailed: failCount,
|
||||
};
|
||||
|
||||
const payload = {
|
||||
generatedAt: now,
|
||||
sourceKey,
|
||||
sourceLabel: label,
|
||||
analysis,
|
||||
products: allProducts,
|
||||
};
|
||||
|
||||
await fs.mkdir(path.dirname(absAggregatedPath), { recursive: true });
|
||||
await fs.writeFile(absAggregatedPath, JSON.stringify(payload, null, 2), "utf8");
|
||||
|
||||
// Append to history
|
||||
let history = [];
|
||||
const absHistoryPath = path.resolve(process.cwd(), runPaths.historyJson);
|
||||
try {
|
||||
const raw = await fs.readFile(absHistoryPath, "utf8");
|
||||
history = JSON.parse(raw);
|
||||
if (!Array.isArray(history)) history = [];
|
||||
} catch {
|
||||
history = [];
|
||||
}
|
||||
history.push(analysis);
|
||||
await fs.writeFile(absHistoryPath, JSON.stringify(history, null, 2), "utf8");
|
||||
|
||||
console.log(`[RETRORIDES] Saved ${allProducts.length} products (${successCount} ok, ${failCount} failed) to ${runPaths.aggregatedJson}`);
|
||||
|
||||
return {
|
||||
analysis,
|
||||
outputJsonPath: absAggregatedPath,
|
||||
historyPath: absHistoryPath,
|
||||
};
|
||||
}
|
||||
|
||||
function convertToShopifyProducts(input, options = {}) {
|
||||
return convertRetroRidesJsonToShopifyProducts(input, {
|
||||
brand: options.brand || label,
|
||||
uploadedImageMap: options.uploadedImageMap,
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
sourceKey,
|
||||
label,
|
||||
defaultBrand: "Retro Rides",
|
||||
defaultImageBaseUrl: "",
|
||||
envImageBaseUrl: "RETRORIDES_IMAGE_BASE_URL",
|
||||
paths,
|
||||
fetchWebsiteData,
|
||||
convertToShopifyProducts,
|
||||
};
|
||||
207
src/business-logic/import-pipeline/sources/retrorides/scraper.js
Normal file
207
src/business-logic/import-pipeline/sources/retrorides/scraper.js
Normal file
@ -0,0 +1,207 @@
|
||||
const FETCH_HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept-Encoding": "identity",
|
||||
"Connection": "keep-alive",
|
||||
"Cache-Control": "no-cache",
|
||||
};
|
||||
|
||||
const SITEMAP_URL = "https://retrorides.co.in/wp-sitemap-posts-product-1.xml";
|
||||
|
||||
const BIKE_MAKES_LOWER = new Set(["bmw", "ducati", "aprilia", "honda", "kawasaki", "triumph", "suzuki", "yamaha", "ktm", "husqvarna"]);
|
||||
// Canonical display names for each bike make
|
||||
const BIKE_MAKE_CANONICAL = { bmw: "BMW", ducati: "Ducati", aprilia: "Aprilia", honda: "Honda", kawasaki: "Kawasaki", triumph: "Triumph", suzuki: "Suzuki", yamaha: "Yamaha", ktm: "KTM", husqvarna: "Husqvarna" };
|
||||
function normalizeBikeMake(name) {
|
||||
const key = String(name || "").toLowerCase().trim();
|
||||
return BIKE_MAKES_LOWER.has(key) ? BIKE_MAKE_CANONICAL[key] : "";
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(function(resolve) { setTimeout(resolve, ms); });
|
||||
}
|
||||
|
||||
async function fetchText(url, maxAttempts) {
|
||||
maxAttempts = maxAttempts || 4;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
const response = await fetch(url, { headers: FETCH_HEADERS });
|
||||
if (response.status === 429) {
|
||||
const waitMs = attempt * 4000;
|
||||
console.log("[RETRY] " + url + " HTTP 429, retrying in " + waitMs + "ms...");
|
||||
await sleep(waitMs);
|
||||
continue;
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new Error("HTTP " + response.status + " " + response.statusText);
|
||||
}
|
||||
return await response.text();
|
||||
} catch (err) {
|
||||
if (attempt === maxAttempts) throw err;
|
||||
await sleep(attempt * 1500);
|
||||
}
|
||||
}
|
||||
throw new Error("fetchText: exhausted retries");
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(str) {
|
||||
const ENT = {
|
||||
"&": "&", "&": "&", "<": "<", ">": ">",
|
||||
""": '"', """: '"', "'": "'",
|
||||
"’": "'", "‘": "'", "“": '"', "”": '"',
|
||||
" ": " ", "’": "'", "‘": "'", "”": '"', "“": '"',
|
||||
};
|
||||
let s = String(str || "");
|
||||
for (let i = 0; i < 2; i++) {
|
||||
s = s.replace(/&[#a-z0-9]+;/gi, function(e) { return ENT[e] !== undefined ? ENT[e] : e; });
|
||||
}
|
||||
return s.trim();
|
||||
}
|
||||
|
||||
function stripHtml(html) {
|
||||
return String(html || "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function slugify(str) {
|
||||
return String(str || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
}
|
||||
|
||||
function extractJsonLd(html) {
|
||||
const blocks = Array.from(html.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi));
|
||||
for (const block of blocks) {
|
||||
try {
|
||||
const parsed = JSON.parse(block[1]);
|
||||
const graph = Array.isArray(parsed && parsed["@graph"]) ? parsed["@graph"] : [parsed];
|
||||
const product = graph.find(function(n) { return n && n["@type"] === "Product"; });
|
||||
const breadcrumb = graph.find(function(n) { return n && n["@type"] === "BreadcrumbList"; });
|
||||
if (product) return { product: product, breadcrumb: breadcrumb };
|
||||
} catch (e) {}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function extractGalleryImages(html) {
|
||||
const matches = Array.from(html.matchAll(/data-large_image=["']([^"']+)["']/g)).map(function(m) { return m[1]; });
|
||||
const seen = new Set();
|
||||
return matches
|
||||
.map(function(raw) {
|
||||
const decoded = decodeHtmlEntities(raw);
|
||||
const cdnMatch = decoded.match(/^https?:\/\/i\d+\.wp\.com\/(.+?)(?:\?.*)?$/);
|
||||
const url = cdnMatch ? "https://" + cdnMatch[1] : decoded.split("?")[0];
|
||||
return url;
|
||||
})
|
||||
.filter(function(url) {
|
||||
if (!url || url.indexOf("wp-content/uploads") === -1) return false;
|
||||
if (seen.has(url)) return false;
|
||||
seen.add(url);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function extractDescription(html) {
|
||||
const panelMatch = html.match(/id=["']tab-description["'][^>]*>([\s\S]*?)<\/div>\s*(?:<\/div>|\s*<div\s+class=["']woocommerce-Tabs)/i);
|
||||
if (panelMatch) {
|
||||
const raw = panelMatch[1].replace(/<h2[^>]*>Description<\/h2>/gi, "").trim();
|
||||
return decodeHtmlEntities(stripHtml(raw)).replace(/\s{2,}/g, " ").trim();
|
||||
}
|
||||
const shortMatch = html.match(/short-description["'][^>]*>([\s\S]*?)<\/div>/i);
|
||||
if (shortMatch) {
|
||||
const raw = decodeHtmlEntities(stripHtml(shortMatch[1])).trim();
|
||||
if (raw && raw !== " ") return raw;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
async function scrapeProductPage(url) {
|
||||
const html = await fetchText(url);
|
||||
const jsonLd = extractJsonLd(html);
|
||||
const jsonLdProduct = jsonLd.product;
|
||||
const breadcrumb = jsonLd.breadcrumb;
|
||||
|
||||
if (!jsonLdProduct) {
|
||||
return { url: url, scrapeError: "No JSON-LD Product found on page" };
|
||||
}
|
||||
|
||||
const title = decodeHtmlEntities(jsonLdProduct.name || "");
|
||||
const sku = String(jsonLdProduct.sku || "").trim();
|
||||
|
||||
const offer = Array.isArray(jsonLdProduct.offers) ? jsonLdProduct.offers[0] : jsonLdProduct.offers;
|
||||
const priceSpec = offer && offer.priceSpecification && offer.priceSpecification[0];
|
||||
const price = Number((priceSpec && priceSpec.price) || (offer && offer.price) || 0);
|
||||
const currency = (priceSpec && priceSpec.priceCurrency) || (offer && offer.priceCurrency) || "INR";
|
||||
const availability = String((offer && offer.availability) || "");
|
||||
const inStock = /InStock|BackOrder/i.test(availability);
|
||||
const quantity = inStock ? 10 : 0;
|
||||
|
||||
const crumbs = ((breadcrumb && breadcrumb.itemListElement) || []).map(function(item) {
|
||||
return decodeHtmlEntities((item && item.item && item.item.name) || (item && item.name) || "");
|
||||
});
|
||||
|
||||
const rawBrand = crumbs[1] || "";
|
||||
const rawModel = crumbs[2] || "";
|
||||
const bikeBrand = normalizeBikeMake(rawBrand);
|
||||
const bikeModel = bikeBrand ? rawModel : "";
|
||||
|
||||
const galleryImages = extractGalleryImages(html);
|
||||
const jsonLdImage = jsonLdProduct.image ? String(jsonLdProduct.image) : "";
|
||||
const images = galleryImages.length > 0 ? galleryImages : (jsonLdImage ? [jsonLdImage] : []);
|
||||
|
||||
const tags = Array.from(html.matchAll(/rel=["']tag["'][^>]*>([^<]+)<\/a>/g))
|
||||
.map(function(m) { return m[1].trim(); })
|
||||
.filter(function(t, i, arr) { return t && arr.indexOf(t) === i; });
|
||||
|
||||
const handleMatch = url.match(/\/product\/([^\/]+)\/?$/);
|
||||
const handle = handleMatch ? handleMatch[1] : slugify(sku || title);
|
||||
const description = extractDescription(html);
|
||||
|
||||
return {
|
||||
url: url, handle: handle, title: title, sku: sku,
|
||||
price: price, currency: currency, quantity: quantity,
|
||||
availability: availability, inStock: inStock, images: images,
|
||||
description: description, bikeBrand: bikeBrand, bikeModel: bikeModel,
|
||||
bikeCategories: crumbs.slice(1, -1).filter(Boolean),
|
||||
tags: tags, brand: bikeBrand || "Retro Rides", scrapeError: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function getAllProductUrls() {
|
||||
const xml = await fetchText(SITEMAP_URL);
|
||||
return Array.from(xml.matchAll(/<loc>([^<]+)<\/loc>/g))
|
||||
.map(function(m) { return m[1].trim(); })
|
||||
.filter(function(u) { return u.indexOf("/product/") !== -1; });
|
||||
}
|
||||
|
||||
async function mapWithConcurrency(items, concurrency, fn) {
|
||||
const results = new Array(items.length);
|
||||
let nextIndex = 0;
|
||||
async function worker() {
|
||||
while (true) {
|
||||
const i = nextIndex++;
|
||||
if (i >= items.length) return;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
|
||||
async function scrapeRetroRidesProducts(options) {
|
||||
const concurrency = (options && options.concurrency) || 3;
|
||||
const urls = await getAllProductUrls();
|
||||
console.log("[RETRORIDES] Found " + urls.length + " product URLs from sitemap");
|
||||
const products = await mapWithConcurrency(urls, concurrency, async function(url, index) {
|
||||
try {
|
||||
const product = await scrapeProductPage(url);
|
||||
if ((index + 1) % 25 === 0 || index === urls.length - 1) {
|
||||
console.log("[RETRORIDES] " + (index + 1) + "/" + urls.length + " scraped");
|
||||
}
|
||||
return product;
|
||||
} catch (err) {
|
||||
console.log("[RETRORIDES] FAIL " + url + ": " + err.message);
|
||||
return { url: url, scrapeError: err.message };
|
||||
}
|
||||
});
|
||||
return products;
|
||||
}
|
||||
|
||||
module.exports = { scrapeRetroRidesProducts: scrapeRetroRidesProducts, getAllProductUrls: getAllProductUrls, scrapeProductPage: scrapeProductPage };
|
||||
Loading…
x
Reference in New Issue
Block a user