admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): fix 2 scraper issues found during test run

Browse Source 
1. parseProductJsonLd: remove sku requirement — out-of-stock products have
   Product JSON-LD but no sku field, causing false "No product JSON-LD found" errors.
   Now picks block with offers.price or description instead.

2. extractProductImages: add fallback for non-standard image paths — products
   with images at /catalog/KTM/... or other custom paths were falling through
   to JSON-LD fallback (1 image). Now also scans the product-img anchor HTML.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-07-13 18:18:35 +05:30
parent 0c66b4b7ee
commit bf207e48a1
1 changed files with 29 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -127,21 +127,25 @@ async function scrapeBrandProductUrls(brand) {
return result;
}
/** Parse the product JSON-LD block that contains sku/offers fields. */
/** Parse the best product JSON-LD block from the page.
* Prefers the block with offers.price (detailed one), then any Product block.
* Out-of-stock products omit the sku field so we don't require it. */
function parseProductJsonLd(html) {
const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
const candidates = [];
for (const b of blocks) {
try {
const data = JSON.parse(b[1]);
// Use the block that has sku field (the more detailed one)
if (data["@type"] === "Product" && data.sku) {
return data;
if (data["@type"] === "Product") {
candidates.push(data);
}
} catch {
// malformed JSON-LD — skip
}
}
return null;
if (!candidates.length) return null;
// Prefer block that has price info (in-stock) or description (detailed one)
return candidates.find((d) => d.offers?.price || d.description) || candidates[0];
}
/** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */
@ -152,31 +156,45 @@ function extractProductId(jsonLd) {
}
/**
* Extract all unique 800x800 gallery images for the current product from the page HTML.
* Falls back to 900x900 or 500x500 if 800x800 not found.
* Extract all unique gallery images for the current product from the page HTML.
* Strategy 1: match data-src on the product-img anchor (the main image carousel).
* Strategy 2 (fallback): JSON-LD image field.
* Skips tiny thumbnails; upgrades 500x500 → 800x800 for consistency.
*/
function extractProductImages(html, productId, jsonLdImage) {
const seen = new Set();
const images = [];
// Strategy 1a: images in /catalog/products/{productId}/ (standard path)
if (productId) {
// Match data-src attributes belonging to this product's images
const re = new RegExp(
`data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`,
"g"
);
for (const m of html.matchAll(re)) {
const url = m[1];
// Prefer large sizes, skip tiny thumbnails
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
seen.add(url);
// Upgrade 500x500 → 800x800
images.push(url.replace("-500x500.", "-800x800."));
}
}
}
// Fallback: use the JSON-LD image if we found nothing
// Strategy 1b: images on the product-img anchor (catches non-standard paths like /catalog/KTM/...)
if (!images.length) {
const productImgBlock = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/);
if (productImgBlock) {
for (const m of productImgBlock[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) {
const url = m[1];
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
seen.add(url);
images.push(url.replace("-500x500.", "-800x800."));
}
}
}
}
// Fallback: JSON-LD image
if (!images.length && jsonLdImage) {
images.push(jsonLdImage);
}