fix(bikegear): fix 2 scraper issues found during test run

1. parseProductJsonLd: remove sku requirement — out-of-stock products have
   Product JSON-LD but no sku field, causing false "No product JSON-LD found" errors.
   Now picks block with offers.price or description instead.

2. extractProductImages: add fallback for non-standard image paths — products
   with images at /catalog/KTM/... or other custom paths were falling through
   to JSON-LD fallback (1 image). Now also scans the product-img anchor HTML.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-07-13 18:18:35 +05:30
parent 0c66b4b7ee
commit bf207e48a1

View File

@ -127,21 +127,25 @@ async function scrapeBrandProductUrls(brand) {
return result; return result;
} }
/** Parse the product JSON-LD block that contains sku/offers fields. */ /** Parse the best product JSON-LD block from the page.
* Prefers the block with offers.price (detailed one), then any Product block.
* Out-of-stock products omit the sku field so we don't require it. */
function parseProductJsonLd(html) { function parseProductJsonLd(html) {
const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)]; const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
const candidates = [];
for (const b of blocks) { for (const b of blocks) {
try { try {
const data = JSON.parse(b[1]); const data = JSON.parse(b[1]);
// Use the block that has sku field (the more detailed one) if (data["@type"] === "Product") {
if (data["@type"] === "Product" && data.sku) { candidates.push(data);
return data;
} }
} catch { } catch {
// malformed JSON-LD — skip // malformed JSON-LD — skip
} }
} }
return null; if (!candidates.length) return null;
// Prefer block that has price info (in-stock) or description (detailed one)
return candidates.find((d) => d.offers?.price || d.description) || candidates[0];
} }
/** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */ /** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */
@ -152,31 +156,45 @@ function extractProductId(jsonLd) {
} }
/** /**
* Extract all unique 800x800 gallery images for the current product from the page HTML. * Extract all unique gallery images for the current product from the page HTML.
* Falls back to 900x900 or 500x500 if 800x800 not found. * Strategy 1: match data-src on the product-img anchor (the main image carousel).
* Strategy 2 (fallback): JSON-LD image field.
* Skips tiny thumbnails; upgrades 500x500 800x800 for consistency.
*/ */
function extractProductImages(html, productId, jsonLdImage) { function extractProductImages(html, productId, jsonLdImage) {
const seen = new Set(); const seen = new Set();
const images = []; const images = [];
// Strategy 1a: images in /catalog/products/{productId}/ (standard path)
if (productId) { if (productId) {
// Match data-src attributes belonging to this product's images
const re = new RegExp( const re = new RegExp(
`data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`, `data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`,
"g" "g"
); );
for (const m of html.matchAll(re)) { for (const m of html.matchAll(re)) {
const url = m[1]; const url = m[1];
// Prefer large sizes, skip tiny thumbnails
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) { if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
seen.add(url); seen.add(url);
// Upgrade 500x500 → 800x800
images.push(url.replace("-500x500.", "-800x800.")); images.push(url.replace("-500x500.", "-800x800."));
} }
} }
} }
// Fallback: use the JSON-LD image if we found nothing // Strategy 1b: images on the product-img anchor (catches non-standard paths like /catalog/KTM/...)
if (!images.length) {
const productImgBlock = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/);
if (productImgBlock) {
for (const m of productImgBlock[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) {
const url = m[1];
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
seen.add(url);
images.push(url.replace("-500x500.", "-800x800."));
}
}
}
}
// Fallback: JSON-LD image
if (!images.length && jsonLdImage) { if (!images.length && jsonLdImage) {
images.push(jsonLdImage); images.push(jsonLdImage);
} }