fix(bikegear): fix 2 scraper issues found during test run
1. parseProductJsonLd: remove sku requirement — out-of-stock products have Product JSON-LD but no sku field, causing false "No product JSON-LD found" errors. Now picks block with offers.price or description instead. 2. extractProductImages: add fallback for non-standard image paths — products with images at /catalog/KTM/... or other custom paths were falling through to JSON-LD fallback (1 image). Now also scans the product-img anchor HTML. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
0c66b4b7ee
commit
bf207e48a1
@ -127,21 +127,25 @@ async function scrapeBrandProductUrls(brand) {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Parse the product JSON-LD block that contains sku/offers fields. */
|
/** Parse the best product JSON-LD block from the page.
|
||||||
|
* Prefers the block with offers.price (detailed one), then any Product block.
|
||||||
|
* Out-of-stock products omit the sku field so we don't require it. */
|
||||||
function parseProductJsonLd(html) {
|
function parseProductJsonLd(html) {
|
||||||
const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
|
const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
|
||||||
|
const candidates = [];
|
||||||
for (const b of blocks) {
|
for (const b of blocks) {
|
||||||
try {
|
try {
|
||||||
const data = JSON.parse(b[1]);
|
const data = JSON.parse(b[1]);
|
||||||
// Use the block that has sku field (the more detailed one)
|
if (data["@type"] === "Product") {
|
||||||
if (data["@type"] === "Product" && data.sku) {
|
candidates.push(data);
|
||||||
return data;
|
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// malformed JSON-LD — skip
|
// malformed JSON-LD — skip
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
if (!candidates.length) return null;
|
||||||
|
// Prefer block that has price info (in-stock) or description (detailed one)
|
||||||
|
return candidates.find((d) => d.offers?.price || d.description) || candidates[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */
|
/** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */
|
||||||
@ -152,31 +156,45 @@ function extractProductId(jsonLd) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extract all unique 800x800 gallery images for the current product from the page HTML.
|
* Extract all unique gallery images for the current product from the page HTML.
|
||||||
* Falls back to 900x900 or 500x500 if 800x800 not found.
|
* Strategy 1: match data-src on the product-img anchor (the main image carousel).
|
||||||
|
* Strategy 2 (fallback): JSON-LD image field.
|
||||||
|
* Skips tiny thumbnails; upgrades 500x500 → 800x800 for consistency.
|
||||||
*/
|
*/
|
||||||
function extractProductImages(html, productId, jsonLdImage) {
|
function extractProductImages(html, productId, jsonLdImage) {
|
||||||
const seen = new Set();
|
const seen = new Set();
|
||||||
const images = [];
|
const images = [];
|
||||||
|
|
||||||
|
// Strategy 1a: images in /catalog/products/{productId}/ (standard path)
|
||||||
if (productId) {
|
if (productId) {
|
||||||
// Match data-src attributes belonging to this product's images
|
|
||||||
const re = new RegExp(
|
const re = new RegExp(
|
||||||
`data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`,
|
`data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`,
|
||||||
"g"
|
"g"
|
||||||
);
|
);
|
||||||
for (const m of html.matchAll(re)) {
|
for (const m of html.matchAll(re)) {
|
||||||
const url = m[1];
|
const url = m[1];
|
||||||
// Prefer large sizes, skip tiny thumbnails
|
|
||||||
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
|
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
|
||||||
seen.add(url);
|
seen.add(url);
|
||||||
// Upgrade 500x500 → 800x800
|
|
||||||
images.push(url.replace("-500x500.", "-800x800."));
|
images.push(url.replace("-500x500.", "-800x800."));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: use the JSON-LD image if we found nothing
|
// Strategy 1b: images on the product-img anchor (catches non-standard paths like /catalog/KTM/...)
|
||||||
|
if (!images.length) {
|
||||||
|
const productImgBlock = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/);
|
||||||
|
if (productImgBlock) {
|
||||||
|
for (const m of productImgBlock[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) {
|
||||||
|
const url = m[1];
|
||||||
|
if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
|
||||||
|
seen.add(url);
|
||||||
|
images.push(url.replace("-500x500.", "-800x800."));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: JSON-LD image
|
||||||
if (!images.length && jsonLdImage) {
|
if (!images.length && jsonLdImage) {
|
||||||
images.push(jsonLdImage);
|
images.push(jsonLdImage);
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user