From bf207e48a13cc087aa31b1997cab9c8981c9afcb Mon Sep 17 00:00:00 2001
From: MOHAN <mohan326856@gmail.com>
Date: Mon, 13 Jul 2026 18:18:35 +0530
Subject: [PATCH] fix(bikegear): fix 2 scraper issues found during test run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

1. parseProductJsonLd: remove sku requirement — out-of-stock products have
   Product JSON-LD but no sku field, causing false "No product JSON-LD found" errors.
   Now picks block with offers.price or description instead.

2. extractProductImages: add fallback for non-standard image paths — products
   with images at /catalog/KTM/... or other custom paths were falling through
   to JSON-LD fallback (1 image). Now also scans the product-img anchor HTML.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../sources/bikegear/scraper.js               | 40 ++++++++++++++-----
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js
index a70579d..dbb8381 100644
--- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js
+++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js
@@ -127,21 +127,25 @@ async function scrapeBrandProductUrls(brand) {
   return result;
 }
 
-/** Parse the product JSON-LD block that contains sku/offers fields. */
+/** Parse the best product JSON-LD block from the page.
+ *  Prefers the block with offers.price (detailed one), then any Product block.
+ *  Out-of-stock products omit the sku field so we don't require it. */
 function parseProductJsonLd(html) {
   const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
+  const candidates = [];
   for (const b of blocks) {
     try {
       const data = JSON.parse(b[1]);
-      // Use the block that has sku field (the more detailed one)
-      if (data["@type"] === "Product" && data.sku) {
-        return data;
+      if (data["@type"] === "Product") {
+        candidates.push(data);
       }
     } catch {
       // malformed JSON-LD — skip
     }
   }
-  return null;
+  if (!candidates.length) return null;
+  // Prefer block that has price info (in-stock) or description (detailed one)
+  return candidates.find((d) => d.offers?.price || d.description) || candidates[0];
 }
 
 /** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */
@@ -152,31 +156,45 @@ function extractProductId(jsonLd) {
 }
 
 /**
- * Extract all unique 800x800 gallery images for the current product from the page HTML.
- * Falls back to 900x900 or 500x500 if 800x800 not found.
+ * Extract all unique gallery images for the current product from the page HTML.
+ * Strategy 1: match data-src on the product-img anchor (the main image carousel).
+ * Strategy 2 (fallback): JSON-LD image field.
+ * Skips tiny thumbnails; upgrades 500x500 → 800x800 for consistency.
  */
 function extractProductImages(html, productId, jsonLdImage) {
   const seen = new Set();
   const images = [];
 
+  // Strategy 1a: images in /catalog/products/{productId}/ (standard path)
   if (productId) {
-    // Match data-src attributes belonging to this product's images
     const re = new RegExp(
       `data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`,
       "g"
     );
     for (const m of html.matchAll(re)) {
       const url = m[1];
-      // Prefer large sizes, skip tiny thumbnails
       if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
         seen.add(url);
-        // Upgrade 500x500 → 800x800
         images.push(url.replace("-500x500.", "-800x800."));
       }
     }
   }
 
-  // Fallback: use the JSON-LD image if we found nothing
+  // Strategy 1b: images on the product-img anchor (catches non-standard paths like /catalog/KTM/...)
+  if (!images.length) {
+    const productImgBlock = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/);
+    if (productImgBlock) {
+      for (const m of productImgBlock[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) {
+        const url = m[1];
+        if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
+          seen.add(url);
+          images.push(url.replace("-500x500.", "-800x800."));
+        }
+      }
+    }
+  }
+
+  // Fallback: JSON-LD image
   if (!images.length && jsonLdImage) {
     images.push(jsonLdImage);
   }