Race-Nation-Shopify-Backend/src/business-logic/import-pipeline/sources/bikegear/scraper.js

/**
 * BikeGear.in scraper (OpenCart / Journal3 theme).
 * 2-phase: listing pages → product URLs, then detail pages → full product data.
 *
 * Prices are in INR (Indian Rupees) as scraped from the site.
 *
 * Fetch modes (set in .env):
 *   BIKEGEAR_FETCH_MODE=browser  — use Playwright Chromium (default; bypasses Cloudflare on VPS IPs)
 *   BIKEGEAR_FETCH_MODE=http     — use axios (works only from residential/whitelisted IPs)
 *   BIKEGEAR_PROXY_URL=http://user:pass@host:port  — proxy for http mode
 */

const axios = require("axios");
const { BRANDS } = require("./brands");

const BASE_URL = "https://bikegear.in";
// Browser mode must run sequentially to avoid Cloudflare 429s — 1 page at a time
const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 3 : 1));
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
// Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";

console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://<redacted>@")})` : ""}`);

// ── Browser pool (Playwright persistent context) ───────────────────────────
// Uses launchPersistentContext so the browser profile (cookies, cache, fingerprint)
// survives across runs on disk — Cloudflare sees the same "user" every time.

const path = require("node:path");
const fs = require("node:fs/promises");

// Profile dir persists between runs — Cloudflare builds trust over multiple visits
const BROWSER_PROFILE_PATH = path.resolve("data/sources/bikegear/.browser-profile");

// Patches injected into every page to hide headless Chromium signals
const STEALTH_SCRIPT = `
  (() => {
    // Hide webdriver flag
    Object.defineProperty(navigator, 'webdriver', { get: () => false });
    // Fake plugins so navigator.plugins.length > 0
    const fakePlugin = (name, file, mimes) => {
      const p = Object.create(Plugin.prototype);
      Object.defineProperty(p, 'name', { get: () => name });
      Object.defineProperty(p, 'filename', { get: () => file });
      Object.defineProperty(p, 'length', { get: () => mimes.length });
      return p;
    };
    const plugins = [
      fakePlugin('Chrome PDF Plugin', 'internal-pdf-viewer', ['application/x-google-chrome-pdf']),
      fakePlugin('Chrome PDF Viewer', 'mhjfbmdgcfjbbpaeojofohoefgiehjai', ['application/pdf']),
      fakePlugin('Native Client', 'internal-nacl-plugin', ['application/x-nacl']),
    ];
    Object.defineProperty(navigator, 'plugins', { get: () => plugins });
    // Fake chrome runtime so window.chrome is defined
    if (!window.chrome) window.chrome = { runtime: {}, loadTimes: () => {}, csi: () => {}, app: {} };
    // Consistent languages
    Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
  })();
`;

let _context = null; // launchPersistentContext = browser + context in one

async function getBrowserContext() {
  if (_context) return _context;

  const { chromium } = require("playwright");
  await fs.mkdir(BROWSER_PROFILE_PATH, { recursive: true });

  _context = await chromium.launchPersistentContext(BROWSER_PROFILE_PATH, {
    headless: true,
    args: [
      "--no-sandbox",
      "--disable-setuid-sandbox",
      "--disable-blink-features=AutomationControlled",
    ],
    userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    locale: "en-US",
    viewport: { width: 1280, height: 800 },
    extraHTTPHeaders: { "Accept-Language": "en-US,en;q=0.9" },
  });

  // Apply stealth patches to every new page opened in this context
  await _context.addInitScript(STEALTH_SCRIPT);

  console.log("[BIKEGEAR] Browser started with persistent profile.");
  return _context;
}

async function closeBrowser() {
  if (_context) {
    await _context.close().catch(() => {});
    _context = null;
    console.log("[BIKEGEAR] Browser closed. Profile saved to disk.");
  }
}

/** Close all open pages to free memory — profile stays alive on disk. */
async function recycleBrowserContext() {
  if (!_context) return;
  console.log("[BIKEGEAR] Cleaning up browser pages to free memory...");
  for (const page of _context.pages()) {
    await page.close().catch(() => {});
  }
}

async function fetchHtmlWithBrowser(url, attempt = 1) {
  const context = await getBrowserContext();
  const page = await context.newPage();
  try {
    const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
    const status = response?.status() ?? 0;

    if (status === 429 || status >= 500) {
      if (attempt <= 4) {
        const wait = attempt * 3000;
        console.log(`[BIKEGEAR] RETRY(browser) ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
        await sleep(wait);
        return fetchHtmlWithBrowser(url, attempt + 1);
      }
      throw new Error(`HTTP ${status} after retries`);
    }
    if (status && status !== 200 && status !== 304) throw new Error(`HTTP ${status}`);

    // Wait for Cloudflare JS challenge to resolve, then add polite delay
    await page.waitForTimeout(1500);
    const html = await page.content();
    if (BROWSER_REQUEST_DELAY > 0) await sleep(BROWSER_REQUEST_DELAY);
    return html;
  } catch (err) {
    if (attempt <= 4 && (err.message?.includes("timeout") || err.message?.includes("net::"))) {
      await sleep(attempt * 2000);
      return fetchHtmlWithBrowser(url, attempt + 1);
    }
    throw err;
  } finally {
    await page.close().catch(() => {});
  }
}

// ── HTTP fetch (axios) ─────────────────────────────────────────────────────

const FETCH_HEADERS = {
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  "Accept-Language": "en-US,en;q=0.5",
  "Cache-Control": "no-cache",
  "Pragma": "no-cache",
  "Upgrade-Insecure-Requests": "1",
  "Sec-Fetch-Dest": "document",
  "Sec-Fetch-Mode": "navigate",
  "Sec-Fetch-Site": "none",
  "Sec-Fetch-User": "?1",
  "Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
  "Sec-CH-UA-Mobile": "?0",
  "Sec-CH-UA-Platform": '"Windows"',
};

function buildProxyConfig(proxyUrl) {
  const u = new URL(proxyUrl);
  const cfg = { host: u.hostname, port: parseInt(u.port, 10), protocol: u.protocol.replace(":", "") };
  if (u.username) cfg.auth = { username: decodeURIComponent(u.username), password: decodeURIComponent(u.password) };
  return cfg;
}

async function fetchHtmlWithHttp(url, attempt = 1) {
  try {
    const opts = { headers: FETCH_HEADERS, timeout: 30000, responseType: "text", maxRedirects: 5 };
    if (PROXY_URL) opts.proxy = buildProxyConfig(PROXY_URL);
    const res = await axios.get(url, opts);
    return res.data;
  } catch (err) {
    const status = err.response?.status;
    if ((status === 429 || status >= 500) && attempt <= 4) {
      const wait = attempt * 2000;
      console.log(`[BIKEGEAR] RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
      await sleep(wait);
      return fetchHtmlWithHttp(url, attempt + 1);
    }
    if (status) throw new Error(`HTTP ${status}`);
    const isNetwork = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"].includes(err.code) ||
      err.message?.includes("timeout") || err.message?.includes("connect");
    if (isNetwork && attempt <= 4) {
      await sleep(attempt * 2000);
      return fetchHtmlWithHttp(url, attempt + 1);
    }
    throw err;
  }
}

// ── Unified fetch entry point ──────────────────────────────────────────────

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function fetchHtml(url, attempt = 1) {
  return FETCH_MODE === "browser"
    ? fetchHtmlWithBrowser(url, attempt)
    : fetchHtmlWithHttp(url, attempt);
}

/** Run up to `limit` tasks from `items` at a time. */
async function runConcurrent(items, fn, limit) {
  const results = [];
  let idx = 0;

  async function worker() {
    while (idx < items.length) {
      const i = idx++;
      results[i] = await fn(items[i], i);
    }
  }

  const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
  await Promise.all(workers);
  return results;
}

/** Extract the max page number from a listing page's HTML. */
function extractMaxPage(html) {
  const pageNums = [...html.matchAll(/\?page=(\d+)/g)].map((m) => parseInt(m[1], 10));
  return pageNums.length ? Math.max(...pageNums) : 1;
}

/** Extract product detail page URLs from a brand listing page HTML. */
function extractProductUrls(html) {
  const urls = new Set();
  // Product links have class="product-img" — these are the card image anchors
  for (const m of html.matchAll(/href="(https:\/\/bikegear\.in\/[^"]+)"\s+class="product-img/g)) {
    const url = m[1];
    // Exclude manufacturer/category/route pages — product slugs are plain paths
    if (!url.includes("route=") && !url.includes("manufacturer") && !url.includes("category")) {
      urls.add(url);
    }
  }
  return [...urls];
}

/** Scrape all listing pages for a brand and collect product detail URLs. */
async function scrapeBrandProductUrls(brand) {
  console.log(`[BIKEGEAR][${brand.name}] Fetching page 1...`);
  let html;
  try {
    html = await fetchHtml(brand.url);
  } catch (err) {
    console.warn(`[BIKEGEAR][${brand.name}] Failed page 1: ${err.message}`);
    return [];
  }

  const maxPage = extractMaxPage(html);
  const urls = new Set(extractProductUrls(html));

  if (maxPage > 1) {
    console.log(`[BIKEGEAR][${brand.name}] Found ${maxPage} pages — fetching pages 2–${maxPage}...`);
    const pageNums = Array.from({ length: maxPage - 1 }, (_, i) => i + 2);

    await runConcurrent(pageNums, async (pageNum) => {
      const pageUrl = `${brand.url}?page=${pageNum}`;
      try {
        const pageHtml = await fetchHtml(pageUrl);
        for (const u of extractProductUrls(pageHtml)) urls.add(u);
      } catch (err) {
        console.warn(`[BIKEGEAR][${brand.name}] Failed page ${pageNum}: ${err.message}`);
      }
    }, LISTING_CONCURRENCY);
  }

  const result = [...urls];
  console.log(`[BIKEGEAR][${brand.name}] Collected ${result.length} product URLs across ${maxPage} page(s).`);
  return result;
}

/** Parse the best product JSON-LD block from the page.
 *  Prefers the block with offers.price (detailed one), then any Product block.
 *  Out-of-stock products omit the sku field so we don't require it. */
function parseProductJsonLd(html) {
  const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
  const candidates = [];
  for (const b of blocks) {
    try {
      const data = JSON.parse(b[1]);
      if (data["@type"] === "Product") {
        candidates.push(data);
      }
    } catch {
      // malformed JSON-LD — skip
    }
  }
  if (!candidates.length) return null;
  // Prefer block that has price info (in-stock) or description (detailed one)
  return candidates.find((d) => d.offers?.price || d.description) || candidates[0];
}

/** Extract product ID from a JSON-LD image URL like /catalog/products/2001191/1-900x900.jpg */
function extractProductId(jsonLd) {
  const imgUrl = jsonLd?.image || "";
  const m = imgUrl.match(/\/catalog\/products\/(\d+)\//);
  return m ? m[1] : null;
}

/**
 * Extract all unique gallery images for the current product from the page HTML.
 * Strategy 1: match data-src on the product-img anchor (the main image carousel).
 * Strategy 2 (fallback): JSON-LD image field.
 * Skips tiny thumbnails; upgrades 500x500 → 800x800 for consistency.
 */
function extractProductImages(html, productId, jsonLdImage) {
  const seen = new Set();
  const images = [];

  // Strategy 1a: images in /catalog/products/{productId}/ (standard path)
  if (productId) {
    const re = new RegExp(
      `data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`,
      "g"
    );
    for (const m of html.matchAll(re)) {
      const url = m[1];
      if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
        seen.add(url);
        images.push(url.replace("-500x500.", "-800x800."));
      }
    }
  }

  // Strategy 1b: images on the product-img anchor (catches non-standard paths like /catalog/KTM/...)
  if (!images.length) {
    const productImgBlock = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/);
    if (productImgBlock) {
      for (const m of productImgBlock[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) {
        const url = m[1];
        if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
          seen.add(url);
          images.push(url.replace("-500x500.", "-800x800."));
        }
      }
    }
  }

  // Fallback: JSON-LD image
  if (!images.length && jsonLdImage) {
    images.push(jsonLdImage);
  }

  return images;
}

/**
 * Extract the feature/description table from the product-extra-description tab-pane section.
 * Specifically targets the `tab-pane` variant (not the popup variant).
 * Returns an HTML string of the table, or empty string if not found.
 */
function extractDescriptionHtml(html) {
  // Match the tab-pane description div — stop at the next product-extra section
  const sectionMatch = html.match(
    /<div[^>]*product-extra-description[^>]*tab-pane[^>]*>([\s\S]*?)(?=<div[^>]*product-extra-(?:attributes|reviews)|<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>)/
  );
  if (!sectionMatch) return "";

  const section = sectionMatch[1];
  const tableMatch = section.match(/<table[^>]*>([\s\S]*?)<\/table>/);
  if (!tableMatch) return "";

  return `<table border="1">${tableMatch[1]}</table>`;
}

/** Parse price from a string like "₹21,497.28" or just "21497.28" */
function parsePrice(str) {
  if (!str) return 0;
  const n = parseFloat(String(str).replace(/[^\d.]/g, ""));
  return Number.isNaN(n) ? 0 : n;
}

/** Scrape a single product detail page and return structured data. */
async function scrapeProductDetail(productUrl, brandName) {
  let html;
  try {
    html = await fetchHtml(productUrl);
  } catch (err) {
    return { url: productUrl, scrapeError: err.message };
  }

  try {
    const jsonLd = parseProductJsonLd(html);
    if (!jsonLd) {
      return { url: productUrl, scrapeError: "No product JSON-LD found" };
    }

    const productId = extractProductId(jsonLd);
    const images = extractProductImages(html, productId, jsonLd.image);
    const descriptionHtml = extractDescriptionHtml(html);
    const price = parsePrice(jsonLd.offers?.price);
    const currency = jsonLd.offers?.priceCurrency || "INR";

    return {
      url: productUrl,
      sku: jsonLd.sku || productUrl.split("/").pop(),
      name: jsonLd.name || "",
      brand: jsonLd.brand?.name || brandName,
      price,
      currency,
      availability: jsonLd.offers?.availability?.replace("https://schema.org/", "") || "Unknown",
      descriptionHtml,
      images,
      productId,
    };
  } catch (err) {
    return { url: productUrl, scrapeError: `Parse error: ${err.message}` };
  }
}

// Recycle the browser context every N brands to prevent memory accumulation
const BROWSER_RECYCLE_EVERY = Number(process.env.BIKEGEAR_RECYCLE_EVERY ?? 10);

/** Main entry point: scrape all configured brands and return flat product array. */
async function scrapeBikeGear() {
  const allProducts = [];
  const totalBrands = BRANDS.length;
  let brandsDone = 0;

  for (const brand of BRANDS) {
    const productUrls = await scrapeBrandProductUrls(brand);
    brandsDone++;

    if (!productUrls.length) {
      console.warn(`[BIKEGEAR][${brand.name}] No product URLs found — skipping.`);
    } else {
      console.log(`[BIKEGEAR][${brand.name}] Scraping ${productUrls.length} product detail pages (concurrency=${DETAIL_CONCURRENCY})...`);

      let done = 0;
      const products = await runConcurrent(productUrls, async (url) => {
        const product = await scrapeProductDetail(url, brand.name);
        done++;
        if (product.scrapeError) {
          console.warn(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} ERR ${url}: ${product.scrapeError}`);
        } else {
          console.log(`[BIKEGEAR][${brand.name}] ${done}/${productUrls.length} OK "${product.name}"`);
        }
        return product;
      }, DETAIL_CONCURRENCY);

      const ok = products.filter((p) => !p.scrapeError).length;
      const fail = products.length - ok;
      console.log(`[BIKEGEAR][${brand.name}] Done: ${ok} ok, ${fail} failed.`);
      allProducts.push(...products);
    }

    // Always delay/recycle between brands — even skipped ones — to avoid rapid-fire requests
    if (brandsDone < totalBrands) {
      if (FETCH_MODE === "browser" && brandsDone % BROWSER_RECYCLE_EVERY === 0) {
        await recycleBrowserContext();
      } else {
        await sleep(1000);
      }
    }
  }

  return allProducts;
}

module.exports = { scrapeBikeGear, closeBrowser };