/**
 * BikeGear.in Standalone Scraper
 * ================================
 * Setup (one time):
 *   npm install axios
 *
 * Run:
 *   node bikegear-scraper.js
 *
 * Pause anytime with Ctrl+C — progress saves automatically.
 * Re-run the same command to resume from where you stopped.
 *
 * Output: bikegear_output.json  (send this file to the server)
 */

const axios = require("axios");
const path = require("node:path");
const fs = require("node:fs/promises");

// ─── Config ───────────────────────────────────────────────────────────────────

const LISTING_CONCURRENCY = 5;   // how many listing pages to fetch at once
const DETAIL_CONCURRENCY  = 5;   // how many product pages to fetch at once
const PROGRESS_FILE       = path.resolve(__dirname, "bikegear_progress.json");
const OUTPUT_FILE         = path.resolve(__dirname, "bikegear_output.json");

// ─── Brand list ───────────────────────────────────────────────────────────────

const BRANDS = [
  { name: "Acerbis",                       url: "https://bikegear.in/acerbis" },
  { name: "Aimshop",                       url: "https://bikegear.in/aimshop" },
  { name: "Airoh Helmets",                 url: "https://bikegear.in/airoh-helmets" },
  { name: "Akrapovic Exhaust Systems",     url: "https://bikegear.in/akrapovic-exhaust-systems" },
  { name: "Alpinestars",                   url: "https://bikegear.in/alpinestars" },
  { name: "Arai Helmet",                   url: "https://bikegear.in/arai-helmet-india" },
  { name: "Arai Helmet USA",               url: "https://bikegear.in/arai-helmet-usa" },
  { name: "Arrow Exhausts",               url: "https://bikegear.in/arrow-exhausts" },
  { name: "Austin Racing",                 url: "https://bikegear.in/austin-racing" },
  { name: "Bassani Exhaust",               url: "https://bikegear.in/bassani-exhaust" },
  { name: "Beeline",                       url: "https://bikegear.in/beeline" },
  { name: "Bell",                          url: "https://bikegear.in/bell-helmet" },
  { name: "Bell Helmets",                  url: "https://bikegear.in/bell-helmets" },
  { name: "Biltwell Inc.",                 url: "https://bikegear.in/biltwell-inc" },
  { name: "Bitubo Suspension",             url: "https://bikegear.in/bitubo-suspension" },
  { name: "Black Widow",                   url: "https://bikegear.in/black-widow" },
  { name: "British Customs",               url: "https://bikegear.in/british-customs" },
  { name: "Brock's Performance",           url: "https://bikegear.in/brocks-performance" },
  { name: "BT Moto",                       url: "https://bikegear.in/bt-moto" },
  { name: "Caberg",                        url: "https://bikegear.in/caberg-helmets" },
  { name: "Cardo Systems",                 url: "https://bikegear.in/cardo-systems" },
  { name: "Chigee",                        url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=403" },
  { name: "Clover",                        url: "https://bikegear.in/clover-online" },
  { name: "CNC Racing",                    url: "https://bikegear.in/cnc-racing" },
  { name: "Cobra Exhaust",                 url: "https://bikegear.in/cobra-exhaust" },
  { name: "Competition Werkes",            url: "https://bikegear.in/competition-werkes" },
  { name: "Daytona",                       url: "https://bikegear.in/daytona" },
  { name: "Dominator",                     url: "https://bikegear.in/dominator-exhaust-systems-in-india" },
  { name: "Ducabike",                      url: "https://bikegear.in/ducabike" },
  { name: "Dynojet Power Commander",       url: "https://bikegear.in/dynojet-power-commander-research-inc" },
  { name: "Evotech Performance",           url: "https://bikegear.in/evotech-performance-online-india" },
  { name: "Falco",                         url: "https://bikegear.in/falco" },
  { name: "Five",                          url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=422" },
  { name: "Five Gloves",                   url: "https://bikegear.in/five-glove-india" },
  { name: "Forcefield",                    url: "https://bikegear.in/forcefield" },
  { name: "Forma Boots",                   url: "https://bikegear.in/forma-boots" },
  { name: "Forma Boots (2)",               url: "https://bikegear.in/forma-boots-1-1" },
  { name: "Fox",                           url: "https://bikegear.in/fox-racing-gear" },
  { name: "Freedom Performance",           url: "https://bikegear.in/freedom-performance-exhaust" },
  { name: "Full Six Carbon",               url: "https://bikegear.in/full-six-carbon" },
  { name: "G.P.R. Exhaust System",        url: "https://bikegear.in/gpr-exhaust-system" },
  { name: "Gaerne",                        url: "https://bikegear.in/gaerne" },
  { name: "GB Racing",                     url: "https://bikegear.in/gb-racing" },
  { name: "Gilles Tooling",               url: "https://bikegear.in/gilles-tooling-india" },
  { name: "Graves Motorsports",            url: "https://bikegear.in/graves-motorsports" },
  { name: "Held",                          url: "https://bikegear.in/held" },
  { name: "Hindle Exhaust",               url: "https://bikegear.in/hindle-exhaust" },
  { name: "HJC Helmets",                  url: "https://bikegear.in/hjc-helmets" },
  { name: "Honda OEM",                     url: "https://bikegear.in/honda-oem" },
  { name: "HP Corse",                      url: "https://bikegear.in/hp-corse-exhaust" },
  { name: "Icon",                          url: "https://bikegear.in/icon-helmet" },
  { name: "Icon (2)",                      url: "https://bikegear.in/icon-helmet-1" },
  { name: "Ilmberger Carbon",              url: "https://bikegear.in/ilmberger-carbon" },
  { name: "IXIL Exhaust",                  url: "https://bikegear.in/ixil-exhaust" },
  { name: "Ixon",                          url: "https://bikegear.in/ixon-motorcycle-gears" },
  { name: "Ixon (2)",                      url: "https://bikegear.in/ixon-motorcycle-gears-1" },
  { name: "IXS",                           url: "https://bikegear.in/ixs" },
  { name: "Jetprime",                      url: "https://bikegear.in/jet-prime" },
  { name: "Just-1",                        url: "https://bikegear.in/just-1" },
  { name: "Kabuto",                        url: "https://bikegear.in/kabuto" },
  { name: "Kineo Wheels",                  url: "https://bikegear.in/kineo-wheels" },
  { name: "KLIM",                          url: "https://bikegear.in/klim" },
  { name: "Knox",                          url: "https://bikegear.in/knox" },
  { name: "KTM",                           url: "https://bikegear.in/ktm" },
  { name: "Leatt",                         url: "https://bikegear.in/leatt-1" },
  { name: "Leo Vince",                     url: "https://bikegear.in/leo-vince-motorcycle-exhausts-system" },
  { name: "Leovince",                      url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=426" },
  { name: "Lextek",                        url: "https://bikegear.in/lextek-motorcycle-products" },
  { name: "Livall",                        url: "https://bikegear.in/livall-helmet-india" },
  { name: "LS2 Helmets",                   url: "https://bikegear.in/ls2-helmet" },
  { name: "M4 Exhaust",                    url: "https://bikegear.in/m4-exhaust" },
  { name: "Macna",                         url: "https://bikegear.in/macna-brand" },
  { name: "Marchesini Wheels",             url: "https://bikegear.in/marchesini-wheels" },
  { name: "Mivv Exhausts",                 url: "https://bikegear.in/mivv-exhausts" },
  { name: "Mosko Moto",                    url: "https://bikegear.in/mosko-moto" },
  { name: "Nexx Helmets",                  url: "https://bikegear.in/nexx-helmets" },
  { name: "Nolan",                         url: "https://bikegear.in/nolan-helmets" },
  { name: "Ohlins Racing",                 url: "https://bikegear.in/ohlins-racing" },
  { name: "Oxford",                        url: "https://bikegear.in/oxford" },
  { name: "OZ Motorbike",                  url: "https://bikegear.in/oz-motorbike" },
  { name: "Premier",                       url: "https://bikegear.in/premier-helmet" },
  { name: "QD Exhausts",                   url: "https://bikegear.in/qd-exhaust" },
  { name: "R&G Racing",                    url: "https://bikegear.in/r-and-g-racing" },
  { name: "Racefit Exhausts",              url: "https://bikegear.in/racefit-exhausts" },
  { name: "Rapid Bike",                    url: "https://bikegear.in/rapid-bike" },
  { name: "Rekluse",                       url: "https://bikegear.in/rekluse-clutch-plates" },
  { name: "Remus Exhausts",               url: "https://bikegear.in/remus-exhaust" },
  { name: "REV'IT",                        url: "https://bikegear.in/revit" },
  { name: "Rizoma",                        url: "https://bikegear.in/rizoma" },
  { name: "Rotobox Wheels",               url: "https://bikegear.in/rotobox-wheels" },
  { name: "RPM Carbon",                    url: "https://bikegear.in/rpm-carbon" },
  { name: "Rukka Motorcycle Clothing",     url: "https://bikegear.in/rukka-motorcycle-clothing" },
  { name: "Ruroc",                         url: "https://bikegear.in/ruroc-helmet" },
  { name: "Samcosport",                    url: "https://bikegear.in/samco-sport" },
  { name: "SC-Project",                    url: "https://bikegear.in/sc-project" },
  { name: "SC-Project (2)",               url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=415" },
  { name: "Schuberth Helmets",             url: "https://bikegear.in/schuberth-helmets" },
  { name: "Scorpion Exhausts",             url: "https://bikegear.in/scorpion-exhaust" },
  { name: "Scorpion EXO",                  url: "https://bikegear.in/scorpion-exo-motorcycle-gears" },
  { name: "Scorpion EXO (2)",             url: "https://bikegear.in/scorpion-exo-motorcycle-gears-1" },
  { name: "Scorpion EXO USA",              url: "https://bikegear.in/scorpion-helmet" },
  { name: "Sena",                          url: "https://bikegear.in/sena-motorcycle-intercom" },
  { name: "Shark",                         url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=423" },
  { name: "Shark Helmets",                 url: "https://bikegear.in/shark-helmets" },
  { name: "Shark Helmets (2)",             url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=412" },
  { name: "Shima",                         url: "https://bikegear.in/shima" },
  { name: "Shoei Helmets",                 url: "https://bikegear.in/shoei-helmets" },
  { name: "Sidi",                          url: "https://bikegear.in/sidi-boots" },
  { name: "Simpson",                       url: "https://bikegear.in/simpson-helmets" },
  { name: "Spark Exhaust",                 url: "https://bikegear.in/spark-exhaust" },
  { name: "Spidi",                         url: "https://bikegear.in/spidi" },
  { name: "Stadler",                       url: "https://bikegear.in/stadler-india" },
  { name: "Suomy Helmets",                 url: "https://bikegear.in/suomy-helmets" },
  { name: "T-Rex Racing",                  url: "https://bikegear.in/t-rex-racing" },
  { name: "TCX Boots",                     url: "https://bikegear.in/tcx-motorcycle-boots" },
  { name: "TEC Bike Parts",               url: "https://bikegear.in/tec-bike-parts" },
  { name: "Termignoni",                    url: "https://bikegear.in/termignoni-exhausts" },
  { name: "Toce Performance",              url: "https://bikegear.in/toce-performance" },
  { name: "Touratech",                     url: "https://bikegear.in/touratech-accessories" },
  { name: "Tractive Suspension",           url: "https://bikegear.in/index.php?route=product/manufacturer/info&manufacturer_id=424" },
  { name: "Two Brothers Racing",           url: "https://bikegear.in/two-brothers-racing" },
  { name: "Vance & Hines",                 url: "https://bikegear.in/vance-and-hines" },
  { name: "Vandemon",                      url: "https://bikegear.in/vandemon-exhaust-and-performance-parts" },
  { name: "Voodoo",                        url: "https://bikegear.in/voodoo-exhaust" },
  { name: "X-Lite Helmets",               url: "https://bikegear.in/x-lite-helmets" },
  { name: "XPD",                           url: "https://bikegear.in/xpd-boots" },
  { name: "Yoshimura",                     url: "https://bikegear.in/yoshimura" },
  { name: "Yoshimura Japan",               url: "https://bikegear.in/yoshimura-japan" },
  { name: "YSS Suspension",               url: "https://bikegear.in/yss-suspension" },
  { name: "Zard Exhausts",                url: "https://bikegear.in/zard-exhausts" },
];

// ─── HTTP fetch (axios) ───────────────────────────────────────────────────────

const HEADERS = {
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
  "Accept-Language": "en-US,en;q=0.5",
  "Cache-Control": "no-cache",
  "Pragma": "no-cache",
  "Upgrade-Insecure-Requests": "1",
  "Sec-Fetch-Dest": "document",
  "Sec-Fetch-Mode": "navigate",
  "Sec-Fetch-Site": "none",
  "Sec-Fetch-User": "?1",
  "Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
  "Sec-CH-UA-Mobile": "?0",
  "Sec-CH-UA-Platform": '"Windows"',
};

function sleep(ms) {
  return new Promise((r) => setTimeout(r, ms));
}

async function fetchHtml(url, attempt = 1) {
  try {
    const res = await axios.get(url, { headers: HEADERS, timeout: 30000, responseType: "text", maxRedirects: 5 });
    return res.data;
  } catch (err) {
    const status = err.response?.status;
    if ((status === 429 || status >= 500) && attempt <= 4) {
      const wait = attempt * 2000;
      console.log(`  RETRY ${url} HTTP ${status} — waiting ${wait}ms (${attempt}/4)`);
      await sleep(wait);
      return fetchHtml(url, attempt + 1);
    }
    if (status) throw new Error(`HTTP ${status}`);
    const isNetwork = ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND"].includes(err.code) ||
      err.message?.includes("timeout") || err.message?.includes("connect");
    if (isNetwork && attempt <= 4) {
      await sleep(attempt * 2000);
      return fetchHtml(url, attempt + 1);
    }
    throw err;
  }
}

// ─── Concurrent runner ────────────────────────────────────────────────────────

async function runConcurrent(items, fn, limit) {
  const results = new Array(items.length);
  let idx = 0;
  async function worker() {
    while (idx < items.length) {
      const i = idx++;
      results[i] = await fn(items[i], i);
    }
  }
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
  return results;
}

// ─── Listing page helpers ─────────────────────────────────────────────────────

function extractMaxPage(html) {
  const nums = [...html.matchAll(/\?page=(\d+)/g)].map((m) => parseInt(m[1], 10));
  return nums.length ? Math.max(...nums) : 1;
}

function extractProductUrls(html) {
  const urls = new Set();
  for (const m of html.matchAll(/href="(https:\/\/bikegear\.in\/[^"]+)"\s+class="product-img/g)) {
    const url = m[1];
    if (!url.includes("route=") && !url.includes("manufacturer") && !url.includes("category")) {
      urls.add(url);
    }
  }
  return [...urls];
}

async function scrapeBrandProductUrls(brand) {
  let html;
  try {
    html = await fetchHtml(brand.url);
  } catch (err) {
    console.warn(`  [${brand.name}] Failed page 1: ${err.message}`);
    return [];
  }

  const maxPage = extractMaxPage(html);
  const urls = new Set(extractProductUrls(html));

  if (maxPage > 1) {
    const pageNums = Array.from({ length: maxPage - 1 }, (_, i) => i + 2);
    await runConcurrent(pageNums, async (pageNum) => {
      try {
        const pageHtml = await fetchHtml(`${brand.url}?page=${pageNum}`);
        for (const u of extractProductUrls(pageHtml)) urls.add(u);
      } catch (err) {
        console.warn(`  [${brand.name}] Failed page ${pageNum}: ${err.message}`);
      }
    }, LISTING_CONCURRENCY);
  }

  const result = [...urls];
  if (result.length) console.log(`  [${brand.name}] Found ${result.length} products across ${maxPage} page(s).`);
  return result;
}

// ─── Product detail helpers ───────────────────────────────────────────────────

function parseProductJsonLd(html) {
  const blocks = [...html.matchAll(/<script\s+type="application\/ld\+json">([\s\S]*?)<\/script>/g)];
  const candidates = [];
  for (const b of blocks) {
    try {
      const data = JSON.parse(b[1]);
      if (data["@type"] === "Product") candidates.push(data);
    } catch {}
  }
  if (!candidates.length) return null;
  return candidates.find((d) => d.offers?.price || d.description) || candidates[0];
}

function extractProductId(jsonLd) {
  const m = (jsonLd?.image || "").match(/\/catalog\/products\/(\d+)\//);
  return m ? m[1] : null;
}

function extractProductImages(html, productId, jsonLdImage) {
  const seen = new Set();
  const images = [];

  if (productId) {
    const re = new RegExp(
      `data-src="(https://bikegear\\.in/image/cache/catalog/products/${productId}/[^"]+\\.(?:jpg|webp|png))"`, "g"
    );
    for (const m of html.matchAll(re)) {
      const url = m[1];
      if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
        seen.add(url);
        images.push(url.replace("-500x500.", "-800x800."));
      }
    }
  }

  if (!images.length) {
    const block = html.match(/class="product-img[^"]*"[^>]*>([\s\S]{0,4000}?)<\/a>/);
    if (block) {
      for (const m of block[1].matchAll(/data-src="(https:\/\/bikegear\.in\/image\/cache\/[^"]+\.(?:jpg|webp|png))"/g)) {
        const url = m[1];
        if (!url.match(/-(50|90|100|180|200)x/) && !seen.has(url)) {
          seen.add(url);
          images.push(url.replace("-500x500.", "-800x800."));
        }
      }
    }
  }

  if (!images.length && jsonLdImage) images.push(jsonLdImage);
  return images;
}

function extractDescriptionHtml(html) {
  const sec = html.match(
    /<div[^>]*product-extra-description[^>]*tab-pane[^>]*>([\s\S]*?)(?=<div[^>]*product-extra-(?:attributes|reviews)|<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>\s*<\/div>)/
  );
  if (!sec) return "";
  const tbl = sec[1].match(/<table[^>]*>([\s\S]*?)<\/table>/);
  return tbl ? `<table border="1">${tbl[1]}</table>` : "";
}

function parsePrice(str) {
  if (!str) return 0;
  const n = parseFloat(String(str).replace(/[^\d.]/g, ""));
  return Number.isNaN(n) ? 0 : n;
}

async function scrapeProductDetail(productUrl, brandName) {
  let html;
  try {
    html = await fetchHtml(productUrl);
  } catch (err) {
    return { url: productUrl, scrapeError: err.message };
  }
  try {
    const jsonLd = parseProductJsonLd(html);
    if (!jsonLd) return { url: productUrl, scrapeError: "No product JSON-LD found" };
    const productId = extractProductId(jsonLd);
    const images = extractProductImages(html, productId, jsonLd.image);
    const descriptionHtml = extractDescriptionHtml(html);
    const price = parsePrice(jsonLd.offers?.price);
    return {
      url: productUrl,
      sku: jsonLd.sku || productUrl.split("/").pop(),
      name: jsonLd.name || "",
      brand: brandName || jsonLd.brand?.name || "BikeGear",
      price,
      currency: jsonLd.offers?.priceCurrency || "INR",
      availability: jsonLd.offers?.availability?.replace("https://schema.org/", "") || "Unknown",
      descriptionHtml,
      images,
      productId,
    };
  } catch (err) {
    return { url: productUrl, scrapeError: `Parse error: ${err.message}` };
  }
}

// ─── Output helpers ───────────────────────────────────────────────────────────

function wrapProduct(product) {
  return {
    productId: product.sku || product.url,
    sourceKey: "bikegear",
    brand: product.brand || "BikeGear",
    bikeModel: "",
    productSummary: {
      id: product.sku,
      name: product.name,
      img: product.images || [],
      cost: { mrp: product.price || 0 },
    },
    scraped: product,
  };
}

async function writeOutput(products) {
  const ok = products.filter((p) => !(p.scraped || p).scrapeError);
  const fail = products.length - ok.length;
  const payload = {
    generatedAt: new Date().toISOString(),
    sourceKey: "bikegear",
    sourceLabel: "BikeGear India",
    analysis: {
      timestamp: new Date().toISOString(),
      totalProducts: products.length,
      totalProductsUnique: ok.length,
      detailSuccess: ok.length,
      detailFailed: fail,
    },
    products,
  };
  await fs.writeFile(OUTPUT_FILE, JSON.stringify(payload, null, 2), "utf8");
  console.log(`\n  Output saved → ${OUTPUT_FILE}  (${ok.length} ok, ${fail} failed)`);
}

// ─── Progress ─────────────────────────────────────────────────────────────────

async function loadProgress() {
  try {
    return JSON.parse(await fs.readFile(PROGRESS_FILE, "utf8"));
  } catch {
    return { completedBrands: [], products: [] };
  }
}

async function saveProgress(p) {
  await fs.writeFile(PROGRESS_FILE, JSON.stringify(p, null, 2), "utf8");
}

// ─── Main ─────────────────────────────────────────────────────────────────────

async function main() {
  console.log("BikeGear Scraper — starting");
  console.log(`${BRANDS.length} brands total | listing concurrency: ${LISTING_CONCURRENCY} | detail concurrency: ${DETAIL_CONCURRENCY}\n`);

  const progress = await loadProgress();
  const done = new Set(progress.completedBrands);
  const remaining = BRANDS.filter((b) => !done.has(b.name));

  if (done.size > 0) {
    console.log(`Resuming — ${done.size} brands already done, ${remaining.length} remaining.\n`);
  }

  let stopping = false;

  async function gracefulExit() {
    if (stopping) return;
    stopping = true;
    console.log("\nCtrl+C — saving progress...");
    await saveProgress(progress);
    await writeOutput(progress.products);
    console.log("Stopped. Re-run to continue.");
    process.exit(0);
  }

  process.on("SIGINT", gracefulExit);
  process.on("SIGTERM", gracefulExit);

  for (const brand of remaining) {
    if (stopping) break;

    process.stdout.write(`[${brand.name}] listing... `);
    const productUrls = await scrapeBrandProductUrls(brand);

    if (!productUrls.length) {
      console.log("no products, skipping.");
      progress.completedBrands.push(brand.name);
      await saveProgress(progress);
      continue;
    }

    console.log(`scraping ${productUrls.length} products...`);
    let done2 = 0;

    const rawProducts = await runConcurrent(productUrls, async (url) => {
      if (stopping) return { url, scrapeError: "interrupted" };
      const p = await scrapeProductDetail(url, brand.name);
      done2++;
      if (p.scrapeError) {
        process.stdout.write(`  ${done2}/${productUrls.length} ERR\n`);
      } else {
        process.stdout.write(`  ${done2}/${productUrls.length} OK: ${p.name}\n`);
      }
      return p;
    }, DETAIL_CONCURRENCY);

    const okCount = rawProducts.filter((p) => !p.scrapeError).length;
    console.log(`[${brand.name}] complete — ${okCount}/${productUrls.length} ok\n`);

    progress.products.push(...rawProducts.map(wrapProduct));
    progress.completedBrands.push(brand.name);

    await saveProgress(progress);
    await writeOutput(progress.products);
  }

  if (!stopping) {
    // ── Retry pass: re-attempt all failed products up to 3 times ──────────────
    const MAX_RETRIES = 3;
    let failedIndexes = progress.products
      .map((p, i) => (p.scraped?.scrapeError ? i : -1))
      .filter((i) => i !== -1);

    if (failedIndexes.length > 0) {
      console.log(`\nRetry pass — ${failedIndexes.length} failed products, up to ${MAX_RETRIES} attempts each...\n`);

      for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
        if (!failedIndexes.length) break;
        console.log(`  Retry attempt ${attempt}/${MAX_RETRIES} — ${failedIndexes.length} products...`);
        await sleep(2000);

        const stillFailed = [];
        for (const idx of failedIndexes) {
          const wrapped = progress.products[idx];
          const url = wrapped.scraped?.url || wrapped.productId;
          const brandName = wrapped.brand;
          const result = await scrapeProductDetail(url, brandName);
          if (result.scrapeError) {
            console.log(`    ERR [${attempt}/${MAX_RETRIES}] ${url}: ${result.scrapeError}`);
            stillFailed.push(idx);
          } else {
            console.log(`    OK  ${result.name}`);
            progress.products[idx] = wrapProduct(result);
          }
        }
        failedIndexes = stillFailed;
      }

      const remaining = failedIndexes.length;
      const fixed = progress.products.filter((p) => !p.scraped?.scrapeError).length;
      console.log(`\nRetry done — ${fixed} ok total, ${remaining} still failed after ${MAX_RETRIES} attempts.`);
    }

    await writeOutput(progress.products);
    await fs.unlink(PROGRESS_FILE).catch(() => {});
    console.log("Progress file removed (clean finish).");
    console.log(`\nSend this file to the server:\n  ${OUTPUT_FILE}\n`);
  }
}

main().catch((err) => {
  console.error("Fatal error:", err);
  process.exit(1);
});