feat(bikegear): add local scraper with resume support for residential IP scraping

Standalone script for running bikegear scrape from a local machine (residential
IP bypasses Cloudflare). Saves progress after every brand so Ctrl+C resumes
cleanly. Output is written to data/sources/bikegear/01_products_aggregated.json
in the exact same format the server pipeline expects — copy the file to the
server and the pipeline uses it as a 24h cache, skipping straight to stage 2.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitignore

@@ -8,6 +8,9 @@ coverage/
 
 # Runtime/generated data
 data/
+
+# Local bikegear scraper progress (not committed — local machine only)
+bikegear-local-scraper/progress.json
 logs/
 *.log
 src/business-logic/import-pipeline/sources/outputs/

bikegear-local-scraper/run.js

@@ -0,0 +1,193 @@
+/**
+ * BikeGear Local Scraper — run this on your LOCAL machine (residential IP).
+ *
+ * Usage (from Race-Nation-Shopify-App-Backend/):
+ *   node bikegear-local-scraper/run.js
+ *
+ * Ctrl+C at any time — progress is saved. Re-run to resume from last brand.
+ *
+ * When done, copy the output file to the server:
+ *   scp data/sources/bikegear/01_products_aggregated.json \
+ *       root@<server>:/home/dev/apps/Race-Nation-Shopify-Backend/data/sources/bikegear/
+ *
+ * Then trigger the pipeline on the server — it will see the 24h cache and
+ * skip straight to image download → watermark → Shopify upload.
+ */
+
+const path = require("node:path");
+const fs = require("node:fs/promises");
+
+// Default to HTTP mode for local runs — residential IP is not blocked
+process.env.BIKEGEAR_FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "http";
+// Higher concurrency is safe from a residential IP
+process.env.BIKEGEAR_LISTING_CONCURRENCY = process.env.BIKEGEAR_LISTING_CONCURRENCY || "5";
+process.env.BIKEGEAR_DETAIL_CONCURRENCY = process.env.BIKEGEAR_DETAIL_CONCURRENCY || "5";
+
+const { BRANDS } = require("../src/business-logic/import-pipeline/sources/bikegear/brands");
+const { scrapeBrandProductUrls, scrapeProductDetail, closeBrowser } = require("../src/business-logic/import-pipeline/sources/bikegear/scraper");
+
+const SOURCE_KEY = "bikegear";
+const SOURCE_LABEL = "BikeGear India";
+
+// Progress file — tracks which brands are done so we can resume
+const PROGRESS_FILE = path.resolve(__dirname, "progress.json");
+// Final output — same path the server pipeline uses (stage 1 cache)
+const OUTPUT_FILE = path.resolve("data/sources/bikegear/01_products_aggregated.json");
+
+async function loadProgress() {
+  try {
+    const raw = await fs.readFile(PROGRESS_FILE, "utf8");
+    return JSON.parse(raw);
+  } catch {
+    return { completedBrands: [], products: [] };
+  }
+}
+
+async function saveProgress(progress) {
+  await fs.mkdir(path.dirname(PROGRESS_FILE), { recursive: true });
+  await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), "utf8");
+}
+
+async function writeOutput(products) {
+  const successfulProducts = products.filter((p) => !(p.scraped || p).scrapeError);
+  const failedProducts = products.filter((p) => (p.scraped || p).scrapeError);
+
+  const payload = {
+    generatedAt: new Date().toISOString(),
+    sourceKey: SOURCE_KEY,
+    sourceLabel: SOURCE_LABEL,
+    analysis: {
+      timestamp: new Date().toISOString(),
+      totalProducts: products.length,
+      totalProductsUnique: successfulProducts.length,
+      detailSuccess: successfulProducts.length,
+      detailFailed: failedProducts.length,
+    },
+    products,
+  };
+
+  await fs.mkdir(path.dirname(OUTPUT_FILE), { recursive: true });
+  await fs.writeFile(OUTPUT_FILE, JSON.stringify(payload, null, 2), "utf8");
+  console.log(`\n[OUTPUT] Written to ${OUTPUT_FILE}`);
+  console.log(`[OUTPUT] ${successfulProducts.length} products OK, ${failedProducts.length} failed.`);
+}
+
+function wrapProduct(product) {
+  return {
+    productId: product.sku || product.url,
+    sourceKey: SOURCE_KEY,
+    brand: product.brand || "BikeGear",
+    bikeModel: "",
+    productSummary: {
+      id: product.sku,
+      name: product.name,
+      img: product.images || [],
+      cost: { mrp: product.price || 0 },
+    },
+    scraped: product,
+  };
+}
+
+async function main() {
+  console.log(`[BIKEGEAR-LOCAL] Fetch mode: ${process.env.BIKEGEAR_FETCH_MODE}`);
+  console.log(`[BIKEGEAR-LOCAL] ${BRANDS.length} brands total.`);
+
+  const progress = await loadProgress();
+  const completedSet = new Set(progress.completedBrands);
+
+  const remaining = BRANDS.filter((b) => !completedSet.has(b.name));
+  console.log(`[BIKEGEAR-LOCAL] ${completedSet.size} brands already done. ${remaining.length} remaining.\n`);
+
+  let shuttingDown = false;
+
+  async function gracefulExit() {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    console.log("\n[BIKEGEAR-LOCAL] Ctrl+C received — saving progress and exiting...");
+    await saveProgress(progress);
+    await writeOutput(progress.products);
+    await closeBrowser();
+    console.log("[BIKEGEAR-LOCAL] Progress saved. Re-run to continue.");
+    process.exit(0);
+  }
+
+  process.on("SIGINT", gracefulExit);
+  process.on("SIGTERM", gracefulExit);
+
+  for (const brand of remaining) {
+    if (shuttingDown) break;
+
+    const productUrls = await scrapeBrandProductUrls(brand);
+
+    if (!productUrls.length) {
+      console.warn(`[${brand.name}] No product URLs — skipping.`);
+      progress.completedBrands.push(brand.name);
+      await saveProgress(progress);
+      continue;
+    }
+
+    console.log(`[${brand.name}] Scraping ${productUrls.length} products...`);
+    let done = 0;
+
+    // Scrape detail pages in batches (concurrency from env)
+    const concurrency = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY || 5);
+    const brandProducts = [];
+
+    async function runConcurrent(items, fn, limit) {
+      let idx = 0;
+      const results = new Array(items.length);
+      async function worker() {
+        while (idx < items.length) {
+          const i = idx++;
+          results[i] = await fn(items[i], i);
+        }
+      }
+      await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
+      return results;
+    }
+
+    const rawProducts = await runConcurrent(productUrls, async (url) => {
+      if (shuttingDown) return { url, scrapeError: "interrupted" };
+      const product = await scrapeProductDetail(url, brand.name);
+      done++;
+      if (product.scrapeError) {
+        console.warn(`  ${done}/${productUrls.length} ERR: ${product.scrapeError}`);
+      } else {
+        console.log(`  ${done}/${productUrls.length} OK: "${product.name}"`);
+      }
+      return product;
+    }, concurrency);
+
+    for (const raw of rawProducts) {
+      brandProducts.push(wrapProduct(raw));
+    }
+
+    const ok = rawProducts.filter((p) => !p.scrapeError).length;
+    console.log(`[${brand.name}] Done: ${ok}/${productUrls.length} OK\n`);
+
+    progress.products.push(...brandProducts);
+    progress.completedBrands.push(brand.name);
+
+    // Save progress after every brand
+    await saveProgress(progress);
+    // Also update the output file incrementally
+    await writeOutput(progress.products);
+  }
+
+  if (!shuttingDown) {
+    console.log("\n[BIKEGEAR-LOCAL] All brands complete!");
+    await writeOutput(progress.products);
+    await closeBrowser();
+
+    // Clean up progress file — scrape is done
+    await fs.unlink(PROGRESS_FILE).catch(() => {});
+    console.log("[BIKEGEAR-LOCAL] Progress file removed (clean run).");
+    console.log(`\nNext step: copy ${OUTPUT_FILE} to the server and trigger the pipeline.`);
+  }
+}
+
+main().catch(async (err) => {
+  console.error("[BIKEGEAR-LOCAL] Fatal error:", err);
+  await closeBrowser();
+  process.exit(1);
+});

src/business-logic/import-pipeline/sources/bikegear/scraper.js

@@ -623,4 +623,4 @@ async function scrapeBikeGear() {
   return allProducts;
 }
 
-module.exports = { scrapeBikeGear, closeBrowser };
+module.exports = { scrapeBikeGear, closeBrowser, scrapeBrandProductUrls, scrapeProductDetail };