From 3d98c873cc7d1328426734402ecaa627489b4f03 Mon Sep 17 00:00:00 2001
From: MOHAN <mohan326856@gmail.com>
Date: Sat, 15 Aug 2026 06:50:52 +0530
Subject: [PATCH] feat(bikegear): add proxy-based image downloader with
 rate-limit delay

Routes image downloads through the webshare residential proxy to avoid
429 rate limiting from bikegear.in CDN on the VPS. Reads delay and
concurrency from BIKEGEAR_IMAGE_DELAY / BIKEGEAR_REQUEST_DELAY env vars.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../import-pipeline/sources/bikegear/index.js | 90 +++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/src/business-logic/import-pipeline/sources/bikegear/index.js b/src/business-logic/import-pipeline/sources/bikegear/index.js
index 7764662..1eac7de 100644
--- a/src/business-logic/import-pipeline/sources/bikegear/index.js
+++ b/src/business-logic/import-pipeline/sources/bikegear/index.js
@@ -1,5 +1,6 @@
 const fs = require("node:fs/promises");
 const path = require("node:path");
+const axios = require("axios");
 const { BRANDS } = require("./brands");
 const { scrapeBikeGear, closeBrowser } = require("./scraper");
 const { convertBikeGearJsonToShopifyProducts } = require("./converter");
@@ -163,6 +164,94 @@ function convertToShopifyProducts(input, options = {}) {
   });
 }
 
+function sanitizeFileName(value) {
+  return String(value || "")
+    .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_")
+    .replace(/\s+/g, " ")
+    .trim()
+    .slice(0, 150);
+}
+
+async function mapWithConcurrency(items, concurrency, worker) {
+  const results = new Array(items.length);
+  let index = 0;
+  async function runWorker() {
+    while (true) {
+      const current = index++;
+      if (current >= items.length) return;
+      results[current] = await worker(items[current], current);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, runWorker));
+  return results;
+}
+
+async function downloadImages({ aggregatedJsonPath, imagesDir }) {
+  const proxyHost = process.env.BIKEGEAR_PROXY_HOST;
+  const proxyPort = Number(process.env.BIKEGEAR_PROXY_PORT || 80);
+  const proxyUser = process.env.BIKEGEAR_PROXY_USERNAME;
+  const proxyPass = process.env.BIKEGEAR_PROXY_PASSWORD;
+  const delayMs = Number(process.env.BIKEGEAR_IMAGE_DELAY || process.env.BIKEGEAR_REQUEST_DELAY || 2000);
+  const concurrency = Number(process.env.BIKEGEAR_IMAGE_CONCURRENCY || 1);
+
+  const axiosConfig = {
+    responseType: "arraybuffer",
+    timeout: 60000,
+    headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36" },
+    ...(proxyHost ? { proxy: { host: proxyHost, port: proxyPort, auth: { username: proxyUser, password: proxyPass } } } : {}),
+  };
+
+  const absJsonPath = path.resolve(process.cwd(), aggregatedJsonPath);
+  const absImagesDir = path.resolve(process.cwd(), imagesDir);
+  const raw = await fs.readFile(absJsonPath, "utf8");
+  const parsed = JSON.parse(raw);
+  const products = Array.isArray(parsed?.products) ? parsed.products : [];
+
+  const usedFolders = new Set();
+  const tasks = [];
+
+  for (const product of products) {
+    const data = product?.scraped || product;
+    if (data?.scrapeError) continue;
+    const imgs = product?.productSummary?.img || data?.images || [];
+    if (!imgs.length) continue;
+
+    const rawName = product?.productSummary?.name || data?.name || "unknown";
+    let folderName = sanitizeFileName(rawName) || product?.productId || "unknown";
+    if (usedFolders.has(folderName)) folderName = `${folderName}__${String(product?.productId || "").slice(0, 8)}`;
+    usedFolders.add(folderName);
+    const productDir = path.join(absImagesDir, folderName);
+
+    for (let i = 0; i < imgs.length; i++) {
+      const imgUrl = imgs[i];
+      let fileName;
+      try { fileName = path.basename(new URL(imgUrl).pathname) || `img_${i + 1}.jpg`; }
+      catch { fileName = `img_${i + 1}.jpg`; }
+      tasks.push({ url: imgUrl, productDir, fileName, name: rawName });
+    }
+  }
+
+  let downloaded = 0, skipped = 0, failed = 0;
+
+  await mapWithConcurrency(tasks, concurrency, async (task) => {
+    await fs.mkdir(task.productDir, { recursive: true });
+    const filePath = path.join(task.productDir, task.fileName);
+    try {
+      try { await fs.access(filePath); skipped++; return; } catch {}
+      const res = await axios.get(task.url, axiosConfig);
+      await fs.writeFile(filePath, Buffer.from(res.data));
+      downloaded++;
+    } catch (e) {
+      failed++;
+      console.log(`[IMAGE-FAIL] ${task.name} | ${task.url} -> ${e.message}`);
+    }
+    if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
+  });
+
+  console.log(`[BIKEGEAR-IMAGES] done — downloaded=${downloaded} skipped=${skipped} failed=${failed} total=${tasks.length}`);
+  return { totalImagesFound: tasks.length, downloaded, skipped, failed, productsCount: products.length };
+}
+
 module.exports = {
   sourceKey,
   label,
@@ -172,5 +261,6 @@ module.exports = {
   envImageBaseUrl: "BIKEGEAR_IMAGE_BASE_URL",
   paths,
   fetchWebsiteData,
+  downloadImages,
   convertToShopifyProducts,
 };