fix(bikegear): support Webshare proxy config and cache-only mode

src/business-logic/import-pipeline/sources/bikegear/index.js

@@ -8,6 +8,12 @@ const sourceKey = "bikegear";
 const label = "BikeGear India";
 const dataDir = path.join("data", "sources", sourceKey);
 
+function readBooleanEnv(name, fallback = false) {
+  const val = process.env[name];
+  if (val == null) return fallback;
+  return ["1", "true", "yes", "y", "on"].includes(String(val).toLowerCase());
+}
+
 function paths() {
   return {
     aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
@@ -21,36 +27,66 @@ function paths() {
   };
 }
 
-async function fetchWebsiteData({ paths: runPaths }) {
+async function readCachedAggregatedData(absAggregatedPath) {
-  // Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.
-  const cacheMaxHours = Number(process.env.BIKEGEAR_CACHE_HOURS ?? 24);
-  const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson);
-
-  if (cacheMaxHours > 0) {
-    try {
   const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
   const products = existing?.products;
   const successCount = Array.isArray(products)
     ? products.filter((p) => !(p.scraped || p).scrapeError).length
     : 0;
-      if (existing?.generatedAt && Array.isArray(products) && successCount > 0) {
+
-        const ageHours = (Date.now() - new Date(existing.generatedAt).getTime()) / (1000 * 60 * 60);
+  return {
-        if (ageHours < cacheMaxHours) {
+    existing,
-          console.log(`[BIKEGEAR] Using cached data (${ageHours.toFixed(1)}h old) — ${successCount} ok products. Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.`);
+    products,
+    successCount,
+  };
+}
+
+function buildCachedSummary(existing, absAggregatedPath, runPaths, cachedReason) {
   return {
     analysis: existing.analysis || {},
     outputJsonPath: absAggregatedPath,
     historyPath: path.resolve(process.cwd(), runPaths.historyJson),
     cached: true,
+    cachedReason,
   };
+}
+
+async function fetchWebsiteData({ paths: runPaths }) {
+  // Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set -1 for any valid cache.
+  // Set BIKEGEAR_CACHE_ONLY=true on VPS hosts that must not scrape BikeGear directly.
+  const cacheMaxHours = Number(process.env.BIKEGEAR_CACHE_HOURS ?? 24);
+  const cacheOnly = readBooleanEnv("BIKEGEAR_CACHE_ONLY", false);
+  const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson);
+
+  try {
+    const { existing, products, successCount } = await readCachedAggregatedData(absAggregatedPath);
+    if (existing?.generatedAt && Array.isArray(products) && successCount > 0) {
+      if (cacheOnly || cacheMaxHours < 0) {
+        const reason = cacheOnly ? "BIKEGEAR_CACHE_ONLY=true" : "BIKEGEAR_CACHE_HOURS=-1";
+        console.log(`[BIKEGEAR] Using cached data (${successCount} ok products) — ${reason}.`);
+        return buildCachedSummary(existing, absAggregatedPath, runPaths, reason);
+      }
+
+      if (cacheMaxHours > 0) {
+        const ageHours = (Date.now() - new Date(existing.generatedAt).getTime()) / (1000 * 60 * 60);
+        if (ageHours < cacheMaxHours) {
+          console.log(`[BIKEGEAR] Using cached data (${ageHours.toFixed(1)}h old) — ${successCount} ok products. Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.`);
+          return buildCachedSummary(existing, absAggregatedPath, runPaths, `${ageHours.toFixed(1)}h old`);
         }
         console.log(`[BIKEGEAR] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`);
+      }
     } else if (Array.isArray(products) && products.length > 0 && successCount === 0) {
       console.log(`[BIKEGEAR] Cached data has 0 successful products — ignoring cache and re-scraping.`);
     }
   } catch {
     // no cache yet
   }
+
+  if (cacheOnly) {
+    throw new Error(
+      `[BIKEGEAR] Cache-only mode is enabled, but no valid cached data was found at ${absAggregatedPath}. ` +
+      "Sync a valid 01_products_aggregated.json into that path or set BIKEGEAR_CACHE_ONLY=false to scrape."
+    );
   }
 
   const now = new Date().toISOString();

src/business-logic/import-pipeline/sources/bikegear/scraper.js

@@ -7,7 +7,12 @@
  * Fetch modes (set in .env):
  *   BIKEGEAR_FETCH_MODE=browser  — use Playwright Chromium (default; bypasses Cloudflare on VPS IPs)
  *   BIKEGEAR_FETCH_MODE=http     — use axios (works only from residential/whitelisted IPs)
- *   BIKEGEAR_PROXY_URL=http://user:pass@host:port  — proxy for http mode
+ *   BIKEGEAR_PROXY_URL=http://user:pass@host:port
+ *   Or Webshare-style parts:
+ *     BIKEGEAR_PROXY_HOST=p.webshare.io
+ *     BIKEGEAR_PROXY_PORT=80
+ *     BIKEGEAR_PROXY_USERNAME=...
+ *     BIKEGEAR_PROXY_PASSWORD=...
  */
 
 const axios = require("axios");
@@ -19,11 +24,41 @@ const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (
 const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
 // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
 const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
-const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
+const PROXY_URL = getProxyUrl();
 // Default to browser mode — it bypasses Cloudflare without needing a proxy
 const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
 
-console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://<redacted>@")})` : ""}`);
+console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`);
+
+function getProxyUrl() {
+  if (process.env.BIKEGEAR_PROXY_URL) {
+    return process.env.BIKEGEAR_PROXY_URL;
+  }
+
+  const host = process.env.BIKEGEAR_PROXY_HOST;
+  const port = process.env.BIKEGEAR_PROXY_PORT;
+  if (!host || !port) {
+    return null;
+  }
+
+  const protocol = String(process.env.BIKEGEAR_PROXY_PROTOCOL || "http").replace(/:$/, "");
+  const username = process.env.BIKEGEAR_PROXY_USERNAME || "";
+  const password = process.env.BIKEGEAR_PROXY_PASSWORD || "";
+  const auth = username
+    ? `${encodeURIComponent(username)}:${encodeURIComponent(password)}@`
+    : "";
+
+  return `${protocol}://${auth}${host}:${port}`;
+}
+
+function maskProxyUrl(proxyUrl) {
+  try {
+    const u = new URL(proxyUrl);
+    return `${u.protocol}//${u.username ? "<redacted>@" : ""}${u.host}`;
+  } catch {
+    return "<invalid proxy url>";
+  }
+}
 
 // ── Browser pool (Playwright persistent context) ───────────────────────────
 // Uses launchPersistentContext so the browser profile (cookies, cache, fingerprint)