diff --git a/src/business-logic/import-pipeline/sources/bikegear/index.js b/src/business-logic/import-pipeline/sources/bikegear/index.js index 286a192..7764662 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/index.js +++ b/src/business-logic/import-pipeline/sources/bikegear/index.js @@ -8,6 +8,12 @@ const sourceKey = "bikegear"; const label = "BikeGear India"; const dataDir = path.join("data", "sources", sourceKey); +function readBooleanEnv(name, fallback = false) { + const val = process.env[name]; + if (val == null) return fallback; + return ["1", "true", "yes", "y", "on"].includes(String(val).toLowerCase()); +} + function paths() { return { aggregatedJson: path.join(dataDir, "01_products_aggregated.json"), @@ -21,36 +27,66 @@ function paths() { }; } +async function readCachedAggregatedData(absAggregatedPath) { + const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); + const products = existing?.products; + const successCount = Array.isArray(products) + ? products.filter((p) => !(p.scraped || p).scrapeError).length + : 0; + + return { + existing, + products, + successCount, + }; +} + +function buildCachedSummary(existing, absAggregatedPath, runPaths, cachedReason) { + return { + analysis: existing.analysis || {}, + outputJsonPath: absAggregatedPath, + historyPath: path.resolve(process.cwd(), runPaths.historyJson), + cached: true, + cachedReason, + }; +} + async function fetchWebsiteData({ paths: runPaths }) { - // Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape. + // Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set -1 for any valid cache. + // Set BIKEGEAR_CACHE_ONLY=true on VPS hosts that must not scrape BikeGear directly. const cacheMaxHours = Number(process.env.BIKEGEAR_CACHE_HOURS ?? 24); + const cacheOnly = readBooleanEnv("BIKEGEAR_CACHE_ONLY", false); const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson); - if (cacheMaxHours > 0) { - try { - const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8")); - const products = existing?.products; - const successCount = Array.isArray(products) - ? products.filter((p) => !(p.scraped || p).scrapeError).length - : 0; - if (existing?.generatedAt && Array.isArray(products) && successCount > 0) { + try { + const { existing, products, successCount } = await readCachedAggregatedData(absAggregatedPath); + if (existing?.generatedAt && Array.isArray(products) && successCount > 0) { + if (cacheOnly || cacheMaxHours < 0) { + const reason = cacheOnly ? "BIKEGEAR_CACHE_ONLY=true" : "BIKEGEAR_CACHE_HOURS=-1"; + console.log(`[BIKEGEAR] Using cached data (${successCount} ok products) — ${reason}.`); + return buildCachedSummary(existing, absAggregatedPath, runPaths, reason); + } + + if (cacheMaxHours > 0) { const ageHours = (Date.now() - new Date(existing.generatedAt).getTime()) / (1000 * 60 * 60); if (ageHours < cacheMaxHours) { console.log(`[BIKEGEAR] Using cached data (${ageHours.toFixed(1)}h old) — ${successCount} ok products. Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.`); - return { - analysis: existing.analysis || {}, - outputJsonPath: absAggregatedPath, - historyPath: path.resolve(process.cwd(), runPaths.historyJson), - cached: true, - }; + return buildCachedSummary(existing, absAggregatedPath, runPaths, `${ageHours.toFixed(1)}h old`); } console.log(`[BIKEGEAR] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`); - } else if (Array.isArray(products) && products.length > 0 && successCount === 0) { - console.log(`[BIKEGEAR] Cached data has 0 successful products — ignoring cache and re-scraping.`); } - } catch { - // no cache yet + } else if (Array.isArray(products) && products.length > 0 && successCount === 0) { + console.log(`[BIKEGEAR] Cached data has 0 successful products — ignoring cache and re-scraping.`); } + } catch { + // no cache yet + } + + if (cacheOnly) { + throw new Error( + `[BIKEGEAR] Cache-only mode is enabled, but no valid cached data was found at ${absAggregatedPath}. ` + + "Sync a valid 01_products_aggregated.json into that path or set BIKEGEAR_CACHE_ONLY=false to scrape." + ); } const now = new Date().toISOString(); diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index 24b06f9..7edd7e8 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -7,7 +7,12 @@ * Fetch modes (set in .env): * BIKEGEAR_FETCH_MODE=browser — use Playwright Chromium (default; bypasses Cloudflare on VPS IPs) * BIKEGEAR_FETCH_MODE=http — use axios (works only from residential/whitelisted IPs) - * BIKEGEAR_PROXY_URL=http://user:pass@host:port — proxy for http mode + * BIKEGEAR_PROXY_URL=http://user:pass@host:port + * Or Webshare-style parts: + * BIKEGEAR_PROXY_HOST=p.webshare.io + * BIKEGEAR_PROXY_PORT=80 + * BIKEGEAR_PROXY_USERNAME=... + * BIKEGEAR_PROXY_PASSWORD=... */ const axios = require("axios"); @@ -19,11 +24,41 @@ const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? ( const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1)); // Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500); -const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null; +const PROXY_URL = getProxyUrl(); // Default to browser mode — it bypasses Cloudflare without needing a proxy const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser"; -console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://@")})` : ""}`); +console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`); + +function getProxyUrl() { + if (process.env.BIKEGEAR_PROXY_URL) { + return process.env.BIKEGEAR_PROXY_URL; + } + + const host = process.env.BIKEGEAR_PROXY_HOST; + const port = process.env.BIKEGEAR_PROXY_PORT; + if (!host || !port) { + return null; + } + + const protocol = String(process.env.BIKEGEAR_PROXY_PROTOCOL || "http").replace(/:$/, ""); + const username = process.env.BIKEGEAR_PROXY_USERNAME || ""; + const password = process.env.BIKEGEAR_PROXY_PASSWORD || ""; + const auth = username + ? `${encodeURIComponent(username)}:${encodeURIComponent(password)}@` + : ""; + + return `${protocol}://${auth}${host}:${port}`; +} + +function maskProxyUrl(proxyUrl) { + try { + const u = new URL(proxyUrl); + return `${u.protocol}//${u.username ? "@" : ""}${u.host}`; + } catch { + return ""; + } +} // ── Browser pool (Playwright persistent context) ─────────────────────────── // Uses launchPersistentContext so the browser profile (cookies, cache, fingerprint)