admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bikegear): support Webshare proxy config and cache-only mode

Browse Source
This commit is contained in:
MOHAN 2026-07-31 01:06:37 +05:30
parent 19fd7f16ea
commit 2b8cf2c6fc
2 changed files with 93 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
src/business-logic/import-pipeline/sources/bikegear/index.js
View File

@ -8,6 +8,12 @@ const sourceKey = "bikegear";
const label = "BikeGear India";
const dataDir = path.join("data", "sources", sourceKey);
function readBooleanEnv(name, fallback = false) {
const val = process.env[name];
if (val == null) return fallback;
return ["1", "true", "yes", "y", "on"].includes(String(val).toLowerCase());
}
function paths() {
return {
aggregatedJson: path.join(dataDir, "01_products_aggregated.json"),
@ -21,36 +27,66 @@ function paths() {
};
}
async function readCachedAggregatedData(absAggregatedPath) {
const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
const products = existing?.products;
const successCount = Array.isArray(products)
? products.filter((p) => !(p.scraped || p).scrapeError).length
: 0;
return {
existing,
products,
successCount,
};
}
function buildCachedSummary(existing, absAggregatedPath, runPaths, cachedReason) {
return {
analysis: existing.analysis || {},
outputJsonPath: absAggregatedPath,
historyPath: path.resolve(process.cwd(), runPaths.historyJson),
cached: true,
cachedReason,
};
}
async function fetchWebsiteData({ paths: runPaths }) {
// Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.
// Cache for BIKEGEAR_CACHE_HOURS (default 24h). Set -1 for any valid cache.
// Set BIKEGEAR_CACHE_ONLY=true on VPS hosts that must not scrape BikeGear directly.
const cacheMaxHours = Number(process.env.BIKEGEAR_CACHE_HOURS ?? 24);
const cacheOnly = readBooleanEnv("BIKEGEAR_CACHE_ONLY", false);
const absAggregatedPath = path.resolve(process.cwd(), runPaths.aggregatedJson);
if (cacheMaxHours > 0) {
try {
const existing = JSON.parse(await fs.readFile(absAggregatedPath, "utf8"));
const products = existing?.products;
const successCount = Array.isArray(products)
? products.filter((p) => !(p.scraped || p).scrapeError).length
: 0;
if (existing?.generatedAt && Array.isArray(products) && successCount > 0) {
try {
const { existing, products, successCount } = await readCachedAggregatedData(absAggregatedPath);
if (existing?.generatedAt && Array.isArray(products) && successCount > 0) {
if (cacheOnly || cacheMaxHours < 0) {
const reason = cacheOnly ? "BIKEGEAR_CACHE_ONLY=true" : "BIKEGEAR_CACHE_HOURS=-1";
console.log(`[BIKEGEAR] Using cached data (${successCount} ok products) — ${reason}.`);
return buildCachedSummary(existing, absAggregatedPath, runPaths, reason);
}
if (cacheMaxHours > 0) {
const ageHours = (Date.now() - new Date(existing.generatedAt).getTime()) / (1000 * 60 * 60);
if (ageHours < cacheMaxHours) {
console.log(`[BIKEGEAR] Using cached data (${ageHours.toFixed(1)}h old) — ${successCount} ok products. Set BIKEGEAR_CACHE_HOURS=0 to force re-scrape.`);
return {
analysis: existing.analysis || {},
outputJsonPath: absAggregatedPath,
historyPath: path.resolve(process.cwd(), runPaths.historyJson),
cached: true,
};
return buildCachedSummary(existing, absAggregatedPath, runPaths, `${ageHours.toFixed(1)}h old`);
}
console.log(`[BIKEGEAR] Cache expired (${ageHours.toFixed(1)}h old). Re-scraping...`);
} else if (Array.isArray(products) && products.length > 0 && successCount === 0) {
console.log(`[BIKEGEAR] Cached data has 0 successful products — ignoring cache and re-scraping.`);
}
} catch {
// no cache yet
} else if (Array.isArray(products) && products.length > 0 && successCount === 0) {
console.log(`[BIKEGEAR] Cached data has 0 successful products — ignoring cache and re-scraping.`);
}
} catch {
// no cache yet
}
if (cacheOnly) {
throw new Error(
`[BIKEGEAR] Cache-only mode is enabled, but no valid cached data was found at ${absAggregatedPath}. ` +
"Sync a valid 01_products_aggregated.json into that path or set BIKEGEAR_CACHE_ONLY=false to scrape."
);
}
const now = new Date().toISOString();

41
src/business-logic/import-pipeline/sources/bikegear/scraper.js
View File

@ -7,7 +7,12 @@
* Fetch modes (set in .env):
* BIKEGEAR_FETCH_MODE=browser — use Playwright Chromium (default; bypasses Cloudflare on VPS IPs)
* BIKEGEAR_FETCH_MODE=http — use axios (works only from residential/whitelisted IPs)
* BIKEGEAR_PROXY_URL=http://user:pass@host:port — proxy for http mode
* BIKEGEAR_PROXY_URL=http://user:pass@host:port
* Or Webshare-style parts:
* BIKEGEAR_PROXY_HOST=p.webshare.io
* BIKEGEAR_PROXY_PORT=80
* BIKEGEAR_PROXY_USERNAME=...
* BIKEGEAR_PROXY_PASSWORD=...
*/
const axios = require("axios");
@ -19,11 +24,41 @@ const LISTING_CONCURRENCY = Number(process.env.BIKEGEAR_LISTING_CONCURRENCY ?? (
const DETAIL_CONCURRENCY = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY ?? (process.env.BIKEGEAR_FETCH_MODE === "http" ? 2 : 1));
// Delay between requests in browser mode (ms) — keeps us under Cloudflare's rate limit
const BROWSER_REQUEST_DELAY = Number(process.env.BIKEGEAR_REQUEST_DELAY ?? 2500);
const PROXY_URL = process.env.BIKEGEAR_PROXY_URL || null;
const PROXY_URL = getProxyUrl();
// Default to browser mode — it bypasses Cloudflare without needing a proxy
const FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "browser";
console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${PROXY_URL.replace(/:\/\/[^@]+@/, "://<redacted>@")})` : ""}`);
console.log(`[BIKEGEAR] Fetch mode: ${FETCH_MODE}${PROXY_URL ? ` (proxy: ${maskProxyUrl(PROXY_URL)})` : ""}`);
function getProxyUrl() {
if (process.env.BIKEGEAR_PROXY_URL) {
return process.env.BIKEGEAR_PROXY_URL;
}
const host = process.env.BIKEGEAR_PROXY_HOST;
const port = process.env.BIKEGEAR_PROXY_PORT;
if (!host || !port) {
return null;
}
const protocol = String(process.env.BIKEGEAR_PROXY_PROTOCOL || "http").replace(/:$/, "");
const username = process.env.BIKEGEAR_PROXY_USERNAME || "";
const password = process.env.BIKEGEAR_PROXY_PASSWORD || "";
const auth = username
? `${encodeURIComponent(username)}:${encodeURIComponent(password)}@`
: "";
return `${protocol}://${auth}${host}:${port}`;
}
function maskProxyUrl(proxyUrl) {
try {
const u = new URL(proxyUrl);
return `${u.protocol}//${u.username ? "<redacted>@" : ""}${u.host}`;
} catch {
return "<invalid proxy url>";
}
}
// ── Browser pool (Playwright persistent context) ───────────────────────────
// Uses launchPersistentContext so the browser profile (cookies, cache, fingerprint)