From f98febc5c5fbb485b4f1e06f582f037df6064279 Mon Sep 17 00:00:00 2001 From: MOHAN Date: Wed, 5 Aug 2026 23:50:13 +0530 Subject: [PATCH] feat(bikegear): add local scraper with resume support for residential IP scraping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standalone script for running bikegear scrape from a local machine (residential IP bypasses Cloudflare). Saves progress after every brand so Ctrl+C resumes cleanly. Output is written to data/sources/bikegear/01_products_aggregated.json in the exact same format the server pipeline expects — copy the file to the server and the pipeline uses it as a 24h cache, skipping straight to stage 2. Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 3 + bikegear-local-scraper/run.js | 193 ++++++++++++++++++ .../sources/bikegear/scraper.js | 2 +- 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 bikegear-local-scraper/run.js diff --git a/.gitignore b/.gitignore index 25cedfd..9a133bf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ coverage/ # Runtime/generated data data/ + +# Local bikegear scraper progress (not committed — local machine only) +bikegear-local-scraper/progress.json logs/ *.log src/business-logic/import-pipeline/sources/outputs/ diff --git a/bikegear-local-scraper/run.js b/bikegear-local-scraper/run.js new file mode 100644 index 0000000..8e5292b --- /dev/null +++ b/bikegear-local-scraper/run.js @@ -0,0 +1,193 @@ +/** + * BikeGear Local Scraper — run this on your LOCAL machine (residential IP). + * + * Usage (from Race-Nation-Shopify-App-Backend/): + * node bikegear-local-scraper/run.js + * + * Ctrl+C at any time — progress is saved. Re-run to resume from last brand. + * + * When done, copy the output file to the server: + * scp data/sources/bikegear/01_products_aggregated.json \ + * root@:/home/dev/apps/Race-Nation-Shopify-Backend/data/sources/bikegear/ + * + * Then trigger the pipeline on the server — it will see the 24h cache and + * skip straight to image download → watermark → Shopify upload. + */ + +const path = require("node:path"); +const fs = require("node:fs/promises"); + +// Default to HTTP mode for local runs — residential IP is not blocked +process.env.BIKEGEAR_FETCH_MODE = process.env.BIKEGEAR_FETCH_MODE || "http"; +// Higher concurrency is safe from a residential IP +process.env.BIKEGEAR_LISTING_CONCURRENCY = process.env.BIKEGEAR_LISTING_CONCURRENCY || "5"; +process.env.BIKEGEAR_DETAIL_CONCURRENCY = process.env.BIKEGEAR_DETAIL_CONCURRENCY || "5"; + +const { BRANDS } = require("../src/business-logic/import-pipeline/sources/bikegear/brands"); +const { scrapeBrandProductUrls, scrapeProductDetail, closeBrowser } = require("../src/business-logic/import-pipeline/sources/bikegear/scraper"); + +const SOURCE_KEY = "bikegear"; +const SOURCE_LABEL = "BikeGear India"; + +// Progress file — tracks which brands are done so we can resume +const PROGRESS_FILE = path.resolve(__dirname, "progress.json"); +// Final output — same path the server pipeline uses (stage 1 cache) +const OUTPUT_FILE = path.resolve("data/sources/bikegear/01_products_aggregated.json"); + +async function loadProgress() { + try { + const raw = await fs.readFile(PROGRESS_FILE, "utf8"); + return JSON.parse(raw); + } catch { + return { completedBrands: [], products: [] }; + } +} + +async function saveProgress(progress) { + await fs.mkdir(path.dirname(PROGRESS_FILE), { recursive: true }); + await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), "utf8"); +} + +async function writeOutput(products) { + const successfulProducts = products.filter((p) => !(p.scraped || p).scrapeError); + const failedProducts = products.filter((p) => (p.scraped || p).scrapeError); + + const payload = { + generatedAt: new Date().toISOString(), + sourceKey: SOURCE_KEY, + sourceLabel: SOURCE_LABEL, + analysis: { + timestamp: new Date().toISOString(), + totalProducts: products.length, + totalProductsUnique: successfulProducts.length, + detailSuccess: successfulProducts.length, + detailFailed: failedProducts.length, + }, + products, + }; + + await fs.mkdir(path.dirname(OUTPUT_FILE), { recursive: true }); + await fs.writeFile(OUTPUT_FILE, JSON.stringify(payload, null, 2), "utf8"); + console.log(`\n[OUTPUT] Written to ${OUTPUT_FILE}`); + console.log(`[OUTPUT] ${successfulProducts.length} products OK, ${failedProducts.length} failed.`); +} + +function wrapProduct(product) { + return { + productId: product.sku || product.url, + sourceKey: SOURCE_KEY, + brand: product.brand || "BikeGear", + bikeModel: "", + productSummary: { + id: product.sku, + name: product.name, + img: product.images || [], + cost: { mrp: product.price || 0 }, + }, + scraped: product, + }; +} + +async function main() { + console.log(`[BIKEGEAR-LOCAL] Fetch mode: ${process.env.BIKEGEAR_FETCH_MODE}`); + console.log(`[BIKEGEAR-LOCAL] ${BRANDS.length} brands total.`); + + const progress = await loadProgress(); + const completedSet = new Set(progress.completedBrands); + + const remaining = BRANDS.filter((b) => !completedSet.has(b.name)); + console.log(`[BIKEGEAR-LOCAL] ${completedSet.size} brands already done. ${remaining.length} remaining.\n`); + + let shuttingDown = false; + + async function gracefulExit() { + if (shuttingDown) return; + shuttingDown = true; + console.log("\n[BIKEGEAR-LOCAL] Ctrl+C received — saving progress and exiting..."); + await saveProgress(progress); + await writeOutput(progress.products); + await closeBrowser(); + console.log("[BIKEGEAR-LOCAL] Progress saved. Re-run to continue."); + process.exit(0); + } + + process.on("SIGINT", gracefulExit); + process.on("SIGTERM", gracefulExit); + + for (const brand of remaining) { + if (shuttingDown) break; + + const productUrls = await scrapeBrandProductUrls(brand); + + if (!productUrls.length) { + console.warn(`[${brand.name}] No product URLs — skipping.`); + progress.completedBrands.push(brand.name); + await saveProgress(progress); + continue; + } + + console.log(`[${brand.name}] Scraping ${productUrls.length} products...`); + let done = 0; + + // Scrape detail pages in batches (concurrency from env) + const concurrency = Number(process.env.BIKEGEAR_DETAIL_CONCURRENCY || 5); + const brandProducts = []; + + async function runConcurrent(items, fn, limit) { + let idx = 0; + const results = new Array(items.length); + async function worker() { + while (idx < items.length) { + const i = idx++; + results[i] = await fn(items[i], i); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker())); + return results; + } + + const rawProducts = await runConcurrent(productUrls, async (url) => { + if (shuttingDown) return { url, scrapeError: "interrupted" }; + const product = await scrapeProductDetail(url, brand.name); + done++; + if (product.scrapeError) { + console.warn(` ${done}/${productUrls.length} ERR: ${product.scrapeError}`); + } else { + console.log(` ${done}/${productUrls.length} OK: "${product.name}"`); + } + return product; + }, concurrency); + + for (const raw of rawProducts) { + brandProducts.push(wrapProduct(raw)); + } + + const ok = rawProducts.filter((p) => !p.scrapeError).length; + console.log(`[${brand.name}] Done: ${ok}/${productUrls.length} OK\n`); + + progress.products.push(...brandProducts); + progress.completedBrands.push(brand.name); + + // Save progress after every brand + await saveProgress(progress); + // Also update the output file incrementally + await writeOutput(progress.products); + } + + if (!shuttingDown) { + console.log("\n[BIKEGEAR-LOCAL] All brands complete!"); + await writeOutput(progress.products); + await closeBrowser(); + + // Clean up progress file — scrape is done + await fs.unlink(PROGRESS_FILE).catch(() => {}); + console.log("[BIKEGEAR-LOCAL] Progress file removed (clean run)."); + console.log(`\nNext step: copy ${OUTPUT_FILE} to the server and trigger the pipeline.`); + } +} + +main().catch(async (err) => { + console.error("[BIKEGEAR-LOCAL] Fatal error:", err); + await closeBrowser(); + process.exit(1); +}); diff --git a/src/business-logic/import-pipeline/sources/bikegear/scraper.js b/src/business-logic/import-pipeline/sources/bikegear/scraper.js index 983fc18..8dfe1fd 100644 --- a/src/business-logic/import-pipeline/sources/bikegear/scraper.js +++ b/src/business-logic/import-pipeline/sources/bikegear/scraper.js @@ -623,4 +623,4 @@ async function scrapeBikeGear() { return allProducts; } -module.exports = { scrapeBikeGear, closeBrowser }; +module.exports = { scrapeBikeGear, closeBrowser, scrapeBrandProductUrls, scrapeProductDetail };