541 lines
18 KiB
JavaScript
541 lines
18 KiB
JavaScript
const { getKytIndiaWebsiteData } = require("./01_get_kytindia_website_data");
|
|
const { downloadProductImagesFromAggregatedJson } = require("./02_download_product_images");
|
|
const { applyWatermarkToDownloadedImages } = require("./03_watermark_downloaded_images");
|
|
const fs = require("node:fs/promises");
|
|
const path = require("node:path");
|
|
const fsSync = require("node:fs");
|
|
const { convertKytJsonToShopifyProducts } = require("./05_kyt_to_shopify_converter");
|
|
const { upsertShopifyProductFull } = require("./06_shopify_product_upsert");
|
|
const { uploadKytWatermarkedImagesToShopifyFiles } = require("./04_shopify_image_file_uploader");
|
|
|
|
const DEFAULT_AGGREGATED_JSON = "data/01_products_aggregated.json";
|
|
const DEFAULT_SHOPIFY_READY_JSON = "data/05_shopify_products_ready.json";
|
|
const DEFAULT_UPLOADED_MAP_JSON = "data/04_shopify_uploaded_images_map.json";
|
|
const DEFAULT_LOGS_DIR = "data/99_run_logs";
|
|
|
|
function nowIsoLocal() {
|
|
const d = new Date();
|
|
return d.toISOString();
|
|
}
|
|
|
|
function initRunLogger(logsDir = DEFAULT_LOGS_DIR) {
|
|
const absLogsDir = path.resolve(process.cwd(), logsDir);
|
|
fsSync.mkdirSync(absLogsDir, { recursive: true });
|
|
|
|
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
const logPath = path.join(absLogsDir, `${stamp}.log`);
|
|
const stream = fsSync.createWriteStream(logPath, { flags: "a" });
|
|
|
|
const original = {
|
|
log: console.log.bind(console),
|
|
info: console.info.bind(console),
|
|
warn: console.warn.bind(console),
|
|
error: console.error.bind(console)
|
|
};
|
|
|
|
function writeLine(level, args) {
|
|
const text = args.map((a) => {
|
|
if (typeof a === "string") return a;
|
|
try {
|
|
return JSON.stringify(a);
|
|
} catch {
|
|
return String(a);
|
|
}
|
|
}).join(" ");
|
|
stream.write(`[${nowIsoLocal()}] [${level}] ${text}\n`);
|
|
}
|
|
|
|
console.log = (...args) => {
|
|
writeLine("LOG", args);
|
|
original.log(...args);
|
|
};
|
|
console.info = (...args) => {
|
|
writeLine("INFO", args);
|
|
original.info(...args);
|
|
};
|
|
console.warn = (...args) => {
|
|
writeLine("WARN", args);
|
|
original.warn(...args);
|
|
};
|
|
console.error = (...args) => {
|
|
writeLine("ERROR", args);
|
|
original.error(...args);
|
|
};
|
|
|
|
console.log(`[RUN-LOG] Writing logs to ${logPath}`);
|
|
return { logPath };
|
|
}
|
|
|
|
function loadDotEnvFile(filePath = ".env") {
|
|
const abs = path.resolve(process.cwd(), filePath);
|
|
if (!fsSync.existsSync(abs)) return;
|
|
|
|
const raw = fsSync.readFileSync(abs, "utf8");
|
|
const lines = raw.split(/\r?\n/);
|
|
for (const line of lines) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
|
|
const eq = trimmed.indexOf("=");
|
|
if (eq <= 0) continue;
|
|
|
|
const key = trimmed.slice(0, eq).trim();
|
|
const value = trimmed.slice(eq + 1).trim();
|
|
if (key && process.env[key] == null) {
|
|
process.env[key] = value;
|
|
}
|
|
}
|
|
}
|
|
|
|
loadDotEnvFile(".env");
|
|
const RUN_LOGGER = initRunLogger(DEFAULT_LOGS_DIR);
|
|
|
|
function parseCliArgs(argv = process.argv.slice(2)) {
|
|
const out = {
|
|
limit: null
|
|
};
|
|
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
const a = argv[i];
|
|
if ((a === "--limit" || a === "-n") && argv[i + 1]) {
|
|
const n = Number.parseInt(argv[i + 1], 10);
|
|
if (Number.isFinite(n) && n > 0) out.limit = n;
|
|
}
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
async function buildLimitedAggregatedJson(inputPath, limit) {
|
|
const absInputPath = path.resolve(process.cwd(), inputPath);
|
|
const raw = await fs.readFile(absInputPath, "utf8");
|
|
const parsed = JSON.parse(raw);
|
|
const products = Array.isArray(parsed?.products) ? parsed.products : [];
|
|
const limited = products.slice(0, limit);
|
|
|
|
const outPath = path.resolve(process.cwd(), `data/01_products_aggregated.limit_${limit}.json`);
|
|
const payload = {
|
|
...parsed,
|
|
generatedAt: new Date().toISOString(),
|
|
products: limited,
|
|
analysis: {
|
|
...(parsed.analysis || {}),
|
|
totalProductsUnique: limited.length,
|
|
limitedFrom: products.length,
|
|
limitApplied: limit
|
|
}
|
|
};
|
|
|
|
await fs.writeFile(outPath, JSON.stringify(payload, null, 2), "utf8");
|
|
return {
|
|
inputPath: absInputPath,
|
|
outputPath: outPath,
|
|
totalBefore: products.length,
|
|
totalAfter: limited.length
|
|
};
|
|
}
|
|
|
|
function readBooleanEnv(name, fallback = false) {
|
|
const val = process.env[name];
|
|
if (val == null) return fallback;
|
|
return ["1", "true", "yes", "y", "on"].includes(String(val).toLowerCase());
|
|
}
|
|
|
|
function logStageSummary(stepKey, summary = {}) {
|
|
const parts = Object.entries(summary)
|
|
.filter(([, value]) => value !== undefined && value !== null && value !== "")
|
|
.map(([key, value]) => `${key}=${typeof value === "object" ? JSON.stringify(value) : value}`);
|
|
|
|
console.log(`[STAGE-SUMMARY] ${stepKey} | ${parts.join(" | ")}`);
|
|
}
|
|
|
|
async function convertAggregatedToShopifyReady({
|
|
inputPath = DEFAULT_AGGREGATED_JSON,
|
|
outputPath = DEFAULT_SHOPIFY_READY_JSON,
|
|
uploadedImageMapPath = DEFAULT_UPLOADED_MAP_JSON,
|
|
imageBaseUrl = process.env.KYT_IMAGE_BASE_URL || "",
|
|
brand = process.env.SHOPIFY_BRAND || "KYT"
|
|
} = {}) {
|
|
const absInputPath = path.resolve(process.cwd(), inputPath);
|
|
const absOutputPath = path.resolve(process.cwd(), outputPath);
|
|
const absUploadedMapPath = path.resolve(process.cwd(), uploadedImageMapPath);
|
|
|
|
const raw = await fs.readFile(absInputPath, "utf8");
|
|
const parsed = JSON.parse(raw);
|
|
let uploadedImageMap = null;
|
|
try {
|
|
const mapRaw = await fs.readFile(absUploadedMapPath, "utf8");
|
|
uploadedImageMap = JSON.parse(mapRaw);
|
|
} catch {
|
|
uploadedImageMap = null;
|
|
}
|
|
|
|
const convertedProducts = convertKytJsonToShopifyProducts(parsed, {
|
|
imageBaseUrl,
|
|
brand,
|
|
uploadedImageMap
|
|
});
|
|
|
|
const payload = {
|
|
generatedAt: new Date().toISOString(),
|
|
sourceFile: absInputPath,
|
|
totalProducts: convertedProducts.length,
|
|
products: convertedProducts
|
|
};
|
|
|
|
await fs.mkdir(path.dirname(absOutputPath), { recursive: true });
|
|
await fs.writeFile(absOutputPath, JSON.stringify(payload, null, 2), "utf8");
|
|
|
|
return {
|
|
inputPath: absInputPath,
|
|
outputPath: absOutputPath,
|
|
uploadedImageMapPath: absUploadedMapPath,
|
|
totalProducts: convertedProducts.length
|
|
};
|
|
}
|
|
|
|
async function upsertShopifyProductsFromConverted({
|
|
convertedPath = DEFAULT_SHOPIFY_READY_JSON,
|
|
shop = process.env.SHOPIFY_SHOP,
|
|
accessToken = process.env.SHOPIFY_ACCESS_TOKEN,
|
|
locationId = process.env.SHOPIFY_LOCATION_ID || null,
|
|
enableSeo = readBooleanEnv("SHOPIFY_ENABLE_SEO", false),
|
|
apiVersion = process.env.SHOPIFY_API_VERSION || "2025-10"
|
|
} = {}) {
|
|
if (!shop) {
|
|
throw new Error("Missing SHOPIFY_SHOP for Shopify upsert stage.");
|
|
}
|
|
if (!accessToken) {
|
|
throw new Error("Missing SHOPIFY_ACCESS_TOKEN for Shopify upsert stage.");
|
|
}
|
|
|
|
const absConvertedPath = path.resolve(process.cwd(), convertedPath);
|
|
const raw = await fs.readFile(absConvertedPath, "utf8");
|
|
const parsed = JSON.parse(raw);
|
|
const products = Array.isArray(parsed?.products) ? parsed.products : [];
|
|
|
|
const startedAtMs = Date.now();
|
|
const metrics = {
|
|
sourcePath: absConvertedPath,
|
|
total: products.length,
|
|
processed: 0,
|
|
created: 0,
|
|
updated: 0,
|
|
failed: 0,
|
|
errors: [],
|
|
startedAt: new Date(startedAtMs).toISOString(),
|
|
finishedAt: null,
|
|
durationSeconds: 0,
|
|
successRate: 0
|
|
};
|
|
|
|
for (let i = 0; i < products.length; i += 1) {
|
|
const product = products[i];
|
|
const label = product?.attributes?.product_name || product?.id || `item-${i + 1}`;
|
|
|
|
try {
|
|
const result = await upsertShopifyProductFull({
|
|
shop,
|
|
accessToken,
|
|
product,
|
|
locationId,
|
|
enableSeo,
|
|
apiVersion
|
|
});
|
|
|
|
metrics.processed += 1;
|
|
if (result.action === "created") metrics.created += 1;
|
|
if (result.action === "updated") metrics.updated += 1;
|
|
|
|
if ((i + 1) % 10 === 0 || i === products.length - 1) {
|
|
console.log(
|
|
`[SHOPIFY] ${i + 1}/${products.length} processed | created=${metrics.created} updated=${metrics.updated} failed=${metrics.failed}`
|
|
);
|
|
}
|
|
} catch (error) {
|
|
metrics.processed += 1;
|
|
metrics.failed += 1;
|
|
metrics.errors.push({
|
|
index: i + 1,
|
|
product: label,
|
|
error: error.message
|
|
});
|
|
console.log(`[SHOPIFY-FAIL] ${label} -> ${error.message}`);
|
|
}
|
|
}
|
|
|
|
const endedAtMs = Date.now();
|
|
metrics.finishedAt = new Date(endedAtMs).toISOString();
|
|
metrics.durationSeconds = Number(((endedAtMs - startedAtMs) / 1000).toFixed(2));
|
|
metrics.successRate = metrics.total > 0
|
|
? Number((((metrics.total - metrics.failed) / metrics.total) * 100).toFixed(2))
|
|
: 0;
|
|
|
|
return metrics;
|
|
}
|
|
|
|
async function runFullKytPipeline(options = {}) {
|
|
const cli = parseCliArgs();
|
|
const onProgress = typeof options.onProgress === "function" ? options.onProgress : null;
|
|
const emitProgress = (stepIndex, stepKey, message) => {
|
|
if (onProgress) {
|
|
onProgress({
|
|
stepIndex,
|
|
totalSteps: 6,
|
|
stepKey,
|
|
message
|
|
});
|
|
}
|
|
};
|
|
|
|
emitProgress(1, "fetchWebsiteData", "Fetching KYT website data");
|
|
console.log("[PIPELINE 1/6] Fetching KYT website data...");
|
|
const dataSummary = await getKytIndiaWebsiteData();
|
|
|
|
let aggregatedPathForRun = DEFAULT_AGGREGATED_JSON;
|
|
let imagesDirForRun = "data/02_downloaded_product_images";
|
|
let limitSummary = null;
|
|
|
|
if (cli.limit) {
|
|
limitSummary = await buildLimitedAggregatedJson(DEFAULT_AGGREGATED_JSON, cli.limit);
|
|
aggregatedPathForRun = path.relative(process.cwd(), limitSummary.outputPath).replace(/\\/g, "/");
|
|
imagesDirForRun = `data/02_downloaded_product_images.limit_${cli.limit}`;
|
|
console.log(
|
|
`[PIPELINE] Limit applied: first ${limitSummary.totalAfter} of ${limitSummary.totalBefore} products -> ${aggregatedPathForRun}`
|
|
);
|
|
}
|
|
|
|
emitProgress(2, "downloadImages", "Downloading product images");
|
|
console.log("\n[PIPELINE 2/6] Downloading product images...");
|
|
const downloadSummary = await downloadProductImagesFromAggregatedJson({
|
|
jsonPath: aggregatedPathForRun,
|
|
outputDir: imagesDirForRun,
|
|
concurrency: Math.max(1, Number.parseInt(process.env.IMAGE_DOWNLOAD_CONCURRENCY || "8", 10) || 8)
|
|
});
|
|
logStageSummary("downloadImages", {
|
|
total: downloadSummary?.totalImagesFound ?? 0,
|
|
downloaded: downloadSummary?.downloaded ?? 0,
|
|
skipped: downloadSummary?.skipped ?? 0,
|
|
failed: downloadSummary?.failed ?? 0,
|
|
products: downloadSummary?.productsCount ?? 0
|
|
});
|
|
|
|
emitProgress(3, "watermarkImages", "Applying watermark to downloaded images");
|
|
console.log("\n[PIPELINE 3/6] Applying watermark in-place...");
|
|
const watermarkPathForRun = process.env.WATERMARK_PATH || "data/watermark.png";
|
|
let watermarkSummary;
|
|
const absWatermarkPath = path.resolve(process.cwd(), watermarkPathForRun);
|
|
if (!fsSync.existsSync(absWatermarkPath)) {
|
|
watermarkSummary = {
|
|
skipped: true,
|
|
reason: `Watermark file not found: ${absWatermarkPath}`,
|
|
imagesDir: path.resolve(process.cwd(), imagesDirForRun),
|
|
watermarkPath: absWatermarkPath,
|
|
totalImagesFound: 0,
|
|
processed: 0,
|
|
skippedCount: 0,
|
|
failed: 0
|
|
};
|
|
console.log(`[PIPELINE 3/6] Watermark stage skipped. File missing: ${absWatermarkPath}`);
|
|
} else {
|
|
const watermarkRequired = readBooleanEnv("WATERMARK_REQUIRED", false);
|
|
try {
|
|
watermarkSummary = await applyWatermarkToDownloadedImages({
|
|
imagesDir: imagesDirForRun,
|
|
watermarkPath: watermarkPathForRun,
|
|
concurrency: Math.max(1, Number.parseInt(process.env.WATERMARK_CONCURRENCY || "4", 10) || 4)
|
|
});
|
|
} catch (error) {
|
|
if (String(error?.message || "").includes("ENOENT") && !watermarkRequired) {
|
|
watermarkSummary = {
|
|
skipped: true,
|
|
reason: `Watermark stage failed with ENOENT and was skipped: ${error.message}`,
|
|
imagesDir: path.resolve(process.cwd(), imagesDirForRun),
|
|
watermarkPath: absWatermarkPath,
|
|
totalImagesFound: 0,
|
|
processed: 0,
|
|
skippedCount: 0,
|
|
failed: 0
|
|
};
|
|
console.log(`[PIPELINE 3/6] Watermark stage skipped after ENOENT: ${error.message}`);
|
|
} else {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
logStageSummary("watermarkImages", {
|
|
total: watermarkSummary?.totalImagesFound ?? 0,
|
|
processed: watermarkSummary?.processed ?? 0,
|
|
skipped: watermarkSummary?.skipped ?? watermarkSummary?.skippedCount ?? 0,
|
|
failed: watermarkSummary?.failed ?? 0,
|
|
concurrency: watermarkSummary?.concurrency ?? ""
|
|
});
|
|
|
|
emitProgress(4, "uploadImagesToShopifyFiles", "Uploading watermarked images to Shopify Files");
|
|
console.log("\n[PIPELINE 4/6] Uploading watermarked images to Shopify Files...");
|
|
const imageUploadEnabled = readBooleanEnv("SHOPIFY_ENABLE_IMAGE_UPLOAD", true);
|
|
const imageUploadRequired = readBooleanEnv("SHOPIFY_IMAGE_UPLOAD_REQUIRED", false);
|
|
let imageUploadSummary;
|
|
|
|
if (!imageUploadEnabled) {
|
|
imageUploadSummary = {
|
|
skipped: true,
|
|
reason: "SHOPIFY_ENABLE_IMAGE_UPLOAD=false",
|
|
totalTasks: 0,
|
|
processed: 0,
|
|
uploaded: 0,
|
|
failed: 0
|
|
};
|
|
console.log("[PIPELINE 4/6] Image upload stage skipped by config.");
|
|
} else {
|
|
try {
|
|
imageUploadSummary = await uploadKytWatermarkedImagesToShopifyFiles({
|
|
shop: process.env.SHOPIFY_SHOP,
|
|
accessToken: process.env.SHOPIFY_ACCESS_TOKEN,
|
|
apiVersion: process.env.SHOPIFY_API_VERSION || "2025-10",
|
|
aggregatedJsonPath: aggregatedPathForRun,
|
|
imagesDir: imagesDirForRun,
|
|
statePath: "data/04_shopify_image_upload_state.json",
|
|
mapPath: DEFAULT_UPLOADED_MAP_JSON,
|
|
concurrency: Math.max(1, Number.parseInt(process.env.SHOPIFY_IMAGE_UPLOAD_CONCURRENCY || "3", 10) || 3)
|
|
});
|
|
} catch (error) {
|
|
const message = String(error?.message || "Unknown image upload error");
|
|
imageUploadSummary = {
|
|
skipped: true,
|
|
reason: message,
|
|
totalTasks: 0,
|
|
processed: 0,
|
|
uploaded: 0,
|
|
failed: 0
|
|
};
|
|
|
|
console.log(`[PIPELINE 4/6] Image upload stage failed: ${message}`);
|
|
if (imageUploadRequired) {
|
|
throw error;
|
|
}
|
|
console.log("[PIPELINE 4/6] Continuing pipeline without image upload (SHOPIFY_IMAGE_UPLOAD_REQUIRED=false).");
|
|
}
|
|
}
|
|
logStageSummary("uploadImagesToShopifyFiles", {
|
|
total: imageUploadSummary?.totalTasks ?? 0,
|
|
processed: imageUploadSummary?.processed ?? 0,
|
|
uploaded: imageUploadSummary?.uploaded ?? 0,
|
|
skipped: imageUploadSummary?.skipped ?? 0,
|
|
failed: imageUploadSummary?.failed ?? 0,
|
|
concurrency: imageUploadSummary?.concurrency ?? ""
|
|
});
|
|
|
|
emitProgress(5, "convertToShopifyReady", "Converting KYT data to Shopify-ready products");
|
|
console.log("\n[PIPELINE 5/6] Converting KYT data to Shopify-ready products...");
|
|
const conversionSummary = await convertAggregatedToShopifyReady({
|
|
inputPath: aggregatedPathForRun,
|
|
outputPath: DEFAULT_SHOPIFY_READY_JSON,
|
|
uploadedImageMapPath: DEFAULT_UPLOADED_MAP_JSON,
|
|
imageBaseUrl: process.env.KYT_IMAGE_BASE_URL || "",
|
|
brand: process.env.SHOPIFY_BRAND || "KYT"
|
|
});
|
|
logStageSummary("convertToShopifyReady", {
|
|
total: conversionSummary?.totalProducts ?? 0
|
|
});
|
|
|
|
emitProgress(6, "upsertToShopify", "Upserting products to Shopify");
|
|
console.log("\n[PIPELINE 6/6] Upserting products to Shopify...");
|
|
const shopifyUpsertSummary = await upsertShopifyProductsFromConverted({
|
|
convertedPath: DEFAULT_SHOPIFY_READY_JSON,
|
|
shop: process.env.SHOPIFY_SHOP,
|
|
accessToken: process.env.SHOPIFY_ACCESS_TOKEN,
|
|
locationId: process.env.SHOPIFY_LOCATION_ID || null,
|
|
enableSeo: readBooleanEnv("SHOPIFY_ENABLE_SEO", false),
|
|
apiVersion: process.env.SHOPIFY_API_VERSION || "2025-10"
|
|
});
|
|
logStageSummary("upsertToShopify", {
|
|
total: shopifyUpsertSummary?.total ?? 0,
|
|
processed: shopifyUpsertSummary?.processed ?? 0,
|
|
created: shopifyUpsertSummary?.created ?? 0,
|
|
updated: shopifyUpsertSummary?.updated ?? 0,
|
|
failed: shopifyUpsertSummary?.failed ?? 0,
|
|
successRate: shopifyUpsertSummary?.successRate ?? 0
|
|
});
|
|
|
|
const summary = {
|
|
completedAt: new Date().toISOString(),
|
|
runLogPath: RUN_LOGGER.logPath,
|
|
limit: cli.limit || null,
|
|
limitSummary,
|
|
steps: {
|
|
fetchWebsiteData: dataSummary,
|
|
downloadImages: downloadSummary,
|
|
watermarkImages: watermarkSummary,
|
|
uploadImagesToShopifyFiles: imageUploadSummary,
|
|
convertToShopifyReady: conversionSummary,
|
|
upsertToShopify: shopifyUpsertSummary
|
|
},
|
|
upcomingSteps: []
|
|
};
|
|
|
|
emitProgress(6, "completed", "KYT pipeline completed");
|
|
console.log("\n=== FULL PIPELINE SUMMARY ===");
|
|
console.log(JSON.stringify(summary, null, 2));
|
|
console.log("\n=== SUMMARY TABLE ===");
|
|
console.table([
|
|
{
|
|
step: "fetchWebsiteData",
|
|
total: dataSummary?.analysis?.totalProductsUnique ?? "",
|
|
processed: dataSummary?.analysis?.detailFetchedNow ?? "",
|
|
skipped: dataSummary?.analysis?.cachedDetailReused ?? "",
|
|
failed: dataSummary?.analysis?.detailFailed ?? ""
|
|
},
|
|
{
|
|
step: "downloadImages",
|
|
total: downloadSummary?.totalImagesFound ?? "",
|
|
processed: downloadSummary?.downloaded ?? "",
|
|
skipped: downloadSummary?.skipped ?? "",
|
|
failed: downloadSummary?.failed ?? ""
|
|
},
|
|
{
|
|
step: "watermarkImages",
|
|
total: watermarkSummary?.totalImagesFound ?? "",
|
|
processed: watermarkSummary?.processed ?? "",
|
|
skipped: watermarkSummary?.skipped ?? "",
|
|
failed: watermarkSummary?.failed ?? ""
|
|
},
|
|
{
|
|
step: "uploadImagesToShopifyFiles",
|
|
total: imageUploadSummary?.totalTasks ?? "",
|
|
processed: imageUploadSummary?.processed ?? "",
|
|
skipped: imageUploadSummary?.skipped ?? "",
|
|
failed: imageUploadSummary?.failed ?? ""
|
|
},
|
|
{
|
|
step: "convertToShopifyReady",
|
|
total: conversionSummary?.totalProducts ?? "",
|
|
processed: conversionSummary?.totalProducts ?? "",
|
|
skipped: "",
|
|
failed: ""
|
|
},
|
|
{
|
|
step: "upsertToShopify",
|
|
total: shopifyUpsertSummary?.total ?? "",
|
|
processed: shopifyUpsertSummary?.processed ?? "",
|
|
skipped: "",
|
|
failed: shopifyUpsertSummary?.failed ?? ""
|
|
}
|
|
]);
|
|
|
|
return summary;
|
|
}
|
|
|
|
module.exports = {
|
|
runFullKytPipeline,
|
|
convertAggregatedToShopifyReady,
|
|
upsertShopifyProductsFromConverted
|
|
};
|
|
|
|
if (require.main === module) {
|
|
runFullKytPipeline().catch((error) => {
|
|
console.error("Full pipeline failed:", error.message);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|