const fs = require("node:fs/promises"); const path = require("node:path"); const DEFAULT_AGGREGATED_JSON = path.join("data", "01_products_aggregated.json"); const DEFAULT_IMAGES_DIR = path.join("data", "02_downloaded_product_images"); const DEFAULT_STATE_PATH = path.join("data", "04_shopify_image_upload_state.json"); const DEFAULT_MAP_PATH = path.join("data", "04_shopify_uploaded_images_map.json"); async function mapWithConcurrency(items, concurrency, worker) { const results = new Array(items.length); let index = 0; async function runWorker() { while (true) { const current = index; index += 1; if (current >= items.length) { return; } results[current] = await worker(items[current], current); } } const workers = Array.from( { length: Math.max(1, Math.min(concurrency, items.length)) }, () => runWorker() ); await Promise.all(workers); return results; } function sanitizeName(value) { return String(value || "") .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_") .replace(/\s+/g, " ") .trim() .slice(0, 150); } function getPathWithoutQuery(value) { const raw = String(value || ""); try { return new URL(raw).pathname; } catch { return raw.split(/[?#]/)[0]; } } function getMimeType(filePath) { const ext = path.extname(filePath).toLowerCase(); if (ext === ".png") return "image/png"; if (ext === ".webp") return "image/webp"; if (ext === ".gif") return "image/gif"; if (ext === ".avif") return "image/avif"; if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg"; if (ext === ".tif" || ext === ".tiff") return "image/tiff"; return "application/octet-stream"; } async function fileFingerprint(filePath) { const stat = await fs.stat(filePath); return `${stat.size}:${Math.trunc(stat.mtimeMs)}`; } function createShopifyClient(shop, accessToken, apiVersion = "2025-10") { return { baseURL: `https://${shop}/admin/api/${apiVersion}/graphql.json`, headers: { "X-Shopify-Access-Token": accessToken, "Content-Type": "application/json" }, timeout: 30000 }; } async function postGraphQL(client, query, variables = {}) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), client.timeout || 30000); try { const response = await fetch(client.baseURL, { method: "POST", headers: client.headers, body: JSON.stringify({ query, variables }), signal: controller.signal }); const json = await response.json(); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${JSON.stringify(json)}`); } if (json.errors?.length) { throw new Error(`GraphQL errors: ${json.errors.map((e) => e.message).join(", ")}`); } return json.data; } finally { clearTimeout(timer); } } async function stagedUploadOneImage(client, localPath) { const stat = await fs.stat(localPath); const filename = path.basename(localPath); const mimeType = getMimeType(localPath); const staged = await postGraphQL( client, `mutation($input: [StagedUploadInput!]!) { stagedUploadsCreate(input: $input) { stagedTargets { url resourceUrl parameters { name value } } userErrors { field message } } }`, { input: [ { filename, mimeType, resource: "FILE", fileSize: String(stat.size), httpMethod: "POST" } ] } ); const stagedErrors = staged.stagedUploadsCreate.userErrors || []; if (stagedErrors.length) { throw new Error(`stagedUploadsCreate failed: ${stagedErrors.map((e) => e.message).join(", ")}`); } const target = staged.stagedUploadsCreate.stagedTargets?.[0]; if (!target?.url || !target?.resourceUrl) { throw new Error("stagedUploadsCreate returned no target."); } const bytes = await fs.readFile(localPath); const form = new FormData(); for (const p of target.parameters || []) { form.append(p.name, p.value); } form.append("file", new Blob([bytes], { type: mimeType }), filename); const uploadRes = await fetch(target.url, { method: "POST", body: form }); if (!uploadRes.ok) { const txt = await uploadRes.text(); throw new Error(`staged binary upload failed: HTTP ${uploadRes.status} ${txt.slice(0, 240)}`); } const created = await postGraphQL( client, `mutation($files: [FileCreateInput!]!) { fileCreate(files: $files) { files { id alt fileStatus ... on MediaImage { image { url } } ... on GenericFile { url } } userErrors { field message } } }`, { files: [ { alt: filename, contentType: "IMAGE", originalSource: target.resourceUrl } ] } ); const createErrors = created.fileCreate.userErrors || []; if (createErrors.length) { throw new Error(`fileCreate failed: ${createErrors.map((e) => e.message).join(", ")}`); } const fileNode = created.fileCreate.files?.[0]; if (!fileNode?.id) { throw new Error("fileCreate returned no file id."); } return { id: fileNode.id, fileStatus: fileNode.fileStatus || "UPLOADED", url: fileNode.image?.url || fileNode.url || "" }; } async function waitForFileReady(client, fileId, maxAttempts = 20, delayMs = 1500) { for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const data = await postGraphQL( client, `query($id: ID!) { node(id: $id) { id ... on File { fileStatus ... on MediaImage { image { url } } ... on GenericFile { url } } } }`, { id: fileId } ); const node = data.node; const status = node?.fileStatus || "UNKNOWN"; const url = node?.image?.url || node?.url || ""; if (status === "READY") { return { status, url }; } if (status === "FAILED") { throw new Error(`File processing failed for ${fileId}`); } if (attempt < maxAttempts) { await new Promise((resolve) => setTimeout(resolve, delayMs)); } } throw new Error(`Timed out waiting for READY status for ${fileId}`); } function buildLocalImageTasks(aggregatedPayload, imagesDir) { const products = Array.isArray(aggregatedPayload?.products) ? aggregatedPayload.products : []; const absImagesDir = path.resolve(process.cwd(), imagesDir); const usedFolderNames = new Set(); const tasks = []; for (let i = 0; i < products.length; i += 1) { const product = products[i] || {}; const productId = product.productId || product?.productDetails?.data?.id || product?.productSummary?.id || `unknown-${i + 1}`; const productNameRaw = product?.productSummary?.name || product?.productDetails?.data?.name || `product-${i + 1}`; let folderName = sanitizeName(productNameRaw) || `product-${i + 1}`; if (usedFolderNames.has(folderName)) { folderName = `${folderName}__${String(productId).slice(0, 8)}`; } usedFolderNames.add(folderName); const productDir = path.join(absImagesDir, folderName); const imagePaths = Array.isArray(product?.productSummary?.img) ? product.productSummary.img : Array.isArray(product?.productDetails?.data?.img) ? product.productDetails.data.img : []; const usedFileNames = new Set(); for (let idx = 0; idx < imagePaths.length; idx += 1) { const sourcePath = imagePaths[idx]; const cleanSourcePath = getPathWithoutQuery(sourcePath); const originalBase = path.basename(cleanSourcePath, path.extname(cleanSourcePath)) || `image_${idx + 1}`; const originalExt = path.extname(cleanSourcePath) || ".png"; const localBase = sanitizeName(originalBase) || `image_${idx + 1}`; let fileName = `${localBase}${originalExt}`; let dupCounter = 2; while (usedFileNames.has(fileName.toLowerCase())) { fileName = `${localBase}_${dupCounter}${originalExt}`; dupCounter += 1; } usedFileNames.add(fileName.toLowerCase()); tasks.push({ productId: String(productId), productName: String(productNameRaw), sourcePath: String(sourcePath), localPath: path.join(productDir, fileName) }); } } return tasks; } async function readJsonOrDefault(filePath, fallback) { try { const raw = await fs.readFile(filePath, "utf8"); const parsed = JSON.parse(raw); return parsed && typeof parsed === "object" ? parsed : fallback; } catch { return fallback; } } async function uploadKytWatermarkedImagesToShopifyFiles(options = {}) { const { shop = process.env.SHOPIFY_SHOP, accessToken = process.env.SHOPIFY_ACCESS_TOKEN, apiVersion = process.env.SHOPIFY_API_VERSION || "2025-10", aggregatedJsonPath = DEFAULT_AGGREGATED_JSON, imagesDir = DEFAULT_IMAGES_DIR, statePath = DEFAULT_STATE_PATH, mapPath = DEFAULT_MAP_PATH, concurrency = Math.max(1, Number.parseInt(process.env.SHOPIFY_IMAGE_UPLOAD_CONCURRENCY || "3", 10) || 3) } = options; if (!shop) throw new Error("Missing shop (or SHOPIFY_SHOP)."); if (!accessToken) throw new Error("Missing accessToken (or SHOPIFY_ACCESS_TOKEN)."); const client = createShopifyClient(shop, accessToken, apiVersion); const absAggregatedPath = path.resolve(process.cwd(), aggregatedJsonPath); const absStatePath = path.resolve(process.cwd(), statePath); const absMapPath = path.resolve(process.cwd(), mapPath); const aggregated = await readJsonOrDefault(absAggregatedPath, { products: [] }); const tasks = buildLocalImageTasks(aggregated, imagesDir); const state = await readJsonOrDefault(absStatePath, { version: 1, updatedAt: null, files: {} }); const stateFiles = state.files && typeof state.files === "object" ? state.files : {}; const byProduct = {}; const bySourcePath = {}; let processed = 0; let uploaded = 0; let skipped = 0; let failed = 0; await mapWithConcurrency(tasks, concurrency, async (task, i) => { processed += 1; try { await fs.access(task.localPath); } catch { failed += 1; return; } try { const fp = await fileFingerprint(task.localPath); const prev = stateFiles[task.localPath]; if (prev && prev.fingerprint === fp && prev.status === "READY" && prev.url) { skipped += 1; console.log(`[IMG-UPLOAD-SKIP] ${task.productId} | ${task.sourcePath} | ${task.localPath}`); if (!byProduct[task.productId]) byProduct[task.productId] = []; byProduct[task.productId].push({ sourcePath: task.sourcePath, localPath: task.localPath, fileId: prev.fileId, status: prev.status, url: prev.url }); bySourcePath[task.sourcePath] = { productId: task.productId, localPath: task.localPath, fileId: prev.fileId, status: prev.status, url: prev.url }; } else { const created = await stagedUploadOneImage(client, task.localPath); const ready = created.fileStatus === "READY" ? { status: "READY", url: created.url || "" } : await waitForFileReady(client, created.id); stateFiles[task.localPath] = { fingerprint: fp, uploadedAt: new Date().toISOString(), fileId: created.id, status: ready.status, url: ready.url }; uploaded += 1; console.log(`[IMG-UPLOAD-OK] ${task.productId} | ${task.sourcePath} | fileId=${created.id} | status=${ready.status}`); if (!byProduct[task.productId]) byProduct[task.productId] = []; byProduct[task.productId].push({ sourcePath: task.sourcePath, localPath: task.localPath, fileId: created.id, status: ready.status, url: ready.url }); bySourcePath[task.sourcePath] = { productId: task.productId, localPath: task.localPath, fileId: created.id, status: ready.status, url: ready.url }; } } catch (error) { failed += 1; stateFiles[task.localPath] = { fingerprint: null, uploadedAt: new Date().toISOString(), status: "FAILED", error: error.message }; console.log(`[IMG-UPLOAD-FAIL] ${task.productName} | ${task.localPath} -> ${error.message}`); } if ((i + 1) % 25 === 0 || i === tasks.length - 1) { console.log(`[IMG-UPLOAD] ${i + 1}/${tasks.length} processed | uploaded=${uploaded} skipped=${skipped} failed=${failed}`); } }); const finalState = { version: 1, updatedAt: new Date().toISOString(), files: stateFiles }; const finalMap = { generatedAt: new Date().toISOString(), sourceAggregatedPath: absAggregatedPath, totalTasks: tasks.length, byProduct, bySourcePath }; await fs.mkdir(path.dirname(absStatePath), { recursive: true }); await fs.mkdir(path.dirname(absMapPath), { recursive: true }); await fs.writeFile(absStatePath, JSON.stringify(finalState, null, 2), "utf8"); await fs.writeFile(absMapPath, JSON.stringify(finalMap, null, 2), "utf8"); return { aggregatedJsonPath: absAggregatedPath, statePath: absStatePath, mapPath: absMapPath, concurrency, totalTasks: tasks.length, processed, uploaded, skipped, failed }; } async function runStandaloneImageUpload() { const summary = await uploadKytWatermarkedImagesToShopifyFiles(); console.log("\nImage upload summary:"); console.log(JSON.stringify(summary, null, 2)); } if (require.main === module) { runStandaloneImageUpload().catch((error) => { console.error("Image upload pipeline failed:", error.message); process.exitCode = 1; }); } module.exports = { uploadKytWatermarkedImagesToShopifyFiles, runStandaloneImageUpload };