473 lines
14 KiB
JavaScript
473 lines
14 KiB
JavaScript
const fs = require("node:fs/promises");
|
|
const path = require("node:path");
|
|
|
|
const DEFAULT_AGGREGATED_JSON = path.join("data", "01_products_aggregated.json");
|
|
const DEFAULT_IMAGES_DIR = path.join("data", "02_downloaded_product_images");
|
|
const DEFAULT_STATE_PATH = path.join("data", "04_shopify_image_upload_state.json");
|
|
const DEFAULT_MAP_PATH = path.join("data", "04_shopify_uploaded_images_map.json");
|
|
|
|
async function mapWithConcurrency(items, concurrency, worker) {
|
|
const results = new Array(items.length);
|
|
let index = 0;
|
|
|
|
async function runWorker() {
|
|
while (true) {
|
|
const current = index;
|
|
index += 1;
|
|
|
|
if (current >= items.length) {
|
|
return;
|
|
}
|
|
|
|
results[current] = await worker(items[current], current);
|
|
}
|
|
}
|
|
|
|
const workers = Array.from(
|
|
{ length: Math.max(1, Math.min(concurrency, items.length)) },
|
|
() => runWorker()
|
|
);
|
|
await Promise.all(workers);
|
|
return results;
|
|
}
|
|
|
|
function sanitizeName(value) {
|
|
return String(value || "")
|
|
.replace(/[<>:"/\\|?*\x00-\x1F]/g, "_")
|
|
.replace(/\s+/g, " ")
|
|
.trim()
|
|
.slice(0, 150);
|
|
}
|
|
|
|
function getPathWithoutQuery(value) {
|
|
const raw = String(value || "");
|
|
try {
|
|
return new URL(raw).pathname;
|
|
} catch {
|
|
return raw.split(/[?#]/)[0];
|
|
}
|
|
}
|
|
|
|
function getMimeType(filePath) {
|
|
const ext = path.extname(filePath).toLowerCase();
|
|
if (ext === ".png") return "image/png";
|
|
if (ext === ".webp") return "image/webp";
|
|
if (ext === ".gif") return "image/gif";
|
|
if (ext === ".avif") return "image/avif";
|
|
if (ext === ".jpg" || ext === ".jpeg") return "image/jpeg";
|
|
if (ext === ".tif" || ext === ".tiff") return "image/tiff";
|
|
return "application/octet-stream";
|
|
}
|
|
|
|
async function fileFingerprint(filePath) {
|
|
const stat = await fs.stat(filePath);
|
|
return `${stat.size}:${Math.trunc(stat.mtimeMs)}`;
|
|
}
|
|
|
|
function createShopifyClient(shop, accessToken, apiVersion = "2025-10") {
|
|
return {
|
|
baseURL: `https://${shop}/admin/api/${apiVersion}/graphql.json`,
|
|
headers: {
|
|
"X-Shopify-Access-Token": accessToken,
|
|
"Content-Type": "application/json"
|
|
},
|
|
timeout: 30000
|
|
};
|
|
}
|
|
|
|
async function postGraphQL(client, query, variables = {}) {
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), client.timeout || 30000);
|
|
try {
|
|
const response = await fetch(client.baseURL, {
|
|
method: "POST",
|
|
headers: client.headers,
|
|
body: JSON.stringify({ query, variables }),
|
|
signal: controller.signal
|
|
});
|
|
|
|
const json = await response.json();
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}: ${JSON.stringify(json)}`);
|
|
}
|
|
if (json.errors?.length) {
|
|
throw new Error(`GraphQL errors: ${json.errors.map((e) => e.message).join(", ")}`);
|
|
}
|
|
return json.data;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
async function stagedUploadOneImage(client, localPath) {
|
|
const stat = await fs.stat(localPath);
|
|
const filename = path.basename(localPath);
|
|
const mimeType = getMimeType(localPath);
|
|
|
|
const staged = await postGraphQL(
|
|
client,
|
|
`mutation($input: [StagedUploadInput!]!) {
|
|
stagedUploadsCreate(input: $input) {
|
|
stagedTargets {
|
|
url
|
|
resourceUrl
|
|
parameters { name value }
|
|
}
|
|
userErrors { field message }
|
|
}
|
|
}`,
|
|
{
|
|
input: [
|
|
{
|
|
filename,
|
|
mimeType,
|
|
resource: "FILE",
|
|
fileSize: String(stat.size),
|
|
httpMethod: "POST"
|
|
}
|
|
]
|
|
}
|
|
);
|
|
|
|
const stagedErrors = staged.stagedUploadsCreate.userErrors || [];
|
|
if (stagedErrors.length) {
|
|
throw new Error(`stagedUploadsCreate failed: ${stagedErrors.map((e) => e.message).join(", ")}`);
|
|
}
|
|
|
|
const target = staged.stagedUploadsCreate.stagedTargets?.[0];
|
|
if (!target?.url || !target?.resourceUrl) {
|
|
throw new Error("stagedUploadsCreate returned no target.");
|
|
}
|
|
|
|
const bytes = await fs.readFile(localPath);
|
|
const form = new FormData();
|
|
for (const p of target.parameters || []) {
|
|
form.append(p.name, p.value);
|
|
}
|
|
form.append("file", new Blob([bytes], { type: mimeType }), filename);
|
|
|
|
const uploadRes = await fetch(target.url, { method: "POST", body: form });
|
|
if (!uploadRes.ok) {
|
|
const txt = await uploadRes.text();
|
|
throw new Error(`staged binary upload failed: HTTP ${uploadRes.status} ${txt.slice(0, 240)}`);
|
|
}
|
|
|
|
const created = await postGraphQL(
|
|
client,
|
|
`mutation($files: [FileCreateInput!]!) {
|
|
fileCreate(files: $files) {
|
|
files {
|
|
id
|
|
alt
|
|
fileStatus
|
|
... on MediaImage {
|
|
image { url }
|
|
}
|
|
... on GenericFile {
|
|
url
|
|
}
|
|
}
|
|
userErrors { field message }
|
|
}
|
|
}`,
|
|
{
|
|
files: [
|
|
{
|
|
alt: filename,
|
|
contentType: "IMAGE",
|
|
originalSource: target.resourceUrl
|
|
}
|
|
]
|
|
}
|
|
);
|
|
|
|
const createErrors = created.fileCreate.userErrors || [];
|
|
if (createErrors.length) {
|
|
throw new Error(`fileCreate failed: ${createErrors.map((e) => e.message).join(", ")}`);
|
|
}
|
|
|
|
const fileNode = created.fileCreate.files?.[0];
|
|
if (!fileNode?.id) {
|
|
throw new Error("fileCreate returned no file id.");
|
|
}
|
|
|
|
return { id: fileNode.id, fileStatus: fileNode.fileStatus || "UPLOADED", url: fileNode.image?.url || fileNode.url || "" };
|
|
}
|
|
|
|
async function waitForFileReady(client, fileId, maxAttempts = 20, delayMs = 1500) {
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
const data = await postGraphQL(
|
|
client,
|
|
`query($id: ID!) {
|
|
node(id: $id) {
|
|
id
|
|
... on File {
|
|
fileStatus
|
|
... on MediaImage {
|
|
image { url }
|
|
}
|
|
... on GenericFile {
|
|
url
|
|
}
|
|
}
|
|
}
|
|
}`,
|
|
{ id: fileId }
|
|
);
|
|
|
|
const node = data.node;
|
|
const status = node?.fileStatus || "UNKNOWN";
|
|
const url = node?.image?.url || node?.url || "";
|
|
|
|
if (status === "READY") {
|
|
return { status, url };
|
|
}
|
|
|
|
if (status === "FAILED") {
|
|
throw new Error(`File processing failed for ${fileId}`);
|
|
}
|
|
|
|
if (attempt < maxAttempts) {
|
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
}
|
|
}
|
|
|
|
throw new Error(`Timed out waiting for READY status for ${fileId}`);
|
|
}
|
|
|
|
function buildLocalImageTasks(aggregatedPayload, imagesDir) {
|
|
const products = Array.isArray(aggregatedPayload?.products) ? aggregatedPayload.products : [];
|
|
const absImagesDir = path.resolve(process.cwd(), imagesDir);
|
|
|
|
const usedFolderNames = new Set();
|
|
const tasks = [];
|
|
|
|
for (let i = 0; i < products.length; i += 1) {
|
|
const product = products[i] || {};
|
|
const productId = product.productId || product?.productDetails?.data?.id || product?.productSummary?.id || `unknown-${i + 1}`;
|
|
const productNameRaw =
|
|
product?.productSummary?.name ||
|
|
product?.productDetails?.data?.name ||
|
|
`product-${i + 1}`;
|
|
|
|
let folderName = sanitizeName(productNameRaw) || `product-${i + 1}`;
|
|
if (usedFolderNames.has(folderName)) {
|
|
folderName = `${folderName}__${String(productId).slice(0, 8)}`;
|
|
}
|
|
usedFolderNames.add(folderName);
|
|
|
|
const productDir = path.join(absImagesDir, folderName);
|
|
const imagePaths = Array.isArray(product?.productSummary?.img)
|
|
? product.productSummary.img
|
|
: Array.isArray(product?.productDetails?.data?.img)
|
|
? product.productDetails.data.img
|
|
: [];
|
|
|
|
const usedFileNames = new Set();
|
|
|
|
for (let idx = 0; idx < imagePaths.length; idx += 1) {
|
|
const sourcePath = imagePaths[idx];
|
|
const cleanSourcePath = getPathWithoutQuery(sourcePath);
|
|
const originalBase = path.basename(cleanSourcePath, path.extname(cleanSourcePath)) || `image_${idx + 1}`;
|
|
const originalExt = path.extname(cleanSourcePath) || ".png";
|
|
|
|
const localBase = sanitizeName(originalBase) || `image_${idx + 1}`;
|
|
let fileName = `${localBase}${originalExt}`;
|
|
let dupCounter = 2;
|
|
|
|
while (usedFileNames.has(fileName.toLowerCase())) {
|
|
fileName = `${localBase}_${dupCounter}${originalExt}`;
|
|
dupCounter += 1;
|
|
}
|
|
usedFileNames.add(fileName.toLowerCase());
|
|
|
|
tasks.push({
|
|
productId: String(productId),
|
|
productName: String(productNameRaw),
|
|
sourcePath: String(sourcePath),
|
|
localPath: path.join(productDir, fileName)
|
|
});
|
|
}
|
|
}
|
|
|
|
return tasks;
|
|
}
|
|
|
|
async function readJsonOrDefault(filePath, fallback) {
|
|
try {
|
|
const raw = await fs.readFile(filePath, "utf8");
|
|
const parsed = JSON.parse(raw);
|
|
return parsed && typeof parsed === "object" ? parsed : fallback;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
async function uploadKytWatermarkedImagesToShopifyFiles(options = {}) {
|
|
const {
|
|
shop = process.env.SHOPIFY_SHOP,
|
|
accessToken = process.env.SHOPIFY_ACCESS_TOKEN,
|
|
apiVersion = process.env.SHOPIFY_API_VERSION || "2025-10",
|
|
aggregatedJsonPath = DEFAULT_AGGREGATED_JSON,
|
|
imagesDir = DEFAULT_IMAGES_DIR,
|
|
statePath = DEFAULT_STATE_PATH,
|
|
mapPath = DEFAULT_MAP_PATH,
|
|
concurrency = Math.max(1, Number.parseInt(process.env.SHOPIFY_IMAGE_UPLOAD_CONCURRENCY || "3", 10) || 3)
|
|
} = options;
|
|
|
|
if (!shop) throw new Error("Missing shop (or SHOPIFY_SHOP).");
|
|
if (!accessToken) throw new Error("Missing accessToken (or SHOPIFY_ACCESS_TOKEN).");
|
|
|
|
const client = createShopifyClient(shop, accessToken, apiVersion);
|
|
const absAggregatedPath = path.resolve(process.cwd(), aggregatedJsonPath);
|
|
const absStatePath = path.resolve(process.cwd(), statePath);
|
|
const absMapPath = path.resolve(process.cwd(), mapPath);
|
|
|
|
const aggregated = await readJsonOrDefault(absAggregatedPath, { products: [] });
|
|
const tasks = buildLocalImageTasks(aggregated, imagesDir);
|
|
|
|
const state = await readJsonOrDefault(absStatePath, {
|
|
version: 1,
|
|
updatedAt: null,
|
|
files: {}
|
|
});
|
|
|
|
const stateFiles = state.files && typeof state.files === "object" ? state.files : {};
|
|
const byProduct = {};
|
|
const bySourcePath = {};
|
|
|
|
let processed = 0;
|
|
let uploaded = 0;
|
|
let skipped = 0;
|
|
let failed = 0;
|
|
|
|
await mapWithConcurrency(tasks, concurrency, async (task, i) => {
|
|
processed += 1;
|
|
|
|
try {
|
|
await fs.access(task.localPath);
|
|
} catch {
|
|
failed += 1;
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const fp = await fileFingerprint(task.localPath);
|
|
const prev = stateFiles[task.localPath];
|
|
|
|
if (prev && prev.fingerprint === fp && prev.status === "READY" && prev.url) {
|
|
skipped += 1;
|
|
console.log(`[IMG-UPLOAD-SKIP] ${task.productId} | ${task.sourcePath} | ${task.localPath}`);
|
|
if (!byProduct[task.productId]) byProduct[task.productId] = [];
|
|
byProduct[task.productId].push({
|
|
sourcePath: task.sourcePath,
|
|
localPath: task.localPath,
|
|
fileId: prev.fileId,
|
|
status: prev.status,
|
|
url: prev.url
|
|
});
|
|
bySourcePath[task.sourcePath] = {
|
|
productId: task.productId,
|
|
localPath: task.localPath,
|
|
fileId: prev.fileId,
|
|
status: prev.status,
|
|
url: prev.url
|
|
};
|
|
} else {
|
|
const created = await stagedUploadOneImage(client, task.localPath);
|
|
const ready = created.fileStatus === "READY"
|
|
? { status: "READY", url: created.url || "" }
|
|
: await waitForFileReady(client, created.id);
|
|
|
|
stateFiles[task.localPath] = {
|
|
fingerprint: fp,
|
|
uploadedAt: new Date().toISOString(),
|
|
fileId: created.id,
|
|
status: ready.status,
|
|
url: ready.url
|
|
};
|
|
|
|
uploaded += 1;
|
|
console.log(`[IMG-UPLOAD-OK] ${task.productId} | ${task.sourcePath} | fileId=${created.id} | status=${ready.status}`);
|
|
if (!byProduct[task.productId]) byProduct[task.productId] = [];
|
|
byProduct[task.productId].push({
|
|
sourcePath: task.sourcePath,
|
|
localPath: task.localPath,
|
|
fileId: created.id,
|
|
status: ready.status,
|
|
url: ready.url
|
|
});
|
|
bySourcePath[task.sourcePath] = {
|
|
productId: task.productId,
|
|
localPath: task.localPath,
|
|
fileId: created.id,
|
|
status: ready.status,
|
|
url: ready.url
|
|
};
|
|
}
|
|
} catch (error) {
|
|
failed += 1;
|
|
stateFiles[task.localPath] = {
|
|
fingerprint: null,
|
|
uploadedAt: new Date().toISOString(),
|
|
status: "FAILED",
|
|
error: error.message
|
|
};
|
|
console.log(`[IMG-UPLOAD-FAIL] ${task.productName} | ${task.localPath} -> ${error.message}`);
|
|
}
|
|
|
|
if ((i + 1) % 25 === 0 || i === tasks.length - 1) {
|
|
console.log(`[IMG-UPLOAD] ${i + 1}/${tasks.length} processed | uploaded=${uploaded} skipped=${skipped} failed=${failed}`);
|
|
}
|
|
});
|
|
|
|
const finalState = {
|
|
version: 1,
|
|
updatedAt: new Date().toISOString(),
|
|
files: stateFiles
|
|
};
|
|
|
|
const finalMap = {
|
|
generatedAt: new Date().toISOString(),
|
|
sourceAggregatedPath: absAggregatedPath,
|
|
totalTasks: tasks.length,
|
|
byProduct,
|
|
bySourcePath
|
|
};
|
|
|
|
await fs.mkdir(path.dirname(absStatePath), { recursive: true });
|
|
await fs.mkdir(path.dirname(absMapPath), { recursive: true });
|
|
await fs.writeFile(absStatePath, JSON.stringify(finalState, null, 2), "utf8");
|
|
await fs.writeFile(absMapPath, JSON.stringify(finalMap, null, 2), "utf8");
|
|
|
|
return {
|
|
aggregatedJsonPath: absAggregatedPath,
|
|
statePath: absStatePath,
|
|
mapPath: absMapPath,
|
|
concurrency,
|
|
totalTasks: tasks.length,
|
|
processed,
|
|
uploaded,
|
|
skipped,
|
|
failed
|
|
};
|
|
}
|
|
|
|
async function runStandaloneImageUpload() {
|
|
const summary = await uploadKytWatermarkedImagesToShopifyFiles();
|
|
console.log("\nImage upload summary:");
|
|
console.log(JSON.stringify(summary, null, 2));
|
|
}
|
|
|
|
if (require.main === module) {
|
|
runStandaloneImageUpload().catch((error) => {
|
|
console.error("Image upload pipeline failed:", error.message);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|
|
|
|
module.exports = {
|
|
uploadKytWatermarkedImagesToShopifyFiles,
|
|
runStandaloneImageUpload
|
|
};
|