From fc2179b343b26753bd34f53a55ba1a94e77f48ab Mon Sep 17 00:00:00 2001 From: metatroncubeswdev Date: Tue, 20 Oct 2026 21:56:54 -0400 Subject: [PATCH] Track platform quotas; wait and retry instead of failing - quota.js: per-platform limits (YouTube 10,000 units/day Pacific per project, Instagram 100 / Threads 250 / TikTok 15 posts per 24 h per account, ...), usage log, .env overrides (QUOTA_) - Publishing checks the budget first and records usage; when a platform is out of quota (or answers quota exceeded / 429) the post stays approved and the queue retries after the reset, without re-posting finished platforms - Drafts that exceed a day's limit get a warning - Connections page shows used/limit and when it frees up - Tests for windows (incl. DST), shared vs per-account limits and the wait-and-retry queue flow; docs for quotas and the Google app setup Co-Authored-By: Claude Opus 5.5 --- connector/src/platforms/index.js | 15 ++- connector/src/platforms/quota.js | 135 ++++++++++++++++++++++++ connector/src/schedule.js | 80 ++++++++++++-- connector/test/platforms.test.mjs | 1 + connector/test/quota.test.mjs | 133 +++++++++++++++++++++++ dashboard/server/index.js | 1 + dashboard/web/src/pages/Connections.jsx | 30 +++++- dashboard/web/src/pages/Publishing.jsx | 4 +- docs/google-app.md | 26 +++++ docs/platforms.md | 24 +++++ 10 files changed, 433 insertions(+), 16 deletions(-) create mode 100644 connector/src/platforms/quota.js create mode 100644 connector/test/quota.test.mjs create mode 100644 docs/google-app.md diff --git a/connector/src/platforms/index.js b/connector/src/platforms/index.js index b232dfa..4926cb2 100644 --- a/connector/src/platforms/index.js +++ b/connector/src/platforms/index.js @@ -9,10 +9,11 @@ import threads from "./threads.js"; import reddit from "./reddit.js"; import * as vault from "./vault.js"; import * as oauth from "./oauth.js"; +import * as quota from "./quota.js"; export const ADAPTERS = { linkedin, gbp, youtube, tiktok, pinterest, threads, reddit }; export const PLATFORM_IDS = Object.keys(ADAPTERS); -export { vault }; +export { vault, quota }; const err = (m, status = 400) => Object.assign(new Error(m), { status }); const FORMAT_WORD = { TEXT: "text", IMAGE: "image", CAROUSEL: "multi-image", REELS: "video", STORY: "story" }; @@ -55,6 +56,7 @@ export async function finishConnect({ platform, code, state, user }) { const app = vault.appCredentials(platform); const tokens = await oauth.exchangeCode(a, app, { code, redirectUri: s.redirectUri, verifier: s.verifier }); const account = await a.identify(tokens); + if (platform === "youtube") quota.record("youtube", null, "read"); const conn = vault.saveConnection({ client: s.client, platform, account, tokens, scopes: (tokens.scope || "").split(/[\s,]+/).filter(Boolean), by: s.by }); // Pick the only target automatically (e.g. one business location). if (a.targets) { @@ -99,7 +101,9 @@ export async function setTarget(connId, targetId) { export async function checkConnection(connId) { const { conn, tokens } = await tokensFor(connId); try { + if (conn.platform === "youtube") quota.ensure("youtube", null, "read"); const account = await adapter(conn.platform).identify(tokens); + if (conn.platform === "youtube") quota.record("youtube", null, "read"); return vault.updateConnection(connId, { account, status: "connected", lastError: null }); } catch (e) { vault.updateConnection(connId, { status: "error", lastError: e.message }); @@ -141,5 +145,12 @@ export async function publish(platform, client, post) { if (!c) throw err(`${a.name} isn't connected for this client.`); const { conn, tokens } = await tokensFor(c.id); const opts = { ...Object.fromEntries((a.fields || []).filter((f) => f.default !== undefined).map((f) => [f.key, f.default])), ...(post.options?.[platform] || {}) }; - return a.publish({ tokens, conn, post, opts, caption: opts.caption ?? post.caption ?? "" }); + quota.ensure(platform, conn.account?.id); + try { + const result = await a.publish({ tokens, conn, post, opts, caption: opts.caption ?? post.caption ?? "" }); + quota.record(platform, conn.account?.id); + return result; + } catch (e) { + throw quota.fromPlatformError(platform, e) || e; + } } diff --git a/connector/src/platforms/quota.js b/connector/src/platforms/quota.js new file mode 100644 index 0000000..b264679 --- /dev/null +++ b/connector/src/platforms/quota.js @@ -0,0 +1,135 @@ +// Platform quotas: how many posts / API units each platform allows, how much Quorla has used, and when it resets. +// Publishing checks the budget first; when it is used up (or the platform itself says so) the post waits and is +// retried automatically after the reset instead of failing. Limits can be raised in .env (QUOTA_=n). +import fs from "node:fs"; +import path from "node:path"; +import { PROJECT_ROOT, loadEnv } from "../env.js"; + +loadEnv(); +const FILE = process.env.QUORLA_QUOTA_FILE || path.join(PROJECT_ROOT, "data", "quota.json"); + +/** + * scope "app" = shared by everything using our developer app (YouTube's daily units are per Google project) + * scope "account" = per connected account (Instagram, Threads, TikTok creator, …) + * window "pt-day" = calendar day in Pacific time (YouTube); "24h" = rolling 24 hours; "minute" = rolling 60 s + */ +export const LIMITS = { + youtube: { scope: "app", window: "pt-day", limit: 10000, unit: "units", cost: { publish: 1600, read: 1 }, note: "Google raises it free after the YouTube API audit." }, + instagram: { scope: "account", window: "24h", limit: 100, unit: "posts", cost: { publish: 1 } }, + facebook: { scope: "account", window: "24h", limit: 200, unit: "posts", cost: { publish: 1 } }, + threads: { scope: "account", window: "24h", limit: 250, unit: "posts", cost: { publish: 1 } }, + tiktok: { scope: "account", window: "24h", limit: 15, unit: "posts", cost: { publish: 1 } }, + linkedin: { scope: "account", window: "24h", limit: 100, unit: "posts", cost: { publish: 1 } }, + pinterest: { scope: "account", window: "24h", limit: 100, unit: "Pins", cost: { publish: 1 } }, + gbp: { scope: "account", window: "24h", limit: 50, unit: "updates", cost: { publish: 1 } }, + reddit: { scope: "account", window: "24h", limit: 10, unit: "posts", cost: { publish: 1 }, note: "Kept low on purpose: subreddits ban accounts that post too often." }, +}; + +export function limitFor(platform) { + const base = LIMITS[platform]; + if (!base) return null; + const override = Number(process.env[`QUOTA_${platform.toUpperCase()}`]); + return { ...base, limit: Number.isFinite(override) && override > 0 ? override : base.limit }; +} + +export class QuotaError extends Error { + constructor(platform, resetAt, message) { + super(message || `${platform}: quota used up until ${new Date(resetAt).toISOString()}.`); + this.platform = platform; this.resetAt = new Date(resetAt).toISOString(); this.quota = true; + } +} + +// ---------- time windows ---------- +function ptOffsetMs(at) { + // Offset of America/Los_Angeles from UTC at that instant (handles DST). + const parts = Object.fromEntries(new Intl.DateTimeFormat("en-US", { timeZone: "America/Los_Angeles", hourCycle: "h23", year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", second: "2-digit" }).formatToParts(at).map((p) => [p.type, p.value])); + const asUtc = Date.UTC(+parts.year, +parts.month - 1, +parts.day, +parts.hour, +parts.minute, +parts.second); + return asUtc - Math.floor(at / 1000) * 1000; +} + +/** [start, end) of the quota window that contains `at`. */ +export function windowAt(lim, at = Date.now()) { + if (lim.window === "minute") return { start: at - 60000, end: at, rolling: true, length: 60000 }; + if (lim.window === "24h") return { start: at - 86400000, end: at, rolling: true, length: 86400000 }; + const local = new Date(at + ptOffsetMs(at)); // wall-clock date in Pacific time, read with UTC getters + const y = local.getUTCFullYear(), m = local.getUTCMonth(), d = local.getUTCDate(); + return { start: ptMidnight(y, m, d), end: ptMidnight(y, m, d + 1), rolling: false }; +} + +/** UTC instant of 00:00 Pacific time on that date (tries PDT and PST; the one that round-trips is right). */ +function ptMidnight(y, m, d) { + for (const hours of [7, 8]) { + const utc = Date.UTC(y, m, d, hours); + if (ptOffsetMs(utc) === -hours * 3600000) return utc; + } + return Date.UTC(y, m, d, 8); +} + +// ---------- usage log ---------- +function read() { try { return JSON.parse(fs.readFileSync(FILE, "utf8")); } catch { return { events: [] }; } } +function write(db) { + fs.mkdirSync(path.dirname(FILE), { recursive: true }); + const cutoff = Date.now() - 3 * 86400000; // keep 3 days, plenty for every window + db.events = db.events.filter((e) => new Date(e.at).getTime() > cutoff); + const tmp = `${FILE}.tmp`; fs.writeFileSync(tmp, JSON.stringify(db)); fs.renameSync(tmp, FILE); +} +const scopeKey = (lim, account) => (lim.scope === "app" ? "app" : String(account || "unknown")); + +/** Units used in the current window, and when room next frees up. */ +export function usage(platform, account, at = Date.now()) { + const lim = limitFor(platform); + if (!lim) return null; + const key = scopeKey(lim, account); + const w = windowAt(lim, at); + const events = read().events.filter((e) => e.platform === platform && e.key === key && new Date(e.at).getTime() >= w.start && new Date(e.at).getTime() <= at); + const used = events.reduce((s, e) => s + e.units, 0); + // Rolling windows free up as old events age out; calendar windows reset at the end of the day. + const resetAt = w.rolling ? (events.length ? new Date(events[0].at).getTime() + w.length : at) : w.end; + return { platform, key, used, limit: lim.limit, remaining: Math.max(0, lim.limit - used), unit: lim.unit, window: lim.window, resetAt: new Date(resetAt).toISOString(), note: lim.note || null }; +} + +/** Throws QuotaError if `kind` (publish/read) doesn't fit in what's left. */ +export function ensure(platform, account, kind = "publish", at = Date.now()) { + const lim = limitFor(platform); + if (!lim) return; + const cost = lim.cost[kind] ?? 1; + const u = usage(platform, account, at); + if (u.used + cost > lim.limit) { + throw new QuotaError(platform, u.resetAt, `${platform}: daily limit reached (${u.used}/${lim.limit} ${lim.unit}). The post will go out automatically after ${u.resetAt}.`); + } +} + +export function record(platform, account, kind = "publish", at = Date.now()) { + const lim = limitFor(platform); + if (!lim) return; + const db = read(); + db.events.push({ platform, key: scopeKey(lim, account), units: lim.cost[kind] ?? 1, kind, at: new Date(at).toISOString() }); + write(db); +} + +/** Does this error from a platform mean "slow down / quota used up"? Returns a QuotaError or null. */ +export function fromPlatformError(platform, e, at = Date.now()) { + const msg = String(e?.message || ""); + const body = JSON.stringify(e?.body || ""); + const status = e?.status; + const quotaLike = status === 429 + || /quotaExceeded|rateLimitExceeded|userRateLimitExceeded|dailyLimitExceeded/i.test(body + msg) + || /rate_limit_exceeded|spam_risk_too_many_posts|spam_risk_user_banned_from_posting/i.test(body + msg) + || /too many requests|request limit reached|application request limit|posting limit|maximum number of posts/i.test(msg) + || /"code":\s*(4|17|32|613)\b/.test(body); + if (!quotaLike) return null; + const lim = limitFor(platform); + const resetAt = lim?.window === "pt-day" ? windowAt(lim, at).end : at + (status === 429 ? 15 * 60000 : 60 * 60000); + return new QuotaError(platform, resetAt, `${platform}: the platform's limit was reached (${msg.slice(0, 160)}). Retrying automatically after ${new Date(resetAt).toISOString()}.`); +} + +/** How many posts are already planned for this platform/account in the window of `when` (for draft warnings). */ +export function plannedWarning(platform, account, plannedCount, when) { + const lim = limitFor(platform); + if (!lim) return null; + const cost = lim.cost.publish ?? 1; + const needed = (plannedCount + 1) * cost; + if (needed <= lim.limit) return null; + const perWindow = Math.floor(lim.limit / cost); + return `${platform}: ${plannedCount + 1} posts are planned in the same ${lim.window === "pt-day" ? "day (Pacific time)" : "24 hours"}, more than the ${perWindow} allowed. Extra posts will wait and go out after the limit resets.`; +} diff --git a/connector/src/schedule.js b/connector/src/schedule.js index e882612..4d0b2b8 100644 --- a/connector/src/schedule.js +++ b/connector/src/schedule.js @@ -7,14 +7,15 @@ import crypto from "node:crypto"; import { PROJECT_ROOT } from "./env.js"; import * as meta from "./meta.js"; import * as extra from "./platforms/index.js"; +import * as quota from "./platforms/quota.js"; const META_PLATFORMS = ["instagram", "facebook"]; export const ALL_PLATFORMS = [...META_PLATFORMS, ...extra.PLATFORM_IDS]; -const DIR = path.join(PROJECT_ROOT, "data", "schedule"); +const DIR = process.env.QUORLA_SCHEDULE_DIR || path.join(PROJECT_ROOT, "data", "schedule"); // override for tests const FILE = path.join(DIR, "queue.json"); const LOCK = path.join(DIR, ".runner.lock"); -const LOG = path.join(PROJECT_ROOT, "data", "publish-log.jsonl"); +const LOG = process.env.QUORLA_SCHEDULE_DIR ? path.join(DIR, "publish-log.jsonl") : path.join(PROJECT_ROOT, "data", "publish-log.jsonl"); const CLIENTS = path.join(PROJECT_ROOT, "clients"); fs.mkdirSync(DIR, { recursive: true }); @@ -77,6 +78,32 @@ export function getPost(id) { return p; } +/** The account a quota is counted against for this client/platform (Page, Instagram or connected account id). */ +function quotaAccount(client, platform) { + if (platform === "instagram") return clientInfo(client).igId || null; + if (platform === "facebook") return clientInfo(client).pageId || null; + return extra.vault.connectionFor(client, platform)?.account?.id || null; +} + +/** Warnings (not errors) when a day's planned posts exceed a platform's limit; extra posts then wait for the reset. */ +function quotaWarnings(post, items) { + const at = new Date(post.publishAt).getTime(); + const out = []; + for (const platform of post.platforms) { + const lim = quota.limitFor(platform); + if (!lim) continue; + const acct = quotaAccount(post.client, platform); + const w = lim.window === "pt-day" ? quota.windowAt(lim, at) : { start: at - 86400000, end: at + 86400000 }; + const planned = items.filter((x) => x.id !== post.id && ["draft", "approved", "publishing"].includes(x.status) && x.platforms.includes(platform) + && (lim.scope === "app" || (x.client === post.client)) + && new Date(x.publishAt).getTime() >= w.start && new Date(x.publishAt).getTime() < w.end).length; + const used = at - Date.now() < 86400000 ? quota.usage(platform, acct)?.used / (lim.cost.publish || 1) || 0 : 0; + const msg = quota.plannedWarning(platform, acct, planned + Math.floor(used), at); + if (msg) out.push(msg); + } + return out; +} + /** * Add a post to the queue as a DRAFT. Nothing is published until a person approves it. * localTime is in the client's time zone, e.g. "2026-10-20 19:00". @@ -105,7 +132,9 @@ export function createPost({ client, platforms = ["instagram"], type = "IMAGE", publishAt: publishAt.toISOString(), status: "draft", createdBy, createdAt: new Date().toISOString(), notes: notes || null, results: {}, errors: {}, }; - const items = read(); items.push(post); write(items); + const items = read(); + post.warnings = quotaWarnings(post, items); + items.push(post); write(items); return post; } @@ -117,6 +146,7 @@ export function updatePost(id, patch, by = "team") { for (const pl of p.platforms) if (!META_PLATFORMS.includes(pl)) extra.validate(pl, p.client, p); if (patch.localTime) { const t = zonedToUtc(patch.localTime, p.timezone); p.publishAt = t.toISOString(); p.localTime = formatInTz(t, p.timezone); } p.status = "draft"; p.updatedBy = by; p.updatedAt = new Date().toISOString(); + p.warnings = quotaWarnings(p, items); write(items); return p; } @@ -138,6 +168,13 @@ export async function approvePost(id, by) { write(items); return p; } +/** Quota use per platform for a client (Connections page): used / limit, and when it resets. */ +export function quotaSummary(client) { + const c = clientInfo(client); + const platforms = [...(c.igId ? ["instagram"] : []), ...(c.pageId ? ["facebook"] : []), ...extra.vault.listConnections({ client }).map((x) => x.platform)]; + return Object.fromEntries(platforms.map((pl) => [pl, quota.usage(pl, quotaAccount(client, pl))]).filter(([, u]) => u)); +} + export async function cancelPost(id, by = "team") { const items = read(); const p = items.find((x) => x.id === id); if (!p) throw err("Post not found.", 404); @@ -150,21 +187,43 @@ export async function cancelPost(id, by = "team") { write(items); return p; } +/** Meta posts go through the Meta connector; quotas are tracked here the same way as for the other platforms. */ +async function publishMeta(platform, a, p) { + const acct = platform === "instagram" ? a.instagram?.id : a.page_id; + quota.ensure(platform, acct); + try { + const r = platform === "instagram" + ? await meta.igPublishAny(a, { type: p.type, media: p.media, caption: p.caption, collaborators: p.collaborators }) + : await meta.fbPublishAny(a, { message: p.fbCaption || p.caption, media: p.media }); + quota.record(platform, acct); + return r; + } catch (e) { throw quota.fromPlatformError(platform, e) || e; } +} + async function publishNow(p) { let a = null; + p.waiting = p.waiting || {}; for (const platform of p.platforms) { if (p.results[platform]?.id && !p.results[platform]?.cancelled) continue; // already done (e.g. FB scheduled natively) try { if (META_PLATFORMS.includes(platform)) a ||= await accountFor(p.client); - p.results[platform] = platform === "instagram" - ? await meta.igPublishAny(a, { type: p.type, media: p.media, caption: p.caption, collaborators: p.collaborators }) - : platform === "facebook" - ? await meta.fbPublishAny(a, { message: p.fbCaption || p.caption, media: p.media }) - : await extra.publish(platform, p.client, p); - delete p.errors[platform]; + p.results[platform] = META_PLATFORMS.includes(platform) ? await publishMeta(platform, a, p) : await extra.publish(platform, p.client, p); + delete p.errors[platform]; delete p.waiting[platform]; log({ action: "published", platform, post: p.id, client: p.client, result: p.results[platform] }); - } catch (e) { p.errors[platform] = e.message; log({ action: "failed", platform, post: p.id, client: p.client, error: e.message }); } + } catch (e) { + p.errors[platform] = e.message; + if (e.quota) { p.waiting[platform] = e.resetAt; log({ action: "waiting_for_quota", platform, post: p.id, client: p.client, until: e.resetAt }); } + else { delete p.waiting[platform]; log({ action: "failed", platform, post: p.id, client: p.client, error: e.message }); } + } } + const waits = Object.values(p.waiting); + if (waits.length) { + // Some platforms are out of quota: keep the post approved and try those again after the earliest reset. + p.status = "approved"; + p.retryAt = waits.sort()[0]; + return; + } + delete p.retryAt; const ok = p.platforms.filter((x) => p.results[x]?.id).length; p.status = ok === p.platforms.length ? "published" : ok ? "partial" : "failed"; p.publishedAt = new Date().toISOString(); @@ -180,6 +239,7 @@ export async function runDue({ now = Date.now() } = {}) { const items = read(); for (const p of items) { if (p.status !== "approved" || new Date(p.publishAt) > now) continue; + if (p.retryAt && new Date(p.retryAt) > now) continue; // waiting for a platform quota to reset p.status = "publishing"; write(items); await publishNow(p); write(items); done.push({ id: p.id, client: p.client, status: p.status, errors: p.errors }); diff --git a/connector/test/platforms.test.mjs b/connector/test/platforms.test.mjs index 0ca4c33..50ce165 100644 --- a/connector/test/platforms.test.mjs +++ b/connector/test/platforms.test.mjs @@ -8,6 +8,7 @@ import path from "node:path"; const TMP = fs.mkdtempSync(path.join(os.tmpdir(), "quorla-test-")); process.env.QUORLA_CONNECTIONS_FILE = path.join(TMP, "connections.json"); +process.env.QUORLA_QUOTA_FILE = path.join(TMP, "quota.json"); process.env.DASHBOARD_SECRET = "test-secret-not-real"; process.env.PUBLIC_URL = "https://app.example.test"; diff --git a/connector/test/quota.test.mjs b/connector/test/quota.test.mjs new file mode 100644 index 0000000..310ea91 --- /dev/null +++ b/connector/test/quota.test.mjs @@ -0,0 +1,133 @@ +// Offline tests for platform quotas and the queue's wait-and-retry behaviour. +// Run: node --test connector/test/quota.test.mjs (temporary files only; no network) +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), "quorla-quota-")); +process.env.QUORLA_QUOTA_FILE = path.join(TMP, "quota.json"); +process.env.QUORLA_CONNECTIONS_FILE = path.join(TMP, "connections.json"); +process.env.QUORLA_SCHEDULE_DIR = path.join(TMP, "schedule"); +process.env.DASHBOARD_SECRET = "test-secret-not-real"; +process.env.ALLOW_PUBLISH = "true"; + +const quota = await import("../src/platforms/quota.js"); +const P = await import("../src/platforms/index.js"); +const sched = await import("../src/schedule.js"); + +const realFetch = globalThis.fetch; +let calls = []; +const json = (b, status = 200, headers = {}) => new Response(JSON.stringify(b), { status, headers: { "content-type": "application/json", ...headers } }); +before(() => { + globalThis.fetch = async (u, init = {}) => { + const url = String(u); + calls.push({ url, method: init.method || "GET" }); + if (url.startsWith("https://api.linkedin.com/rest/posts")) return new Response("", { status: 201, headers: { "x-restli-id": `urn:li:share:${calls.length}` } }); + return json({ error: { message: `no fake route ${url}` } }, 599); + }; +}); +after(() => { globalThis.fetch = realFetch; fs.rmSync(TMP, { recursive: true, force: true }); }); + +const H = 3600000; +const reset = () => { try { fs.rmSync(process.env.QUORLA_QUOTA_FILE); } catch { /* none yet */ } }; + +test("YouTube's day runs midnight to midnight Pacific time, including the DST change", () => { + const lim = quota.limitFor("youtube"); + const noonPT = Date.parse("2026-10-21T19:00:00Z"); // 12:00 PDT (UTC-7) + const w = quota.windowAt(lim, noonPT); + assert.equal(new Date(w.start).toISOString(), "2026-10-21T07:00:00.000Z"); + assert.equal(new Date(w.end).toISOString(), "2026-10-22T07:00:00.000Z"); + const dst = quota.windowAt(lim, Date.parse("2026-11-01T20:00:00Z")); // Nov 1: clocks go back, 25-hour day + assert.equal(new Date(dst.start).toISOString(), "2026-11-01T07:00:00.000Z"); + assert.equal(new Date(dst.end).toISOString(), "2026-11-02T08:00:00.000Z"); +}); + +test("YouTube allows 6 uploads (6 x 1,600 of 10,000 units), then waits until midnight Pacific", () => { + reset(); + const at = Date.parse("2026-10-21T19:00:00Z"); + for (let i = 0; i < 6; i++) { quota.ensure("youtube", null, "publish", at); quota.record("youtube", null, "publish", at); } + const u = quota.usage("youtube", null, at); + assert.deepEqual([u.used, u.remaining], [9600, 400]); + assert.throws(() => quota.ensure("youtube", null, "publish", at), (e) => e.quota && e.resetAt === "2026-10-22T07:00:00.000Z"); + assert.doesNotThrow(() => quota.ensure("youtube", null, "publish", Date.parse("2026-10-22T07:00:01Z")), "new day, new budget"); + assert.doesNotThrow(() => quota.ensure("youtube", "someone-else", "read", at), "reads (1 unit) still fit"); +}); + +test("YouTube's budget is shared by all clients; per-account platforms are separate", () => { + reset(); + const at = Date.now(); + for (let i = 0; i < 6; i++) quota.record("youtube", `client-${i}`, "publish", at); + assert.throws(() => quota.ensure("youtube", "client-x", "publish", at), /daily limit reached/); + for (let i = 0; i < 15; i++) quota.record("tiktok", "acct-a", "publish", at - (15 - i) * 60000); + assert.throws(() => quota.ensure("tiktok", "acct-a", "publish", at), (e) => e.quota); + assert.doesNotThrow(() => quota.ensure("tiktok", "acct-b", "publish", at), "another TikTok account has its own limit"); +}); + +test("rolling 24 h windows free up when the oldest post ages out", () => { + reset(); + const at = Date.parse("2026-10-21T12:00:00Z"); + const first = at - 20 * H; + quota.record("tiktok", "acct", "publish", first); + for (let i = 1; i < 15; i++) quota.record("tiktok", "acct", "publish", at - H); + let err; + try { quota.ensure("tiktok", "acct", "publish", at); } catch (e) { err = e; } + assert.equal(err.resetAt, new Date(first + 24 * H).toISOString()); + assert.doesNotThrow(() => quota.ensure("tiktok", "acct", "publish", first + 24 * H + 1000)); +}); + +test("limits can be changed in .env", () => { + reset(); + process.env.QUOTA_REDDIT = "2"; + quota.record("reddit", "u", "publish"); quota.record("reddit", "u", "publish"); + assert.throws(() => quota.ensure("reddit", "u", "publish"), /2\/2 posts/); + delete process.env.QUOTA_REDDIT; +}); + +test("platform 'slow down' errors become a retry, other errors don't", () => { + const at = Date.parse("2026-10-21T19:00:00Z"); + const yt = quota.fromPlatformError("youtube", { message: "YouTube: upload failed", status: 403, body: { error: { errors: [{ reason: "quotaExceeded" }] } } }, at); + assert.equal(yt.resetAt, "2026-10-22T07:00:00.000Z"); + const tt = quota.fromPlatformError("tiktok", { message: "TikTok: spam_risk_too_many_posts", status: 403 }, at); + assert.ok(tt.quota); + assert.equal(quota.fromPlatformError("reddit", { message: "Reddit: Too Many Requests", status: 429 }, at).resetAt, new Date(at + 15 * 60000).toISOString()); + assert.equal(quota.fromPlatformError("linkedin", { message: "LinkedIn: invalid image", status: 400 }), null); +}); + +test("drafts beyond the daily limit get a warning, not an error", () => { + reset(); + process.env.QUOTA_LINKEDIN = "1"; + P.vault.saveConnection({ client: "falcon-fashions", platform: "linkedin", account: { id: "abc", name: "Pat" }, tokens: { accessToken: "t", expiresAt: new Date(Date.now() + 864e5).toISOString() }, by: "t" }); + const when = new Date(Date.now() + 3 * 864e5).toISOString().slice(0, 10) + " 10:00"; + const first = sched.createPost({ client: "falcon-fashions", platforms: ["linkedin"], type: "TEXT", caption: "one", localTime: when }); + assert.deepEqual(first.warnings, []); + const second = sched.createPost({ client: "falcon-fashions", platforms: ["linkedin"], type: "TEXT", caption: "two", localTime: when.replace("10:00", "11:00") }); + assert.equal(second.warnings.length, 1); + assert.match(second.warnings[0], /more than the 1 allowed/); + delete process.env.QUOTA_LINKEDIN; +}); + +test("the queue waits for the reset and then publishes, without failing the post", async () => { + reset(); + process.env.QUOTA_LINKEDIN = "1"; + const post = sched.createPost({ client: "falcon-fashions", platforms: ["linkedin"], type: "TEXT", caption: "Hello", localTime: new Date(Date.now() + 2 * 60000).toLocaleString("sv-SE", { timeZone: "America/Toronto" }).slice(0, 16) }); + await sched.approvePost(post.id, "Tester"); + quota.record("linkedin", "abc", "publish"); // the day's single post is already used + calls = []; + const t1 = Date.now() + 3 * 60000; + await sched.runDue({ now: t1 }); + let p = sched.getPost(post.id); + assert.equal(p.status, "approved", "still approved, not failed"); + assert.ok(p.retryAt && p.waiting.linkedin, "remembers when to retry"); + assert.equal(calls.length, 0, "nothing was sent to LinkedIn"); + await sched.runDue({ now: t1 + 60000 }); + assert.equal(calls.length, 0, "not retried before the reset"); + reset(); // the 24 h window has passed + await sched.runDue({ now: new Date(p.retryAt).getTime() + 1000 }); + p = sched.getPost(post.id); + assert.equal(p.status, "published"); + assert.ok(p.results.linkedin.id); + assert.ok(!p.retryAt && !p.waiting.linkedin); + delete process.env.QUOTA_LINKEDIN; +}); diff --git a/dashboard/server/index.js b/dashboard/server/index.js index 2afd37f..9da0a95 100644 --- a/dashboard/server/index.js +++ b/dashboard/server/index.js @@ -100,6 +100,7 @@ app.get("/api/platforms", h(async (req) => ({ app.put("/api/platforms/:id/app", requireAdmin, h(async (req) => { extra.adapter(req.params.id); extra.vault.saveApp(req.params.id, req.body || {}, req.user.name || req.user.email); return { ok: true }; })); app.delete("/api/platforms/:id/app", requireAdmin, h(async (req) => { extra.adapter(req.params.id); extra.vault.removeApp(req.params.id); return { ok: true }; })); app.get("/api/connections", h(async (req) => ({ connections: extra.vault.listConnections({ client: req.query.client || undefined }) }))); +app.get("/api/quota", h(async (req) => ({ quota: sched.quotaSummary(clients.safeSlug(req.query.client) && String(req.query.client)) }))); app.get("/api/oauth/:platform/start", requireAdmin, h(async (req) => { const client = String(req.query.client || ""); if (!clients.getClient(client)) throw Object.assign(new Error("Pick a client first."), { status: 400 }); diff --git a/dashboard/web/src/pages/Connections.jsx b/dashboard/web/src/pages/Connections.jsx index 37fd56f..31e2640 100644 --- a/dashboard/web/src/pages/Connections.jsx +++ b/dashboard/web/src/pages/Connections.jsx @@ -7,6 +7,19 @@ import { Badge, Button, cx, Dialog, ErrorNote, Field, fmt, Panel, Spinner, timeA const FORMAT_LABEL = { TEXT: "text", IMAGE: "image", CAROUSEL: "multi-image", REELS: "video", STORY: "story" }; +/** "3 of 15 posts used · resets 9:12 PM" (YouTube: units per day, shared by all clients). */ +function QuotaLine({ q }) { + if (!q) return null; + const reset = new Date(q.resetAt).toLocaleString("en-CA", { weekday: "short", hour: "numeric", minute: "2-digit" }); + const full = q.remaining <= 0; + return ( +
+ {fmt(q.used)} of {fmt(q.limit)} {q.unit} used {q.window === "pt-day" ? "today (all clients)" : "in the last 24 h"} + {(q.used > 0 || full) && <> · {full ? "full, posts wait until" : "frees up"} {reset}} +
+ ); +} + function CopyText({ text }) { const [done, setDone] = useState(false); const copy = () => navigator.clipboard?.writeText(text).then(() => { setDone(true); setTimeout(() => setDone(false), 1400); }).catch(() => {}); @@ -64,7 +77,7 @@ function AppDialog({ platform, onClose, onSaved }) { ); } -function PlatformCard({ platform, conn, client, isAdmin, mediaLinks, onSetup, reload, setError }) { +function PlatformCard({ platform, conn, client, isAdmin, mediaLinks, onSetup, reload, setError, quota }) { const [busy, setBusy] = useState(null); const [targets, setTargets] = useState(null); const needsMediaLinks = ["tiktok", "threads", "gbp"].includes(platform.id) && !mediaLinks; @@ -108,6 +121,7 @@ function PlatformCard({ platform, conn, client, isAdmin, mediaLinks, onSetup, re Connected {timeAgo(conn.connectedAt)} by {conn.connectedBy} {conn.account?.url && <> · View profile} + {conn.lastError &&
{conn.lastError}
} )} @@ -147,12 +161,16 @@ export default function Connections() { const [conns, setConns] = useState([]); const [error, setError] = useState(null); const [setup, setSetup] = useState(null); + const [quota, setQuota] = useState({}); const client = params.get("client") || ""; const notice = params.get("connected"); const oauthError = params.get("error"); const loadPlatforms = useCallback(() => api("/api/platforms").then(setPlatforms).catch(setError), []); - const loadConns = useCallback(() => (client ? api(`/api/connections?client=${encodeURIComponent(client)}`).then((r) => setConns(r.connections)).catch(setError) : Promise.resolve()), [client]); + const loadConns = useCallback(() => (client ? Promise.all([ + api(`/api/connections?client=${encodeURIComponent(client)}`).then((r) => setConns(r.connections)), + api(`/api/quota?client=${encodeURIComponent(client)}`).then((r) => setQuota(r.quota)).catch(() => setQuota({})), + ]).catch(setError) : Promise.resolve()), [client]); useEffect(() => { api("/api/overview").then((r) => setClients(r.clients)).catch(setError); loadPlatforms(); }, [loadPlatforms]); useEffect(() => { loadConns(); }, [loadConns]); @@ -196,12 +214,18 @@ export default function Connections() { Facebook: {current?.pageId ? "Page linked" : current?.igOnly ? none (Instagram only) : not linked} {!current?.linked && Link it from the client's page.} + {(quota.instagram || quota.facebook) && ( +
+ {quota.instagram &&
Instagram
} + {quota.facebook &&
Facebook
} +
+ )}
{platforms.platforms.map((p) => ( c.platform === p.id)} client={client} isAdmin={isAdmin} - mediaLinks={platforms.mediaLinks} onSetup={() => setSetup(p)} reload={loadConns} setError={setError} /> + mediaLinks={platforms.mediaLinks} onSetup={() => setSetup(p)} reload={loadConns} setError={setError} quota={quota[p.id]} /> ))}
diff --git a/dashboard/web/src/pages/Publishing.jsx b/dashboard/web/src/pages/Publishing.jsx index 4650a3a..567e7be 100644 --- a/dashboard/web/src/pages/Publishing.jsx +++ b/dashboard/web/src/pages/Publishing.jsx @@ -74,7 +74,9 @@ export default function Publishing() {
{p.localTime} ({p.timezone.split("/").pop().replace("_", " ")} time)

{p.caption || No caption}

{p.collaborators?.length > 0 &&
Collab: {p.collaborators.map((c) => "@" + c).join(", ")}
} - {Object.entries(p.errors || {}).map(([k, v]) =>
{k}: {v}
)} + {Object.entries(p.errors || {}).filter(([k]) => !p.waiting?.[k]).map(([k, v]) =>
{k}: {v}
)} + {Object.entries(p.waiting || {}).map(([k, until]) =>
{k}: daily limit reached. Goes out automatically after {new Date(until).toLocaleString("en-CA", { weekday: "short", hour: "numeric", minute: "2-digit" })}.
)} + {p.status === "draft" && (p.warnings || []).map((w) =>
{w}
)} {Object.entries(p.results || {}).filter(([, v]) => v?.permalink).map(([k, v]) => View on {k})} {p.results?.facebook?.scheduledOnFacebook && !p.results.facebook.cancelled &&
Facebook part is scheduled in Meta Planner.
}
by {p.createdBy}{p.approvedBy ? ` · approved by ${p.approvedBy}` : ""}
diff --git a/docs/google-app.md b/docs/google-app.md new file mode 100644 index 0000000..6024cd3 --- /dev/null +++ b/docs/google-app.md @@ -0,0 +1,26 @@ +# Google app: Quorla (YouTube + Google Business Profile) + +- Google Cloud project **Quorla** (ID `quorla`) in the **metatroncubesolutions.com** organization, created with + dev@metatroncubesolutions.com. +- APIs enabled: YouTube Data API v3, My Business Account Management API, My Business Business Information API, + Google My Business API. No billing account needed; all are free within quota. +- OAuth consent screen (Google Auth Platform): app name Quorla, logo `branding/quorla-icon-1024.png`, support and + developer contact dev@metatroncubesolutions.com, home https://app.quorlasocial.com, privacy /privacy, terms /terms, + authorized domain quorlasocial.com. Audience **External**, publishing status **Testing**. +- Scopes: openid, userinfo.email, business.manage (non-sensitive); youtube.upload, youtube.readonly (sensitive). + No restricted scopes, so no paid security assessment is needed. +- Test users (only these can connect while in Testing; max 100): dev@metatroncubesolutions.com, + metatroncubesoftwaresolutions@gmail.com. Add each client's Google account here until the app is verified. +- OAuth web client "Quorla web (app.quorlasocial.com)" with redirect URIs + https://app.quorlasocial.com/oauth/youtube/callback and https://app.quorlasocial.com/oauth/gbp/callback. + Its ID and secret are in the PC's `.env` as YOUTUBE_CLIENT_ID/SECRET and GBP_CLIENT_ID/SECRET; copy those four lines + to the server's `.env`, or enter them in Quorla → Connections → Set up. Google shows the secret only once; if it is + lost, add a new secret on the client in Google Cloud. + +## Still to do +1. **Business Profile API access**: request it (Google form) for project number of `quorla`; until approved the + Business Profile APIs have a quota of 0. +2. **Verify quorlasocial.com in Google Search Console** (DNS TXT record) — required before Google verifies the app. +3. After deploy: **publish the app** (Audience → Publish app) and submit **verification** (sensitive YouTube scopes): + Google asks for a demo video of the sign-in and how each scope is used. +4. **YouTube API audit** (free form) to lift the 10,000 units/day quota and the private-only upload limit. diff --git a/docs/platforms.md b/docs/platforms.md index 7b244d4..7ed3c1c 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -56,6 +56,30 @@ TikTok also needs the domain verified in its developer portal. - **Not built yet:** comments/inbox and analytics for these platforms (only follower counts are shown), Reddit image posts, Pinterest video cover images, LinkedIn document (PDF) posts, X (paid API). +## Quotas and limits + +Quorla tracks each platform's limit (`connector/src/platforms/quota.js`, usage in `data/quota.json`): + +| Platform | Limit tracked | Window | +|---|---|---| +| YouTube | 10,000 units per Google project, shared by all clients (upload = 1,600, read = 1) → about 6 uploads/day until the YouTube audit raises it | calendar day, Pacific time | +| Instagram | 100 API posts per account | rolling 24 h | +| Facebook | 200 posts per Page | rolling 24 h | +| Threads | 250 posts per profile | rolling 24 h | +| TikTok | 15 posts per creator | rolling 24 h | +| LinkedIn | 100 posts per member/Page | rolling 24 h | +| Pinterest | 100 Pins per account | rolling 24 h | +| Google Business Profile | 50 updates per account | rolling 24 h | +| Reddit | 10 posts per account (kept low: subreddits ban frequent posters) | rolling 24 h | + +- Before publishing, Quorla checks the budget. If it's used up, or the platform answers "quota exceeded" / "too many + requests", the post **stays approved and waits**; the queue retries it automatically after the reset. Platforms that + already went out are not posted twice. +- Saving a draft that would exceed a day's limit shows a warning on the draft (not an error). +- The Connections page shows used / limit and when it frees up. +- Raise a limit after a platform approves more with `QUOTA_=n` in `.env` (e.g. `QUOTA_YOUTUBE=1000000`). +- Tests: `node --test connector/test/quota.test.mjs`. + ## Suggested order after merging 1. Deploy (`git pull && docker compose up -d --build`), then Connections → Set up for one platform at a time.