quorlasocial/connector/src/competitors.js

// Competitors per client: Instagram Business/Creator accounts the team tracks (added by hand or suggested by the AI).
// Each refresh stores one snapshot per account per day (followers, posts, recent media), so growth can be charted.
// Uses Instagram business_discovery through the client's (or the agency's) connected Instagram account.
import fs from "node:fs";
import path from "node:path";
import { PROJECT_ROOT } from "./env.js";
import * as meta from "./meta.js";
import * as places from "./places.js";

const FILE = process.env.QUORLA_COMPETITORS_FILE || path.join(PROJECT_ROOT, "data", "competitors.json");
const CLIENTS = path.join(PROJECT_ROOT, "clients");
const KEEP_DAYS = 180;
fs.mkdirSync(path.dirname(FILE), { recursive: true });

const read = () => { try { return JSON.parse(fs.readFileSync(FILE, "utf8")); } catch { return {}; } };
const write = (db) => { const tmp = FILE + ".tmp"; fs.writeFileSync(tmp, JSON.stringify(db)); fs.renameSync(tmp, FILE); };
const err = (msg, status = 400) => Object.assign(new Error(msg), { status });
const today = () => new Date().toISOString().slice(0, 10);
const handle = (u) => String(u || "").trim().replace(/^@/, "").replace(/^https?:\/\/(www\.)?instagram\.com\//i, "").replace(/[/?#].*$/, "").toLowerCase();

function checkClient(slug) {
  if (!/^[a-z0-9][a-z0-9-]*$/.test(String(slug || "")) || !fs.existsSync(path.join(CLIENTS, slug))) throw err("Unknown client.", 404);
  return slug;
}
function clientLink(slug) { try { return JSON.parse(fs.readFileSync(path.join(CLIENTS, slug, "client.json"), "utf8")); } catch { return {}; } }

/** An Instagram account id to run business_discovery from: the client's own, else any the agency manages. */
export async function viewerIgId(slug) {
  const link = clientLink(slug);
  if (link.igId) return link.igId;
  const any = (await meta.listAccounts()).find((a) => a.instagram?.id);
  if (!any) throw err("No Instagram account is connected to the agency's Meta account, so competitors can't be looked up.");
  return any.instagram.id;
}

async function lookup(igId, username) {
  try {
    const b = await meta.igBusinessDiscovery(igId, username, { posts: 25 });
    if (!b?.username) throw new Error("not found");
    return b;
  } catch (e) {
    throw err(`Couldn't read @${username}. Instagram only shares public Business and Creator accounts; check the handle and that the account is public and professional. (${e.message})`);
  }
}

function snapshotOf(b) {
  return {
    date: today(), followers: b.followers_count ?? null, following: b.follows_count ?? null, mediaCount: b.media_count ?? null,
    name: b.name || null, bio: (b.biography || "").slice(0, 300),
    media: (b.media || []).map(mediaOf),
  };
}
const tagsOf = (caption) => [...new Set((String(caption || "").match(/#[\p{L}\p{N}_]+/gu) || []).map((t) => t.toLowerCase()))].slice(0, 30);
const typeOf = (m) => (m.media_product_type === "REELS" ? "Reel" : m.media_type === "CAROUSEL_ALBUM" ? "Carousel" : m.media_type === "VIDEO" ? "Video" : "Image");
const mediaOf = (m) => ({ id: m.id, at: m.timestamp, likes: m.like_count ?? null, comments: m.comments_count ?? null, type: typeOf(m), permalink: m.permalink, caption: (m.caption || "").slice(0, 200), tags: tagsOf(m.caption) });
function addSnapshot(entry, snap) {
  entry.snapshots = (entry.snapshots || []).filter((s) => s.date !== snap.date);
  // Keep full media only on the latest snapshot; older ones keep the counts for the growth chart.
  entry.snapshots = entry.snapshots.map(({ media, ...rest }) => rest);
  entry.snapshots.push(snap);
  const cutoff = new Date(Date.now() - KEEP_DAYS * 86400000).toISOString().slice(0, 10);
  entry.snapshots = entry.snapshots.filter((s) => s.date >= cutoff);
}

export function listTracked(client) {
  checkClient(client);
  return (read()[client]?.accounts || []).map(({ snapshots, ...a }) => ({ ...a, lastChecked: snapshots?.at(-1)?.date || null }));
}

export async function add(client, username, { by = "team", source = "manual", note = "", kind = "local", google = null } = {}) {
  checkClient(client);
  const u = handle(username);
  if (!/^[a-z0-9._]{1,30}$/.test(u)) throw err("Enter an Instagram handle, like @bakery_name.");
  const db = read();
  const c = (db[client] ||= { accounts: [] });
  if (c.accounts.some((a) => a.username === u)) throw err(`@${u} is already tracked for this client.`);
  if (c.accounts.length >= 10) throw err("Track up to 10 competitors per client. Remove one first.");
  const own = clientLink(client);
  const b = await lookup(await viewerIgId(client), u);
  if (own.igId && String(b.id) === String(own.igId)) throw err("That's the client's own account.");
  const entry = { username: u, name: b.name || null, source, kind: KINDS.includes(kind) ? kind : "local", note: String(note).slice(0, 300), google, addedBy: by, addedAt: new Date().toISOString(), snapshots: [] };
  addSnapshot(entry, snapshotOf(b));
  c.accounts.push(entry);
  write(db);
  return { username: u, name: entry.name };
}

export function remove(client, username) {
  checkClient(client);
  const db = read();
  const u = handle(username);
  const before = db[client]?.accounts?.length || 0;
  if (db[client]) db[client].accounts = db[client].accounts.filter((a) => a.username !== u);
  if ((db[client]?.accounts?.length || 0) === before) throw err("Not tracked.", 404);
  write(db);
}

/** Refresh every tracked account for one client (or all clients). One snapshot per account per day. */
export async function refresh(client) {
  const db = read();
  const slugs = client ? [checkClient(client)] : Object.keys(db);
  const errors = [];
  for (const slug of slugs) {
    const c = db[slug];
    if (!c?.accounts?.length) continue;
    let igId;
    try { igId = await viewerIgId(slug); } catch (e) { errors.push(`${slug}: ${e.message}`); continue; }
    for (const a of c.accounts) {
      try { const b = await lookup(igId, a.username); a.name = b.name || a.name; addSnapshot(a, snapshotOf(b)); }
      catch (e) { errors.push(`${slug} @${a.username}: ${e.message}`); }
      // Cached for 30 days, so this calls Google about once a month per account.
      if (a.google?.placeId && places.enabled()) { try { a.google = await places.ratingOf(a.google.placeId); } catch {} }
    }
  }
  write(db);
  return { refreshed: slugs.length, errors };
}

const engagement = (m) => (m.likes ?? 0) + (m.comments ?? 0);
/** Numbers for one account from its latest posts and snapshot history. */
function stats({ followers, media, history }) {
  const now = Date.now();
  const recent = media.filter((m) => m.at && now - new Date(m.at) <= 28 * 86400000);
  const last12 = media.slice(0, 12);
  // Median, not mean: one viral post would otherwise make a typical post look many times better than it is.
  const avgEng = median(last12.map(engagement));
  const mix = {}, byType = {};
  for (const m of media.slice(0, 25)) { mix[m.type] = (mix[m.type] || 0) + 1; (byType[m.type] ||= []).push(engagement(m)); }
  // Hashtags on the account's 8 best posts: what the rivals that win actually use.
  const tags = {};
  for (const m of [...media].sort((a, b) => engagement(b) - engagement(a)).slice(0, 8)) for (const t of m.tags || []) tags[t] = (tags[t] || 0) + 1;
  const monthAgo = new Date(now - 30 * 86400000).toISOString().slice(0, 10);
  const base = history.find((h) => h.date >= monthAgo && h.followers != null) || null;
  return {
    followers,
    growth30: base && followers != null && base.date !== today() ? followers - base.followers : null,
    growthSince: base && base.date !== today() ? base.date : null,
    postsPerWeek: media.length ? Math.round((recent.length / 4) * 10) / 10 : null,
    avgEngagement: avgEng,
    engagementRate: avgEng != null && followers ? Math.round((avgEng / followers) * 10000) / 100 : null,
    formatMix: mix,
    engagementByFormat: Object.fromEntries(Object.entries(byType).map(([k, v]) => [k, median(v)])),
    topTags: Object.entries(tags).sort((a, b) => b[1] - a[1]).slice(0, 12).map(([t]) => t),
    allTags: [...new Set(media.flatMap((m) => m.tags || []))],
    topPosts: [...media].sort((a, b) => engagement(b) - engagement(a)).slice(0, 3),
  };
}
function median(xs) {
  const s = [...xs].sort((x, y) => x - y);
  return s.length ? Math.round(s.length % 2 ? s[(s.length - 1) / 2] : (s[s.length / 2 - 1] + s[s.length / 2]) / 2) : null;
}

/**
 * Comparison for a client: their own account (from the dashboard snapshot passed in) plus every tracked competitor.
 * own: { username, followers, posts:[{id,timestamp,like_count,comments_count,media_type,media_product_type,permalink,caption}], history:[{date, followers}] }
 */
export function compare(client, own) {
  checkClient(client);
  const accounts = (read()[client]?.accounts || []).map((a) => {
    const latest = a.snapshots?.at(-1) || {};
    return {
      username: a.username, name: a.name, source: a.source, kind: a.kind || "local", google: a.google || null, note: a.note, addedBy: a.addedBy, addedAt: a.addedAt, lastChecked: latest.date || null,
      ...stats({ followers: latest.followers ?? null, media: latest.media || [], history: a.snapshots || [] }),
      series: (a.snapshots || []).filter((s) => s.followers != null).map((s) => ({ date: s.date, followers: s.followers })),
    };
  });
  let you = null;
  if (own) {
    const media = (own.posts || []).map(mediaOf);
    you = { username: own.username, own: true, google: read()[client]?.self?.google || null, ...stats({ followers: own.followers ?? null, media, history: own.history || [] }), series: (own.history || []).filter((h) => h.followers != null) };
  }
  return { you, competitors: accounts };
}

// ---------- AI suggestions: accounts a run found; a person adds or dismisses each one ----------
const KINDS = ["local", "benchmark"];
const clip = (s, n) => String(s || "").slice(0, n);
const suggestionView = ({ snapshot, ...s }) => ({ ...s, ...stats({ followers: snapshot?.followers ?? null, media: snapshot?.media || [], history: [] }), bio: snapshot?.bio || "" });

/** Google rating for a candidate: checked through Places when a place id is given, else what the AI read on the web (unverified). */
async function googleOf(s) {
  if (s.placeId && places.enabled()) { try { return await places.ratingOf(s.placeId); } catch {} }
  return s.google && Number.isFinite(+s.google.rating) ? { rating: +s.google.rating, reviews: Number.isFinite(+s.google.reviews) ? +s.google.reviews : null, verified: false } : null;
}

/** The client's own Google listing (by place id), stored so reports can put the client in the Google comparison. */
export async function setClientPlace(client, placeId) {
  checkClient(client);
  const google = await places.ratingOf(placeId);
  const db = read();
  (db[client] ||= { accounts: [] }).self = { google };
  write(db);
  return google;
}

/**
 * Check each candidate on Instagram and store the real ones as suggestions.
 * list: [{ username, kind: "local" | "benchmark", why, placeId?, google?: { rating, reviews } }]
 * Returns the verified numbers (so the report quotes Instagram, not guesses) and why the rest were skipped.
 */
export async function suggest(client, list, { by = "agent" } = {}) {
  checkClient(client);
  const igId = await viewerIgId(client);
  const own = clientLink(client);
  const known = read()[client] || {};
  const checked = [], skipped = [];
  for (const s of (list || []).slice(0, 15)) {
    const u = handle(s.username);
    if (!/^[a-z0-9._]{1,30}$/.test(u)) { skipped.push({ username: s.username, reason: "not an Instagram handle" }); continue; }
    if (known.accounts?.some((a) => a.username === u)) { skipped.push({ username: u, reason: "already tracked" }); continue; }
    if (known.suggestions?.some((x) => x.username === u && x.status === "dismissed")) { skipped.push({ username: u, reason: "the team dismissed it before" }); continue; }
    try {
      const b = await lookup(igId, u);
      if (own.igId && String(b.id) === String(own.igId)) { skipped.push({ username: u, reason: "the client's own account" }); continue; }
      const g = await googleOf(s);
      checked.push({ username: u, name: b.name || null, kind: KINDS.includes(s.kind) ? s.kind : "local", why: clip(s.why, 300), google: g, by, at: new Date().toISOString(), status: "new", snapshot: snapshotOf(b) });
    } catch (e) {
      skipped.push({ username: u, reason: e.message });
    }
  }
  // Re-read before writing: lookups take a while and the nightly refresh may have saved in between.
  const db = read();
  const c = (db[client] ||= { accounts: [] });
  c.suggestions = [...(c.suggestions || []).filter((x) => !checked.some((n) => n.username === x.username)), ...checked];
  write(db);
  return { checked: checked.map(suggestionView).map(({ allTags, ...v }) => v), skipped };
}

export function listSuggestions(client) {
  checkClient(client);
  return (read()[client]?.suggestions || []).filter((s) => s.status === "new").map(suggestionView);
}

function setSuggestion(client, username, status) {
  const db = read(), u = handle(username);
  const s = db[client]?.suggestions?.find((x) => x.username === u);
  if (!s) throw err("Suggestion not found.", 404);
  s.status = status;
  write(db);
  return s;
}

export function dismissSuggestion(client, username) {
  checkClient(client);
  setSuggestion(client, username, "dismissed");
  return { ok: true };
}

export async function acceptSuggestion(client, username, { by = "team" } = {}) {
  checkClient(client);
  const s = (read()[client]?.suggestions || []).find((x) => x.username === handle(username));
  if (!s) throw err("Suggestion not found.", 404);
  const added = await add(client, s.username, { by, source: "suggested", note: s.why, kind: s.kind, google: s.google });
  setSuggestion(client, s.username, "added");
  return added;
}