From c404ea2ef9f26d39f3341c89e80343fc80d54cd5 Mon Sep 17 00:00:00 2001 From: metatroncubeswdev Date: Tue, 27 Oct 2026 18:51:54 -0400 Subject: [PATCH] Research: Google Places ratings, Google Trends, Instagram hashtag top posts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Places API (New): competitor scans find local rivals on Google Maps and the server reads real ratings/review counts by place id; client's own listing fills the "You" row; recent reviews for praise/complaint themes; 30-day cache - Google Trends via DataForSEO: rising/top related searches and % change per keyword in the client's region; used by trend-scan and to check gap demand - Instagram Hashtag Search: top/recent posts under a niche hashtag (tool + Competitors page panel), counts Instagram's 30 tags/week limit - Server settings → Research: keys plus on/off switches for each source (hashtag search off until Meta approves Public Content Access) - Reports mark website-sourced ratings as unverified; skill takes handles only from a business's own site Co-Authored-By: Claude Opus 5.5 --- .claude/skills/competitor-scan/SKILL.md | 13 ++-- .claude/skills/trend-scan/SKILL.md | 4 +- .env.example | 6 ++ connector/src/competitors.js | 27 ++++++-- connector/src/hashtags.js | 77 +++++++++++++++++++++ connector/src/places.js | 76 +++++++++++++++++++++ connector/src/server.js | 28 +++++++- connector/src/trends.js | 89 +++++++++++++++++++++++++ dashboard/server/index.js | 6 ++ dashboard/server/jobs.js | 1 + dashboard/server/settings.js | 8 +++ dashboard/web/src/pages/Competitors.jsx | 62 ++++++++++++++++- tools/report-pdf/blocks.mjs | 8 ++- 13 files changed, 391 insertions(+), 14 deletions(-) create mode 100644 connector/src/hashtags.js create mode 100644 connector/src/places.js create mode 100644 connector/src/trends.js diff --git a/.claude/skills/competitor-scan/SKILL.md b/.claude/skills/competitor-scan/SKILL.md index d33bcc3..cb0441f 100644 --- a/.claude/skills/competitor-scan/SKILL.md +++ b/.claude/skills/competitor-scan/SKILL.md @@ -7,18 +7,21 @@ Goal: tell the client who they're actually up against, who's winning and *why*, 1. **Know the client.** Read `clients//client.json`, `brand.md`, `goals.md`, `constraints.md` and any `audit-*` file. Note: niche, exact offer, city/area (and radius that matters: a clinic competes within ~10 km, an online brand nationally), price level, ideal customer. Call `competitors` to see who's already tracked, so you don't suggest them again. -2. **Search for candidates** with WebSearch (8–15 searches, no more): - - Local rivals: " ", "best near ", Google Maps / Yelp / directory listings, " instagram". For each business note its Google rating and review count if the search result shows them. +2. **Search for candidates** (8–15 searches in total, no more): + - Local rivals, Google Maps first: `google_places` with query " " (2–4 variations, e.g. by neighbourhood or sub-service). It returns what customers see on Maps, with real ratings, review counts, website links and place ids. Keep each rival's `placeId`. If it says Google Places isn't set up, use WebSearch (" ", Yelp / directory listings) and note any rating the result shows. + - Then WebSearch for rivals Maps misses (online-only businesses, " instagram"). - Benchmarks: accounts in the same niche (other cities or countries) known for strong content, e.g. "best instagram accounts", industry roundups. - - Find each business's Instagram handle from its website or listing (WebFetch the site's homepage and look for instagram.com links). Never guess a handle. + - Find each business's Instagram handle from its own website or official listing (WebFetch the site's homepage and look for instagram.com links), benchmarks included; a handle seen only in a roundup article doesn't count. Never guess a handle. + - The client's own Google listing: find it with `google_places` (business name + city), check the name and address match, then save it with `client_google_listing`. Without Google Places: read `google` ({"rating": 4.8, "reviews": 120}) from `clients//client.json`, or from the `gbp` screenshot in `clients//screenshots/` and save it into client.json. If none of these works, list it under open questions. + - For the 3–5 strongest local rivals, `google_places` with place_id and reviews=true: note what customers praise and complain about (themes only; in the report, paraphrase and never quote reviewers by name). - Aim for 6–10 local rivals and 3–5 benchmarks. -3. **Verify and save** with `competitor_suggest` (one call, up to 15 accounts): `kind` local or benchmark, a one-line `why` a business owner would understand ("Same street, same price range, 4.8★ from 310 reviews"), and `google` rating/reviews when you found them. The tool checks each on Instagram and returns the real numbers; skipped ones (private, personal or wrong handle) are listed with the reason. Don't retry a skipped handle with variations. +3. **Verify and save** with `competitor_suggest` (one call, up to 15 accounts): `kind` local or benchmark, a one-line `why` a business owner would understand ("Same street, same price range, 4.8★ from 310 reviews"), and the rival's `placeId` when you have it (the server then reads the real Google rating; pass `google` rating/reviews only when there's no place id). The tool checks each on Instagram and returns the real numbers; skipped ones (private, personal or wrong handle) are listed with the reason. Don't retry a skipped handle with variations. 4. **Analyse** using only what the tool returned (and `competitors` for already-tracked ones) plus the client's own numbers (`instagram_insights`, `instagram_recent_posts`): - Who's winning: engagement rate (engagement per follower) matters more than follower count; a small account with high engagement beats a big quiet one. Posting rhythm. Follower growth where tracked. - Why they win: format that works for them (`engagementByFormat`), what their top posts have in common (topic, hook, people on camera, offers, local references), hashtags their best posts use (`topTags`) that the client doesn't. - - Gaps: topics, formats or angles none of the rivals cover well, which fit the client's strengths. These are the most valuable part of the report. + - Gaps: topics, formats or angles none of the rivals cover well, which fit the client's strengths. These are the most valuable part of the report. When `google_trends` is set up, check that each gap has real search demand in the client's region (one call, gap keywords together) and say whether it's growing. - Local businesses: compare Google rating and review counts too; that's often where customers decide. 5. **Write** `clients//competitors-YYYY-MM.md`, client-ready (plain language, no tools or file names, no other clients). Use the visual blocks described in `.claude/skills/account-audit/SKILL.md` (front matter, callouts, `mc:score`, `mc:actions`) plus these competitor blocks, all drawn from the verified data (empty fenced blocks, never type numbers in them): diff --git a/.claude/skills/trend-scan/SKILL.md b/.claude/skills/trend-scan/SKILL.md index 22c1d84..010c0f1 100644 --- a/.claude/skills/trend-scan/SKILL.md +++ b/.claude/skills/trend-scan/SKILL.md @@ -14,7 +14,9 @@ Works only with official data: the `competitor_lookup` tool (Instagram Business 4. For each outlier note: format (Reel/carousel/image), first line of caption (the hook), topic, any visible trend (audio name if in caption, meme format, "POV", before/after, day-in-the-life, ranking, myth-vs-fact), length cues, CTA. Open the permalink in the browser only if the user wants frames reviewed. 5. Group into 3–5 patterns. A pattern needs 2+ outliers from different accounts — one viral post is luck, not a trend. 6. Check the client's own last 30 posts the same way (`instagram_recent_posts`) to show which of their posts were outliers and why. -7. Optional: web search " Instagram trend " and Meta's own creator/trending pages for audio and formats this month; mark anything you could not verify. +6b. **Hashtag top posts** (`hashtag_top_posts`, skip if it says it's switched off): 3–5 specific niche or local hashtags (from the client's best posts and rivals' `topTags`; never generic tags like #food), kind=top. Instagram allows 30 different hashtags per account per week, so reuse last week's tags where you can. Treat a Reel here as an outlier when it clearly beats the other posts under the same tag; group them with the account outliers in step 5. These posts come without a username: link them, don't credit or name the poster. +7. **What people are searching for** (`google_trends`, skip if it says it isn't set up): 2–3 calls of up to 5 keywords each, in the client's region (location from `client.json`, e.g. "Ontario,Canada"), `past_12_months`. Use the words customers type ("emergency dentist", "odoo erp"), not industry jargon. Note keywords rising (positive `change`), the rising related searches (new questions to answer in content) and the peak months (post before the peak, not during it). Put these in their own "Search trends" section; they're Google searches, not Instagram trends. +8. Optional: web search " Instagram trend " and Meta's own creator/trending pages for audio and formats this month; mark anything you could not verify. ## Output → `clients//trends/YYYY-MM-DD.md` - Patterns table: pattern · example links · why it works · how this client does it (on-brand version). diff --git a/.env.example b/.env.example index eefb4c5..af1910d 100644 --- a/.env.example +++ b/.env.example @@ -79,3 +79,9 @@ DASHBOARD_NIGHTLY_REFRESH=true # YouTube comments in the Inbox (public comments, read with an API key, not the channel's sign-in). # Google Cloud (project Quorla) → APIs & Services → Credentials → Create API key → restrict to YouTube Data API v3. # YOUTUBE_API_KEY= +# Google Places: real Google ratings/reviews for competitor scans. Google Cloud → enable Places API (New) → +# Create API key → restrict to Places API (New) + this server's IP. Can also be set in Server settings → Research. +# GOOGLE_PLACES_API_KEY= +# Google Trends via DataForSEO (dataforseo.com → API Access). Can also be set in Server settings → Research. +# DATAFORSEO_LOGIN= +# DATAFORSEO_PASSWORD= diff --git a/connector/src/competitors.js b/connector/src/competitors.js index f71418f..19081dd 100644 --- a/connector/src/competitors.js +++ b/connector/src/competitors.js @@ -5,6 +5,7 @@ import fs from "node:fs"; import path from "node:path"; import { PROJECT_ROOT } from "./env.js"; import * as meta from "./meta.js"; +import * as places from "./places.js"; const FILE = process.env.QUORLA_COMPETITORS_FILE || path.join(PROJECT_ROOT, "data", "competitors.json"); const CLIENTS = path.join(PROJECT_ROOT, "clients"); @@ -24,7 +25,7 @@ function checkClient(slug) { function clientLink(slug) { try { return JSON.parse(fs.readFileSync(path.join(CLIENTS, slug, "client.json"), "utf8")); } catch { return {}; } } /** An Instagram account id to run business_discovery from: the client's own, else any the agency manages. */ -async function viewerIgId(slug) { +export async function viewerIgId(slug) { const link = clientLink(slug); if (link.igId) return link.igId; const any = (await meta.listAccounts()).find((a) => a.instagram?.id); @@ -107,6 +108,8 @@ export async function refresh(client) { for (const a of c.accounts) { try { const b = await lookup(igId, a.username); a.name = b.name || a.name; addSnapshot(a, snapshotOf(b)); } catch (e) { errors.push(`${slug} @${a.username}: ${e.message}`); } + // Cached for 30 days, so this calls Google about once a month per account. + if (a.google?.placeId && places.enabled()) { try { a.google = await places.ratingOf(a.google.placeId); } catch {} } } } write(db); @@ -164,7 +167,7 @@ export function compare(client, own) { let you = null; if (own) { const media = (own.posts || []).map(mediaOf); - you = { username: own.username, own: true, ...stats({ followers: own.followers ?? null, media, history: own.history || [] }), series: (own.history || []).filter((h) => h.followers != null) }; + you = { username: own.username, own: true, google: read()[client]?.self?.google || null, ...stats({ followers: own.followers ?? null, media, history: own.history || [] }), series: (own.history || []).filter((h) => h.followers != null) }; } return { you, competitors: accounts }; } @@ -174,9 +177,25 @@ const KINDS = ["local", "benchmark"]; const clip = (s, n) => String(s || "").slice(0, n); const suggestionView = ({ snapshot, ...s }) => ({ ...s, ...stats({ followers: snapshot?.followers ?? null, media: snapshot?.media || [], history: [] }), bio: snapshot?.bio || "" }); +/** Google rating for a candidate: checked through Places when a place id is given, else what the AI read on the web (unverified). */ +async function googleOf(s) { + if (s.placeId && places.enabled()) { try { return await places.ratingOf(s.placeId); } catch {} } + return s.google && Number.isFinite(+s.google.rating) ? { rating: +s.google.rating, reviews: Number.isFinite(+s.google.reviews) ? +s.google.reviews : null, verified: false } : null; +} + +/** The client's own Google listing (by place id), stored so reports can put the client in the Google comparison. */ +export async function setClientPlace(client, placeId) { + checkClient(client); + const google = await places.ratingOf(placeId); + const db = read(); + (db[client] ||= { accounts: [] }).self = { google }; + write(db); + return google; +} + /** * Check each candidate on Instagram and store the real ones as suggestions. - * list: [{ username, kind: "local" | "benchmark", why, google: { rating, reviews } }] + * list: [{ username, kind: "local" | "benchmark", why, placeId?, google?: { rating, reviews } }] * Returns the verified numbers (so the report quotes Instagram, not guesses) and why the rest were skipped. */ export async function suggest(client, list, { by = "agent" } = {}) { @@ -193,7 +212,7 @@ export async function suggest(client, list, { by = "agent" } = {}) { try { const b = await lookup(igId, u); if (own.igId && String(b.id) === String(own.igId)) { skipped.push({ username: u, reason: "the client's own account" }); continue; } - const g = s.google && Number.isFinite(+s.google.rating) ? { rating: +s.google.rating, reviews: Number.isFinite(+s.google.reviews) ? +s.google.reviews : null } : null; + const g = await googleOf(s); checked.push({ username: u, name: b.name || null, kind: KINDS.includes(s.kind) ? s.kind : "local", why: clip(s.why, 300), google: g, by, at: new Date().toISOString(), status: "new", snapshot: snapshotOf(b) }); } catch (e) { skipped.push({ username: u, reason: e.message }); diff --git a/connector/src/hashtags.js b/connector/src/hashtags.js new file mode 100644 index 0000000..853366e --- /dev/null +++ b/connector/src/hashtags.js @@ -0,0 +1,77 @@ +// Instagram Hashtag Search: top and recent public posts under a hashtag, to spot Reels and formats taking off in a +// client's niche from accounts we don't track yet. Needs Meta's "Instagram Public Content Access" feature and the +// Server settings switch RESEARCH_IG_HASHTAGS=true. Instagram allows 30 different hashtags per Instagram account +// per 7 days; we count them here so a run never trips the limit, and keep answers for 24 hours. +import fs from "node:fs"; +import path from "node:path"; +import { PROJECT_ROOT } from "./env.js"; +import * as meta from "./meta.js"; + +const FILE = path.join(PROJECT_ROOT, "data", "hashtag-search.json"); +const WEEK = 7 * 86400000, DAY = 86400000, LIMIT = 30; +const FIELDS = "id,media_type,media_product_type,like_count,comments_count,permalink,caption,timestamp"; + +const err = (msg, status = 400) => Object.assign(new Error(msg), { status }); +export const enabled = () => process.env.RESEARCH_IG_HASHTAGS === "true"; +const read = () => { try { return JSON.parse(fs.readFileSync(FILE, "utf8")); } catch { return { used: {}, ids: {}, cache: {} }; } }; +const write = (db) => { fs.mkdirSync(path.dirname(FILE), { recursive: true }); fs.writeFileSync(FILE + ".tmp", JSON.stringify(db)); fs.renameSync(FILE + ".tmp", FILE); }; +const clean = (t) => String(t || "").trim().replace(/^#/, "").toLowerCase(); +const typeOf = (m) => (m.media_product_type === "REELS" ? "Reel" : m.media_type === "CAROUSEL_ALBUM" ? "Carousel" : m.media_type === "VIDEO" ? "Video" : "Image"); + +/** Hashtags this Instagram account searched in the last 7 days (Instagram's own count). */ +function usedThisWeek(db, igId) { + const now = Date.now(); + return (db.used[igId] || []).filter((u) => now - u.at < WEEK); +} + +export function quota(igId) { + const used = usedThisWeek(read(), igId); + return { used: used.length, left: Math.max(0, LIMIT - used.length), tags: used.map((u) => u.tag), resetsAt: used.length ? new Date(Math.min(...used.map((u) => u.at)) + WEEK).toISOString() : null }; +} + +/** + * Top (most engaged) or recent posts under one hashtag, searched through igId (the client's own Instagram). + * Returns posts with type, likes, comments, link and caption; Instagram doesn't say who posted them. + */ +export async function topPosts(igId, tag, { kind = "top", limit = 25 } = {}) { + if (!enabled()) throw err("Hashtag search is switched off. Turn it on in Server settings → Research once Meta has approved Instagram Public Content Access."); + const t = clean(tag); + if (!/^[\p{L}\p{N}_]{2,100}$/u.test(t)) throw err("Not a hashtag."); + const edge = kind === "recent" ? "recent_media" : "top_media"; + let db = read(); + const key = `${igId}:${t}:${edge}`; + const hit = db.cache[key]; + if (hit && Date.now() - hit.at < DAY) return { ...hit.data, cached: true }; + + const used = usedThisWeek(db, igId); + if (!used.some((u) => u.tag === t) && used.length >= LIMIT) + throw err(`Instagram allows ${LIMIT} different hashtags per account per week and this account has used them all (${used.map((u) => "#" + u.tag).join(", ")}). Try again after ${new Date(Math.min(...used.map((u) => u.at)) + WEEK).toISOString().slice(0, 10)}, or reuse one of those.`, 429); + + let id = db.ids[t]; + if (!id) { + const r = await meta.graph("ig_hashtag_search", { user_id: igId, q: t }); + id = r.data?.[0]?.id; + if (!id) throw err(`Instagram has no hashtag #${t}.`, 404); + } + const r = await meta.graph(`${id}/${edge}`, { user_id: igId, fields: FIELDS, limit: Math.min(Math.max(+limit || 25, 1), 50) }); + const posts = (r.data || []).map((m) => ({ + type: typeOf(m), likes: m.like_count ?? null, comments: m.comments_count ?? null, at: m.timestamp || null, link: m.permalink, caption: (m.caption || "").slice(0, 300), + })); + const reels = posts.filter((p) => p.type === "Reel"); + const data = { + hashtag: t, kind: edge === "top_media" ? "top" : "recent", checkedAt: new Date().toISOString(), posts, + formatMix: Object.fromEntries(["Reel", "Carousel", "Image", "Video"].map((k) => [k, posts.filter((p) => p.type === k).length]).filter(([, n]) => n)), + note: "Instagram doesn't say who posted these. Likes can be hidden by the poster (shown as null).", + reelsShare: posts.length ? Math.round((reels.length / posts.length) * 100) : null, + }; + + db = read(); // re-read: another run may have searched meanwhile + db.ids[t] = id; + const list = usedThisWeek(db, igId); + if (!list.some((u) => u.tag === t)) list.push({ tag: t, at: Date.now() }); + db.used[igId] = list; + db.cache[key] = { at: Date.now(), data }; + for (const [k, v] of Object.entries(db.cache)) if (Date.now() - v.at > DAY) delete db.cache[k]; + write(db); + return { ...data, quota: { used: list.length, left: LIMIT - list.length } }; +} diff --git a/connector/src/places.js b/connector/src/places.js new file mode 100644 index 0000000..aec0683 --- /dev/null +++ b/connector/src/places.js @@ -0,0 +1,76 @@ +// Google Places API (New): real Google ratings, review counts and recent reviews for any public business. +// Key: GOOGLE_PLACES_API_KEY (Server settings → Research). Answers are cached for up to 30 days (Google's limit +// for keeping Places content), so re-running a scan or re-exporting a report doesn't call Google again. +import fs from "node:fs"; +import path from "node:path"; +import { PROJECT_ROOT } from "./env.js"; + +const API = "https://places.googleapis.com/v1"; +const CACHE = path.join(PROJECT_ROOT, "data", "places-cache.json"); +const KEEP_MS = 30 * 86400000; +const BASIC = ["id", "displayName", "formattedAddress", "rating", "userRatingCount", "websiteUri", "googleMapsUri", "primaryTypeDisplayName", "businessStatus"]; +const REVIEWS = ["reviews.rating", "reviews.text", "reviews.relativePublishTimeDescription", "reviews.publishTime", "reviews.authorAttribution.displayName"]; + +const err = (msg, status = 400) => Object.assign(new Error(msg), { status }); +export const enabled = () => Boolean(process.env.GOOGLE_PLACES_API_KEY) && process.env.RESEARCH_GOOGLE_PLACES !== "false"; + +function readCache() { + try { + const all = JSON.parse(fs.readFileSync(CACHE, "utf8")), now = Date.now(); + return Object.fromEntries(Object.entries(all).filter(([, v]) => now - v.at < KEEP_MS)); + } catch { return {}; } +} +function cached(key, fn) { + if (process.env.RESEARCH_GOOGLE_PLACES === "false") return Promise.reject(err("Google Places is switched off in Server settings → Research.")); + const hit = readCache()[key]; + if (hit) return Promise.resolve(hit.data); + return fn().then((data) => { + const all = readCache(); + all[key] = { at: Date.now(), data }; + fs.mkdirSync(path.dirname(CACHE), { recursive: true }); + fs.writeFileSync(CACHE + ".tmp", JSON.stringify(all)); + fs.renameSync(CACHE + ".tmp", CACHE); + return data; + }); +} + +async function call(url, fields, body) { + if (!enabled()) throw err("Google Places isn't set up: add the Google Places API key in Server settings → Research."); + const res = await fetch(url, { + method: body ? "POST" : "GET", + headers: { "X-Goog-Api-Key": process.env.GOOGLE_PLACES_API_KEY, "X-Goog-FieldMask": fields.join(","), ...(body ? { "Content-Type": "application/json" } : {}) }, + body: body ? JSON.stringify(body) : undefined, + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw err(`Google Places: ${data.error?.message || res.statusText}`, res.status === 429 ? 429 : 502); + return data; +} + +const view = (p) => ({ + placeId: p.id, name: p.displayName?.text || null, address: p.formattedAddress || null, type: p.primaryTypeDisplayName?.text || null, + rating: p.rating ?? null, reviews: p.userRatingCount ?? 0, website: p.websiteUri || null, mapsUrl: p.googleMapsUri || null, + closed: p.businessStatus && p.businessStatus !== "OPERATIONAL" ? p.businessStatus : undefined, + ...(p.reviews ? { recentReviews: p.reviews.map((r) => ({ rating: r.rating, when: r.relativePublishTimeDescription, at: r.publishTime, author: r.authorAttribution?.displayName || null, text: (r.text?.text || "").slice(0, 600) })) } : {}), +}); + +/** Businesses matching a text search, e.g. "digital marketing agency Waterloo ON" (up to 20). */ +export function search(query, { max = 10 } = {}) { + const q = String(query || "").trim(); + if (!q) throw err("Search text is required."); + const n = Math.min(Math.max(+max || 10, 1), 20); + return cached(`s:${q.toLowerCase()}:${n}`, async () => + ((await call(`${API}/places:searchText`, BASIC.map((f) => `places.${f}`), { textQuery: q, pageSize: n })).places || []).map(view)); +} + +/** One business by its place id; reviews=true adds up to 5 recent reviews. */ +export function details(placeId, { reviews = false } = {}) { + const id = String(placeId || "").trim().replace(/^places\//, ""); + if (!/^[\w-]{10,}$/.test(id)) throw err("Not a Google place id."); + return cached(`d:${id}:${reviews ? 1 : 0}`, async () => view(await call(`${API}/places/${id}`, reviews ? [...BASIC, ...REVIEWS] : BASIC))); +} + +/** Rating snapshot stored with a competitor: checked by the server, not typed by the AI. */ +export async function ratingOf(placeId) { + const d = await details(placeId); + return { rating: d.rating, reviews: d.reviews, placeId: d.placeId, mapsUrl: d.mapsUrl, verified: true, checkedAt: new Date().toISOString().slice(0, 10) }; +} diff --git a/connector/src/server.js b/connector/src/server.js index 6326542..612f82a 100644 --- a/connector/src/server.js +++ b/connector/src/server.js @@ -12,6 +12,9 @@ import * as sched from "./schedule.js"; import * as lib from "./library.js"; import * as plan from "./plan.js"; import * as competitors from "./competitors.js"; +import * as places from "./places.js"; +import * as trends from "./trends.js"; +import * as hashtags from "./hashtags.js"; import * as brandkit from "./brandkit.js"; import * as extra from "./platforms/index.js"; @@ -216,9 +219,32 @@ server.tool("competitor_add", "Start tracking an Instagram competitor for a clie wrap(async ({ client, username, note }) => competitors.add(client, username, { by: "agent", source: "suggested", note }))); server.tool("competitor_suggest", "Suggest Instagram competitors for a client. Each handle is checked on Instagram (public Business/Creator only) and saved as a suggestion the team adds or dismisses on the Competitors page; nothing is tracked until a person adds it. Returns the real numbers for each (followers, posts/week, median engagement, engagement rate, format mix, engagement by format, top hashtags, top posts) and why any were skipped. kind: local = a business the client competes with for the same customers; benchmark = same niche elsewhere, worth learning from.", - { client: z.string(), accounts: z.array(z.object({ username: z.string(), kind: z.enum(["local", "benchmark"]), why: z.string(), google: z.object({ rating: z.number(), reviews: z.number().optional() }).optional() })).max(15) }, + { client: z.string(), accounts: z.array(z.object({ username: z.string(), kind: z.enum(["local", "benchmark"]), why: z.string(), + placeId: z.string().optional().describe("Google place id from google_places; the server then reads the real Google rating and review count"), + google: z.object({ rating: z.number(), reviews: z.number().optional() }).optional().describe("Only when Google Places isn't set up: rating/reviews you read on the web (stored as unverified)") })).max(15) }, wrap(async ({ client, accounts }) => competitors.suggest(client, accounts, { by: "agent" }))); +server.tool("google_places", "Google Maps businesses with their real Google rating and review count. Give query to search (e.g. \"digital marketing agency Waterloo ON\": the local rivals customers see on Maps, with website links to find their Instagram), or place_id for one business; reviews=true adds up to 5 recent reviews (for what customers praise or complain about). Answers are cached for 30 days.", + { query: z.string().optional(), place_id: z.string().optional(), reviews: z.boolean().default(false), max: z.number().int().min(1).max(20).default(10) }, + wrap(async ({ query, place_id, reviews, max }) => { + if (place_id) return places.details(place_id, { reviews }); + if (!query) throw new Error("Give query or place_id."); + return places.search(query, { max }); + })); + +server.tool("google_trends", "Google search interest for 1–5 keywords in a region (Google Trends via DataForSEO): average and recent interest, % change (last quarter of the period vs the one before), peak period, and the rising and top related searches per keyword. Use it to find what people in the client's market are searching for now and what's coming up seasonally. It's Google search behaviour, not Instagram trends. Cached for 7 days.", + { keywords: z.array(z.string()).min(1).max(5), location: z.string().default("Canada").describe("DataForSEO location name, e.g. \"Canada\", \"Ontario,Canada\", \"India\", \"Tamil Nadu,India\", \"New York,United States\""), + range: z.enum(trends.RANGES).default("past_12_months") }, + wrap(async (a) => trends.explore(a))); + +server.tool("hashtag_top_posts", "Top (most engaged) or recent public Instagram posts under a hashtag, searched through the client's own Instagram: type (Reel/Carousel/Image), likes, comments, link and caption, plus the format mix. Finds Reels taking off in the niche from accounts nobody tracks yet. Instagram doesn't say who posted them. Instagram allows 30 different hashtags per account per week, so pick 3–5 specific niche or local tags (#waterlooeats, not #food); the answer shows how many are left. Off unless switched on in Server settings.", + { client: z.string(), hashtag: z.string(), kind: z.enum(["top", "recent"]).default("top"), limit: z.number().int().min(1).max(50).default(25) }, + wrap(async ({ client, hashtag, kind, limit }) => hashtags.topPosts(await competitors.viewerIgId(client), hashtag, { kind, limit }))); + +server.tool("client_google_listing", "Save which Google Maps listing is the client's own (place id from google_places). The server stores its real rating and review count so reports can compare the client with rivals on Google.", + { client: z.string(), place_id: z.string() }, + wrap(async ({ client, place_id }) => competitors.setClientPlace(client, place_id))); + server.tool("brand_kit", "A client's saved hashtag sets, approved phrases/CTAs and words to avoid. Read it before writing captions for that client.", { client: z.string() }, wrap(async ({ client }) => brandkit.get(client))); diff --git a/connector/src/trends.js b/connector/src/trends.js new file mode 100644 index 0000000..7bbdc96 --- /dev/null +++ b/connector/src/trends.js @@ -0,0 +1,89 @@ +// Google Trends through DataForSEO (keywords_data/google_trends/explore/live): how search interest in a niche's +// keywords moves in a region, plus the rising and top related searches. Login: DATAFORSEO_LOGIN / DATAFORSEO_PASSWORD +// (Server settings → Research). Answers are cached for 7 days, so repeat runs in the same week cost nothing. +import fs from "node:fs"; +import path from "node:path"; +import { PROJECT_ROOT } from "./env.js"; + +const API = "https://api.dataforseo.com/v3/keywords_data/google_trends/explore/live"; +const CACHE = path.join(PROJECT_ROOT, "data", "trends-cache.json"); +const KEEP_MS = 7 * 86400000; +export const RANGES = ["past_30_days", "past_90_days", "past_12_months", "past_5_years"]; + +const err = (msg, status = 400) => Object.assign(new Error(msg), { status }); +export const enabled = () => Boolean(process.env.DATAFORSEO_LOGIN && process.env.DATAFORSEO_PASSWORD) && process.env.RESEARCH_GOOGLE_TRENDS !== "false"; + +function readCache() { + try { + const all = JSON.parse(fs.readFileSync(CACHE, "utf8")), now = Date.now(); + return Object.fromEntries(Object.entries(all).filter(([, v]) => now - v.at < KEEP_MS)); + } catch { return {}; } +} +function remember(key, data) { + const all = readCache(); + all[key] = { at: Date.now(), data }; + fs.mkdirSync(path.dirname(CACHE), { recursive: true }); + fs.writeFileSync(CACHE + ".tmp", JSON.stringify(all)); + fs.renameSync(CACHE + ".tmp", CACHE); +} + +const avg = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null); +const round = (n) => (n == null ? null : Math.round(n)); + +/** Per keyword: average interest, recent vs earlier (the trend), peak period; from DataForSEO's graph item. */ +function summarise(graph, keywords) { + const points = (graph?.data || []).filter((p) => !p.missing_data); + return keywords.map((k, i) => { + const series = points.map((p) => ({ from: p.date_from, value: p.values?.[i] ?? null })).filter((p) => p.value != null); + const vals = series.map((p) => p.value), q = Math.max(1, Math.floor(vals.length / 4)); + const recent = avg(vals.slice(-q)), before = avg(vals.slice(-2 * q, -q)); + const peak = series.reduce((m, p) => (!m || p.value > m.value ? p : m), null); + return { + keyword: k, average: round(avg(vals)), recent: round(recent), + change: recent != null && before ? Math.round(((recent - before) / before) * 100) : null, + peak: peak && { from: peak.from, value: peak.value }, + series: series.map((p) => [p.from, p.value]), + }; + }); +} + +/** + * keywords: 1–5 search terms compared on one 0–100 scale. location: DataForSEO location name, e.g. "Canada", + * "Ontario,Canada", "India". range: one of RANGES. + */ +export async function explore({ keywords, location = "Canada", range = "past_12_months" } = {}) { + if (process.env.RESEARCH_GOOGLE_TRENDS === "false") throw err("Google Trends is switched off in Server settings → Research."); + const kws = [...new Set((keywords || []).map((k) => String(k).trim().toLowerCase()).filter(Boolean))].slice(0, 5); + if (!kws.length) throw err("Give 1–5 keywords."); + if (!RANGES.includes(range)) throw err(`range must be one of ${RANGES.join(", ")}.`); + const key = JSON.stringify([kws, location.toLowerCase(), range]); + const hit = readCache()[key]; + if (hit) return { ...hit.data, cached: true }; + if (!enabled()) throw err("Google Trends isn't set up: add the DataForSEO login and password in Server settings → Research."); + + const res = await fetch(API, { + method: "POST", + headers: { Authorization: `Basic ${Buffer.from(`${process.env.DATAFORSEO_LOGIN}:${process.env.DATAFORSEO_PASSWORD}`).toString("base64")}`, "Content-Type": "application/json" }, + body: JSON.stringify([{ keywords: kws, location_name: location, time_range: range, type: "web", item_types: ["google_trends_graph", "google_trends_queries_list"] }]), + }); + const body = await res.json().catch(() => ({})); + const task = body.tasks?.[0]; + if (!res.ok || body.status_code !== 20000 || !task || task.status_code !== 20000) + throw err(`Google Trends (DataForSEO): ${task?.status_message || body.status_message || res.statusText}`, 502); + + const items = task.result?.[0]?.items || []; + const lists = items.filter((x) => x.type === "google_trends_queries_list"); + const related = (k) => { + const l = lists.find((x) => (x.keywords || []).map((w) => w.toLowerCase()).includes(k)) || (kws.length === 1 ? lists[0] : null); + const pick = (a) => (a || []).slice(0, 10).map((r) => ({ query: r.query, value: r.value })); + return { rising: pick(l?.data?.rising), top: pick(l?.data?.top) }; + }; + const data = { + location, range, checkedAt: new Date().toISOString().slice(0, 10), + note: "Interest is relative (0–100, 100 = the busiest point for any of these keywords in this period), not a search count. Rising values are % growth; very large values mean 'breakout'.", + keywords: summarise(items.find((x) => x.type === "google_trends_graph"), kws).map((s) => ({ ...s, ...related(s.keyword) })), + cost: task.cost ?? body.cost ?? null, + }; + remember(key, data); + return data; +} diff --git a/dashboard/server/index.js b/dashboard/server/index.js index 36f76f4..a7c8c1c 100644 --- a/dashboard/server/index.js +++ b/dashboard/server/index.js @@ -28,6 +28,7 @@ import * as sched from "../../connector/src/schedule.js"; import * as lib from "../../connector/src/library.js"; import * as plan from "../../connector/src/plan.js"; import * as competitors from "../../connector/src/competitors.js"; +import * as hashtags from "../../connector/src/hashtags.js"; import * as brandkit from "../../connector/src/brandkit.js"; import * as extra from "../../connector/src/platforms/index.js"; import { publicBaseUrl, resolveMediaToken } from "../../connector/src/platforms/media.js"; @@ -496,6 +497,11 @@ app.delete("/api/competitors/:client/:username", h(async (req) => { competitors. app.get("/api/competitors/:client/suggestions", h(async (req) => ({ suggestions: competitors.listSuggestions(req.params.client) }))); app.post("/api/competitors/:client/suggestions/:username/add", h(async (req) => ({ added: await competitors.acceptSuggestion(req.params.client, req.params.username, { by: who(req) }) }))); app.post("/api/competitors/:client/suggestions/:username/dismiss", h(async (req) => competitors.dismissSuggestion(req.params.client, req.params.username))); +app.get("/api/competitors/:client/hashtag", h(async (req) => { + const igId = await competitors.viewerIgId(req.params.client); + if (!req.query.tag) return { enabled: hashtags.enabled(), quota: hashtags.quota(igId) }; + return hashtags.topPosts(igId, String(req.query.tag), { kind: req.query.kind === "recent" ? "recent" : "top" }); +})); app.post("/api/competitors/refresh", h(async (req) => competitors.refresh(String(req.body?.client || "") || undefined))); // ---------- published posts and their numbers ---------- diff --git a/dashboard/server/jobs.js b/dashboard/server/jobs.js index 4ac3e90..8a43965 100644 --- a/dashboard/server/jobs.js +++ b/dashboard/server/jobs.js @@ -42,6 +42,7 @@ const READ_TOOLS = [ "mcp__social-manager__list_accounts", "mcp__social-manager__instagram_insights", "mcp__social-manager__instagram_recent_posts", "mcp__social-manager__instagram_comments", "mcp__social-manager__facebook_page_insights", "mcp__social-manager__facebook_recent_posts", "mcp__social-manager__audience_demographics", "mcp__social-manager__instagram_mentions", "mcp__social-manager__competitor_lookup", "mcp__social-manager__competitors", "mcp__social-manager__competitor_suggest", + "mcp__social-manager__google_places", "mcp__social-manager__client_google_listing", "mcp__social-manager__google_trends", "mcp__social-manager__hashtag_top_posts", "mcp__social-manager__list_connections", "mcp__social-manager__suggested_post_times", "mcp__social-manager__content_library", "mcp__social-manager__brand_kit", "mcp__social-manager__caption_check", "mcp__social-manager__content_plan", // Calendar items are plans for the team (idea → brief → …), never posts: nothing is scheduled or published from a run. diff --git a/dashboard/server/settings.js b/dashboard/server/settings.js index 0de74a5..afb2b40 100644 --- a/dashboard/server/settings.js +++ b/dashboard/server/settings.js @@ -31,6 +31,14 @@ const GROUPS = [ { key: "ANTHROPIC_BASE_URL", label: "Model server address", type: "url", placeholder: "http://172.17.0.1:11434", hint: "Leave empty to use Anthropic. For your own Ollama, enter its address and set Model to the Ollama model name (e.g. qwen3.6:35b-a3b)." }, { key: "ANTHROPIC_AUTH_TOKEN", label: "Model server token", secret: true, hint: "Only with a model server address. Ollama accepts any text, e.g. ollama." }, ] }, + { name: "Research", fields: [ + { key: "RESEARCH_GOOGLE_PLACES", label: "Google ratings & reviews (Places)", hint: "Competitor scans read real Google ratings and reviews. Needs the key below. Off: scans use what websites show (marked unverified).", type: "bool", default: true }, + { key: "RESEARCH_GOOGLE_TRENDS", label: "Google search trends (DataForSEO)", hint: "Trend scans and competitor scans check what people search for in the client's region. Needs the DataForSEO login below.", type: "bool", default: true }, + { key: "RESEARCH_IG_HASHTAGS", label: "Instagram hashtag top posts", hint: "Trend scans find top Reels under niche hashtags. Turn on only after Meta approves Instagram Public Content Access. 30 hashtags per client account per week.", type: "bool", default: false }, + { key: "GOOGLE_PLACES_API_KEY", label: "Google Places API key", hint: "Real Google ratings and reviews for clients and competitors. Restrict the key to Places API (New) and this server's IP.", secret: true }, + { key: "DATAFORSEO_LOGIN", label: "DataForSEO API login", hint: "For Google Trends (rising searches in a client's niche). DataForSEO dashboard → API Access." }, + { key: "DATAFORSEO_PASSWORD", label: "DataForSEO API password", secret: true }, + ] }, { name: "Email (SMTP)", scope: "self", fields: [ { key: "SMTP_HOST", label: "Host", placeholder: "mail.example.com" }, { key: "SMTP_PORT", label: "Port", type: "number", placeholder: "587" }, diff --git a/dashboard/web/src/pages/Competitors.jsx b/dashboard/web/src/pages/Competitors.jsx index a5ef3ea..e3e3649 100644 --- a/dashboard/web/src/pages/Competitors.jsx +++ b/dashboard/web/src/pages/Competitors.jsx @@ -1,7 +1,7 @@ import { useCallback, useEffect, useState } from "react"; import { useNavigate, useSearchParams } from "react-router-dom"; import { Bar, BarChart, CartesianGrid, Cell, LabelList, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts"; -import { Check, ExternalLink, Plus, RefreshCw, Sparkles, Swords, Trash2, X } from "lucide-react"; +import { Check, ExternalLink, Hash, Plus, RefreshCw, Sparkles, Swords, Trash2, X } from "lucide-react"; import { api } from "../api.js"; import { Badge, Button, cx, Empty, ErrorNote, Field, fmt, fmtDate, Panel, Spinner, timeAgo } from "../ui.jsx"; @@ -98,6 +98,64 @@ function Suggestions({ items, busy, onAdd, onDismiss, full }) { ); } +/** Top public posts under a hashtag (Instagram Hashtag Search). Hidden unless switched on in Server settings → Research. */ +function HashtagPosts({ client }) { + const [status, setStatus] = useState(null); + const [tag, setTag] = useState(""); + const [kind, setKind] = useState("top"); + const [result, setResult] = useState(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + useEffect(() => { + setStatus(null); setResult(null); setError(null); + if (client) api(`/api/competitors/${encodeURIComponent(client)}/hashtag`).then(setStatus).catch(() => setStatus(null)); + }, [client]); + if (!status?.enabled) return null; + const search = async (e) => { + e.preventDefault(); + setBusy(true); setError(null); + try { + const r = await api(`/api/competitors/${encodeURIComponent(client)}/hashtag?tag=${encodeURIComponent(tag.replace(/^#/, ""))}&kind=${kind}`); + setResult(r); + if (r.quota) setStatus((s) => ({ ...s, quota: { ...s.quota, used: r.quota.used, left: r.quota.left } })); + } catch (err) { setError(err); } + setBusy(false); + }; + const posts = result ? [...result.posts].sort((a, b) => (b.likes ?? 0) + (b.comments ?? 0) - ((a.likes ?? 0) + (a.comments ?? 0))) : []; + return ( + +
+ setTag(e.target.value)} /> + + +
+ + {result && ( +
+

+ #{result.hashtag} · {result.posts.length} posts{result.reelsShare != null ? ` · ${result.reelsShare}% Reels` : ""}{result.cached ? " · saved from the last 24 hours" : ""}. Instagram doesn't show who posted them. +

+
    + {posts.map((p) => ( +
  • +
    + {p.type} + {p.likes == null ? "likes hidden" : `${fmt(p.likes)} likes`} · {fmt(p.comments)} comments +
    + {p.caption &&

    {p.caption}

    } + Open on Instagram +
    • + ))} +
+
+ )} + + ); +} + export default function Competitors() { const [params, setParams] = useSearchParams(); const [clients, setClients] = useState(null); @@ -151,6 +209,8 @@ export default function Competitors() { = 10} /> + + {!data ? (!error && ) : !data.competitors.length ? (
}> diff --git a/tools/report-pdf/blocks.mjs b/tools/report-pdf/blocks.mjs index 2fd0a39..386427e 100644 --- a/tools/report-pdf/blocks.mjs +++ b/tools/report-pdf/blocks.mjs @@ -68,6 +68,9 @@ function competitionData(slug, snap) { formatMix: a.formatMix || {}, engagementByFormat: a.engagementByFormat || {}, topTags: a.topTags || [], allTags: a.allTags || [], topPosts: (a.topPosts || []).map((p) => ({ at: p.at, type: p.type, likes: p.likes, comments: p.comments, link: p.permalink, caption: plain(p.caption).slice(0, 140) })), }); + if (you && !you.google) { + try { you.google = JSON.parse(readFileSync(join(root, "clients", slug, "client.json"), "utf8")).google || null; } catch {} + } const rows = [...(you ? [keep(you, "you")] : []), ...competitors.map((c) => keep(c, "tracked")), ...listSuggestions(slug).map((s) => keep(s, "suggested"))]; return rows.length > 1 ? rows : null; } catch { @@ -309,9 +312,10 @@ const BLOCKS = { const rows = (d?.competition || []).filter((r) => r.google?.rating); if (!rows.length) return noData("Google ratings"); const sorted = [...rows].sort((a, b) => (b.google.reviews || 0) - (a.google.reviews || 0)), maxRev = Math.max(...rows.map((r) => r.google.reviews || 0), 1); - return card("", "Google rating and reviews", "where local customers often decide", + const unverified = rows.some((r) => r.google.verified === false); + return card("", "Google rating and reviews", unverified ? "where local customers often decide · * as shown on the business's own website" : "where local customers often decide · from Google Maps", `
${sorted.map((r) => `
${r.role === "you" ? "You" : esc(r.name || `@${r.username}`)} - ★ ${r.google.rating.toFixed(1)}
${num(r.google.reviews)} reviews
`).join("")}
`); + ★ ${r.google.rating.toFixed(1)}${r.google.verified === false ? "*" : ""}
${r.google.reviews == null ? "—" : `${num(r.google.reviews)} reviews`}`).join("")}`); }, };