MCP activation + usage PostHog events, North Star dashboard prompt (#369)

* Add MCP activation + usage PostHog events and North Star dashboard prompt

- mcp:authorize_success (server) on OAuth consent completion
- mcp:tool_call (server) on every MCP tool invocation, with tool/success/
  error_code/client_id and mcp_client vs in_app_agent source
- mcp:consent_viewed / mcp:consent_denied on the OAuth consent page
- mcp:setup_url_copy / mcp:setup_command_copy intent events on /ai
- docs/posthog-north-star-dashboard.md: prompt to configure the PostHog
  core-actions dashboard (activation rate, MCP setup funnel, daily MCP users)

* Fix codex review findings: count schema-rejected MCP calls as failures, wrap audit tools

- mcp:tool_call now fires after output validation; schema mismatches report
  success:false with MCP_OUTPUT_VALIDATION (the SDK surfaces them to the
  client as JSON-RPC errors)
- run_site_audit/get_audit_status/get_audit_issues/get_audit_pages were
  registered without instrumentMcpToolHandler, so their usage and failures
  were invisible

* Remove dashboard prompt doc

src/client/features/ai-mcp/SetupControls.tsx

@@ -57,7 +57,13 @@ export function Collapsible({
   );
 }
 
-export function CodeBlock({ code }: { code: string }) {
+export function CodeBlock({
+  code,
+  onCopy,
+}: {
+  code: string;
+  onCopy?: () => void;
+}) {
   return (
     <div className="flex items-stretch overflow-hidden rounded-md border border-base-300 bg-base-100">
       <pre className="min-w-0 flex-1 overflow-x-auto p-3 text-xs leading-relaxed text-base-content">
@@ -68,6 +74,7 @@ export function CodeBlock({ code }: { code: string }) {
           value={code}
           successMessage="Copied to clipboard"
           iconOnly
+          onCopy={onCopy}
         />
       </div>
     </div>
@@ -78,10 +85,12 @@ export function CopyButton({
   value,
   successMessage,
   iconOnly = false,
+  onCopy,
 }: {
   value: string;
   successMessage: string;
   iconOnly?: boolean;
+  onCopy?: () => void;
 }) {
   const [copied, setCopied] = useState(false);
 
@@ -95,6 +104,7 @@ export function CopyButton({
       toast.success(successMessage);
       setCopied(true);
       setTimeout(() => setCopied(false), 2000);
+      onCopy?.();
     } catch {
       toast.error("Could not copy to clipboard");
     }

src/routes/_app/ai.tsx

@@ -1,5 +1,6 @@
 import { createFileRoute } from "@tanstack/react-router";
 import { ArrowUpRight } from "lucide-react";
+import { captureClientEvent } from "@/client/lib/posthog";
 import { ClaudeIcon, CodexIcon } from "@/client/features/ai-mcp/AgentIcons";
 import { AvailableTools } from "@/client/features/ai-mcp/AvailableTools";
 import {
@@ -59,7 +60,11 @@ function AiPage() {
               <p className="text-xs font-medium uppercase tracking-wide text-base-content/50">
                 MCP server URL
               </p>
-              <CopyButton value={mcpUrl} successMessage="MCP URL copied" />
+              <CopyButton
+                value={mcpUrl}
+                successMessage="MCP URL copied"
+                onCopy={() => captureClientEvent("mcp:setup_url_copy")}
+              />
             </div>
             <code className="mt-2 block break-all font-mono text-sm text-base-content">
               {mcpUrl}
@@ -89,6 +94,11 @@ function AiPage() {
               </p>
               <CodeBlock
                 code={`claude mcp add --transport http --scope user openseo ${mcpUrl}`}
+                onCopy={() =>
+                  captureClientEvent("mcp:setup_command_copy", {
+                    agent: "claude-code",
+                  })
+                }
               />
               <p className="text-sm text-base-content/70">
                 Approve the login when prompted.
@@ -141,7 +151,14 @@ function AiPage() {
               <p className="text-sm text-base-content/70">
                 Run this in your terminal:
               </p>
-              <CodeBlock code={`codex mcp add openseo --url ${mcpUrl}`} />
+              <CodeBlock
+                code={`codex mcp add openseo --url ${mcpUrl}`}
+                onCopy={() =>
+                  captureClientEvent("mcp:setup_command_copy", {
+                    agent: "codex",
+                  })
+                }
+              />
               <p className="text-sm text-base-content/70">
                 Approve the login when prompted.
               </p>

src/routes/_authenticated.oauth-consent.tsx

@@ -1,7 +1,8 @@
 import { createFileRoute } from "@tanstack/react-router";
 import { Check, Database, KeyRound, User } from "lucide-react";
-import { useState } from "react";
+import { useEffect, useState } from "react";
 import { useSession } from "@/lib/auth-client";
+import { captureClientEvent } from "@/client/lib/posthog";
 
 export const Route = createFileRoute("/_authenticated/oauth-consent")({
   component: OAuthConsentPage,
@@ -27,9 +28,16 @@ function OAuthConsentPage() {
 
   const userEmail = session?.user?.email ?? null;
 
+  useEffect(() => {
+    captureClientEvent("mcp:consent_viewed");
+  }, []);
+
   async function respond(accept: boolean) {
     setError(null);
     setIsSubmitting(true);
+    if (!accept) {
+      captureClientEvent("mcp:consent_denied");
+    }
 
     const response = await fetch("/api/oauth/consent", {
       method: "POST",

src/server/mcp/instrumentation.test.ts

@@ -1,11 +1,16 @@
 import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 import { z } from "zod";
-import type { ToolExtra } from "@/server/mcp/context";
+import {
+  runWithMcpToolAuthContext,
+  type McpToolAuthContext,
+  type ToolExtra,
+} from "@/server/mcp/context";
 import { AppError } from "@/server/lib/errors";
 
 const mocks = vi.hoisted(() => ({
   captureServerError: vi.fn(),
+  captureServerEvent: vi.fn(),
 }));
 
 // waitUntil runs the capture promise inline so assertions see the call.
@@ -15,6 +20,7 @@ vi.mock("cloudflare:workers", () => ({
 
 vi.mock("@/server/lib/posthog", () => ({
   captureServerError: mocks.captureServerError,
+  captureServerEvent: mocks.captureServerEvent,
 }));
 
 const toolExtra: ToolExtra = {
@@ -30,9 +36,21 @@ function okResult(structuredContent: Record<string, unknown>): CallToolResult {
   return { content: [{ type: "text", text: "ok" }], structuredContent };
 }
 
+const authContext: McpToolAuthContext = {
+  userId: "user-1",
+  userEmail: "user@example.com",
+  organizationId: "org-1",
+  clientId: "client-1",
+  scopes: ["mcp"],
+  audience: "https://app.openseo.so/mcp",
+  subject: "user-1",
+  baseUrl: "https://app.openseo.so",
+};
+
 describe("instrumentMcpToolHandler", () => {
   beforeEach(() => {
     mocks.captureServerError.mockReset();
+    mocks.captureServerEvent.mockReset();
   });
 
   it("passes a valid result through without reporting", async () => {
@@ -85,4 +103,71 @@ describe("instrumentMcpToolHandler", () => {
     await expect(wrapped({}, toolExtra)).rejects.toThrow("NOT_FOUND");
     expect(mocks.captureServerError).not.toHaveBeenCalled();
   });
+
+  it("captures a usage event when auth context is present", async () => {
+    const { instrumentMcpToolHandler } = await import("./instrumentation");
+    const wrapped = instrumentMcpToolHandler("demo", outputSchema, async () =>
+      okResult({ items: [] }),
+    );
+
+    await runWithMcpToolAuthContext(authContext, () =>
+      wrapped({}, toolExtra),
+    );
+
+    expect(mocks.captureServerEvent).toHaveBeenCalledTimes(1);
+    expect(mocks.captureServerEvent.mock.calls[0][0]).toMatchObject({
+      distinctId: "user-1",
+      event: "mcp:tool_call",
+      organizationId: "org-1",
+      properties: {
+        tool: "demo",
+        success: true,
+        client_id: "client-1",
+        source: "mcp_client",
+      },
+    });
+  });
+
+  it("marks schema-rejected results as failed usage", async () => {
+    const { instrumentMcpToolHandler } = await import("./instrumentation");
+    const wrapped = instrumentMcpToolHandler("demo", outputSchema, async () =>
+      okResult({ items: "not-an-array" }),
+    );
+
+    await runWithMcpToolAuthContext(authContext, () =>
+      wrapped({}, toolExtra),
+    );
+
+    expect(mocks.captureServerEvent.mock.calls[0][0]).toMatchObject({
+      event: "mcp:tool_call",
+      properties: { success: false, error_code: "MCP_OUTPUT_VALIDATION" },
+    });
+  });
+
+  it("captures a failed usage event with the error code", async () => {
+    const { instrumentMcpToolHandler } = await import("./instrumentation");
+    const wrapped = instrumentMcpToolHandler("demo", outputSchema, async () => {
+      throw new AppError("NOT_FOUND");
+    });
+
+    await expect(
+      runWithMcpToolAuthContext(authContext, () => wrapped({}, toolExtra)),
+    ).rejects.toThrow("NOT_FOUND");
+
+    expect(mocks.captureServerEvent.mock.calls[0][0]).toMatchObject({
+      event: "mcp:tool_call",
+      properties: { success: false, error_code: "NOT_FOUND" },
+    });
+  });
+
+  it("skips the usage event when auth context is missing", async () => {
+    const { instrumentMcpToolHandler } = await import("./instrumentation");
+    const wrapped = instrumentMcpToolHandler("demo", outputSchema, async () =>
+      okResult({ items: [] }),
+    );
+
+    await wrapped({}, toolExtra);
+
+    expect(mocks.captureServerEvent).not.toHaveBeenCalled();
+  });
 });

src/server/mcp/instrumentation.ts

@@ -8,15 +8,48 @@ import {
   type ZodRawShapeCompat,
 } from "@modelcontextprotocol/sdk/server/zod-compat.js";
 import { asAppError } from "@/server/lib/errors";
-import { captureServerError } from "@/server/lib/posthog";
+import { captureServerError, captureServerEvent } from "@/server/lib/posthog";
 import { shouldCaptureAppErrorCode } from "@/shared/error-codes";
-import type { ToolExtra } from "@/server/mcp/context";
+import { getAuth, type ToolExtra } from "@/server/mcp/context";
 
 type ToolHandler<TArgs> = (
   args: TArgs,
   extra: ToolExtra,
 ) => CallToolResult | Promise<CallToolResult>;
 
+/**
+ * Usage analytics for every MCP tool invocation. `clientId` distinguishes
+ * external MCP clients (OAuth) from the in-app agent (first-party auth, null
+ * clientId); self-hosted installs never report because captureServerEvent is
+ * gated to hosted mode. Analytics must never affect the tool call, so a
+ * missing auth context (e.g. in tests) is swallowed.
+ */
+function captureMcpToolCall(
+  toolName: string,
+  extra: ToolExtra,
+  outcome: { success: boolean; errorCode?: string },
+) {
+  try {
+    const auth = getAuth(extra);
+    waitUntil(
+      captureServerEvent({
+        distinctId: auth.userId,
+        event: "mcp:tool_call",
+        organizationId: auth.organizationId,
+        properties: {
+          tool: toolName,
+          success: outcome.success,
+          error_code: outcome.errorCode,
+          client_id: auth.clientId,
+          source: auth.clientId ? "mcp_client" : "in_app_agent",
+        },
+      }),
+    );
+  } catch {
+    // no auth context — skip analytics
+  }
+}
+
 /**
  * Wraps an MCP tool handler so failures reach PostHog. Unlike TanStack server
  * functions (covered by errorHandlingMiddleware), the MCP route has no error
@@ -41,6 +74,9 @@ export function instrumentMcpToolHandler<TArgs>(
   return async (args, extra) => {
     try {
       const result = await handler(args, extra);
+      // The SDK converts an output-schema mismatch into a client-visible
+      // JSON-RPC error, so count it as a failed call, not a success.
+      let outputValidationFailed = false;
       if (
         normalizedOutputSchema &&
         !result.isError &&
@@ -51,6 +87,7 @@ export function instrumentMcpToolHandler<TArgs>(
           result.structuredContent,
         );
         if (!validation.success) {
+          outputValidationFailed = true;
           // getParseErrorMessage reports type-level mismatches (expected vs
           // received *types*), so it carries no row data. Keep it that way:
           // output schemas must not gain value-echoing refinements (enums on
@@ -67,9 +104,20 @@ export function instrumentMcpToolHandler<TArgs>(
           );
         }
       }
+      captureMcpToolCall(
+        toolName,
+        extra,
+        outputValidationFailed
+          ? { success: false, errorCode: "MCP_OUTPUT_VALIDATION" }
+          : { success: !result.isError },
+      );
       return result;
     } catch (error) {
       const appError = asAppError(error);
+      captureMcpToolCall(toolName, extra, {
+        success: false,
+        errorCode: appError?.code ?? "INTERNAL_ERROR",
+      });
       if (shouldCaptureAppErrorCode(appError?.code)) {
         console.error(`mcp.tool error (${toolName}):`, error);
         waitUntil(

src/server/mcp/oauth-provider.ts

@@ -1,3 +1,4 @@
+import { waitUntil } from "cloudflare:workers";
 import {
   OAuthProvider,
   type AuthRequest,
@@ -12,6 +13,7 @@ import {
   MCP_SCOPE,
 } from "@/lib/oauth-resource";
 import { asAppError } from "@/server/lib/errors";
+import { captureServerEvent } from "@/server/lib/posthog";
 import {
   createWorkersOAuthMcpProps,
   MCP_ROUTE,
@@ -364,6 +366,18 @@ async function handleOAuthConsentResponse(
     props,
   });
 
+  waitUntil(
+    captureServerEvent({
+      distinctId: context.userId,
+      event: "mcp:authorize_success",
+      organizationId: context.organizationId,
+      properties: {
+        client_id: authRequest.clientId,
+        scopes: scopes.join(" "),
+      },
+    }),
+  );
+
   return jsonResponse({ redirectTo });
 }
 

src/server/mcp/server.ts

@@ -210,21 +210,37 @@ export function registerOpenSeoMcpTools(server: McpServer) {
   server.registerTool(
     runSiteAuditTool.name,
     runSiteAuditTool.config,
-    runSiteAuditTool.handler,
+    instrumentMcpToolHandler(
+      runSiteAuditTool.name,
+      runSiteAuditTool.config.outputSchema,
+      runSiteAuditTool.handler,
+    ),
   );
   server.registerTool(
     getAuditStatusTool.name,
     getAuditStatusTool.config,
-    getAuditStatusTool.handler,
+    instrumentMcpToolHandler(
+      getAuditStatusTool.name,
+      getAuditStatusTool.config.outputSchema,
+      getAuditStatusTool.handler,
+    ),
   );
   server.registerTool(
     getAuditIssuesTool.name,
     getAuditIssuesTool.config,
-    getAuditIssuesTool.handler,
+    instrumentMcpToolHandler(
+      getAuditIssuesTool.name,
+      getAuditIssuesTool.config.outputSchema,
+      getAuditIssuesTool.handler,
+    ),
   );
   server.registerTool(
     getAuditPagesTool.name,
     getAuditPagesTool.config,
-    getAuditPagesTool.handler,
+    instrumentMcpToolHandler(
+      getAuditPagesTool.name,
+      getAuditPagesTool.config.outputSchema,
+      getAuditPagesTool.handler,
+    ),
   );
 }