Merge main: add the real public quick-audit endpoint for crawlerx.co

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/routeTree.gen.ts

@@ -38,6 +38,7 @@ import { Route as Char91DotwellKnownChar93OpenaiAppsChallengeRouteImport } from
 import { Route as AuthenticatedOnboardingIndexRouteImport } from './routes/_authenticated.onboarding.index'
 import { Route as AppSettingsIndexRouteImport } from './routes/_app/settings/index'
 import { Route as ApiStripeWebhookRouteImport } from './routes/api/stripe/webhook'
+import { Route as ApiPublicQuickAuditRouteImport } from './routes/api/public/quick-audit'
 import { Route as ApiAutumnSplatRouteImport } from './routes/api/autumn/$'
 import { Route as ApiAuthSplatRouteImport } from './routes/api/auth/$'
 import { Route as AuthenticatedOnboardingChatRouteImport } from './routes/_authenticated.onboarding.chat'
@@ -214,6 +215,11 @@ const ApiStripeWebhookRoute = ApiStripeWebhookRouteImport.update({
   path: '/api/stripe/webhook',
   getParentRoute: () => rootRouteImport,
 } as any)
+const ApiPublicQuickAuditRoute = ApiPublicQuickAuditRouteImport.update({
+  id: '/api/public/quick-audit',
+  path: '/api/public/quick-audit',
+  getParentRoute: () => rootRouteImport,
+} as any)
 const ApiAutumnSplatRoute = ApiAutumnSplatRouteImport.update({
   id: '/api/autumn/$',
   path: '/api/autumn/$',
@@ -417,6 +423,7 @@ export interface FileRoutesByFullPath {
   '/onboarding/chat': typeof AuthenticatedOnboardingChatRoute
   '/api/auth/$': typeof ApiAuthSplatRoute
   '/api/autumn/$': typeof ApiAutumnSplatRoute
+  '/api/public/quick-audit': typeof ApiPublicQuickAuditRoute
   '/api/stripe/webhook': typeof ApiStripeWebhookRoute
   '/settings/': typeof AppSettingsIndexRoute
   '/onboarding/': typeof AuthenticatedOnboardingIndexRoute
@@ -473,6 +480,7 @@ export interface FileRoutesByTo {
   '/onboarding/chat': typeof AuthenticatedOnboardingChatRoute
   '/api/auth/$': typeof ApiAuthSplatRoute
   '/api/autumn/$': typeof ApiAutumnSplatRoute
+  '/api/public/quick-audit': typeof ApiPublicQuickAuditRoute
   '/api/stripe/webhook': typeof ApiStripeWebhookRoute
   '/settings': typeof AppSettingsIndexRoute
   '/onboarding': typeof AuthenticatedOnboardingIndexRoute
@@ -533,6 +541,7 @@ export interface FileRoutesById {
   '/_authenticated/onboarding/chat': typeof AuthenticatedOnboardingChatRoute
   '/api/auth/$': typeof ApiAuthSplatRoute
   '/api/autumn/$': typeof ApiAutumnSplatRoute
+  '/api/public/quick-audit': typeof ApiPublicQuickAuditRoute
   '/api/stripe/webhook': typeof ApiStripeWebhookRoute
   '/_app/settings/': typeof AppSettingsIndexRoute
   '/_authenticated/onboarding/': typeof AuthenticatedOnboardingIndexRoute
@@ -593,6 +602,7 @@ export interface FileRouteTypes {
     | '/onboarding/chat'
     | '/api/auth/$'
     | '/api/autumn/$'
+    | '/api/public/quick-audit'
     | '/api/stripe/webhook'
     | '/settings/'
     | '/onboarding/'
@@ -649,6 +659,7 @@ export interface FileRouteTypes {
     | '/onboarding/chat'
     | '/api/auth/$'
     | '/api/autumn/$'
+    | '/api/public/quick-audit'
     | '/api/stripe/webhook'
     | '/settings'
     | '/onboarding'
@@ -708,6 +719,7 @@ export interface FileRouteTypes {
     | '/_authenticated/onboarding/chat'
     | '/api/auth/$'
     | '/api/autumn/$'
+    | '/api/public/quick-audit'
     | '/api/stripe/webhook'
     | '/_app/settings/'
     | '/_authenticated/onboarding/'
@@ -752,6 +764,7 @@ export interface RootRouteChildren {
   ApiTeamSetupRoute: typeof ApiTeamSetupRoute
   ApiAuthSplatRoute: typeof ApiAuthSplatRoute
   ApiAutumnSplatRoute: typeof ApiAutumnSplatRoute
+  ApiPublicQuickAuditRoute: typeof ApiPublicQuickAuditRoute
   ApiStripeWebhookRoute: typeof ApiStripeWebhookRoute
   ApiGa4OauthCallbackRoute: typeof ApiGa4OauthCallbackRoute
   ApiGscOauthCallbackRoute: typeof ApiGscOauthCallbackRoute
@@ -962,6 +975,13 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof ApiStripeWebhookRouteImport
       parentRoute: typeof rootRouteImport
     }
+    '/api/public/quick-audit': {
+      id: '/api/public/quick-audit'
+      path: '/api/public/quick-audit'
+      fullPath: '/api/public/quick-audit'
+      preLoaderRoute: typeof ApiPublicQuickAuditRouteImport
+      parentRoute: typeof rootRouteImport
+    }
     '/api/autumn/$': {
       id: '/api/autumn/$'
       path: '/api/autumn/$'
@@ -1385,6 +1405,7 @@ const rootRouteChildren: RootRouteChildren = {
   ApiTeamSetupRoute: ApiTeamSetupRoute,
   ApiAuthSplatRoute: ApiAuthSplatRoute,
   ApiAutumnSplatRoute: ApiAutumnSplatRoute,
+  ApiPublicQuickAuditRoute: ApiPublicQuickAuditRoute,
   ApiStripeWebhookRoute: ApiStripeWebhookRoute,
   ApiGa4OauthCallbackRoute: ApiGa4OauthCallbackRoute,
   ApiGscOauthCallbackRoute: ApiGscOauthCallbackRoute,

src/routes/api/public/quick-audit.ts

@@ -0,0 +1,125 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { env } from "cloudflare:workers";
+import { selectQuickAuditFinding } from "@/server/features/publicAudit/quickAuditFindings";
+import { normalizeUrl } from "@/server/lib/audit/url-utils";
+import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";
+
+// Public, unauthenticated endpoint backing the "Run Live Audit" widget on the
+// crawlerx.co marketing site. It crawls exactly one real page — the same
+// crawler + HTML analyzer the authenticated product uses for every audit —
+// and returns a finding derived from that crawl. It must never fall back to
+// a canned/fabricated result: an unreachable or blocked site is reported as
+// an honest error instead.
+//
+// Anonymous and unmetered, so it is rate-limited per IP and globally via KV
+// (same pattern as invitation-send-limit.ts) to bound the outbound-crawl
+// cost of public abuse.
+
+const ALLOWED_ORIGINS = new Set([
+  "https://crawlerx.co",
+  "https://www.crawlerx.co",
+  "http://localhost:5173",
+]);
+
+const PER_IP_DAILY_LIMIT = 8;
+const GLOBAL_DAILY_LIMIT = 500;
+const WINDOW_TTL_SECONDS = 60 * 60 * 24;
+
+function corsHeaders(origin: string | null): Record<string, string> {
+  const allowOrigin = origin && ALLOWED_ORIGINS.has(origin) ? origin : "https://crawlerx.co";
+  return {
+    "Access-Control-Allow-Origin": allowOrigin,
+    "Access-Control-Allow-Methods": "POST, OPTIONS",
+    "Access-Control-Allow-Headers": "Content-Type",
+    Vary: "Origin",
+  };
+}
+
+async function consumeDailyBudget(key: string, limit: number): Promise<boolean> {
+  const count = Number((await env.KV.get(key)) ?? "0");
+  if (count >= limit) return false;
+  await env.KV.put(key, String(count + 1), {
+    expirationTtl: WINDOW_TTL_SECONDS,
+  });
+  return true;
+}
+
+function parseRequestedUrl(body: unknown): string | null {
+  if (typeof body !== "object" || body === null || !("url" in body)) return null;
+  const rawUrl = (body as { url: unknown }).url;
+  if (typeof rawUrl !== "string" || rawUrl.trim().length === 0) return null;
+
+  const withScheme = /^https?:\/\//i.test(rawUrl.trim())
+    ? rawUrl.trim()
+    : `https://${rawUrl.trim()}`;
+  return normalizeUrl(withScheme);
+}
+
+async function handleQuickAudit(request: Request): Promise<Response> {
+  const headers = corsHeaders(request.headers.get("origin"));
+
+  if (request.method === "OPTIONS") {
+    return new Response(null, { headers });
+  }
+
+  let body: unknown;
+  try {
+    body = await request.json();
+  } catch {
+    return Response.json(
+      { error: "Invalid request body" },
+      { status: 400, headers },
+    );
+  }
+
+  const normalizedUrl = parseRequestedUrl(body);
+  if (!normalizedUrl) {
+    return Response.json(
+      { error: "That doesn't look like a valid URL." },
+      { status: 400, headers },
+    );
+  }
+
+  const day = new Date().toISOString().slice(0, 10);
+  const ip = request.headers.get("cf-connecting-ip") ?? "unknown";
+  if (!(await consumeDailyBudget(`quick-audit:global:${day}`, GLOBAL_DAILY_LIMIT))) {
+    return Response.json(
+      { error: "Quick audit is at capacity right now — please try again in a bit." },
+      { status: 429, headers },
+    );
+  }
+  if (!(await consumeDailyBudget(`quick-audit:ip:${ip}:${day}`, PER_IP_DAILY_LIMIT))) {
+    return Response.json(
+      {
+        error:
+          "You've reached today's quick-audit limit. Sign up for unlimited audits.",
+      },
+      { status: 429, headers },
+    );
+  }
+
+  const domain = new URL(normalizedUrl).hostname.replace(/^www\./, "");
+  const page = await crawlPage(normalizedUrl, null, false);
+
+  if (page.fetchClass !== "ok" || page.statusCode >= 400) {
+    const reason =
+      page.fetchClass === "blocked"
+        ? "That site's security settings blocked our crawler from reading it."
+        : page.statusCode >= 400
+          ? `That site returned an HTTP ${page.statusCode} error.`
+          : "We couldn't reach that site — check the URL and try again.";
+    return Response.json({ error: reason }, { status: 422, headers });
+  }
+
+  const finding = selectQuickAuditFinding(domain, page);
+  return Response.json(finding, { headers });
+}
+
+export const Route = createFileRoute("/api/public/quick-audit")({
+  server: {
+    handlers: {
+      POST: ({ request }: { request: Request }) => handleQuickAudit(request),
+      OPTIONS: ({ request }: { request: Request }) => handleQuickAudit(request),
+    },
+  },
+});

src/server/features/publicAudit/quickAuditFindings.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, it } from "vitest";
+import type { CrawledPageResult } from "@/server/lib/audit/types";
+import { selectQuickAuditFinding } from "@/server/features/publicAudit/quickAuditFindings";
+
+function basePage(overrides: Partial<CrawledPageResult> = {}): CrawledPageResult {
+  return {
+    id: "page-1",
+    url: "https://example.com/",
+    statusCode: 200,
+    fetchClass: "ok",
+    redirectUrl: null,
+    title: "Example | A Great Business",
+    metaDescription: "A description of the business, well under the limit.",
+    canonicalUrl: "https://example.com/",
+    robotsMeta: null,
+    xRobotsTag: null,
+    headerCanonicalUrl: null,
+    ogTitle: null,
+    ogDescription: null,
+    ogImage: null,
+    h1Count: 1,
+    h2Count: 0,
+    h3Count: 0,
+    h4Count: 0,
+    h5Count: 0,
+    h6Count: 0,
+    headingOrder: [1],
+    wordCount: 400,
+    contentHash: null,
+    isHtml: true,
+    htmlBytes: 1000,
+    imagesTotal: 2,
+    imagesMissingAlt: 0,
+    images: [],
+    links: [],
+    hasStructuredData: true,
+    hreflangTags: [],
+    isIndexable: true,
+    responseTimeMs: 120,
+    crawlDepth: null,
+    inSitemap: false,
+    ...overrides,
+  };
+}
+
+describe("selectQuickAuditFinding", () => {
+  it("reports a missing title as the headline issue, never a fabricated one", () => {
+    const finding = selectQuickAuditFinding("example.com", basePage({ title: "" }));
+
+    expect(finding.priority).toBe("Critical Impact");
+    expect(finding.action).toBe("Add a title tag");
+    expect(finding.current).toContain("no <title>");
+    expect(finding.suggested).toContain("example.com");
+  });
+
+  it("reports a healthy page as healthy instead of inventing an issue", () => {
+    const finding = selectQuickAuditFinding("example.com", basePage());
+
+    expect(finding.priority).toBe("Good Health");
+    expect(finding.score).toBeGreaterThan(90);
+  });
+});

src/server/features/publicAudit/quickAuditFindings.ts

@@ -0,0 +1,141 @@
+import type { CrawledPageResult } from "@/server/lib/audit/types";
+
+export type QuickAuditPriority = "Critical Impact" | "High Priority" | "Good Health";
+
+export interface QuickAuditFinding {
+  domain: string;
+  score: number;
+  action: string;
+  issue: string;
+  priority: QuickAuditPriority;
+  current: string;
+  suggested: string;
+}
+
+const TITLE_MAX_LENGTH = 60;
+const DESCRIPTION_MAX_LENGTH = 160;
+
+type Issue = Omit<QuickAuditFinding, "domain" | "score"> & { weight: number };
+
+/**
+ * Picks the single most-impactful real issue on a crawled page, plus a
+ * composite score from every issue found. Used by the public marketing-site
+ * "quick audit" widget, which crawls exactly one real page (via
+ * {@link import("@/server/workflows/site-audit-workflow-helpers").crawlPage})
+ * — this never fabricates a finding for a page it hasn't actually fetched.
+ */
+export function selectQuickAuditFinding(
+  domain: string,
+  page: CrawledPageResult,
+): QuickAuditFinding {
+  const title = page.title.trim();
+  const description = page.metaDescription.trim();
+  const issues: Issue[] = [];
+
+  if (!title) {
+    issues.push({
+      weight: 30,
+      action: "Add a title tag",
+      issue:
+        "This page has no <title> tag — search engines fall back to a generic or URL-based title in results.",
+      priority: "Critical Impact",
+      current: "<!-- no <title> tag found -->",
+      suggested: `<title>${domain} | Describe this page in 50-60 characters</title>`,
+    });
+  } else if (title.length > TITLE_MAX_LENGTH) {
+    issues.push({
+      weight: 15,
+      action: "Shorten the title tag",
+      issue: `Title tag is ${title.length} characters — Google truncates titles past ~${TITLE_MAX_LENGTH} characters in search results.`,
+      priority: "High Priority",
+      current: `<title>${title}</title>`,
+      suggested: `<title>${title.slice(0, TITLE_MAX_LENGTH - 1).trim()}…</title>`,
+    });
+  }
+
+  if (!description) {
+    issues.push({
+      weight: 20,
+      action: "Add a meta description",
+      issue:
+        "No meta description found — Google will auto-generate a snippet from page content instead.",
+      priority: "High Priority",
+      current: '<!-- no <meta name="description"> tag found -->',
+      suggested: `<meta name="description" content="Describe what ${domain} offers in one or two sentences, under ${DESCRIPTION_MAX_LENGTH} characters." />`,
+    });
+  } else if (description.length > DESCRIPTION_MAX_LENGTH) {
+    issues.push({
+      weight: 8,
+      action: "Shorten the meta description",
+      issue: `Meta description is ${description.length} characters — Google truncates snippets past ~${DESCRIPTION_MAX_LENGTH} characters.`,
+      priority: "High Priority",
+      current: `<meta name="description" content="${description}" />`,
+      suggested: `<meta name="description" content="${description.slice(0, DESCRIPTION_MAX_LENGTH - 1).trim()}…" />`,
+    });
+  }
+
+  if (page.h1Count === 0) {
+    issues.push({
+      weight: 18,
+      action: "Add an H1 heading",
+      issue:
+        "No <h1> tag found on this page — a single clear H1 helps both users and search engines understand the page topic.",
+      priority: "High Priority",
+      current: "<!-- no <h1> tag found -->",
+      suggested: "<h1>State the page's main topic here</h1>",
+    });
+  } else if (page.h1Count > 1) {
+    issues.push({
+      weight: 10,
+      action: "Use a single H1 per page",
+      issue: `${page.h1Count} <h1> tags found — multiple H1s dilute the page's topical signal.`,
+      priority: "High Priority",
+      current: `${page.h1Count}× <h1>…</h1>`,
+      suggested: "Keep exactly one <h1>; demote the rest to <h2>.",
+    });
+  }
+
+  if (!page.canonicalUrl) {
+    issues.push({
+      weight: 10,
+      action: "Add a canonical tag",
+      issue:
+        "No canonical URL found — without one, duplicate or parameterized versions of this page can compete with each other in search results.",
+      priority: "High Priority",
+      current: '<!-- no <link rel="canonical"> tag found -->',
+      suggested: `<link rel="canonical" href="${page.url}" />`,
+    });
+  }
+
+  if (page.imagesMissingAlt > 0) {
+    issues.push({
+      weight: 6,
+      action: "Add alt text to images",
+      issue: `${page.imagesMissingAlt} of ${page.imagesTotal} images on this page are missing alt text.`,
+      priority: "Good Health",
+      current: '<img src="...">',
+      suggested: '<img src="..." alt="Describe what the image shows">',
+    });
+  }
+
+  if (issues.length === 0) {
+    return {
+      domain,
+      score: 96,
+      action: "No critical issues found on this quick scan",
+      issue:
+        "Title, meta description, H1, and canonical tag all look healthy on this page.",
+      priority: "Good Health",
+      current: `<title>${title}</title>`,
+      suggested:
+        "Run a full crawl in CrawlerX to check Core Web Vitals, internal linking, and every other page on the site.",
+    };
+  }
+
+  issues.sort((a, b) => b.weight - a.weight);
+  const totalPenalty = issues.reduce((sum, issue) => sum + issue.weight, 0);
+  const score = Math.max(35, 100 - totalPenalty);
+  const { weight: _weight, ...top } = issues[0];
+
+  return { domain, score, ...top };
+}