Onboarding chat: let read_website fetch specific user-named URLs (#300)

src/server/features/onboarding/OnboardingChatAgent.ts

@@ -48,7 +48,7 @@ function buildSystemPrompt(domain: string | null): string {
     "When a request is beyond your preview tools, don't conclude OpenSEO can't do it — describe what the full product does per the fact sheet, and don't claim capabilities the fact sheet doesn't list.",
     "You have tools to pull real search data. Never state a metric, search volume, keyword difficulty, ranking, or competitor figure you did not get from a tool.",
     "Core tools for THIS user's own site — use these freely whenever the user asks you to analyze their site, recommend a strategy, or for any site-specific advice:",
-    "- read_website: reads their pages as plain text. Always available.",
+    "- read_website: reads web pages as plain text. With no arguments it reads the user's own site; when the user names or pastes specific page URLs (their own pages or a competitor's), pass those as `urls` to read exactly those pages. Always available, no credits — use it whenever the user points you at specific URLs.",
     "- get_seo_metrics: their estimated organic traffic, ranking-keyword count, and the keywords they already rank for (each with real search volume and difficulty). May report it's unavailable for brand-new sites or unsupported markets.",
     "- research_keywords: given one seed topic from their site, returns related keywords each with real monthly search volume and difficulty (KD). Use it to ground keyword suggestions in real data — especially when get_seo_metrics shows no rankings. Seed it with the site's primary topic; call it again only for a clearly distinct second theme.",
     "Market & competitor tools — these cost more credits, so use them SPARINGLY and only when the user's question is specifically about competitors, the live SERP, or backlinks. Do NOT call them just to enrich a routine strategy, and never call more than one or two per reply. The core site tools above answer most questions on their own.",

src/server/features/onboarding/onboardingChatTools.ts

@@ -1,7 +1,11 @@
 import { tool, type ToolSet } from "ai";
 import { z } from "zod";
 import { AppError } from "@/server/lib/errors";
-import { readSite } from "@/server/features/onboarding/scrape";
+import {
+  MAX_PAGES,
+  readPages,
+  readSite,
+} from "@/server/features/onboarding/scrape";
 import { DomainService } from "@/server/features/domain/services/DomainService";
 import { KeywordResearchService } from "@/server/features/keywords/services/KeywordResearchService";
 import { createDataforseoClient } from "@/server/lib/dataforseo";
@@ -75,18 +79,34 @@ function coreSiteTools(ctx: ToolContext): ToolSet {
   return {
     read_website: tool({
       description:
-        "Read the user's own website (their pages, as plain text) to ground site-specific advice and strategy. Uses the project's saved domain.",
+        "Read web pages as plain text to ground advice. With no arguments, reads the user's own site (homepage plus a few pages from its sitemap) from the project's saved domain. Pass `urls` to read specific pages the user names instead — particular pages of their site, or a competitor's page to compare. Always available; uses no credits.",
-      inputSchema: z.object({}),
+      inputSchema: z.object({
-      execute: async () => {
+        urls: z
-        if (!project.domain) {
+          .array(z.string().url())
-          throw new AppError("VALIDATION_ERROR", "Set a website domain first");
+          .max(MAX_PAGES)
+          .optional()
+          .describe(
+            `Specific page URLs to read (max ${MAX_PAGES}). Omit to read the user's own site from its saved domain.`,
+          ),
+      }),
+      execute: async ({ urls }) => {
+        const site =
+          urls && urls.length > 0
+            ? await readPages(urls)
+            : project.domain
+              ? await readSite(project.domain)
+              : null;
+        if (!site) {
+          throw new AppError(
+            "VALIDATION_ERROR",
+            "Provide page URLs to read, or set a website domain first.",
+          );
         }
-        const site = await readSite(project.domain);
         if (site.blocked) {
           return {
             blocked: true,
             pages: [],
-            note: "Could not read the site's pages. Ask the user to describe what they do, and keep the advice high-level.",
+            note: "Could not read the requested page(s). Ask the user to describe what they cover, and keep the advice high-level.",
           };
         }
         return {

src/server/features/onboarding/scrape.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { readSite } from "@/server/features/onboarding/scrape";
+import { readPages, readSite } from "@/server/features/onboarding/scrape";
 
 describe("readSite SSRF guard", () => {
   beforeEach(() => {
@@ -26,3 +26,33 @@ describe("readSite SSRF guard", () => {
     expect(fetch).not.toHaveBeenCalled();
   });
 });
+
+describe("readPages SSRF guard", () => {
+  beforeEach(() => {
+    vi.stubGlobal("fetch", vi.fn());
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  it("skips private/metadata URLs without fetching them", async () => {
+    const result = await readPages([
+      "http://169.254.169.254/latest/meta-data/",
+      "http://localhost:3000/admin",
+    ]);
+
+    expect(result.blocked).toBe(true);
+    expect(result.pages).toEqual([]);
+    // Every URL is validated before any outbound fetch.
+    expect(fetch).not.toHaveBeenCalled();
+  });
+
+  it("returns blocked for an empty URL list without fetching", async () => {
+    const result = await readPages([]);
+
+    expect(result.blocked).toBe(true);
+    expect(result.pages).toEqual([]);
+    expect(fetch).not.toHaveBeenCalled();
+  });
+});

src/server/features/onboarding/scrape.ts

@@ -6,7 +6,7 @@
 
 import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
 
-const MAX_PAGES = 5;
+export const MAX_PAGES = 5;
 const PER_PAGE_CHAR_LIMIT = 4000;
 const FETCH_TIMEOUT_MS = 10_000;
 const MAX_RESPONSE_BYTES = 2_000_000;
@@ -19,7 +19,6 @@ type ScrapedPage = {
 };
 
 type SiteReadResult = {
-  rootUrl: string;
   pages: ScrapedPage[];
   /** True when we couldn't read any page (blocked, offline, etc.). */
   blocked: boolean;
@@ -139,37 +138,61 @@ function decodeEntities(value: string): string {
     .replace(/ /g, " ");
 }
 
-/** Discovers and reads up to MAX_PAGES pages of a site as plain text. */
+/** Fetches one (already-validated) URL and shapes it as a page, or null if it
+ * couldn't be read or yielded no text. */
+async function scrapePage(url: string): Promise<ScrapedPage | null> {
+  const html = await fetchText(url);
+  if (!html) {
+    return null;
+  }
+  const text = htmlToText(html).slice(0, PER_PAGE_CHAR_LIMIT);
+  if (text.length === 0) {
+    return null;
+  }
+  return { url, title: extractTitle(html), text };
+}
+
+/**
+ * Reads a specific list of page URLs as plain text — used when the user names
+ * exact pages (their own or a competitor's) rather than asking us to discover a
+ * site. Each URL is independently run through the SSRF guard, so a blocked or
+ * unreachable URL is skipped rather than failing the batch.
+ */
+export async function readPages(urls: string[]): Promise<SiteReadResult> {
+  const pages: ScrapedPage[] = [];
+  for (const rawUrl of urls.slice(0, MAX_PAGES)) {
+    let url: string;
+    try {
+      // Re-validates host, blocks private/metadata IPs, does DoH DNS resolution.
+      url = await normalizeAndValidateStartUrl(rawUrl);
+    } catch {
+      continue; // blocked or unparseable URL
+    }
+    const page = await scrapePage(url);
+    if (page) {
+      pages.push(page);
+    }
+  }
+
+  return { pages, blocked: pages.length === 0 };
+}
+
+/**
+ * Discovers a site's representative URLs (homepage + sitemap) and reads them.
+ * Just URL discovery on top of readPages, which does the validated fetching.
+ */
 export async function readSite(domain: string): Promise<SiteReadResult> {
   let rootUrl: string;
   try {
     rootUrl = await normalizeAndValidateStartUrl(domain);
   } catch {
     // Blocked (private/metadata host) or unparseable domain — nothing to read.
-    return { rootUrl: `https://${domain}`, pages: [], blocked: true };
+    return { pages: [], blocked: true };
   }
   const origin = new URL(rootUrl).origin;
 
   // Prefer the sitemap for representative URLs; always include the homepage.
   const sitemap = await fetchText(`${origin}/sitemap.xml`);
   const discovered = sitemap ? parseSitemapUrls(sitemap, origin) : [];
-  const targets = [
+  return readPages([rootUrl, ...discovered.filter((url) => url !== rootUrl)]);
-    rootUrl,
-    ...discovered.filter((url) => url !== rootUrl),
-  ].slice(0, MAX_PAGES);
-
-  const pages: ScrapedPage[] = [];
-  for (const url of targets) {
-    const html = await fetchText(url);
-    if (!html) {
-      continue;
-    }
-    const text = htmlToText(html).slice(0, PER_PAGE_CHAR_LIMIT);
-    if (text.length === 0) {
-      continue;
-    }
-    pages.push({ url, title: extractTitle(html), text });
-  }
-
-  return { rootUrl, pages, blocked: pages.length === 0 };
 }