From 616b6727fa45cd7a0831df84f87e9205e2b770dd Mon Sep 17 00:00:00 2001 From: Ben Senescu <44480372+bensenescu@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:57:50 -0400 Subject: [PATCH] Onboarding chat: let read_website fetch specific user-named URLs (#300) --- .../onboarding/OnboardingChatAgent.ts | 2 +- .../onboarding/onboardingChatTools.ts | 36 +++++++--- src/server/features/onboarding/scrape.test.ts | 32 ++++++++- src/server/features/onboarding/scrape.ts | 69 ++++++++++++------- 4 files changed, 106 insertions(+), 33 deletions(-) diff --git a/src/server/features/onboarding/OnboardingChatAgent.ts b/src/server/features/onboarding/OnboardingChatAgent.ts index 0d186a1..4dbb141 100644 --- a/src/server/features/onboarding/OnboardingChatAgent.ts +++ b/src/server/features/onboarding/OnboardingChatAgent.ts @@ -48,7 +48,7 @@ function buildSystemPrompt(domain: string | null): string { "When a request is beyond your preview tools, don't conclude OpenSEO can't do it — describe what the full product does per the fact sheet, and don't claim capabilities the fact sheet doesn't list.", "You have tools to pull real search data. Never state a metric, search volume, keyword difficulty, ranking, or competitor figure you did not get from a tool.", "Core tools for THIS user's own site — use these freely whenever the user asks you to analyze their site, recommend a strategy, or for any site-specific advice:", - "- read_website: reads their pages as plain text. Always available.", + "- read_website: reads web pages as plain text. With no arguments it reads the user's own site; when the user names or pastes specific page URLs (their own pages or a competitor's), pass those as `urls` to read exactly those pages. Always available, no credits — use it whenever the user points you at specific URLs.", "- get_seo_metrics: their estimated organic traffic, ranking-keyword count, and the keywords they already rank for (each with real search volume and difficulty). May report it's unavailable for brand-new sites or unsupported markets.", "- research_keywords: given one seed topic from their site, returns related keywords each with real monthly search volume and difficulty (KD). Use it to ground keyword suggestions in real data — especially when get_seo_metrics shows no rankings. Seed it with the site's primary topic; call it again only for a clearly distinct second theme.", "Market & competitor tools — these cost more credits, so use them SPARINGLY and only when the user's question is specifically about competitors, the live SERP, or backlinks. Do NOT call them just to enrich a routine strategy, and never call more than one or two per reply. The core site tools above answer most questions on their own.", diff --git a/src/server/features/onboarding/onboardingChatTools.ts b/src/server/features/onboarding/onboardingChatTools.ts index 205ab0a..5a20860 100644 --- a/src/server/features/onboarding/onboardingChatTools.ts +++ b/src/server/features/onboarding/onboardingChatTools.ts @@ -1,7 +1,11 @@ import { tool, type ToolSet } from "ai"; import { z } from "zod"; import { AppError } from "@/server/lib/errors"; -import { readSite } from "@/server/features/onboarding/scrape"; +import { + MAX_PAGES, + readPages, + readSite, +} from "@/server/features/onboarding/scrape"; import { DomainService } from "@/server/features/domain/services/DomainService"; import { KeywordResearchService } from "@/server/features/keywords/services/KeywordResearchService"; import { createDataforseoClient } from "@/server/lib/dataforseo"; @@ -75,18 +79,34 @@ function coreSiteTools(ctx: ToolContext): ToolSet { return { read_website: tool({ description: - "Read the user's own website (their pages, as plain text) to ground site-specific advice and strategy. Uses the project's saved domain.", - inputSchema: z.object({}), - execute: async () => { - if (!project.domain) { - throw new AppError("VALIDATION_ERROR", "Set a website domain first"); + "Read web pages as plain text to ground advice. With no arguments, reads the user's own site (homepage plus a few pages from its sitemap) from the project's saved domain. Pass `urls` to read specific pages the user names instead — particular pages of their site, or a competitor's page to compare. Always available; uses no credits.", + inputSchema: z.object({ + urls: z + .array(z.string().url()) + .max(MAX_PAGES) + .optional() + .describe( + `Specific page URLs to read (max ${MAX_PAGES}). Omit to read the user's own site from its saved domain.`, + ), + }), + execute: async ({ urls }) => { + const site = + urls && urls.length > 0 + ? await readPages(urls) + : project.domain + ? await readSite(project.domain) + : null; + if (!site) { + throw new AppError( + "VALIDATION_ERROR", + "Provide page URLs to read, or set a website domain first.", + ); } - const site = await readSite(project.domain); if (site.blocked) { return { blocked: true, pages: [], - note: "Could not read the site's pages. Ask the user to describe what they do, and keep the advice high-level.", + note: "Could not read the requested page(s). Ask the user to describe what they cover, and keep the advice high-level.", }; } return { diff --git a/src/server/features/onboarding/scrape.test.ts b/src/server/features/onboarding/scrape.test.ts index 0cb3e3b..68782b5 100644 --- a/src/server/features/onboarding/scrape.test.ts +++ b/src/server/features/onboarding/scrape.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { readSite } from "@/server/features/onboarding/scrape"; +import { readPages, readSite } from "@/server/features/onboarding/scrape"; describe("readSite SSRF guard", () => { beforeEach(() => { @@ -26,3 +26,33 @@ describe("readSite SSRF guard", () => { expect(fetch).not.toHaveBeenCalled(); }); }); + +describe("readPages SSRF guard", () => { + beforeEach(() => { + vi.stubGlobal("fetch", vi.fn()); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("skips private/metadata URLs without fetching them", async () => { + const result = await readPages([ + "http://169.254.169.254/latest/meta-data/", + "http://localhost:3000/admin", + ]); + + expect(result.blocked).toBe(true); + expect(result.pages).toEqual([]); + // Every URL is validated before any outbound fetch. + expect(fetch).not.toHaveBeenCalled(); + }); + + it("returns blocked for an empty URL list without fetching", async () => { + const result = await readPages([]); + + expect(result.blocked).toBe(true); + expect(result.pages).toEqual([]); + expect(fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/src/server/features/onboarding/scrape.ts b/src/server/features/onboarding/scrape.ts index 1a637b2..7b389c1 100644 --- a/src/server/features/onboarding/scrape.ts +++ b/src/server/features/onboarding/scrape.ts @@ -6,7 +6,7 @@ import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy"; -const MAX_PAGES = 5; +export const MAX_PAGES = 5; const PER_PAGE_CHAR_LIMIT = 4000; const FETCH_TIMEOUT_MS = 10_000; const MAX_RESPONSE_BYTES = 2_000_000; @@ -19,7 +19,6 @@ type ScrapedPage = { }; type SiteReadResult = { - rootUrl: string; pages: ScrapedPage[]; /** True when we couldn't read any page (blocked, offline, etc.). */ blocked: boolean; @@ -139,37 +138,61 @@ function decodeEntities(value: string): string { .replace(/ /g, " "); } -/** Discovers and reads up to MAX_PAGES pages of a site as plain text. */ +/** Fetches one (already-validated) URL and shapes it as a page, or null if it + * couldn't be read or yielded no text. */ +async function scrapePage(url: string): Promise { + const html = await fetchText(url); + if (!html) { + return null; + } + const text = htmlToText(html).slice(0, PER_PAGE_CHAR_LIMIT); + if (text.length === 0) { + return null; + } + return { url, title: extractTitle(html), text }; +} + +/** + * Reads a specific list of page URLs as plain text — used when the user names + * exact pages (their own or a competitor's) rather than asking us to discover a + * site. Each URL is independently run through the SSRF guard, so a blocked or + * unreachable URL is skipped rather than failing the batch. + */ +export async function readPages(urls: string[]): Promise { + const pages: ScrapedPage[] = []; + for (const rawUrl of urls.slice(0, MAX_PAGES)) { + let url: string; + try { + // Re-validates host, blocks private/metadata IPs, does DoH DNS resolution. + url = await normalizeAndValidateStartUrl(rawUrl); + } catch { + continue; // blocked or unparseable URL + } + const page = await scrapePage(url); + if (page) { + pages.push(page); + } + } + + return { pages, blocked: pages.length === 0 }; +} + +/** + * Discovers a site's representative URLs (homepage + sitemap) and reads them. + * Just URL discovery on top of readPages, which does the validated fetching. + */ export async function readSite(domain: string): Promise { let rootUrl: string; try { rootUrl = await normalizeAndValidateStartUrl(domain); } catch { // Blocked (private/metadata host) or unparseable domain — nothing to read. - return { rootUrl: `https://${domain}`, pages: [], blocked: true }; + return { pages: [], blocked: true }; } const origin = new URL(rootUrl).origin; // Prefer the sitemap for representative URLs; always include the homepage. const sitemap = await fetchText(`${origin}/sitemap.xml`); const discovered = sitemap ? parseSitemapUrls(sitemap, origin) : []; - const targets = [ - rootUrl, - ...discovered.filter((url) => url !== rootUrl), - ].slice(0, MAX_PAGES); - - const pages: ScrapedPage[] = []; - for (const url of targets) { - const html = await fetchText(url); - if (!html) { - continue; - } - const text = htmlToText(html).slice(0, PER_PAGE_CHAR_LIMIT); - if (text.length === 0) { - continue; - } - pages.push({ url, title: extractTitle(html), text }); - } - - return { rootUrl, pages, blocked: pages.length === 0 }; + return readPages([rootUrl, ...discovered.filter((url) => url !== rootUrl)]); }