/**
* HTML page analyzer using htmlparser2's streaming tokenizer.
*
* Extracts SEO-relevant data from a page's HTML: title, meta description,
* headings, images, links, canonical, OG tags, structured data, robots meta,
* word count, hreflang.
*
* Deliberately NOT a DOM parser: the previous cheerio implementation built a
* full DOM (~5-10x the HTML's size) per page, and with 25 concurrent parses
* on a 128MB isolate that was the audit engine's dominant OOM cause. The
* tokenizer keeps only the accumulated text and extracted fields in memory.
*/
import { Parser } from "htmlparser2";
import { normalizeUrl, isSameOrigin } from "./url-utils";
import type { PageAnalysis, PageLink } from "./types";
const SKIPPED_LINK_PROTOCOLS = /^(javascript:|mailto:|tel:|#)/;
/** Subtrees whose text is not visible content. */
const NON_CONTENT_TAGS = new Set(["script", "style", "noscript", "svg"]);
const HEADING_LEVELS: Record = {
h1: 1,
h2: 2,
h3: 3,
h4: 4,
h5: 5,
h6: 6,
};
const MAX_ANCHOR_CHARS = 200;
interface OpenAnchor {
href: string;
rel: string;
text: string[];
}
/**
* Analyze an HTML string and extract all SEO-relevant data.
*/
export function analyzeHtml(
html: string,
pageUrl: string,
statusCode: number,
responseTimeMs: number,
redirectUrl: string | null = null,
): PageAnalysis {
let title: string | null = null;
let titleDepth = 0;
let titleDone = false;
// parse5 (the old DOM path) treats content as raw text when
// scripting is enabled; skip element extraction inside it to match.
let noscriptDepth = 0;
let metaDescription: string | null = null;
let canonical: string | null = null;
let robotsMeta: string | null = null;
let ogTitle: string | null = null;
let ogDescription: string | null = null;
let ogImage: string | null = null;
let hasStructuredData = false;
const hreflangTags: string[] = [];
const h1s: string[] = [];
const headingOrder: number[] = [];
let openH1: string[] | null = null;
const images: Array<{ src: string | null; alt: string | null }> = [];
const linksByTarget = new Map();
let openAnchor: OpenAnchor | null = null;
// Visible text: prefer text inside an explicit ; when the document
// never opens one (fragments), fall back to all non-head text. Both
// exclude NON_CONTENT_TAGS subtrees.
let suppressDepth = 0;
let bodyDepth = 0;
let headDepth = 0;
let sawBody = false;
const bodyParts: string[] = [];
const fallbackParts: string[] = [];
const handleMetaTag = (attribs: Record) => {
const content = attribs["content"];
if (attribs["name"] === "description") {
metaDescription ??= content?.trim() ?? "";
} else if (attribs["name"] === "robots") {
robotsMeta ??= content ?? null;
} else if (attribs["property"] === "og:title") {
ogTitle ??= content ?? null;
} else if (attribs["property"] === "og:description") {
ogDescription ??= content ?? null;
} else if (attribs["property"] === "og:image") {
ogImage ??= content ?? null;
}
};
const handleLinkTag = (attribs: Record) => {
if (attribs["rel"] === "canonical") {
canonical ??= attribs["href"] ?? null;
} else if (attribs["rel"] === "alternate" && attribs["hreflang"]) {
hreflangTags.push(attribs["hreflang"]);
}
};
const closeAnchor = () => {
if (!openAnchor) return;
const { href, rel, text } = openAnchor;
openAnchor = null;
const resolved = normalizeUrl(href, pageUrl);
if (!resolved || linksByTarget.has(resolved)) return;
const anchor = text
.join("")
.replace(/\s+/g, " ")
.trim()
.slice(0, MAX_ANCHOR_CHARS);
linksByTarget.set(resolved, {
targetUrl: resolved,
anchor: anchor || null,
isInternal: isSameOrigin(resolved, pageUrl),
isNofollow: rel.split(/\s+/).includes("nofollow"),
});
};
const parser = new Parser(
{
onopentag(name, attribs) {
if (NON_CONTENT_TAGS.has(name)) {
suppressDepth += 1;
}
if (name === "noscript") noscriptDepth += 1;
if (noscriptDepth > 0) return;
switch (name) {
case "title":
// Ignore inside <svg> — only the document title counts.
if (!titleDone && suppressDepth === 0) {
titleDepth += 1;
if (title === null) title = "";
}
break;
case "head":
headDepth += 1;
break;
case "body":
bodyDepth += 1;
sawBody = true;
break;
case "meta":
handleMetaTag(attribs);
break;
case "link":
handleLinkTag(attribs);
break;
case "img":
images.push({
src: attribs["src"] ?? null,
alt: "alt" in attribs ? attribs["alt"] : null,
});
break;
case "script":
if (attribs["type"] === "application/ld+json") {
hasStructuredData = true;
}
break;
case "a": {
// HTML forbids nested <a>; browsers implicitly close the open
// one, and the tokenizer has no tree correction, so mirror that.
closeAnchor();
const href = attribs["href"];
if (href && !SKIPPED_LINK_PROTOCOLS.test(href)) {
openAnchor = {
href,
rel: attribs["rel"]?.toLowerCase() ?? "",
text: [],
};
}
break;
}
}
const headingLevel = HEADING_LEVELS[name];
if (headingLevel !== undefined) {
headingOrder.push(headingLevel);
if (headingLevel === 1 && openH1 === null) openH1 = [];
}
},
ontext(text) {
if (suppressDepth > 0) return;
if (titleDepth > 0) {
if (title !== null) title += text;
return;
}
if (openH1) openH1.push(text);
if (openAnchor) openAnchor.text.push(text);
if (bodyDepth > 0) {
bodyParts.push(text);
} else if (headDepth === 0) {
fallbackParts.push(text);
}
},
onclosetag(name) {
if (NON_CONTENT_TAGS.has(name) && suppressDepth > 0) {
suppressDepth -= 1;
}
if (name === "noscript" && noscriptDepth > 0) {
noscriptDepth -= 1;
return;
}
if (noscriptDepth > 0) return;
if (name === "title" && titleDepth > 0) {
titleDepth -= 1;
if (titleDepth === 0) titleDone = true;
}
if (name === "head" && headDepth > 0) headDepth -= 1;
if (name === "body" && bodyDepth > 0) bodyDepth -= 1;
if (name === "a") closeAnchor();
if (name === "h1" && openH1) {
h1s.push(openH1.join("").trim());
openH1 = null;
}
},
},
// Defaults (non-XML mode): lowercased tag/attribute names, decoded
// entities — matching what the DOM-based implementation saw.
);
parser.write(html);
parser.end();
const rawText = (sawBody ? bodyParts : fallbackParts).join("");
const bodyText = rawText.replace(/\s+/g, " ").trim();
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
return {
url: pageUrl,
statusCode,
redirectUrl,
responseTimeMs,
title: (title ?? "").trim(),
metaDescription: metaDescription ?? "",
canonical,
robotsMeta,
ogTitle,
ogDescription,
ogImage,
h1s,
headingOrder,
wordCount,
bodyText,
images,
links: Array.from(linksByTarget.values()),
hasStructuredData,
hreflangTags,
};
}