250 lines
7.6 KiB
TypeScript
250 lines
7.6 KiB
TypeScript
/**
|
|
* HTML page analyzer using htmlparser2's streaming tokenizer.
|
|
*
|
|
* Extracts SEO-relevant data from a page's HTML: title, meta description,
|
|
* headings, images, links, canonical, OG tags, structured data, robots meta,
|
|
* word count, hreflang.
|
|
*
|
|
* Deliberately NOT a DOM parser: the previous cheerio implementation built a
|
|
* full DOM (~5-10x the HTML's size) per page, and with 25 concurrent parses
|
|
* on a 128MB isolate that was the audit engine's dominant OOM cause. The
|
|
* tokenizer keeps only the accumulated text and extracted fields in memory.
|
|
*/
|
|
import { Parser } from "htmlparser2";
|
|
import { normalizeUrl, isSameOrigin } from "./url-utils";
|
|
import type { PageAnalysis, PageLink } from "./types";
|
|
|
|
const SKIPPED_LINK_PROTOCOLS = /^(javascript:|mailto:|tel:|#)/;
|
|
/** Subtrees whose text is not visible content. */
|
|
const NON_CONTENT_TAGS = new Set(["script", "style", "noscript", "svg"]);
|
|
const HEADING_LEVELS: Record<string, number> = {
|
|
h1: 1,
|
|
h2: 2,
|
|
h3: 3,
|
|
h4: 4,
|
|
h5: 5,
|
|
h6: 6,
|
|
};
|
|
const MAX_ANCHOR_CHARS = 200;
|
|
|
|
interface OpenAnchor {
|
|
href: string;
|
|
rel: string;
|
|
text: string[];
|
|
}
|
|
|
|
/**
|
|
* Analyze an HTML string and extract all SEO-relevant data.
|
|
*/
|
|
export function analyzeHtml(
|
|
html: string,
|
|
pageUrl: string,
|
|
statusCode: number,
|
|
responseTimeMs: number,
|
|
redirectUrl: string | null = null,
|
|
): PageAnalysis {
|
|
let title: string | null = null;
|
|
let titleDepth = 0;
|
|
let titleDone = false;
|
|
// parse5 (the old DOM path) treats <noscript> content as raw text when
|
|
// scripting is enabled; skip element extraction inside it to match.
|
|
let noscriptDepth = 0;
|
|
let metaDescription: string | null = null;
|
|
let canonical: string | null = null;
|
|
let robotsMeta: string | null = null;
|
|
let ogTitle: string | null = null;
|
|
let ogDescription: string | null = null;
|
|
let ogImage: string | null = null;
|
|
let hasStructuredData = false;
|
|
const hreflangTags: string[] = [];
|
|
|
|
const h1s: string[] = [];
|
|
const headingOrder: number[] = [];
|
|
let openH1: string[] | null = null;
|
|
|
|
const images: Array<{ src: string | null; alt: string | null }> = [];
|
|
const linksByTarget = new Map<string, PageLink>();
|
|
let openAnchor: OpenAnchor | null = null;
|
|
|
|
// Visible text: prefer text inside an explicit <body>; when the document
|
|
// never opens one (fragments), fall back to all non-head text. Both
|
|
// exclude NON_CONTENT_TAGS subtrees.
|
|
let suppressDepth = 0;
|
|
let bodyDepth = 0;
|
|
let headDepth = 0;
|
|
let sawBody = false;
|
|
const bodyParts: string[] = [];
|
|
const fallbackParts: string[] = [];
|
|
|
|
const handleMetaTag = (attribs: Record<string, string>) => {
|
|
const content = attribs["content"];
|
|
if (attribs["name"] === "description") {
|
|
metaDescription ??= content?.trim() ?? "";
|
|
} else if (attribs["name"] === "robots") {
|
|
robotsMeta ??= content ?? null;
|
|
} else if (attribs["property"] === "og:title") {
|
|
ogTitle ??= content ?? null;
|
|
} else if (attribs["property"] === "og:description") {
|
|
ogDescription ??= content ?? null;
|
|
} else if (attribs["property"] === "og:image") {
|
|
ogImage ??= content ?? null;
|
|
}
|
|
};
|
|
|
|
const handleLinkTag = (attribs: Record<string, string>) => {
|
|
if (attribs["rel"] === "canonical") {
|
|
canonical ??= attribs["href"] ?? null;
|
|
} else if (attribs["rel"] === "alternate" && attribs["hreflang"]) {
|
|
hreflangTags.push(attribs["hreflang"]);
|
|
}
|
|
};
|
|
|
|
const closeAnchor = () => {
|
|
if (!openAnchor) return;
|
|
const { href, rel, text } = openAnchor;
|
|
openAnchor = null;
|
|
const resolved = normalizeUrl(href, pageUrl);
|
|
if (!resolved || linksByTarget.has(resolved)) return;
|
|
const anchor = text
|
|
.join("")
|
|
.replace(/\s+/g, " ")
|
|
.trim()
|
|
.slice(0, MAX_ANCHOR_CHARS);
|
|
linksByTarget.set(resolved, {
|
|
targetUrl: resolved,
|
|
anchor: anchor || null,
|
|
isInternal: isSameOrigin(resolved, pageUrl),
|
|
isNofollow: rel.split(/\s+/).includes("nofollow"),
|
|
});
|
|
};
|
|
|
|
const parser = new Parser(
|
|
{
|
|
onopentag(name, attribs) {
|
|
if (NON_CONTENT_TAGS.has(name)) {
|
|
suppressDepth += 1;
|
|
}
|
|
if (name === "noscript") noscriptDepth += 1;
|
|
if (noscriptDepth > 0) return;
|
|
switch (name) {
|
|
case "title":
|
|
// Ignore <title> inside <svg> — only the document title counts.
|
|
if (!titleDone && suppressDepth === 0) {
|
|
titleDepth += 1;
|
|
if (title === null) title = "";
|
|
}
|
|
break;
|
|
case "head":
|
|
headDepth += 1;
|
|
break;
|
|
case "body":
|
|
bodyDepth += 1;
|
|
sawBody = true;
|
|
break;
|
|
case "meta":
|
|
handleMetaTag(attribs);
|
|
break;
|
|
case "link":
|
|
handleLinkTag(attribs);
|
|
break;
|
|
case "img":
|
|
images.push({
|
|
src: attribs["src"] ?? null,
|
|
alt: "alt" in attribs ? attribs["alt"] : null,
|
|
});
|
|
break;
|
|
case "script":
|
|
if (attribs["type"] === "application/ld+json") {
|
|
hasStructuredData = true;
|
|
}
|
|
break;
|
|
case "a": {
|
|
// HTML forbids nested <a>; browsers implicitly close the open
|
|
// one, and the tokenizer has no tree correction, so mirror that.
|
|
closeAnchor();
|
|
const href = attribs["href"];
|
|
if (href && !SKIPPED_LINK_PROTOCOLS.test(href)) {
|
|
openAnchor = {
|
|
href,
|
|
rel: attribs["rel"]?.toLowerCase() ?? "",
|
|
text: [],
|
|
};
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
const headingLevel = HEADING_LEVELS[name];
|
|
if (headingLevel !== undefined) {
|
|
headingOrder.push(headingLevel);
|
|
if (headingLevel === 1 && openH1 === null) openH1 = [];
|
|
}
|
|
},
|
|
ontext(text) {
|
|
if (suppressDepth > 0) return;
|
|
if (titleDepth > 0) {
|
|
if (title !== null) title += text;
|
|
return;
|
|
}
|
|
if (openH1) openH1.push(text);
|
|
if (openAnchor) openAnchor.text.push(text);
|
|
if (bodyDepth > 0) {
|
|
bodyParts.push(text);
|
|
} else if (headDepth === 0) {
|
|
fallbackParts.push(text);
|
|
}
|
|
},
|
|
onclosetag(name) {
|
|
if (NON_CONTENT_TAGS.has(name) && suppressDepth > 0) {
|
|
suppressDepth -= 1;
|
|
}
|
|
if (name === "noscript" && noscriptDepth > 0) {
|
|
noscriptDepth -= 1;
|
|
return;
|
|
}
|
|
if (noscriptDepth > 0) return;
|
|
if (name === "title" && titleDepth > 0) {
|
|
titleDepth -= 1;
|
|
if (titleDepth === 0) titleDone = true;
|
|
}
|
|
if (name === "head" && headDepth > 0) headDepth -= 1;
|
|
if (name === "body" && bodyDepth > 0) bodyDepth -= 1;
|
|
if (name === "a") closeAnchor();
|
|
if (name === "h1" && openH1) {
|
|
h1s.push(openH1.join("").trim());
|
|
openH1 = null;
|
|
}
|
|
},
|
|
},
|
|
// Defaults (non-XML mode): lowercased tag/attribute names, decoded
|
|
// entities — matching what the DOM-based implementation saw.
|
|
);
|
|
parser.write(html);
|
|
parser.end();
|
|
|
|
const rawText = (sawBody ? bodyParts : fallbackParts).join("");
|
|
const bodyText = rawText.replace(/\s+/g, " ").trim();
|
|
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
|
|
|
return {
|
|
url: pageUrl,
|
|
statusCode,
|
|
redirectUrl,
|
|
responseTimeMs,
|
|
title: (title ?? "").trim(),
|
|
metaDescription: metaDescription ?? "",
|
|
canonical,
|
|
robotsMeta,
|
|
ogTitle,
|
|
ogDescription,
|
|
ogImage,
|
|
h1s,
|
|
headingOrder,
|
|
wordCount,
|
|
bodyText,
|
|
images,
|
|
links: Array.from(linksByTarget.values()),
|
|
hasStructuredData,
|
|
hreflangTags,
|
|
};
|
|
}
|