21 lines
599 B
JavaScript
21 lines
599 B
JavaScript
import Sitemapper from "sitemapper";
|
|
import normalizeUrl from "normalize-url";
|
|
|
|
/** Return an array of normalized URLs from <origin>/sitemap.xml (or [] if none). */
|
|
export async function getSitemapUrls(startUrl) {
|
|
const origin = new URL(startUrl).origin;
|
|
const sitemapUrl = `${origin}/sitemap.xml`;
|
|
const sm = new Sitemapper({ url: sitemapUrl, timeout: 15000 });
|
|
|
|
try {
|
|
const { sites } = await sm.fetch();
|
|
const out = [];
|
|
for (const u of sites || []) {
|
|
try { out.push(normalizeUrl(u, { stripHash: true })); } catch {}
|
|
}
|
|
return out;
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|