first commit
This commit is contained in:
Alaguraj0361
commit
6c345df1c2
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
# .gitignore
|
||||
node_modules
|
||||
.env
|
||||
reports
|
||||
11
config/db.js
Normal file
11
config/db.js
Normal file
@ -0,0 +1,11 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
export async function connectDB() {
|
||||
try {
|
||||
await mongoose.connect(process.env.MONGODB_URI, { dbName: 'crawlerX' });
|
||||
console.log('✅ MongoDB connected');
|
||||
} catch (err) {
|
||||
console.error('❌ MongoDB connection error:', err);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
150
controllers/auth.controller.js
Normal file
150
controllers/auth.controller.js
Normal file
@ -0,0 +1,150 @@
|
||||
import bcrypt from "bcrypt";
|
||||
import jwt from "jsonwebtoken";
|
||||
import User from "../models/user.model.js";
|
||||
import { sendResetPasswordMail, sendSignupMail, } from "../utils/mailer.js";
|
||||
import crypto from "crypto";
|
||||
|
||||
|
||||
export async function signup(req, res) {
|
||||
try {
|
||||
const { email, password } = req.body;
|
||||
if (!email || !password)
|
||||
return res.status(400).json({ error: "Email and password required" });
|
||||
|
||||
const exists = await User.findOne({ email });
|
||||
if (exists) return res.status(400).json({ error: "User already exists" });
|
||||
|
||||
const passwordHash = await bcrypt.hash(password, 10);
|
||||
const user = await User.create({ email, passwordHash });
|
||||
|
||||
// ✅ send confirmation email (non-blocking)
|
||||
sendSignupMail(email)
|
||||
.then(() => console.log("Signup email sent to", email))
|
||||
.catch(err => console.error("Email send failed:", err));
|
||||
|
||||
res.status(201).json({ message: "Signup success, email sent", id: user._id });
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
res.status(500).json({ error: "Signup failed" });
|
||||
}
|
||||
}
|
||||
|
||||
export async function login(req, res) {
|
||||
try {
|
||||
const { email, password } = req.body;
|
||||
const user = await User.findOne({ email });
|
||||
if (!user) return res.status(401).json({ error: "Invalid credentials" });
|
||||
|
||||
const match = await bcrypt.compare(password, user.passwordHash);
|
||||
if (!match) return res.status(401).json({ error: "Invalid credentials" });
|
||||
|
||||
const token = jwt.sign(
|
||||
{ id: user._id, email: user.email },
|
||||
process.env.JWT_SECRET,
|
||||
{ expiresIn: "1h" }
|
||||
);
|
||||
|
||||
res.json({ message: "Login success", token });
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
res.status(500).json({ error: "Login failed" });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/auth/change-password
|
||||
* Body: { currentPassword, newPassword }
|
||||
* Header: Authorization: Bearer <token>
|
||||
*/
|
||||
export async function changePassword(req, res) {
|
||||
try {
|
||||
const { currentPassword, newPassword } = req.body;
|
||||
|
||||
// if using FormData, fields come from req.body AFTER a multipart parser
|
||||
if (!currentPassword || !newPassword) {
|
||||
return res.status(400).json({ error: "Current password and new password are required" });
|
||||
}
|
||||
|
||||
const user = await User.findById(req.user.id);
|
||||
if (!user) return res.status(404).json({ error: "User not found" });
|
||||
|
||||
const isMatch = await bcrypt.compare(currentPassword, user.passwordHash);
|
||||
if (!isMatch)
|
||||
return res.status(401).json({ error: "Current password is incorrect" });
|
||||
|
||||
user.passwordHash = await bcrypt.hash(newPassword, 10);
|
||||
await user.save();
|
||||
|
||||
res.json({ message: "Password updated successfully" });
|
||||
} catch (err) {
|
||||
console.error("changePassword error:", err); // ✅ show actual error
|
||||
res.status(500).json({ error: "Failed to change password" });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/auth/forgot-password
|
||||
* Body: { email }
|
||||
*/
|
||||
export async function forgotPassword(req, res) {
|
||||
try {
|
||||
const { email } = req.body;
|
||||
if (!email) return res.status(400).json({ error: "Email is required" });
|
||||
|
||||
const user = await User.findOne({ email });
|
||||
|
||||
if (!user)
|
||||
return res.json({
|
||||
message: "If the email is registered, a reset link has been sent.",
|
||||
verificationCode: null, // user not found
|
||||
});
|
||||
|
||||
// Generate 4-digit numeric verification code
|
||||
const verificationCode = Math.floor(1000 + Math.random() * 9000).toString();
|
||||
|
||||
// Save code and expiry in DB
|
||||
user.resetPasswordToken = verificationCode;
|
||||
user.resetPasswordExpires = Date.now() + 60 * 60 * 1000; // 1 hour
|
||||
await user.save();
|
||||
|
||||
// Send code via email
|
||||
await sendResetPasswordMail(email, verificationCode);
|
||||
|
||||
// ✅ Return verification code in response
|
||||
res.json({
|
||||
message: "If the email is registered, a reset link has been sent.",
|
||||
verificationCode, // This is the 4-digit code
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("forgotPassword error:", err);
|
||||
res.status(500).json({ error: "Failed to send reset link" });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/auth/reset-password
|
||||
* Body: { token, newPassword }
|
||||
*/
|
||||
export async function resetPassword(req, res) {
|
||||
try {
|
||||
const { token, newPassword } = req.body;
|
||||
if (!token || !newPassword)
|
||||
return res.status(400).json({ error: "Token and new password are required" });
|
||||
|
||||
const user = await User.findOne({
|
||||
resetPasswordToken: token,
|
||||
resetPasswordExpires: { $gt: Date.now() },
|
||||
});
|
||||
if (!user) return res.status(400).json({ error: "Invalid or expired token" });
|
||||
|
||||
user.passwordHash = await bcrypt.hash(newPassword, 10);
|
||||
user.resetPasswordToken = undefined;
|
||||
user.resetPasswordExpires = undefined;
|
||||
await user.save();
|
||||
|
||||
res.json({ message: "Password has been reset successfully" });
|
||||
} catch (err) {
|
||||
console.error("resetPassword error:", err);
|
||||
res.status(500).json({ error: "Failed to reset password" });
|
||||
}
|
||||
}
|
||||
111
controllers/blog.controller.js
Normal file
111
controllers/blog.controller.js
Normal file
@ -0,0 +1,111 @@
|
||||
import Blog from '../models/blog.model.js';
|
||||
import Category from '../models/category.model.js';
|
||||
import slugify from 'slugify';
|
||||
|
||||
// ✅ Create Blog for particular project
|
||||
export const createBlog = async (req, res) => {
|
||||
try {
|
||||
const { projectId, title, description, categoryId, tags } = req.body;
|
||||
if (!projectId) return res.status(400).json({ message: 'projectId is required' });
|
||||
|
||||
const slug = slugify(title, { lower: true, strict: true });
|
||||
|
||||
const blog = await Blog.create({
|
||||
projectId,
|
||||
title,
|
||||
description,
|
||||
slug,
|
||||
category: categoryId,
|
||||
tags,
|
||||
imageUrl: req.files?.imageUrl ? `/uploads/${req.files.imageUrl[0].filename}` : '',
|
||||
bigImageUrl: req.files?.bigImageUrl ? `/uploads/${req.files.bigImageUrl[0].filename}` : ''
|
||||
});
|
||||
|
||||
res.status(201).json(blog);
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
|
||||
// ✅ Get All Blogs for a particular project
|
||||
export const getAllBlogs = async (req, res) => {
|
||||
try {
|
||||
const { page = 1, limit = 10, search = '', category, projectId } = req.query;
|
||||
if (!projectId) return res.status(400).json({ message: 'projectId is required' });
|
||||
|
||||
const query = {
|
||||
projectId,
|
||||
title: { $regex: search, $options: 'i' }
|
||||
};
|
||||
|
||||
if (category) {
|
||||
const cat = await Category.findOne({ slug: category });
|
||||
if (cat) query.category = cat._id;
|
||||
}
|
||||
|
||||
const blogs = await Blog.find(query)
|
||||
.populate('category', 'name slug')
|
||||
.sort({ createdAt: -1 })
|
||||
.skip((page - 1) * limit)
|
||||
.limit(parseInt(limit));
|
||||
|
||||
const total = await Blog.countDocuments(query);
|
||||
|
||||
res.json({ total, page: parseInt(page), blogs });
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
|
||||
// ✅ Get Single Blog by Slug + projectId
|
||||
export const getBlogBySlug = async (req, res) => {
|
||||
try {
|
||||
const { projectId } = req.query; // 👈 query param மூலம்
|
||||
if (!projectId) return res.status(400).json({ message: 'projectId is required' });
|
||||
|
||||
const blog = await Blog.findOne({ slug: req.params.slug, projectId })
|
||||
.populate('category', 'name slug');
|
||||
|
||||
if (!blog) return res.status(404).json({ message: 'Blog not found' });
|
||||
res.json(blog);
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
|
||||
// ✅ Add Comment to Blog (projectId check optional – id already unique)
|
||||
export const addComment = async (req, res) => {
|
||||
try {
|
||||
const { text, name } = req.body;
|
||||
const blog = await Blog.findById(req.params.id);
|
||||
if (!blog) return res.status(404).json({ message: 'Blog not found' });
|
||||
|
||||
blog.comments.push({
|
||||
user: req.user?._id,
|
||||
name: name || 'Anonymous',
|
||||
text
|
||||
});
|
||||
|
||||
await blog.save();
|
||||
res.json(blog.comments);
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
|
||||
// ✅ Like / Unlike
|
||||
export const likeBlog = async (req, res) => {
|
||||
try {
|
||||
const blog = await Blog.findById(req.params.id);
|
||||
if (!blog) return res.status(404).json({ message: 'Blog not found' });
|
||||
|
||||
const userId = req.user._id;
|
||||
if (blog.likes.includes(userId)) blog.likes.pull(userId);
|
||||
else blog.likes.push(userId);
|
||||
|
||||
await blog.save();
|
||||
res.json({ likesCount: blog.likes.length });
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
46
controllers/category.controller.js
Normal file
46
controllers/category.controller.js
Normal file
@ -0,0 +1,46 @@
|
||||
import Category from '../models/category.model.js';
|
||||
import slugify from "slugify";
|
||||
|
||||
// Create a new category (Admin only)
|
||||
export const createCategory = async (req, res) => {
|
||||
try {
|
||||
const { name, projectId } = req.body;
|
||||
if (!projectId) return res.status(400).json({ message: "projectId is required" });
|
||||
|
||||
const slug = slugify(name, { lower: true, strict: true });
|
||||
|
||||
const category = await Category.create({
|
||||
name,
|
||||
slug,
|
||||
projectId
|
||||
});
|
||||
|
||||
res.status(201).json(category);
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
|
||||
// Get all categories for a project
|
||||
export const getCategories = async (req, res) => {
|
||||
try {
|
||||
const { projectId } = req.query;
|
||||
if (!projectId) return res.status(400).json({ message: "projectId is required" });
|
||||
|
||||
const categories = await Category.find({ projectId }).sort({ name: 1 });
|
||||
res.json(categories);
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
|
||||
// Delete a category (Admin only)
|
||||
export const deleteCategory = async (req, res) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
await Category.findByIdAndDelete(id);
|
||||
res.json({ message: "Category deleted" });
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
54
controllers/comment.controller.js
Normal file
54
controllers/comment.controller.js
Normal file
@ -0,0 +1,54 @@
|
||||
import Blog from "../models/blog.model.js";
|
||||
|
||||
// Add comment to a blog
|
||||
export const addComment = async (req, res) => {
|
||||
try {
|
||||
const { blogId } = req.params;
|
||||
const { text, name } = req.body;
|
||||
|
||||
const blog = await Blog.findById(blogId);
|
||||
if (!blog) return res.status(404).json({ message: "Blog not found" });
|
||||
|
||||
blog.comments.push({
|
||||
user: req.user?._id || null,
|
||||
name: name || "Anonymous",
|
||||
text
|
||||
});
|
||||
|
||||
await blog.save();
|
||||
res.status(201).json(blog.comments);
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
|
||||
// Get all comments for a blog
|
||||
export const getComments = async (req, res) => {
|
||||
try {
|
||||
const { blogId } = req.params;
|
||||
|
||||
const blog = await Blog.findById(blogId);
|
||||
if (!blog) return res.status(404).json({ message: "Blog not found" });
|
||||
|
||||
res.json(blog.comments.sort((a,b) => b.createdAt - a.createdAt));
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
|
||||
// Delete a comment (Admin only)
|
||||
export const deleteComment = async (req, res) => {
|
||||
try {
|
||||
const { blogId, commentId } = req.params;
|
||||
|
||||
const blog = await Blog.findById(blogId);
|
||||
if (!blog) return res.status(404).json({ message: "Blog not found" });
|
||||
|
||||
blog.comments.id(commentId)?.remove();
|
||||
await blog.save();
|
||||
|
||||
res.json({ message: "Comment deleted" });
|
||||
} catch (err) {
|
||||
res.status(500).json({ message: err.message });
|
||||
}
|
||||
};
|
||||
115
controllers/crawl.controller.js
Normal file
115
controllers/crawl.controller.js
Normal file
@ -0,0 +1,115 @@
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { crawl } from "../crawler.js";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const ts = () =>
|
||||
new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-");
|
||||
|
||||
function attachJson(res, filename, obj) {
|
||||
const json = JSON.stringify(obj, null, 2);
|
||||
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||
res.setHeader("Content-Disposition", `attachment; filename="${filename}"`);
|
||||
return res.send(json);
|
||||
}
|
||||
function isAbs(p) {
|
||||
try {
|
||||
return path.isAbsolute(p);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function crawlHandler(req, res) {
|
||||
try {
|
||||
const { url, max, stream, download, nostore } = req.query;
|
||||
if (!url) return res.status(400).json({ error: "Missing url param" });
|
||||
|
||||
const target = new URL(String(url));
|
||||
const limit = Math.min(Math.max(parseInt(max ?? "50", 10), 1), 500);
|
||||
const wantsStream =
|
||||
String(stream) === "1" ||
|
||||
(req.get("accept") || "").includes("text/event-stream");
|
||||
|
||||
/* ---------- SSE mode ---------- */
|
||||
if (wantsStream) {
|
||||
if (String(download) === "1") {
|
||||
return res.status(400).json({ error: "download not supported with stream=1" });
|
||||
}
|
||||
|
||||
res.setHeader("Content-Type", "text/event-stream");
|
||||
res.setHeader("Cache-Control", "no-cache, no-transform");
|
||||
res.setHeader("Connection", "keep-alive");
|
||||
res.flushHeaders?.();
|
||||
|
||||
const send = (obj, evt) => {
|
||||
if (evt) res.write(`event: ${evt}\n`);
|
||||
res.write(`data: ${JSON.stringify(obj)}\n\n`);
|
||||
};
|
||||
|
||||
const heartbeat = setInterval(() => res.write(":\n\n"), 15000);
|
||||
let finished = false;
|
||||
|
||||
req.on("close", () => {
|
||||
clearInterval(heartbeat);
|
||||
if (!finished) console.warn("SSE client disconnected.");
|
||||
});
|
||||
|
||||
const onProgress = (tick) => send(tick, "tick");
|
||||
send({ ok: true, message: "Crawl started", url: target.toString(), limit }, "started");
|
||||
|
||||
const result = await crawl(target.toString(), limit, onProgress, {
|
||||
persistReports: false,
|
||||
collectPages: true,
|
||||
});
|
||||
|
||||
finished = true;
|
||||
clearInterval(heartbeat);
|
||||
send({ ok: true, done: true, result }, "done");
|
||||
return res.end();
|
||||
}
|
||||
|
||||
/* ---------- Non-streaming mode ---------- */
|
||||
const preferMemory = String(nostore) === "1" || String(download) === "1";
|
||||
const result = await crawl(
|
||||
target.toString(),
|
||||
limit,
|
||||
undefined,
|
||||
preferMemory
|
||||
? { persistReports: false, collectPages: true }
|
||||
: { persistReports: true, collectPages: true }
|
||||
);
|
||||
|
||||
if (String(download) === "1") {
|
||||
const filename = `crawl-${ts()}.json`;
|
||||
|
||||
if (Array.isArray(result?.results)) {
|
||||
return attachJson(res, filename, result.results);
|
||||
}
|
||||
|
||||
const jsonPath = result?.files?.json;
|
||||
if (jsonPath) {
|
||||
const abs = isAbs(jsonPath) ? jsonPath : path.join(__dirname, jsonPath);
|
||||
if (fs.existsSync(abs)) {
|
||||
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||
res.setHeader("Content-Disposition", `attachment; filename="${filename}"`);
|
||||
return fs.createReadStream(abs).pipe(res);
|
||||
}
|
||||
}
|
||||
return attachJson(res, filename, result ?? {});
|
||||
}
|
||||
|
||||
return res.json({
|
||||
ok: true,
|
||||
message: "Crawl completed",
|
||||
url: target.toString(),
|
||||
limit,
|
||||
...result,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("Crawl error:", err);
|
||||
res.status(500).json({ error: "Crawl failed", details: String(err?.message ?? err) });
|
||||
}
|
||||
}
|
||||
113
controllers/lighthouseController.js
Normal file
113
controllers/lighthouseController.js
Normal file
@ -0,0 +1,113 @@
|
||||
import lighthouse from 'lighthouse';
|
||||
import { launch } from 'chrome-launcher';
|
||||
import PageSpeedTest from '../models/pageSpeedTest.model.js';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
const reportsDir = path.join(process.cwd(), 'public', 'lighthouse-treemap');
|
||||
// Ensure folder exists
|
||||
if (!fs.existsSync(reportsDir)) fs.mkdirSync(reportsDir, { recursive: true });
|
||||
|
||||
const launchChromeAndRunLighthouse = async (url, device = 'mobile') => {
|
||||
const chrome = await launch({ chromeFlags: ['--headless'] });
|
||||
|
||||
const options = {
|
||||
port: chrome.port,
|
||||
emulatedFormFactor: device,
|
||||
throttlingMethod: device === 'mobile' ? 'simulate' : 'devtools',
|
||||
output: 'json', // JSON for metrics
|
||||
};
|
||||
|
||||
const runnerResult = await lighthouse(url, options);
|
||||
const lhr = runnerResult.lhr;
|
||||
|
||||
// Create HTML treemap report (only once, for mobile)
|
||||
let treemapFile = null;
|
||||
if (device === 'mobile') {
|
||||
const fileName = `treemap-${Date.now()}.html`;
|
||||
treemapFile = `/lighthouse-treemap/${fileName}`;
|
||||
|
||||
// Generate HTML report
|
||||
const htmlReport = await lighthouse(url, {
|
||||
port: chrome.port,
|
||||
emulatedFormFactor: device,
|
||||
throttlingMethod: 'simulate',
|
||||
output: 'html',
|
||||
});
|
||||
|
||||
fs.writeFileSync(path.join(reportsDir, fileName), htmlReport.report);
|
||||
}
|
||||
|
||||
await chrome.kill();
|
||||
|
||||
// Structured result
|
||||
const result = {
|
||||
url,
|
||||
device,
|
||||
scores: {
|
||||
performance: Math.round(lhr.categories.performance?.score * 100),
|
||||
accessibility: Math.round(lhr.categories.accessibility?.score * 100),
|
||||
bestPractices: Math.round(lhr.categories['best-practices']?.score * 100),
|
||||
seo: Math.round(lhr.categories.seo?.score * 100),
|
||||
pwa: lhr.categories.pwa?.score ? Math.round(lhr.categories.pwa.score * 100) : null,
|
||||
},
|
||||
metrics: {
|
||||
firstContentfulPaint: lhr.audits['first-contentful-paint']?.displayValue || null,
|
||||
largestContentfulPaint: lhr.audits['largest-contentful-paint']?.displayValue || null,
|
||||
totalBlockingTime: lhr.audits['total-blocking-time']?.displayValue || null,
|
||||
timeToInteractive: lhr.audits['interactive']?.displayValue || null,
|
||||
speedIndex: lhr.audits['speed-index']?.displayValue || null,
|
||||
cumulativeLayoutShift: lhr.audits['cumulative-layout-shift']?.displayValue || null,
|
||||
},
|
||||
opportunities: Object.values(lhr.audits)
|
||||
.filter(a => a.details?.type === 'opportunity')
|
||||
.map(a => ({
|
||||
title: a.title,
|
||||
description: a.description,
|
||||
estimatedSavings: a.details?.overallSavingsMs
|
||||
? `${Math.round(a.details.overallSavingsMs)} ms`
|
||||
: null,
|
||||
})),
|
||||
diagnostics: {
|
||||
usesHTTPS: lhr.audits['is-on-https']?.score === 1,
|
||||
usesEfficientCachePolicy: lhr.audits['uses-long-cache-ttl']?.score === 1,
|
||||
imageCompression: lhr.audits['uses-optimized-images']?.score === 1,
|
||||
},
|
||||
failedAudits: Object.values(lhr.audits)
|
||||
.filter(a => a.score !== null && a.score !== 1 && a.scoreDisplayMode !== 'notApplicable')
|
||||
.map(a => ({ title: a.title, description: a.description })),
|
||||
passedAudits: Object.values(lhr.audits)
|
||||
.filter(a => a.score === 1 && a.scoreDisplayMode !== 'notApplicable' && !a.details?.type)
|
||||
.map(a => a.title),
|
||||
notApplicableAudits: Object.values(lhr.audits)
|
||||
.filter(a => a.scoreDisplayMode === 'notApplicable')
|
||||
.map(a => a.title),
|
||||
screenshot: lhr.audits['final-screenshot']?.details?.data || null,
|
||||
createdAt: new Date(),
|
||||
treemapPath: treemapFile,
|
||||
};
|
||||
|
||||
const report = await PageSpeedTest.create(result);
|
||||
return { report };
|
||||
};
|
||||
|
||||
export const runAudit = async (req, res, next) => {
|
||||
try {
|
||||
const { url } = req.body;
|
||||
if (!url) return res.status(400).json({ message: 'URL is required' });
|
||||
|
||||
const mobileResult = await launchChromeAndRunLighthouse(url, 'mobile');
|
||||
const desktopResult = await launchChromeAndRunLighthouse(url, 'desktop');
|
||||
|
||||
res.status(200).json({
|
||||
message: 'Audit completed successfully',
|
||||
results: {
|
||||
mobile: mobileResult.report,
|
||||
desktop: desktopResult.report,
|
||||
treemap: mobileResult.report.treemapPath, // HTML report
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
};
|
||||
39
controllers/maisondetreats/cakeOrder.controller.js
Normal file
39
controllers/maisondetreats/cakeOrder.controller.js
Normal file
@ -0,0 +1,39 @@
|
||||
import { CakeOrder } from "../../models/maisondetreats/cakeOrder.model.js";
|
||||
import { sendCakeOrderMail } from "../../utils/mailer.js";
|
||||
|
||||
export const createCakeOrder = async (req, res) => {
|
||||
try {
|
||||
const { order, email } = req.body;
|
||||
|
||||
if (!order || typeof order !== "object") {
|
||||
return res.status(400).json({ message: "Order data is required" });
|
||||
}
|
||||
|
||||
const newOrder = await CakeOrder.create({ order, email });
|
||||
|
||||
// ✅ send confirmation email (non-blocking)
|
||||
if (email) {
|
||||
sendCakeOrderMail(email, order)
|
||||
.then(() => console.log("Cake order email sent to", email))
|
||||
.catch((err) => console.error("Email send failed:", err));
|
||||
}
|
||||
|
||||
res.status(201).json({
|
||||
message: "Cake order created successfully",
|
||||
data: newOrder,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("Error creating cake order:", err);
|
||||
res.status(500).json({ message: "Server error", error: err.message });
|
||||
}
|
||||
};
|
||||
// GET /api/cake-orders → List all orders
|
||||
export const getAllCakeOrders = async (_req, res) => {
|
||||
try {
|
||||
const orders = await CakeOrder.find().sort({ createdAt: -1 });
|
||||
res.json({ data: orders });
|
||||
} catch (err) {
|
||||
console.error("Error fetching cake orders:", err);
|
||||
res.status(500).json({ message: "Server error", error: err.message });
|
||||
}
|
||||
};
|
||||
68
controllers/message.controller.js
Normal file
68
controllers/message.controller.js
Normal file
@ -0,0 +1,68 @@
|
||||
// message.controller.js
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import axios from "axios";
|
||||
import Message from "../models/message.model.js";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
export const sendMessage = async (req, res) => {
|
||||
try {
|
||||
const { project, name, email, message } = req.body;
|
||||
|
||||
if (!project) return res.status(400).json({ success: false, error: "Project is required" });
|
||||
if (!message) return res.status(400).json({ success: false, error: "Message is required" });
|
||||
|
||||
// Save message to MongoDB
|
||||
const newMessage = await Message.create({ project, name, email, message });
|
||||
|
||||
// Send WhatsApp Template Message
|
||||
const url = `https://graph.facebook.com/v22.0/774121419125441/messages`;
|
||||
const payload = {
|
||||
messaging_product: "whatsapp",
|
||||
to: 917871207631,
|
||||
type: "template",
|
||||
template: {
|
||||
name: "new_message_alert",
|
||||
language: { code: "en_US" },
|
||||
components: [
|
||||
{
|
||||
type: "body",
|
||||
parameters: [
|
||||
{ type: "text", text: project || "Project" },
|
||||
{ type: "text", text: name || "Guest" },
|
||||
{ type: "text", text: email || "N/A" },
|
||||
{ type: "text", text: message || "No message" },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const headers = {
|
||||
Authorization: `Bearer EAALKxEMPlp0BPkmoTAJlZAZAymtgqzcUuGVdZAZAKSZAw1csXR5Xy2DodBUC2zXckOYvQ2jOV4aFlZAeCo4IuJCyMb5aFt2UfNRQ1pDGk08QlbCjjCTMsZALipZCMNYyNVwN2pTDwUcYeNZByOrweVVdXD1ErZAbzjc04wmR8ilhQXink4it05BatwkZBf3xCLyy3k6R0tgx9JoymQTn83iZANBWDzvmX3vW5dx6Pud6xNEfqYNsjwZDZD`,
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
|
||||
const response = await axios.post(url, payload, { headers });
|
||||
console.log("✅ WhatsApp API Response:", response.data);
|
||||
|
||||
return res.status(201).json({ success: true, data: newMessage });
|
||||
} catch (err) {
|
||||
console.error("❌ WhatsApp API Error:", err.response?.data || err.message);
|
||||
return res.status(500).json({ success: false, error: "Server Error" });
|
||||
}
|
||||
};
|
||||
|
||||
// ✅ Add this function and export it
|
||||
export const getMessages = async (req, res) => {
|
||||
try {
|
||||
const { project } = req.query;
|
||||
if (!project) return res.status(400).json({ success: false, error: "Project is required" });
|
||||
|
||||
const messages = await Message.find({ project }).sort({ createdAt: -1 });
|
||||
return res.status(200).json({ success: true, data: messages });
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
return res.status(500).json({ success: false, error: "Server Error" });
|
||||
}
|
||||
};
|
||||
132
controllers/payment.controller.js
Normal file
132
controllers/payment.controller.js
Normal file
@ -0,0 +1,132 @@
|
||||
// controllers/payment.controller.js
|
||||
import Stripe from "stripe";
|
||||
import { Payment } from "../models/payment.model.js";
|
||||
|
||||
// ✅ Load Stripe Secret Key from .env
|
||||
const stripe = new Stripe("sk_test_51SB8SnIFk8fh986GkYaNPVSfZzh6gcuXhq3tOa5hyE4U4vYIqrHwyGRu2OE1N5TNW39tJmfFOyYfsh4HcZOjlsj100xIeM46zU", {
|
||||
apiVersion: "2022-11-15",
|
||||
});
|
||||
|
||||
/**
|
||||
* 🔹 Option 1: PaymentIntent API (client uses clientSecret)
|
||||
*/
|
||||
export async function createPaymentIntent(req, res) {
|
||||
try {
|
||||
const { amount } = req.body;
|
||||
if (!amount) return res.status(400).json({ error: "amount is required" });
|
||||
|
||||
const paymentIntent = await stripe.paymentIntents.create({
|
||||
amount: Math.round(amount * 100), // dollars → cents
|
||||
currency: "usd",
|
||||
automatic_payment_methods: { enabled: true },
|
||||
});
|
||||
|
||||
await Payment.create({
|
||||
amount: Math.round(amount * 100),
|
||||
stripePaymentIntentId: paymentIntent.id,
|
||||
status: "pending",
|
||||
});
|
||||
|
||||
res.json({ clientSecret: paymentIntent.client_secret });
|
||||
} catch (err) {
|
||||
console.error("❌ Error creating PaymentIntent:", err);
|
||||
res.status(500).json({ error: "Internal Server Error" });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 🔹 Option 2: Stripe Checkout Session (redirect flow)
|
||||
*/
|
||||
export async function createCheckoutSession(req, res) {
|
||||
try {
|
||||
const { email, amount, planId } = req.body;
|
||||
if (!email || !amount) {
|
||||
return res.status(400).json({ error: "email and amount are required" });
|
||||
}
|
||||
|
||||
const session = await stripe.checkout.sessions.create({
|
||||
payment_method_types: ["card"],
|
||||
mode: "payment",
|
||||
customer_email: email,
|
||||
line_items: [
|
||||
{
|
||||
price_data: {
|
||||
currency: "usd",
|
||||
product_data: { name: planId || "SEO Plan" },
|
||||
unit_amount: Math.round(amount * 100),
|
||||
},
|
||||
quantity: 1,
|
||||
},
|
||||
],
|
||||
success_url: "https://app.crawlerx.co/success",
|
||||
cancel_url: "https://app.crawlerx.co/cancel",
|
||||
});
|
||||
|
||||
// Save to DB using stripeSessionId instead of stripePaymentIntentId
|
||||
await Payment.create({
|
||||
email,
|
||||
amount: Math.round(amount * 100),
|
||||
stripeSessionId: session.id, // ✅ use session id
|
||||
status: "pending",
|
||||
});
|
||||
|
||||
res.json({ sessionId: session.id });
|
||||
} catch (err) {
|
||||
console.error("❌ Error creating checkout session:", err);
|
||||
res.status(500).json({ error: "Internal Server Error" });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 🔹 Stripe Webhook
|
||||
* Stripe requires `express.raw({ type: "application/json" })` in route
|
||||
*/
|
||||
export async function handleWebhook(req, res) {
|
||||
const sig = req.headers["stripe-signature"];
|
||||
let event;
|
||||
|
||||
try {
|
||||
event = stripe.webhooks.constructEvent(
|
||||
req.rawBody, // Must be raw body
|
||||
sig,
|
||||
process.env.STRIPE_WEBHOOK_SECRET
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("❌ Webhook signature verification failed:", err.message);
|
||||
return res.status(400).send(`Webhook Error: ${err.message}`);
|
||||
}
|
||||
|
||||
switch (event.type) {
|
||||
case "payment_intent.succeeded": {
|
||||
const paymentIntent = event.data.object;
|
||||
console.log("✅ PaymentIntent succeeded:", paymentIntent.id);
|
||||
|
||||
await Payment.findOneAndUpdate(
|
||||
{ stripePaymentIntentId: paymentIntent.id },
|
||||
{ status: "succeeded" }
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
case "checkout.session.completed": {
|
||||
const session = event.data.object;
|
||||
console.log("✅ Checkout session completed:", session.id);
|
||||
|
||||
// Update DB record created earlier
|
||||
await Payment.findOneAndUpdate(
|
||||
{ email: session.customer_email, status: "pending" },
|
||||
{
|
||||
stripePaymentIntentId: session.payment_intent,
|
||||
status: "succeeded",
|
||||
}
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
console.log(`Unhandled event type ${event.type}`);
|
||||
}
|
||||
|
||||
res.json({ received: true });
|
||||
}
|
||||
20
controllers/sitemap.controller.js
Normal file
20
controllers/sitemap.controller.js
Normal file
@ -0,0 +1,20 @@
|
||||
import { getSitemapUrls } from "../utils/sitemap.js";
|
||||
|
||||
export async function sitemapHandler(req, res) {
|
||||
try {
|
||||
const { u } = req.query;
|
||||
if (!u) return res.status(400).json({ error: "Missing ?u=https://site.com" });
|
||||
|
||||
const origin = new URL(String(u));
|
||||
const urls = await getSitemapUrls(origin.toString());
|
||||
res.json({
|
||||
ok: true,
|
||||
origin: origin.origin,
|
||||
count: urls.length,
|
||||
urls,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("sitemap error:", err);
|
||||
res.status(500).json({ error: "Failed to fetch sitemap", details: String(err?.message ?? err) });
|
||||
}
|
||||
}
|
||||
709
crawler copy.js
Normal file
709
crawler copy.js
Normal file
@ -0,0 +1,709 @@
|
||||
import got from "got";
|
||||
import * as cheerio from "cheerio";
|
||||
import normalizeUrl from "normalize-url";
|
||||
import { isInternal } from "./utils/urlHelpers.js";
|
||||
import { getSitemapUrls } from "./utils/sitemap.js";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { chromium } from "playwright";
|
||||
|
||||
// NEW libs
|
||||
import pixelWidth from "string-pixel-width";
|
||||
import * as readability from "text-readability";
|
||||
import stringSimilarity from "string-similarity";
|
||||
|
||||
/* ------------------------------ globals --------------------------------- */
|
||||
const visited = new Set();
|
||||
const queue = [];
|
||||
const results = [];
|
||||
|
||||
// Link provenance: every discovered edge (source -> target)
|
||||
const edges = []; // { from, raw_href, to, discovered_by }
|
||||
|
||||
// Quick referrer map for error report
|
||||
const referrers = new Map(); // url -> Array<{from, raw_href, discovered_by}>
|
||||
|
||||
const REAL_UA =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
||||
const REAL_HEADERS = {
|
||||
"user-agent": REAL_UA,
|
||||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"accept-language": "en-US,en;q=0.9",
|
||||
"upgrade-insecure-requests": "1",
|
||||
};
|
||||
|
||||
/* ------------------------------ utils ----------------------------------- */
|
||||
function csvEscape(v) {
|
||||
if (v === undefined || v === null) return "";
|
||||
const s = String(v);
|
||||
return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
|
||||
}
|
||||
function ensureDir(dir) {
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
function writePageReports(results) {
|
||||
ensureDir("reports");
|
||||
const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
|
||||
const base = path.join("reports", `crawl-${stamp}`);
|
||||
|
||||
fs.writeFileSync(`${base}.json`, JSON.stringify(results, null, 2), "utf8");
|
||||
console.log(`\n📝 Full JSON report saved: ${base}.json`);
|
||||
|
||||
// Columns (a Screaming-Frog-ish shape with our extras)
|
||||
const headers = [
|
||||
"url", "status", "status_text", "time_ms", "bytes", "content_type", "http_version",
|
||||
"title", "title_length", "title_pixel_width",
|
||||
"meta_description", "meta_description_length", "meta_description_pixel_width",
|
||||
"h1_1", "h1_1_length", "h1_1_pixel_width", "h1_2", "h1_2_length", "h1_2_pixel_width",
|
||||
"h2_1", "h2_2",
|
||||
"canonical", "robots_meta", "x_robots_tag", "noindex", "nofollow",
|
||||
"lang", "word_count", "flesch_reading_ease", "flesch_kincaid_grade",
|
||||
"gunning_fog", "coleman_liau", "ari", "smog",
|
||||
"schema_types", "inlinks", "outlinks", "render_mode",
|
||||
"last_modified", "set_cookie", "crawl_timestamp",
|
||||
"duplicate_title_exact", "nearest_title_similarity", "nearest_title_url",
|
||||
"duplicate_description_exact", "nearest_description_similarity", "nearest_description_url"
|
||||
];
|
||||
const lines = [headers.join(",")];
|
||||
for (const r of results) {
|
||||
lines.push([
|
||||
r.url,
|
||||
r.status,
|
||||
r.status_text ?? "",
|
||||
r.time_ms,
|
||||
r.bytes,
|
||||
r.content_type,
|
||||
r.http_version ?? "",
|
||||
r.title,
|
||||
r.title_length,
|
||||
r.title_pixel_width,
|
||||
r.meta_description,
|
||||
r.meta_description_length,
|
||||
r.meta_description_pixel_width,
|
||||
r.h1_1 ?? "",
|
||||
r.h1_1_length ?? 0,
|
||||
r.h1_1_pixel_width ?? "",
|
||||
r.h1_2 ?? "",
|
||||
r.h1_2_length ?? 0,
|
||||
r.h1_2_pixel_width ?? "",
|
||||
r.h2_1 ?? "",
|
||||
r.h2_2 ?? "",
|
||||
r.canonical,
|
||||
r.robots_meta,
|
||||
r.x_robots_tag ?? "",
|
||||
r.noindex,
|
||||
r.nofollow,
|
||||
r.lang ?? "",
|
||||
r.word_count ?? "",
|
||||
r.flesch_reading_ease ?? "",
|
||||
r.flesch_kincaid_grade ?? "",
|
||||
r.gunning_fog ?? "",
|
||||
r.coleman_liau ?? "",
|
||||
r.ari ?? "",
|
||||
r.smog ?? "",
|
||||
Array.isArray(r.schema_types) ? r.schema_types.join("|") : "",
|
||||
r.inlinks ?? 0,
|
||||
r.outlinks ?? 0,
|
||||
r.render_mode,
|
||||
r.last_modified ?? "",
|
||||
r.set_cookie ? "yes" : "no",
|
||||
r.crawl_timestamp ?? "",
|
||||
r.duplicate_title_exact ?? "",
|
||||
r.nearest_title_similarity ?? "",
|
||||
r.nearest_title_url ?? "",
|
||||
r.duplicate_description_exact ?? "",
|
||||
r.nearest_description_similarity ?? "",
|
||||
r.nearest_description_url ?? ""
|
||||
].map(csvEscape).join(","));
|
||||
}
|
||||
//fs.writeFileSync(`${base}.csv`, lines.join("\n"), "utf8");
|
||||
//console.log(`\n📝 Page reports saved:\n - ${base}.csv\n - ${base}.json`);
|
||||
}
|
||||
function writeLinkEdges(edges) {
|
||||
ensureDir("reports");
|
||||
const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
|
||||
const file = path.join("reports", `links-${stamp}.csv`);
|
||||
const headers = ["from", "raw_href", "to", "discovered_by"];
|
||||
const lines = [headers.join(",")];
|
||||
for (const e of edges) {
|
||||
lines.push([e.from, e.raw_href, e.to, e.discovered_by].map(csvEscape).join(","));
|
||||
}
|
||||
fs.writeFileSync(file, lines.join("\n"), "utf8");
|
||||
console.log(`🔗 Link provenance saved: ${file}`);
|
||||
}
|
||||
function writeErrors(results) {
|
||||
ensureDir("reports");
|
||||
const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
|
||||
const file = path.join("reports", `errors-${stamp}.csv`);
|
||||
const headers = ["url", "status", "title", "from_page", "raw_href", "discovered_by"];
|
||||
const lines = [headers.join(",")];
|
||||
|
||||
for (const r of results) {
|
||||
if (r && r.status !== null && r.status >= 400) {
|
||||
const refs = referrers.get(r.url) || [];
|
||||
if (refs.length === 0) {
|
||||
lines.push([r.url, r.status, r.title, "", "", ""].map(csvEscape).join(","));
|
||||
} else {
|
||||
for (const ref of refs) {
|
||||
lines.push([r.url, r.status, r.title, ref.from, ref.raw_href, ref.discovered_by].map(csvEscape).join(","));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fs.writeFileSync(file, lines.join("\n"), "utf8");
|
||||
console.log(`❗ Error report saved: ${file}`);
|
||||
}
|
||||
function addEdge(from, rawHref, to, discovered_by) {
|
||||
edges.push({ from, raw_href: rawHref || "", to, discovered_by });
|
||||
if (!referrers.has(to)) referrers.set(to, []);
|
||||
referrers.get(to).push({ from, raw_href: rawHref || "", discovered_by });
|
||||
}
|
||||
|
||||
/* ---------------------- parse HTML without JS --------------------------- */
|
||||
function safeJsonParse(txt) {
|
||||
try { return JSON.parse(txt); } catch { return null; }
|
||||
}
|
||||
function parseSchemaTypes($) {
|
||||
const types = new Set();
|
||||
$('script[type="application/ld+json"]').each((_, el) => {
|
||||
const raw = $(el).contents().text();
|
||||
const parsed = safeJsonParse(raw);
|
||||
if (!parsed) return;
|
||||
const collect = (obj) => {
|
||||
if (!obj) return;
|
||||
if (Array.isArray(obj)) { obj.forEach(collect); return; }
|
||||
if (typeof obj === "object") {
|
||||
const t = obj["@type"];
|
||||
if (typeof t === "string") types.add(t);
|
||||
else if (Array.isArray(t)) t.forEach(x => typeof x === "string" && types.add(x));
|
||||
// nested
|
||||
Object.values(obj).forEach(collect);
|
||||
}
|
||||
};
|
||||
collect(parsed);
|
||||
});
|
||||
return [...types];
|
||||
}
|
||||
function parseHtml(html, url) {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
let title = ($("title").first().text() || "").trim();
|
||||
const ogTitle = $('meta[property="og:title"]').attr("content") || "";
|
||||
const twTitle = $('meta[name="twitter:title"]').attr("content") || "";
|
||||
|
||||
// Headings (capture top two H1s and H2s)
|
||||
const h1s = $("h1").map((_, el) => $(el).text().trim()).get();
|
||||
const h2s = $("h2").map((_, el) => $(el).text().trim()).get();
|
||||
|
||||
const h1_1 = h1s[0] || "";
|
||||
const h1_2 = h1s[1] || "";
|
||||
const h2_1 = h2s[0] || "";
|
||||
const h2_2 = h2s[1] || "";
|
||||
|
||||
const totalHeadings = $("h1,h2,h3,h4,h5,h6,[role='heading']").length;
|
||||
|
||||
if (!title) title = (ogTitle || twTitle || h1_1 || "").trim();
|
||||
|
||||
const metaDesc = ($('meta[name="description"]').attr("content") || "").trim();
|
||||
const canonical = ($('link[rel="canonical"]').attr("href") || "").trim();
|
||||
const robotsMeta = ($('meta[name="robots"]').attr("content") || "").trim();
|
||||
const robotsLower = robotsMeta.toLowerCase();
|
||||
const noindex = /(^|[,;\s])noindex([,;\s]|$)/.test(robotsLower);
|
||||
const nofollow = /(^|[,;\s])nofollow([,;\s]|$)/.test(robotsLower);
|
||||
|
||||
const lang = ($("html").attr("lang") || "").trim();
|
||||
|
||||
// Basic text body for word count / readability
|
||||
const bodyText = ($("main").text() || $("body").text() || "").replace(/\s+/g, " ").trim();
|
||||
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
||||
|
||||
// Internal links + raw href
|
||||
const internalLinks = new Set();
|
||||
const rawLinks = [];
|
||||
$("a[href]").each((_, el) => {
|
||||
const href = $(el).attr("href");
|
||||
if (!href) return;
|
||||
try {
|
||||
const abs = new URL(href, url).toString();
|
||||
rawLinks.push({ raw: href, abs });
|
||||
internalLinks.add(abs);
|
||||
} catch { }
|
||||
});
|
||||
|
||||
// Schema.org JSON-LD types
|
||||
const schemaTypes = parseSchemaTypes($);
|
||||
|
||||
return {
|
||||
title,
|
||||
metaDesc,
|
||||
h1_1, h1_2, h2_1, h2_2,
|
||||
totalHeadings,
|
||||
canonical, robotsMeta, noindex, nofollow,
|
||||
internalLinks, rawLinks,
|
||||
lang,
|
||||
wordCount,
|
||||
schemaTypes,
|
||||
bodyText
|
||||
};
|
||||
}
|
||||
|
||||
/* ------------------------------ fetchers -------------------------------- */
|
||||
async function fetchWithGot(url) {
|
||||
const t0 = Date.now();
|
||||
const res = await got(url, {
|
||||
timeout: { request: 20000 },
|
||||
throwHttpErrors: false,
|
||||
headers: REAL_HEADERS,
|
||||
http2: false
|
||||
});
|
||||
const dt = Date.now() - t0;
|
||||
const contentType = (res.headers["content-type"] || "").toLowerCase();
|
||||
const bytes = res.headers["content-length"]
|
||||
? Number(res.headers["content-length"])
|
||||
: Buffer.byteLength(res.body || "", "utf8");
|
||||
|
||||
return {
|
||||
status: res.statusCode ?? null,
|
||||
status_text: res.statusMessage ?? "",
|
||||
time_ms: dt,
|
||||
contentType,
|
||||
body: res.body,
|
||||
bytes,
|
||||
render_mode: "http",
|
||||
httpVersion: res.httpVersion ?? "",
|
||||
headers: res.headers
|
||||
};
|
||||
}
|
||||
|
||||
async function createBrowserContext() {
|
||||
const browser = await chromium.launch({ headless: true, args: ["--disable-blink-features=AutomationControlled"] });
|
||||
const context = await browser.newContext({
|
||||
ignoreHTTPSErrors: true, // Ignore SSL certificate errors
|
||||
userAgent: REAL_UA,
|
||||
viewport: { width: 1366, height: 768 },
|
||||
deviceScaleFactor: 1,
|
||||
isMobile: false,
|
||||
locale: "en-US",
|
||||
extraHTTPHeaders: REAL_HEADERS
|
||||
});
|
||||
await context.addInitScript(() => {
|
||||
Object.defineProperty(navigator, "webdriver", { get: () => false });
|
||||
Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3] });
|
||||
Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
|
||||
});
|
||||
return { browser: context.browser(), context };
|
||||
}
|
||||
|
||||
async function fetchWithPlaywrightAndExtract(url, shared) {
|
||||
const page = await shared.context.newPage();
|
||||
const t0 = Date.now();
|
||||
let status = null, mainHeaders = {}, statusText = "";
|
||||
|
||||
try {
|
||||
const resp = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
status = resp?.status() ?? null;
|
||||
statusText = resp?.statusText() ?? "";
|
||||
try { mainHeaders = resp ? await resp.headers() : {}; } catch { }
|
||||
|
||||
try { await page.waitForLoadState("networkidle", { timeout: 12000 }); } catch { }
|
||||
try {
|
||||
await page.waitForFunction(() => {
|
||||
const main = document.querySelector("main") || document.body;
|
||||
const textLen = (main?.innerText || "").replace(/\s+/g, " ").trim().length;
|
||||
const hasHeading = !!document.querySelector("h1, h2, [role='heading'], [class*='title'], [class*='heading'], [class*='hero'], [class*='banner']");
|
||||
return textLen > 160 || hasHeading;
|
||||
}, { timeout: 8000 });
|
||||
} catch { }
|
||||
|
||||
const dom = await page.evaluate(() => {
|
||||
const clean = s => (s || "").replace(/\s+/g, " ").trim();
|
||||
const getTextList = sel => Array.from(document.querySelectorAll(sel))
|
||||
.map(el => clean(el.textContent)).filter(Boolean);
|
||||
|
||||
const title = document.title || "";
|
||||
const ogTitle = document.querySelector('meta[property="og:title"]')?.content || "";
|
||||
const twTitle = document.querySelector('meta[name="twitter:title"]')?.content || "";
|
||||
const metaDesc = document.querySelector('meta[name="description"]')?.content || "";
|
||||
const canonical = document.querySelector('link[rel="canonical"]')?.href || "";
|
||||
const robotsMeta = document.querySelector('meta[name="robots"]')?.content || "";
|
||||
const lang = document.documentElement.getAttribute("lang") || "";
|
||||
|
||||
const h1 = getTextList("h1");
|
||||
const h2 = getTextList("h2");
|
||||
const h3 = getTextList("h3");
|
||||
const totalHeadings = document.querySelectorAll("h1,h2,h3,h4,h5,h6,[role='heading']").length;
|
||||
|
||||
const links = Array.from(document.querySelectorAll("a[href]"))
|
||||
.map(a => {
|
||||
const raw = a.getAttribute("href");
|
||||
try { return { raw, abs: new URL(raw, location.href).toString() }; }
|
||||
catch { return null; }
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
const firstHeading = h1[0] || h2[0] || "";
|
||||
const bodyText = clean((document.querySelector("main") || document.body).innerText || "");
|
||||
|
||||
const schemaScripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map(s => s.textContent || "");
|
||||
|
||||
return {
|
||||
htmlLen: (document.documentElement.outerHTML || "").length,
|
||||
title, ogTitle, twTitle, metaDesc, canonical, robotsMeta, lang,
|
||||
h1, h2, totalHeadings,
|
||||
links,
|
||||
bodyText,
|
||||
schemaScripts
|
||||
};
|
||||
});
|
||||
|
||||
// Parse schema types from strings (outside of page)
|
||||
const schemaTypes = [];
|
||||
for (const raw of dom.schemaScripts || []) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
const collect = (obj) => {
|
||||
if (!obj) return;
|
||||
if (Array.isArray(obj)) { obj.forEach(collect); return; }
|
||||
if (typeof obj === "object") {
|
||||
const t = obj["@type"];
|
||||
if (typeof t === "string") schemaTypes.push(t);
|
||||
else if (Array.isArray(t)) t.forEach(x => typeof x === "string" && schemaTypes.push(x));
|
||||
Object.values(obj).forEach(collect);
|
||||
}
|
||||
};
|
||||
collect(parsed);
|
||||
} catch { }
|
||||
}
|
||||
|
||||
const dt = Date.now() - t0;
|
||||
const robotsLower = (dom.robotsMeta || "").toLowerCase();
|
||||
const noindex = /(^|[,;\s])noindex([,;\s]|$)/.test(robotsLower);
|
||||
const nofollow = /(^|[,;\s])nofollow([,;\s]|$)/.test(robotsLower);
|
||||
const finalTitle = (dom.title || dom.ogTitle || dom.twTitle || dom.h1?.[0] || "").trim();
|
||||
|
||||
return {
|
||||
status,
|
||||
status_text: statusText,
|
||||
time_ms: dt,
|
||||
contentType: "text/html",
|
||||
bytes: dom.htmlLen || 0,
|
||||
render_mode: "rendered",
|
||||
headers: mainHeaders,
|
||||
domExtract: {
|
||||
title: finalTitle,
|
||||
metaDesc: dom.metaDesc || "",
|
||||
canonical: dom.canonical || "",
|
||||
robotsMeta: dom.robotsMeta || "",
|
||||
lang: dom.lang || "",
|
||||
noindex, nofollow,
|
||||
h1_1: dom.h1?.[0] || "",
|
||||
h1_2: dom.h1?.[1] || "",
|
||||
h2_1: dom.h2?.[0] || "",
|
||||
h2_2: dom.h2?.[1] || "",
|
||||
totalHeadings: dom.totalHeadings || 0,
|
||||
links: new Set((dom.links || []).map(l => l.abs)),
|
||||
rawLinks: dom.links || [],
|
||||
bodyText: dom.bodyText || "",
|
||||
schemaTypes: Array.from(new Set(schemaTypes))
|
||||
}
|
||||
};
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------- render decision ------------------------------ */
|
||||
function shouldRender(currentUrl, httpRes, parsed, homeTitle) {
|
||||
const { pathname } = new URL(currentUrl);
|
||||
if ((httpRes.bytes ?? 0) < 4000) return true; // tiny HTML shell
|
||||
if (parsed.totalHeadings === 0) return true;
|
||||
if (homeTitle && parsed.title && parsed.title === homeTitle && pathname !== "/") return true;
|
||||
return false;
|
||||
}
|
||||
function withWWW(urlStr) {
|
||||
try { const u = new URL(urlStr); if (!u.hostname.startsWith("www.")) u.hostname = "www." + u.hostname; return u.toString(); }
|
||||
catch { return urlStr; }
|
||||
}
|
||||
|
||||
/* ------------------------ per-page enrichers ---------------------------- */
|
||||
function measurePixelWidth(text, size = 16, font = "arial") {
|
||||
if (!text) return 0;
|
||||
try { return pixelWidth(text, { font, size }); } catch { return Math.round(text.length * size * 0.5); }
|
||||
}
|
||||
function computeReadability(text) {
|
||||
if (!text) return {};
|
||||
const safe = text.slice(0, 200000); // cap
|
||||
const out = {};
|
||||
try { out.flesch_reading_ease = readability.fleschReadingEase(safe); } catch { }
|
||||
try { out.flesch_kincaid_grade = readability.fleschKincaidGrade(safe); } catch { }
|
||||
try { out.gunning_fog = readability.gunningFog(safe); } catch { }
|
||||
try { out.coleman_liau = readability.colemanLiauIndex(safe); } catch { }
|
||||
try { out.ari = readability.automatedReadabilityIndex(safe); } catch { }
|
||||
try { out.smog = readability.smogIndex(safe); } catch { }
|
||||
return out;
|
||||
}
|
||||
|
||||
/* -------------------------------- main ---------------------------------- */
|
||||
// async function crawl(startUrl, maxPages = 50) {
|
||||
|
||||
export async function crawl(startUrl, maxPages = 50) {
|
||||
const start = normalizeUrl(startUrl, { stripHash: true });
|
||||
queue.push(start);
|
||||
|
||||
// Seed from sitemap.xml + record provenance
|
||||
try {
|
||||
const sitemapUrls = await getSitemapUrls(start);
|
||||
for (const u of sitemapUrls) {
|
||||
queue.push(u);
|
||||
addEdge("sitemap.xml", u, u, "sitemap");
|
||||
}
|
||||
console.log(`📌 Seeded ${sitemapUrls.length} URL(s) from sitemap.xml`);
|
||||
} catch (e) {
|
||||
console.log("⚠️ Sitemap step skipped:", e.message);
|
||||
}
|
||||
|
||||
let shared = null;
|
||||
async function getShared() { if (!shared) shared = await createBrowserContext(); return shared; }
|
||||
|
||||
let homeTitle = null;
|
||||
|
||||
while (queue.length > 0 && visited.size < maxPages) {
|
||||
const url = queue.shift();
|
||||
if (!url) continue;
|
||||
|
||||
const normUrl = normalizeUrl(url, { stripHash: true });
|
||||
if (visited.has(normUrl)) continue;
|
||||
visited.add(normUrl);
|
||||
|
||||
let attemptUrls = [normUrl];
|
||||
let usedWWWRetry = false;
|
||||
|
||||
for (let attempt = 0; attempt < attemptUrls.length; attempt++) {
|
||||
const currentUrl = attemptUrls[attempt];
|
||||
try {
|
||||
// 1) HTTP fetch
|
||||
let pageRes = await fetchWithGot(currentUrl);
|
||||
|
||||
let parsed = {
|
||||
title: "", metaDesc: "", h1_1: "", h1_2: "", h2_1: "", h2_2: "",
|
||||
totalHeadings: 0, canonical: "", robotsMeta: "", noindex: false, nofollow: false,
|
||||
internalLinks: new Set(), rawLinks: [],
|
||||
lang: "", wordCount: 0, bodyText: "", schemaTypes: []
|
||||
};
|
||||
if (pageRes.contentType.includes("text/html")) {
|
||||
const p = parseHtml(pageRes.body || "", currentUrl);
|
||||
parsed = { ...parsed, ...p };
|
||||
}
|
||||
|
||||
if (!homeTitle && new URL(currentUrl).pathname === "/") {
|
||||
homeTitle = parsed.title || "";
|
||||
}
|
||||
|
||||
// 2) Render if needed
|
||||
if (pageRes.contentType.includes("text/html") && shouldRender(currentUrl, pageRes, parsed, homeTitle)) {
|
||||
const s = await getShared();
|
||||
const rendered = await fetchWithPlaywrightAndExtract(currentUrl, s);
|
||||
if (rendered.domExtract) {
|
||||
pageRes = { ...rendered, body: null };
|
||||
parsed = {
|
||||
...parsed,
|
||||
title: rendered.domExtract.title,
|
||||
metaDesc: rendered.domExtract.metaDesc,
|
||||
h1_1: rendered.domExtract.h1_1,
|
||||
h1_2: rendered.domExtract.h1_2,
|
||||
h2_1: rendered.domExtract.h2_1,
|
||||
h2_2: rendered.domExtract.h2_2,
|
||||
totalHeadings: rendered.domExtract.totalHeadings,
|
||||
canonical: rendered.domExtract.canonical,
|
||||
robotsMeta: rendered.domExtract.robotsMeta,
|
||||
noindex: rendered.domExtract.noindex,
|
||||
nofollow: rendered.domExtract.nofollow,
|
||||
internalLinks: rendered.domExtract.links,
|
||||
rawLinks: rendered.domExtract.rawLinks,
|
||||
lang: rendered.domExtract.lang || parsed.lang,
|
||||
bodyText: rendered.domExtract.bodyText || parsed.bodyText,
|
||||
wordCount: (rendered.domExtract.bodyText || "").split(/\s+/).filter(Boolean).length,
|
||||
schemaTypes: rendered.domExtract.schemaTypes
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// If still looks empty, try www once
|
||||
if (!usedWWWRetry && parsed.totalHeadings === 0 && !parsed.h1_1) {
|
||||
attemptUrls.push(withWWW(currentUrl));
|
||||
usedWWWRetry = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Enqueue internal links + record provenance
|
||||
for (const link of parsed.internalLinks) {
|
||||
if (isInternal(start, link)) {
|
||||
const ln = normalizeUrl(link, { stripHash: true });
|
||||
const rawMatch = (parsed.rawLinks || []).find(r => r.abs === link)?.raw ?? "";
|
||||
addEdge(currentUrl, rawMatch, ln, pageRes.render_mode);
|
||||
if (!visited.has(ln)) queue.push(ln);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Per-page metrics & enrichers ----
|
||||
const title = parsed.title || "";
|
||||
const metaDesc = parsed.metaDesc || "";
|
||||
const h1_1 = parsed.h1_1 || "";
|
||||
const h1_2 = parsed.h1_2 || "";
|
||||
const lang = parsed.lang || "";
|
||||
const bodyText = parsed.bodyText || "";
|
||||
const wordCount = parsed.wordCount || (bodyText ? bodyText.split(/\s+/).filter(Boolean).length : 0);
|
||||
|
||||
const titlePx = measurePixelWidth(title, 16, "arial");
|
||||
const descPx = measurePixelWidth(metaDesc, 14, "arial");
|
||||
const h1_1_px = measurePixelWidth(h1_1, 24, "arial");
|
||||
const h1_2_px = measurePixelWidth(h1_2, 24, "arial");
|
||||
|
||||
const read = computeReadability(bodyText);
|
||||
|
||||
const headers = pageRes.headers || {};
|
||||
const xRobots = (headers["x-robots-tag"] || headers["x-robots-tag".toLowerCase()]) ?? "";
|
||||
const lastModified = headers["last-modified"] ?? headers["Last-Modified"] ?? "";
|
||||
const setCookie = !!headers["set-cookie"];
|
||||
|
||||
const outlinks = parsed.internalLinks.size;
|
||||
const inlinks = (referrers.get(currentUrl) || []).length;
|
||||
|
||||
// Save page row
|
||||
results.push({
|
||||
url: currentUrl,
|
||||
status: pageRes.status,
|
||||
status_text: pageRes.status_text ?? "",
|
||||
time_ms: pageRes.time_ms,
|
||||
bytes: pageRes.bytes,
|
||||
content_type: pageRes.contentType,
|
||||
http_version: pageRes.httpVersion ?? "",
|
||||
title,
|
||||
title_length: title.length,
|
||||
title_pixel_width: titlePx,
|
||||
meta_description: metaDesc,
|
||||
meta_description_length: metaDesc.length,
|
||||
meta_description_pixel_width: descPx,
|
||||
h1_1,
|
||||
h1_1_length: h1_1.length,
|
||||
h1_1_pixel_width: h1_1_px,
|
||||
h1_2,
|
||||
h1_2_length: h1_2.length,
|
||||
h1_2_pixel_width: h1_2_px,
|
||||
h2_1: parsed.h2_1 || "",
|
||||
h2_2: parsed.h2_2 || "",
|
||||
canonical: parsed.canonical,
|
||||
robots_meta: parsed.robotsMeta,
|
||||
x_robots_tag: Array.isArray(xRobots) ? xRobots.join("; ") : xRobots,
|
||||
noindex: parsed.noindex,
|
||||
nofollow: parsed.nofollow,
|
||||
lang,
|
||||
word_count: wordCount,
|
||||
flesch_reading_ease: read.flesch_reading_ease ?? "",
|
||||
flesch_kincaid_grade: read.flesch_kincaid_grade ?? "",
|
||||
gunning_fog: read.gunning_fog ?? "",
|
||||
coleman_liau: read.coleman_liau ?? "",
|
||||
ari: read.ari ?? "",
|
||||
smog: read.smog ?? "",
|
||||
schema_types: parsed.schemaTypes || [],
|
||||
inlinks,
|
||||
outlinks,
|
||||
render_mode: pageRes.render_mode,
|
||||
last_modified: lastModified,
|
||||
set_cookie: setCookie,
|
||||
crawl_timestamp: new Date().toISOString()
|
||||
});
|
||||
|
||||
console.log(
|
||||
`[${pageRes.status ?? "ERR"}] ${pageRes.time_ms}ms ${String(pageRes.render_mode).padEnd(8)} H:${parsed.totalHeadings} ${currentUrl} ${title || h1_1}`
|
||||
);
|
||||
break; // success for this URL; stop attempts
|
||||
} catch (err) {
|
||||
console.error(`[ERROR] ${currentUrl} -> ${err.message}`);
|
||||
results.push({
|
||||
url: currentUrl,
|
||||
status: null, status_text: "", time_ms: null, bytes: null, content_type: "",
|
||||
http_version: "", title: "", title_length: 0, title_pixel_width: "",
|
||||
meta_description: "", meta_description_length: 0, meta_description_pixel_width: "",
|
||||
h1_1: "", h1_1_length: 0, h1_1_pixel_width: "", h1_2: "", h1_2_length: 0, h1_2_pixel_width: "",
|
||||
h2_1: "", h2_2: "",
|
||||
canonical: "", robots_meta: "", x_robots_tag: "", noindex: false, nofollow: false,
|
||||
lang: "", word_count: "", flesch_reading_ease: "", flesch_kincaid_grade: "",
|
||||
gunning_fog: "", coleman_liau: "", ari: "", smog: "",
|
||||
schema_types: [], inlinks: 0, outlinks: 0, render_mode: "error",
|
||||
last_modified: "", set_cookie: "", crawl_timestamp: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (shared) await shared.browser.close();
|
||||
|
||||
// -------------------- Post-process: duplicates & similarity -------------
|
||||
// Titles
|
||||
const titleMap = new Map();
|
||||
for (const r of results) {
|
||||
const key = (r.title || "").trim();
|
||||
if (!titleMap.has(key)) titleMap.set(key, []);
|
||||
titleMap.get(key).push(r);
|
||||
}
|
||||
for (const [t, arr] of titleMap.entries()) {
|
||||
if (!t) continue;
|
||||
const isDup = arr.length > 1;
|
||||
for (const row of arr) row.duplicate_title_exact = isDup ? "yes" : "no";
|
||||
}
|
||||
|
||||
// Meta descriptions
|
||||
const descMap = new Map();
|
||||
for (const r of results) {
|
||||
const key = (r.meta_description || "").trim();
|
||||
if (!descMap.has(key)) descMap.set(key, []);
|
||||
descMap.get(key).push(r);
|
||||
}
|
||||
for (const [d, arr] of descMap.entries()) {
|
||||
if (!d) continue;
|
||||
const isDup = arr.length > 1;
|
||||
for (const row of arr) row.duplicate_description_exact = isDup ? "yes" : "no";
|
||||
}
|
||||
|
||||
// Nearest neighbor similarities (within site, lightweight)
|
||||
const titleList = results.map(r => ({ url: r.url, text: (r.title || "").trim() }));
|
||||
const descList = results.map(r => ({ url: r.url, text: (r.meta_description || "").trim() }));
|
||||
for (const r of results) {
|
||||
// titles
|
||||
const others = titleList.filter(x => x.url !== r.url && x.text);
|
||||
let bestT = { rating: 0, target: "" };
|
||||
if (r.title && others.length) {
|
||||
const ratings = stringSimilarity.findBestMatch(r.title, others.map(x => x.text));
|
||||
const best = ratings.bestMatch;
|
||||
bestT.rating = best.rating;
|
||||
const idx = ratings.ratings.findIndex(x => x.rating === best.rating);
|
||||
bestT.target = others[idx]?.url || "";
|
||||
}
|
||||
r.nearest_title_similarity = bestT.rating ? bestT.rating.toFixed(3) : "";
|
||||
r.nearest_title_url = bestT.target;
|
||||
|
||||
// descriptions
|
||||
const othersD = descList.filter(x => x.url !== r.url && x.text);
|
||||
let bestD = { rating: 0, target: "" };
|
||||
if (r.meta_description && othersD.length) {
|
||||
const ratingsD = stringSimilarity.findBestMatch(r.meta_description, othersD.map(x => x.text));
|
||||
const best = ratingsD.bestMatch;
|
||||
bestD.rating = best.rating;
|
||||
const idx = ratingsD.ratings.findIndex(x => x.rating === best.rating);
|
||||
bestD.target = othersD[idx]?.url || "";
|
||||
}
|
||||
r.nearest_description_similarity = bestD.rating ? bestD.rating.toFixed(3) : "";
|
||||
r.nearest_description_url = bestD.target;
|
||||
}
|
||||
|
||||
console.log(`\n✅ Crawl finished. Total pages: ${visited.size}`);
|
||||
writePageReports(results);
|
||||
writeLinkEdges(edges);
|
||||
writeErrors(results);
|
||||
}
|
||||
|
||||
// // CLI: node crawler.js https://site.com 200
|
||||
// const START_URL = process.argv[2] || "https://example.com";
|
||||
// const MAX_PAGES = Number(process.argv[3] || 100);
|
||||
// crawl(START_URL, MAX_PAGES);
|
||||
921
crawler.js
Normal file
921
crawler.js
Normal file
@ -0,0 +1,921 @@
|
||||
// crawler.js
|
||||
import got from "got";
|
||||
import * as cheerio from "cheerio";
|
||||
import normalizeUrl from "normalize-url";
|
||||
import { isInternal } from "./utils/urlHelpers.js";
|
||||
import { getSitemapUrls } from "./utils/sitemap.js";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { chromium } from "playwright";
|
||||
|
||||
// NEW libs
|
||||
import pixelWidth from "string-pixel-width";
|
||||
import * as readability from "text-readability";
|
||||
import stringSimilarity from "string-similarity";
|
||||
|
||||
/* ------------------------------ globals --------------------------------- */
|
||||
// NOTE: We'll reset these at the start of crawl() so repeated runs don't share state.
|
||||
const visited = new Set();
|
||||
const queue = [];
|
||||
const results = [];
|
||||
|
||||
// Link provenance: every discovered edge (source -> target)
|
||||
const edges = []; // { from, raw_href, to, discovered_by }
|
||||
|
||||
// Quick referrer map for error report
|
||||
const referrers = new Map(); // url -> Array<{from, raw_href, discovered_by}>
|
||||
|
||||
const REAL_UA =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
||||
const REAL_HEADERS = {
|
||||
"user-agent": REAL_UA,
|
||||
accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"accept-language": "en-US,en;q=0.9",
|
||||
"upgrade-insecure-requests": "1",
|
||||
};
|
||||
|
||||
/* ------------------------------ utils ----------------------------------- */
|
||||
function csvEscape(v) {
|
||||
if (v === undefined || v === null) return "";
|
||||
const s = String(v);
|
||||
return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
|
||||
}
|
||||
function ensureDir(dir) {
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
function writePageReports(rows) {
|
||||
ensureDir("reports");
|
||||
const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
|
||||
const base = path.join("reports", `crawl-${stamp}`);
|
||||
|
||||
fs.writeFileSync(`${base}.json`, JSON.stringify(rows, null, 2), "utf8");
|
||||
console.log(`\n📝 Full JSON report saved: ${base}.json`);
|
||||
|
||||
// Columns (a Screaming-Frog-ish shape with our extras)
|
||||
const headers = [
|
||||
"url",
|
||||
"status",
|
||||
"status_text",
|
||||
"time_ms",
|
||||
"bytes",
|
||||
"content_type",
|
||||
"http_version",
|
||||
"title",
|
||||
"title_length",
|
||||
"title_pixel_width",
|
||||
"meta_description",
|
||||
"meta_description_length",
|
||||
"meta_description_pixel_width",
|
||||
"h1_1",
|
||||
"h1_1_length",
|
||||
"h1_1_pixel_width",
|
||||
"h1_2",
|
||||
"h1_2_length",
|
||||
"h1_2_pixel_width",
|
||||
"h2_1",
|
||||
"h2_2",
|
||||
"canonical",
|
||||
"robots_meta",
|
||||
"x_robots_tag",
|
||||
"noindex",
|
||||
"nofollow",
|
||||
"lang",
|
||||
"word_count",
|
||||
"flesch_reading_ease",
|
||||
"flesch_kincaid_grade",
|
||||
"gunning_fog",
|
||||
"coleman_liau",
|
||||
"ari",
|
||||
"smog",
|
||||
"schema_types",
|
||||
"inlinks",
|
||||
"outlinks",
|
||||
"render_mode",
|
||||
"last_modified",
|
||||
"set_cookie",
|
||||
"crawl_timestamp",
|
||||
"duplicate_title_exact",
|
||||
"nearest_title_similarity",
|
||||
"nearest_title_url",
|
||||
"duplicate_description_exact",
|
||||
"nearest_description_similarity",
|
||||
"nearest_description_url",
|
||||
];
|
||||
const lines = [headers.join(",")];
|
||||
for (const r of rows) {
|
||||
lines.push(
|
||||
[
|
||||
r.url,
|
||||
r.status,
|
||||
r.status_text ?? "",
|
||||
r.time_ms,
|
||||
r.bytes,
|
||||
r.content_type,
|
||||
r.http_version ?? "",
|
||||
r.title,
|
||||
r.title_length,
|
||||
r.title_pixel_width,
|
||||
r.meta_description,
|
||||
r.meta_description_length,
|
||||
r.meta_description_pixel_width,
|
||||
r.h1_1 ?? "",
|
||||
r.h1_1_length ?? 0,
|
||||
r.h1_1_pixel_width ?? "",
|
||||
r.h1_2 ?? "",
|
||||
r.h1_2_length ?? 0,
|
||||
r.h1_2_pixel_width ?? "",
|
||||
r.h2_1 ?? "",
|
||||
r.h2_2 ?? "",
|
||||
r.canonical,
|
||||
r.robots_meta,
|
||||
r.x_robots_tag ?? "",
|
||||
r.noindex,
|
||||
r.nofollow,
|
||||
r.lang ?? "",
|
||||
r.word_count ?? "",
|
||||
r.flesch_reading_ease ?? "",
|
||||
r.flesch_kincaid_grade ?? "",
|
||||
r.gunning_fog ?? "",
|
||||
r.coleman_liau ?? "",
|
||||
r.ari ?? "",
|
||||
r.smog ?? "",
|
||||
Array.isArray(r.schema_types) ? r.schema_types.join("|") : "",
|
||||
r.inlinks ?? 0,
|
||||
r.outlinks ?? 0,
|
||||
r.render_mode,
|
||||
r.last_modified ?? "",
|
||||
r.set_cookie ? "yes" : "no",
|
||||
r.crawl_timestamp ?? "",
|
||||
r.duplicate_title_exact ?? "",
|
||||
r.nearest_title_similarity ?? "",
|
||||
r.nearest_title_url ?? "",
|
||||
r.duplicate_description_exact ?? "",
|
||||
r.nearest_description_similarity ?? "",
|
||||
r.nearest_description_url ?? "",
|
||||
]
|
||||
.map(csvEscape)
|
||||
.join(",")
|
||||
);
|
||||
}
|
||||
// If you also want CSV persisted, uncomment:
|
||||
// fs.writeFileSync(`${base}.csv`, lines.join("\n"), "utf8");
|
||||
// console.log(`📝 CSV report saved: ${base}.csv`);
|
||||
|
||||
return { json: path.resolve(`${base}.json`) /*, csv: path.resolve(`${base}.csv`)*/ };
|
||||
}
|
||||
function writeLinkEdges(edges) {
|
||||
ensureDir("reports");
|
||||
const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
|
||||
const file = path.join("reports", `links-${stamp}.csv`);
|
||||
const headers = ["from", "raw_href", "to", "discovered_by"];
|
||||
const lines = [headers.join(",")];
|
||||
for (const e of edges) {
|
||||
lines.push([e.from, e.raw_href, e.to, e.discovered_by].map(csvEscape).join(","));
|
||||
}
|
||||
fs.writeFileSync(file, lines.join("\n"), "utf8");
|
||||
console.log(`🔗 Link provenance saved: ${file}`);
|
||||
return { linksCsv: path.resolve(file) };
|
||||
}
|
||||
function writeErrors(rows) {
|
||||
ensureDir("reports");
|
||||
const stamp = new Date().toISOString().replace(/[:T]/g, "-").slice(0, 19);
|
||||
const file = path.join("reports", `errors-${stamp}.csv`);
|
||||
const headers = ["url", "status", "title", "from_page", "raw_href", "discovered_by"];
|
||||
const lines = [headers.join(",")];
|
||||
|
||||
for (const r of rows) {
|
||||
if (r && r.status !== null && r.status >= 400) {
|
||||
const refs = referrers.get(r.url) || [];
|
||||
if (refs.length === 0) {
|
||||
lines.push([r.url, r.status, r.title, "", "", ""].map(csvEscape).join(","));
|
||||
} else {
|
||||
for (const ref of refs) {
|
||||
lines.push([r.url, r.status, r.title, ref.from, ref.raw_href, ref.discovered_by].map(csvEscape).join(","));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fs.writeFileSync(file, lines.join("\n"), "utf8");
|
||||
console.log(`❗ Error report saved: ${file}`);
|
||||
return { errorsCsv: path.resolve(file) };
|
||||
}
|
||||
function addEdge(from, rawHref, to, discovered_by) {
|
||||
edges.push({ from, raw_href: rawHref || "", to, discovered_by });
|
||||
if (!referrers.has(to)) referrers.set(to, []);
|
||||
referrers.get(to).push({ from, raw_href: rawHref || "", discovered_by });
|
||||
}
|
||||
|
||||
/* ---------------------- parse HTML without JS --------------------------- */
|
||||
function safeJsonParse(txt) {
|
||||
try {
|
||||
return JSON.parse(txt);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
function parseSchemaTypes($) {
|
||||
const types = new Set();
|
||||
$('script[type="application/ld+json"]').each((_, el) => {
|
||||
const raw = $(el).contents().text();
|
||||
const parsed = safeJsonParse(raw);
|
||||
if (!parsed) return;
|
||||
const collect = (obj) => {
|
||||
if (!obj) return;
|
||||
if (Array.isArray(obj)) {
|
||||
obj.forEach(collect);
|
||||
return;
|
||||
}
|
||||
if (typeof obj === "object") {
|
||||
const t = obj["@type"];
|
||||
if (typeof t === "string") types.add(t);
|
||||
else if (Array.isArray(t)) t.forEach((x) => typeof x === "string" && types.add(x));
|
||||
// nested
|
||||
Object.values(obj).forEach(collect);
|
||||
}
|
||||
};
|
||||
collect(parsed);
|
||||
});
|
||||
return [...types];
|
||||
}
|
||||
function parseHtml(html, url) {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
let title = ($("title").first().text() || "").trim();
|
||||
const ogTitle = $('meta[property="og:title"]').attr("content") || "";
|
||||
const twTitle = $('meta[name="twitter:title"]').attr("content") || "";
|
||||
|
||||
// Headings (capture top two H1s and H2s)
|
||||
const h1s = $("h1")
|
||||
.map((_, el) => $(el).text().trim())
|
||||
.get();
|
||||
const h2s = $("h2")
|
||||
.map((_, el) => $(el).text().trim())
|
||||
.get();
|
||||
|
||||
const h1_1 = h1s[0] || "";
|
||||
const h1_2 = h1s[1] || "";
|
||||
const h2_1 = h2s[0] || "";
|
||||
const h2_2 = h2s[1] || "";
|
||||
|
||||
const totalHeadings = $("h1,h2,h3,h4,h5,h6,[role='heading']").length;
|
||||
|
||||
if (!title) title = (ogTitle || twTitle || h1_1 || "").trim();
|
||||
|
||||
const metaDesc = ($('meta[name="description"]').attr("content") || "").trim();
|
||||
const canonical = ($('link[rel="canonical"]').attr("href") || "").trim();
|
||||
const robotsMeta = ($('meta[name="robots"]').attr("content") || "").trim();
|
||||
const robotsLower = robotsMeta.toLowerCase();
|
||||
const noindex = /(^|[,;\s])noindex([,;\s]|$)/.test(robotsLower);
|
||||
const nofollow = /(^|[,;\s])nofollow([,;\s]|$)/.test(robotsLower);
|
||||
|
||||
const lang = ($("html").attr("lang") || "").trim();
|
||||
|
||||
// Basic text body for word count / readability
|
||||
const bodyText = ($("main").text() || $("body").text() || "").replace(/\s+/g, " ").trim();
|
||||
const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;
|
||||
|
||||
// Internal links + raw href
|
||||
const internalLinks = new Set();
|
||||
const rawLinks = [];
|
||||
$("a[href]").each((_, el) => {
|
||||
const href = $(el).attr("href");
|
||||
if (!href) return;
|
||||
try {
|
||||
const abs = new URL(href, url).toString();
|
||||
rawLinks.push({ raw: href, abs });
|
||||
internalLinks.add(abs);
|
||||
} catch {}
|
||||
});
|
||||
|
||||
// Schema.org JSON-LD types
|
||||
const schemaTypes = parseSchemaTypes($);
|
||||
|
||||
return {
|
||||
title,
|
||||
metaDesc,
|
||||
h1_1,
|
||||
h1_2,
|
||||
h2_1,
|
||||
h2_2,
|
||||
totalHeadings,
|
||||
canonical,
|
||||
robotsMeta,
|
||||
noindex,
|
||||
nofollow,
|
||||
internalLinks,
|
||||
rawLinks,
|
||||
lang,
|
||||
wordCount,
|
||||
schemaTypes,
|
||||
bodyText,
|
||||
};
|
||||
}
|
||||
|
||||
/* ------------------------------ fetchers -------------------------------- */
|
||||
async function fetchWithGot(url) {
|
||||
const t0 = Date.now();
|
||||
const res = await got(url, {
|
||||
timeout: { request: 20000 },
|
||||
throwHttpErrors: false,
|
||||
headers: REAL_HEADERS,
|
||||
http2: false,
|
||||
});
|
||||
const dt = Date.now() - t0;
|
||||
const contentType = (res.headers["content-type"] || "").toLowerCase();
|
||||
const bytes = res.headers["content-length"]
|
||||
? Number(res.headers["content-length"])
|
||||
: Buffer.byteLength(res.body || "", "utf8");
|
||||
|
||||
return {
|
||||
status: res.statusCode ?? null,
|
||||
status_text: res.statusMessage ?? "",
|
||||
time_ms: dt,
|
||||
contentType,
|
||||
body: res.body,
|
||||
bytes,
|
||||
render_mode: "http",
|
||||
httpVersion: res.httpVersion ?? "",
|
||||
headers: res.headers,
|
||||
};
|
||||
}
|
||||
|
||||
async function createBrowserContext() {
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ["--disable-blink-features=AutomationControlled"],
|
||||
});
|
||||
const context = await browser.newContext({
|
||||
ignoreHTTPSErrors: true, // Ignore SSL certificate errors
|
||||
userAgent: REAL_UA,
|
||||
viewport: { width: 1366, height: 768 },
|
||||
deviceScaleFactor: 1,
|
||||
isMobile: false,
|
||||
locale: "en-US",
|
||||
extraHTTPHeaders: REAL_HEADERS,
|
||||
});
|
||||
await context.addInitScript(() => {
|
||||
Object.defineProperty(navigator, "webdriver", { get: () => false });
|
||||
Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3] });
|
||||
Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
|
||||
});
|
||||
return { browser: context.browser(), context };
|
||||
}
|
||||
|
||||
async function fetchWithPlaywrightAndExtract(url, shared) {
|
||||
const page = await shared.context.newPage();
|
||||
const t0 = Date.now();
|
||||
let status = null,
|
||||
mainHeaders = {},
|
||||
statusText = "";
|
||||
|
||||
try {
|
||||
const resp = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
status = resp?.status() ?? null;
|
||||
statusText = resp?.statusText() ?? "";
|
||||
try {
|
||||
mainHeaders = resp ? await resp.headers() : {};
|
||||
} catch {}
|
||||
|
||||
try {
|
||||
await page.waitForLoadState("networkidle", { timeout: 12000 });
|
||||
} catch {}
|
||||
try {
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const main = document.querySelector("main") || document.body;
|
||||
const textLen = (main?.innerText || "").replace(/\s+/g, " ").trim().length;
|
||||
const hasHeading = !!document.querySelector(
|
||||
"h1, h2, [role='heading'], [class*='title'], [class*='heading'], [class*='hero'], [class*='banner']"
|
||||
);
|
||||
return textLen > 160 || hasHeading;
|
||||
},
|
||||
{ timeout: 8000 }
|
||||
);
|
||||
} catch {}
|
||||
|
||||
const dom = await page.evaluate(() => {
|
||||
const clean = (s) => (s || "").replace(/\s+/g, " ").trim();
|
||||
const getTextList = (sel) =>
|
||||
Array.from(document.querySelectorAll(sel))
|
||||
.map((el) => clean(el.textContent))
|
||||
.filter(Boolean);
|
||||
|
||||
const title = document.title || "";
|
||||
const ogTitle = document.querySelector('meta[property="og:title"]')?.content || "";
|
||||
const twTitle = document.querySelector('meta[name="twitter:title"]')?.content || "";
|
||||
const metaDesc = document.querySelector('meta[name="description"]')?.content || "";
|
||||
const canonical = document.querySelector('link[rel="canonical"]')?.href || "";
|
||||
const robotsMeta = document.querySelector('meta[name="robots"]')?.content || "";
|
||||
const lang = document.documentElement.getAttribute("lang") || "";
|
||||
|
||||
const h1 = getTextList("h1");
|
||||
const h2 = getTextList("h2");
|
||||
const totalHeadings = document.querySelectorAll("h1,h2,h3,h4,h5,h6,[role='heading']").length;
|
||||
|
||||
const links = Array.from(document.querySelectorAll("a[href]"))
|
||||
.map((a) => {
|
||||
const raw = a.getAttribute("href");
|
||||
try {
|
||||
return { raw, abs: new URL(raw, location.href).toString() };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
const bodyText = clean((document.querySelector("main") || document.body).innerText || "");
|
||||
|
||||
const schemaScripts = Array.from(
|
||||
document.querySelectorAll('script[type="application/ld+json"]')
|
||||
).map((s) => s.textContent || "");
|
||||
|
||||
return {
|
||||
htmlLen: (document.documentElement.outerHTML || "").length,
|
||||
title,
|
||||
ogTitle,
|
||||
twTitle,
|
||||
metaDesc,
|
||||
canonical,
|
||||
robotsMeta,
|
||||
lang,
|
||||
h1,
|
||||
h2,
|
||||
totalHeadings,
|
||||
links,
|
||||
bodyText,
|
||||
schemaScripts,
|
||||
};
|
||||
});
|
||||
|
||||
// Parse schema types from strings (outside of page)
|
||||
const schemaTypes = [];
|
||||
for (const raw of dom.schemaScripts || []) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
const collect = (obj) => {
|
||||
if (!obj) return;
|
||||
if (Array.isArray(obj)) {
|
||||
obj.forEach(collect);
|
||||
return;
|
||||
}
|
||||
if (typeof obj === "object") {
|
||||
const t = obj["@type"];
|
||||
if (typeof t === "string") schemaTypes.push(t);
|
||||
else if (Array.isArray(t)) t.forEach((x) => typeof x === "string" && schemaTypes.push(x));
|
||||
Object.values(obj).forEach(collect);
|
||||
}
|
||||
};
|
||||
collect(parsed);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const dt = Date.now() - t0;
|
||||
const robotsLower = (dom.robotsMeta || "").toLowerCase();
|
||||
const noindex = /(^|[,;\s])noindex([,;\s]|$)/.test(robotsLower);
|
||||
const nofollow = /(^|[,;\s])nofollow([,;\s]|$)/.test(robotsLower);
|
||||
const finalTitle = (dom.title || dom.ogTitle || dom.twTitle || dom.h1?.[0] || "").trim();
|
||||
|
||||
return {
|
||||
status,
|
||||
status_text: statusText,
|
||||
time_ms: dt,
|
||||
contentType: "text/html",
|
||||
bytes: dom.htmlLen || 0,
|
||||
render_mode: "rendered",
|
||||
headers: mainHeaders,
|
||||
domExtract: {
|
||||
title: finalTitle,
|
||||
metaDesc: dom.metaDesc || "",
|
||||
canonical: dom.canonical || "",
|
||||
robotsMeta: dom.robotsMeta || "",
|
||||
lang: dom.lang || "",
|
||||
noindex,
|
||||
nofollow,
|
||||
h1_1: dom.h1?.[0] || "",
|
||||
h1_2: dom.h1?.[1] || "",
|
||||
h2_1: dom.h2?.[0] || "",
|
||||
h2_2: dom.h2?.[1] || "",
|
||||
totalHeadings: dom.totalHeadings || 0,
|
||||
links: new Set((dom.links || []).map((l) => l.abs)),
|
||||
rawLinks: dom.links || [],
|
||||
bodyText: dom.bodyText || "",
|
||||
schemaTypes: Array.from(new Set(schemaTypes)),
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------- render decision ------------------------------ */
|
||||
function shouldRender(currentUrl, httpRes, parsed, homeTitle) {
|
||||
const { pathname } = new URL(currentUrl);
|
||||
if ((httpRes.bytes ?? 0) < 4000) return true; // tiny HTML shell
|
||||
if (parsed.totalHeadings === 0) return true;
|
||||
if (homeTitle && parsed.title && parsed.title === homeTitle && pathname !== "/") return true;
|
||||
return false;
|
||||
}
|
||||
function withWWW(urlStr) {
|
||||
try {
|
||||
const u = new URL(urlStr);
|
||||
if (!u.hostname.startsWith("www.")) u.hostname = "www." + u.hostname;
|
||||
return u.toString();
|
||||
} catch {
|
||||
return urlStr;
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------ per-page enrichers ---------------------------- */
|
||||
function measurePixelWidth(text, size = 16, font = "arial") {
|
||||
if (!text) return 0;
|
||||
try {
|
||||
return pixelWidth(text, { font, size });
|
||||
} catch {
|
||||
return Math.round(text.length * size * 0.5);
|
||||
}
|
||||
}
|
||||
function computeReadability(text) {
|
||||
if (!text) return {};
|
||||
const safe = text.slice(0, 200000); // cap
|
||||
const out = {};
|
||||
try {
|
||||
out.flesch_reading_ease = readability.fleschReadingEase(safe);
|
||||
} catch {}
|
||||
try {
|
||||
out.flesch_kincaid_grade = readability.fleschKincaidGrade(safe);
|
||||
} catch {}
|
||||
try {
|
||||
out.gunning_fog = readability.gunningFog(safe);
|
||||
} catch {}
|
||||
try {
|
||||
out.coleman_liau = readability.colemanLiauIndex(safe);
|
||||
} catch {}
|
||||
try {
|
||||
out.ari = readability.automatedReadabilityIndex(safe);
|
||||
} catch {}
|
||||
try {
|
||||
out.smog = readability.smogIndex(safe);
|
||||
} catch {}
|
||||
return out;
|
||||
}
|
||||
|
||||
/* -------------------------------- main ---------------------------------- */
|
||||
/**
|
||||
* Crawl a site and return a structured report.
|
||||
* @param {string} startUrl
|
||||
* @param {number} maxPages
|
||||
* @param {(tick:any)=>void} [onProgress] optional callback for progress events
|
||||
* @param {{persistReports?: boolean, collectPages?: boolean}} [options]
|
||||
* @returns {{ results: any[], files: Record<string,string>, total: number }}
|
||||
*/
|
||||
export async function crawl(startUrl, maxPages = 50, onProgress, options = {}) {
|
||||
const persistReports = options.persistReports !== false; // default true
|
||||
|
||||
// Reset global state per run
|
||||
visited.clear();
|
||||
queue.length = 0;
|
||||
results.length = 0;
|
||||
edges.length = 0;
|
||||
referrers.clear();
|
||||
|
||||
const start = normalizeUrl(startUrl, { stripHash: true });
|
||||
queue.push(start);
|
||||
|
||||
// Seed from sitemap.xml + record provenance
|
||||
try {
|
||||
const sitemapUrls = await getSitemapUrls(start);
|
||||
for (const u of sitemapUrls) {
|
||||
queue.push(u);
|
||||
addEdge("sitemap.xml", u, u, "sitemap");
|
||||
}
|
||||
console.log(`📌 Seeded ${sitemapUrls.length} URL(s) from sitemap.xml`);
|
||||
} catch (e) {
|
||||
console.log("⚠️ Sitemap step skipped:", e.message);
|
||||
}
|
||||
|
||||
let shared = null;
|
||||
async function getShared() {
|
||||
if (!shared) shared = await createBrowserContext();
|
||||
return shared;
|
||||
}
|
||||
|
||||
let homeTitle = null;
|
||||
|
||||
while (queue.length > 0 && visited.size < maxPages) {
|
||||
const url = queue.shift();
|
||||
if (!url) continue;
|
||||
|
||||
const normUrl = normalizeUrl(url, { stripHash: true });
|
||||
if (visited.has(normUrl)) continue;
|
||||
visited.add(normUrl);
|
||||
|
||||
const attemptUrls = [normUrl];
|
||||
let usedWWWRetry = false;
|
||||
|
||||
for (let attempt = 0; attempt < attemptUrls.length; attempt++) {
|
||||
const currentUrl = attemptUrls[attempt];
|
||||
try {
|
||||
// 1) HTTP fetch
|
||||
let pageRes = await fetchWithGot(currentUrl);
|
||||
|
||||
let parsed = {
|
||||
title: "",
|
||||
metaDesc: "",
|
||||
h1_1: "",
|
||||
h1_2: "",
|
||||
h2_1: "",
|
||||
h2_2: "",
|
||||
totalHeadings: 0,
|
||||
canonical: "",
|
||||
robotsMeta: "",
|
||||
noindex: false,
|
||||
nofollow: false,
|
||||
internalLinks: new Set(),
|
||||
rawLinks: [],
|
||||
lang: "",
|
||||
wordCount: 0,
|
||||
bodyText: "",
|
||||
schemaTypes: [],
|
||||
};
|
||||
if (pageRes.contentType.includes("text/html")) {
|
||||
const p = parseHtml(pageRes.body || "", currentUrl);
|
||||
parsed = { ...parsed, ...p };
|
||||
}
|
||||
|
||||
if (!homeTitle && new URL(currentUrl).pathname === "/") {
|
||||
homeTitle = parsed.title || "";
|
||||
}
|
||||
|
||||
// 2) Render if needed
|
||||
if (pageRes.contentType.includes("text/html") && shouldRender(currentUrl, pageRes, parsed, homeTitle)) {
|
||||
const s = await getShared();
|
||||
const rendered = await fetchWithPlaywrightAndExtract(currentUrl, s);
|
||||
if (rendered.domExtract) {
|
||||
pageRes = { ...rendered, body: null };
|
||||
parsed = {
|
||||
...parsed,
|
||||
title: rendered.domExtract.title,
|
||||
metaDesc: rendered.domExtract.metaDesc,
|
||||
h1_1: rendered.domExtract.h1_1,
|
||||
h1_2: rendered.domExtract.h1_2,
|
||||
h2_1: rendered.domExtract.h2_1,
|
||||
h2_2: rendered.domExtract.h2_2,
|
||||
totalHeadings: rendered.domExtract.totalHeadings,
|
||||
canonical: rendered.domExtract.canonical,
|
||||
robotsMeta: rendered.domExtract.robotsMeta,
|
||||
noindex: rendered.domExtract.noindex,
|
||||
nofollow: rendered.domExtract.nofollow,
|
||||
internalLinks: rendered.domExtract.links,
|
||||
rawLinks: rendered.domExtract.rawLinks,
|
||||
lang: rendered.domExtract.lang || parsed.lang,
|
||||
bodyText: rendered.domExtract.bodyText || parsed.bodyText,
|
||||
wordCount: (rendered.domExtract.bodyText || "")
|
||||
.split(/\s+/)
|
||||
.filter(Boolean).length,
|
||||
schemaTypes: rendered.domExtract.schemaTypes,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// If still looks empty, try www once
|
||||
if (!usedWWWRetry && parsed.totalHeadings === 0 && !parsed.h1_1) {
|
||||
attemptUrls.push(withWWW(currentUrl));
|
||||
usedWWWRetry = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Enqueue internal links + record provenance
|
||||
for (const link of parsed.internalLinks) {
|
||||
if (isInternal(start, link)) {
|
||||
const ln = normalizeUrl(link, { stripHash: true });
|
||||
const rawMatch = (parsed.rawLinks || []).find((r) => r.abs === link)?.raw ?? "";
|
||||
addEdge(currentUrl, rawMatch, ln, pageRes.render_mode);
|
||||
if (!visited.has(ln)) queue.push(ln);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Per-page metrics & enrichers ----
|
||||
const title = parsed.title || "";
|
||||
const metaDesc = parsed.metaDesc || "";
|
||||
const h1_1 = parsed.h1_1 || "";
|
||||
const h1_2 = parsed.h1_2 || "";
|
||||
const lang = parsed.lang || "";
|
||||
const bodyText = parsed.bodyText || "";
|
||||
const wordCount = parsed.wordCount || (bodyText ? bodyText.split(/\s+/).filter(Boolean).length : 0);
|
||||
|
||||
const titlePx = measurePixelWidth(title, 16, "arial");
|
||||
const descPx = measurePixelWidth(metaDesc, 14, "arial");
|
||||
const h1_1_px = measurePixelWidth(h1_1, 24, "arial");
|
||||
const h1_2_px = measurePixelWidth(h1_2, 24, "arial");
|
||||
|
||||
const read = computeReadability(bodyText);
|
||||
|
||||
const headers = pageRes.headers || {};
|
||||
const xRobots = (headers["x-robots-tag"] || headers["x-robots-tag".toLowerCase()]) ?? "";
|
||||
const lastModified = headers["last-modified"] ?? headers["Last-Modified"] ?? "";
|
||||
const setCookie = !!headers["set-cookie"];
|
||||
|
||||
const outlinks = parsed.internalLinks.size;
|
||||
const inlinks = (referrers.get(currentUrl) || []).length;
|
||||
|
||||
// Save page row
|
||||
results.push({
|
||||
url: currentUrl,
|
||||
status: pageRes.status,
|
||||
status_text: pageRes.status_text ?? "",
|
||||
time_ms: pageRes.time_ms,
|
||||
bytes: pageRes.bytes,
|
||||
content_type: pageRes.contentType,
|
||||
http_version: pageRes.httpVersion ?? "",
|
||||
title,
|
||||
title_length: title.length,
|
||||
title_pixel_width: titlePx,
|
||||
meta_description: metaDesc,
|
||||
meta_description_length: metaDesc.length,
|
||||
meta_description_pixel_width: descPx,
|
||||
h1_1,
|
||||
h1_1_length: h1_1.length,
|
||||
h1_1_pixel_width: h1_1_px,
|
||||
h1_2,
|
||||
h1_2_length: h1_2.length,
|
||||
h1_2_pixel_width: h1_2_px,
|
||||
h2_1: parsed.h2_1 || "",
|
||||
h2_2: parsed.h2_2 || "",
|
||||
canonical: parsed.canonical,
|
||||
robots_meta: parsed.robotsMeta,
|
||||
x_robots_tag: Array.isArray(xRobots) ? xRobots.join("; ") : xRobots,
|
||||
noindex: parsed.noindex,
|
||||
nofollow: parsed.nofollow,
|
||||
lang,
|
||||
word_count: wordCount,
|
||||
flesch_reading_ease: read.flesch_reading_ease ?? "",
|
||||
flesch_kincaid_grade: read.flesch_kincaid_grade ?? "",
|
||||
gunning_fog: read.gunning_fog ?? "",
|
||||
coleman_liau: read.coleman_liau ?? "",
|
||||
ari: read.ari ?? "",
|
||||
smog: read.smog ?? "",
|
||||
schema_types: parsed.schemaTypes || [],
|
||||
inlinks,
|
||||
outlinks,
|
||||
render_mode: pageRes.render_mode,
|
||||
last_modified: lastModified,
|
||||
set_cookie: setCookie,
|
||||
crawl_timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
console.log(
|
||||
`[${pageRes.status ?? "ERR"}] ${pageRes.time_ms}ms ${String(pageRes.render_mode).padEnd(8)} H:${parsed.totalHeadings} ${currentUrl} ${
|
||||
title || h1_1
|
||||
}`
|
||||
);
|
||||
|
||||
// optional progress callback (non-fatal)
|
||||
try {
|
||||
onProgress?.({
|
||||
url: currentUrl,
|
||||
status: pageRes.status,
|
||||
title,
|
||||
inlinks,
|
||||
outlinks,
|
||||
visited: visited.size,
|
||||
queued: queue.length,
|
||||
});
|
||||
} catch {}
|
||||
|
||||
break; // success for this URL; stop attempts
|
||||
} catch (err) {
|
||||
console.error(`[ERROR] ${currentUrl} -> ${err.message}`);
|
||||
results.push({
|
||||
url: currentUrl,
|
||||
status: null,
|
||||
status_text: "",
|
||||
time_ms: null,
|
||||
bytes: null,
|
||||
content_type: "",
|
||||
http_version: "",
|
||||
title: "",
|
||||
title_length: 0,
|
||||
title_pixel_width: "",
|
||||
meta_description: "",
|
||||
meta_description_length: 0,
|
||||
meta_description_pixel_width: "",
|
||||
h1_1: "",
|
||||
h1_1_length: 0,
|
||||
h1_1_pixel_width: "",
|
||||
h1_2: "",
|
||||
h1_2_length: 0,
|
||||
h1_2_pixel_width: "",
|
||||
h2_1: "",
|
||||
h2_2: "",
|
||||
canonical: "",
|
||||
robots_meta: "",
|
||||
x_robots_tag: "",
|
||||
noindex: false,
|
||||
nofollow: false,
|
||||
lang: "",
|
||||
word_count: "",
|
||||
flesch_reading_ease: "",
|
||||
flesch_kincaid_grade: "",
|
||||
gunning_fog: "",
|
||||
coleman_liau: "",
|
||||
ari: "",
|
||||
smog: "",
|
||||
schema_types: [],
|
||||
inlinks: 0,
|
||||
outlinks: 0,
|
||||
render_mode: "error",
|
||||
last_modified: "",
|
||||
set_cookie: "",
|
||||
crawl_timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
try {
|
||||
onProgress?.({
|
||||
url: currentUrl,
|
||||
error: String(err?.message || err),
|
||||
visited: visited.size,
|
||||
queued: queue.length,
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (shared) await shared.browser.close();
|
||||
|
||||
// -------------------- Post-process: duplicates & similarity -------------
|
||||
// Titles
|
||||
const titleMap = new Map();
|
||||
for (const r of results) {
|
||||
const key = (r.title || "").trim();
|
||||
if (!titleMap.has(key)) titleMap.set(key, []);
|
||||
titleMap.get(key).push(r);
|
||||
}
|
||||
for (const [t, arr] of titleMap.entries()) {
|
||||
if (!t) continue;
|
||||
const isDup = arr.length > 1;
|
||||
for (const row of arr) row.duplicate_title_exact = isDup ? "yes" : "no";
|
||||
}
|
||||
|
||||
// Meta descriptions
|
||||
const descMap = new Map();
|
||||
for (const r of results) {
|
||||
const key = (r.meta_description || "").trim();
|
||||
if (!descMap.has(key)) descMap.set(key, []);
|
||||
descMap.get(key).push(r);
|
||||
}
|
||||
for (const [d, arr] of descMap.entries()) {
|
||||
if (!d) continue;
|
||||
const isDup = arr.length > 1;
|
||||
for (const row of arr) row.duplicate_description_exact = isDup ? "yes" : "no";
|
||||
}
|
||||
|
||||
// Nearest neighbor similarities (within site, lightweight)
|
||||
const titleList = results.map((r) => ({ url: r.url, text: (r.title || "").trim() }));
|
||||
const descList = results.map((r) => ({ url: r.url, text: (r.meta_description || "").trim() }));
|
||||
for (const r of results) {
|
||||
// titles
|
||||
const others = titleList.filter((x) => x.url !== r.url && x.text);
|
||||
let bestT = { rating: 0, target: "" };
|
||||
if (r.title && others.length) {
|
||||
const ratings = stringSimilarity.findBestMatch(r.title, others.map((x) => x.text));
|
||||
const best = ratings.bestMatch;
|
||||
bestT.rating = best.rating;
|
||||
const idx = ratings.ratings.findIndex((x) => x.rating === best.rating);
|
||||
bestT.target = others[idx]?.url || "";
|
||||
}
|
||||
r.nearest_title_similarity = bestT.rating ? bestT.rating.toFixed(3) : "";
|
||||
r.nearest_title_url = bestT.target;
|
||||
|
||||
// descriptions
|
||||
const othersD = descList.filter((x) => x.url !== r.url && x.text);
|
||||
let bestD = { rating: 0, target: "" };
|
||||
if (r.meta_description && othersD.length) {
|
||||
const ratingsD = stringSimilarity.findBestMatch(r.meta_description, othersD.map((x) => x.text));
|
||||
const best = ratingsD.bestMatch;
|
||||
bestD.rating = best.rating;
|
||||
const idx = ratingsD.ratings.findIndex((x) => x.rating === best.rating);
|
||||
bestD.target = othersD[idx]?.url || "";
|
||||
}
|
||||
r.nearest_description_similarity = bestD.rating ? bestD.rating.toFixed(3) : "";
|
||||
r.nearest_description_url = bestD.target;
|
||||
}
|
||||
|
||||
console.log(`\n✅ Crawl finished. Total pages: ${visited.size}`);
|
||||
|
||||
let files = {};
|
||||
if (persistReports) {
|
||||
const a = writePageReports(results);
|
||||
const b = writeLinkEdges(edges);
|
||||
const c = writeErrors(results);
|
||||
files = { ...a, ...b, ...c };
|
||||
}
|
||||
|
||||
return { results, files, total: results.length };
|
||||
}
|
||||
|
||||
// // CLI: node crawler.js https://site.com 200
|
||||
// const START_URL = process.argv[2] || "https://example.com";
|
||||
// const MAX_PAGES = Number(process.argv[3] || 100);
|
||||
// crawl(START_URL, MAX_PAGES);
|
||||
19
middlewares/auth.middleware.js
Normal file
19
middlewares/auth.middleware.js
Normal file
@ -0,0 +1,19 @@
|
||||
import jwt from "jsonwebtoken";
|
||||
|
||||
export function authMiddleware(req, res, next) {
|
||||
const header = req.headers.authorization;
|
||||
|
||||
if (!header?.startsWith("Bearer ")) {
|
||||
return res.status(401).json({ error: "Missing token" });
|
||||
}
|
||||
|
||||
const token = header.split(" ")[1];
|
||||
|
||||
try {
|
||||
req.user = jwt.verify(token, process.env.JWT_SECRET);
|
||||
next();
|
||||
} catch (err) {
|
||||
console.error("JWT verification failed:", err.message);
|
||||
return res.status(401).json({ error: "Invalid or expired token" });
|
||||
}
|
||||
}
|
||||
4
middlewares/pageSpeedErrorHandler.js
Normal file
4
middlewares/pageSpeedErrorHandler.js
Normal file
@ -0,0 +1,4 @@
|
||||
export const errorHandler = (err, req, res, next) => {
|
||||
console.error(err);
|
||||
res.status(500).json({ message: err.message || 'Internal Server Error' });
|
||||
};
|
||||
41
models/blog.model.js
Normal file
41
models/blog.model.js
Normal file
@ -0,0 +1,41 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const commentSchema = new mongoose.Schema({
|
||||
user: { type: mongoose.Schema.Types.ObjectId, ref: 'User' },
|
||||
name: String,
|
||||
text: { type: String, required: true },
|
||||
createdAt: { type: Date, default: Date.now }
|
||||
});
|
||||
|
||||
const blogSchema = new mongoose.Schema({
|
||||
projectId: { type: String, required: true, index: true },
|
||||
title: { type: String, required: true },
|
||||
slug: { type: String, required: true, unique: false },
|
||||
description: { type: String, required: true },
|
||||
imageUrl: String,
|
||||
bigImageUrl: String, // ✅ New field
|
||||
category: { type: mongoose.Schema.Types.ObjectId, ref: 'Category' },
|
||||
tags: [String],
|
||||
comments: [commentSchema],
|
||||
likes: [{ type: mongoose.Schema.Types.ObjectId, ref: 'User' }],
|
||||
author: { type: mongoose.Schema.Types.ObjectId, ref: 'User' }
|
||||
}, { timestamps: true });
|
||||
|
||||
// 👇 projectId + slug combo unique
|
||||
blogSchema.index({ projectId: 1, slug: 1 }, { unique: true });
|
||||
|
||||
// 👇 Add base URL when converting to JSON
|
||||
blogSchema.set('toJSON', {
|
||||
transform: (doc, ret) => {
|
||||
const baseUrl = process.env.BACKEND_URL || 'http://localhost:3010';
|
||||
if (ret.imageUrl && !ret.imageUrl.startsWith('http')) {
|
||||
ret.imageUrl = `${baseUrl}${ret.imageUrl}`;
|
||||
}
|
||||
if (ret.bigImageUrl && !ret.bigImageUrl.startsWith('http')) {
|
||||
ret.bigImageUrl = `${baseUrl}${ret.bigImageUrl}`;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
});
|
||||
|
||||
export default mongoose.model('Blog', blogSchema);
|
||||
21
models/category.model.js
Normal file
21
models/category.model.js
Normal file
@ -0,0 +1,21 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const categorySchema = new mongoose.Schema(
|
||||
{
|
||||
name: {
|
||||
type: String,
|
||||
required: true,
|
||||
unique: true,
|
||||
trim: true,
|
||||
},
|
||||
projectId: {
|
||||
type: String, // For multi-project support
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
const Category = mongoose.model("Category", categorySchema);
|
||||
|
||||
export default Category;
|
||||
10
models/comments.model.js
Normal file
10
models/comments.model.js
Normal file
@ -0,0 +1,10 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const commentSchema = new mongoose.Schema({
|
||||
blog: { type: mongoose.Schema.Types.ObjectId, ref: "Blog" },
|
||||
name: String,
|
||||
text: String,
|
||||
createdAt: { type: Date, default: Date.now }
|
||||
});
|
||||
|
||||
export default mongoose.model("Comment", commentSchema);
|
||||
18
models/maisondetreats/cakeOrder.model.js
Normal file
18
models/maisondetreats/cakeOrder.model.js
Normal file
@ -0,0 +1,18 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const CakeOrderSchema = new mongoose.Schema(
|
||||
{
|
||||
order: {
|
||||
type: Object,
|
||||
required: true,
|
||||
// Example format:
|
||||
// {
|
||||
// "Mini Cakes": { "Thandai Cake": 1, "Mango Cardamom": 1 },
|
||||
// "Mithai-Inspired Macarons": { "Mango macarons (pack of 6)": 1, "Pista (pack of 6)": 10 }
|
||||
// }
|
||||
},
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
export const CakeOrder = mongoose.model("CakeOrder", CakeOrderSchema);
|
||||
12
models/message.model.js
Normal file
12
models/message.model.js
Normal file
@ -0,0 +1,12 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const MessageSchema = new mongoose.Schema({
|
||||
project: { type: String, required: true }, // new field to identify project
|
||||
name: { type: String, default: "Guest" },
|
||||
email: { type: String },
|
||||
message: { type: String, required: true },
|
||||
}, { timestamps: true });
|
||||
|
||||
const Message = mongoose.models.Message || mongoose.model("Message", MessageSchema);
|
||||
|
||||
export default Message;
|
||||
42
models/pageSpeedTest.model.js
Normal file
42
models/pageSpeedTest.model.js
Normal file
@ -0,0 +1,42 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const pageSpeedTestSchema = new mongoose.Schema({
|
||||
url: { type: String, required: true },
|
||||
device: { type: String, enum: ['mobile', 'desktop'], required: true },
|
||||
scores: {
|
||||
performance: Number,
|
||||
accessibility: Number,
|
||||
bestPractices: Number,
|
||||
seo: Number,
|
||||
pwa: Number,
|
||||
},
|
||||
metrics: {
|
||||
firstContentfulPaint: String,
|
||||
largestContentfulPaint: String,
|
||||
totalBlockingTime: String,
|
||||
timeToInteractive: String,
|
||||
speedIndex: String,
|
||||
cumulativeLayoutShift: String,
|
||||
},
|
||||
opportunities: [
|
||||
{
|
||||
title: String,
|
||||
description: String,
|
||||
estimatedSavings: String,
|
||||
},
|
||||
],
|
||||
diagnostics: Object,
|
||||
failedAudits: [
|
||||
{
|
||||
title: String,
|
||||
description: String,
|
||||
},
|
||||
],
|
||||
passedAudits: [String],
|
||||
notApplicableAudits: [String],
|
||||
screenshot: String,
|
||||
treemapPath: { type: String },
|
||||
createdAt: { type: Date, default: Date.now },
|
||||
});
|
||||
|
||||
export default mongoose.model('PageSpeedTest', pageSpeedTestSchema);
|
||||
12
models/payment.model.js
Normal file
12
models/payment.model.js
Normal file
@ -0,0 +1,12 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const paymentSchema = new mongoose.Schema({
|
||||
email: { type: String, required: true },
|
||||
amount: { type: Number, required: true }, // store in cents
|
||||
currency: { type: String, default: "usd" },
|
||||
stripePaymentIntentId: { type: String }, // ❌ remove required: true
|
||||
stripeSessionId: { type: String }, // ✅ store Checkout Session ID
|
||||
status: { type: String, default: "pending" }, // pending, succeeded, failed
|
||||
}, { timestamps: true });
|
||||
|
||||
export const Payment = mongoose.model("Payment", paymentSchema);
|
||||
14
models/user.model.js
Normal file
14
models/user.model.js
Normal file
@ -0,0 +1,14 @@
|
||||
import mongoose from "mongoose";
|
||||
|
||||
const userSchema = new mongoose.Schema(
|
||||
{
|
||||
email: { type: String, required: true, unique: true, lowercase: true },
|
||||
passwordHash: { type: String, required: true },
|
||||
// ➡️ Add these two lines
|
||||
resetPasswordToken: { type: String },
|
||||
resetPasswordExpires: { type: Date },
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
export default mongoose.model("User", userSchema);
|
||||
5093
package-lock.json
generated
Normal file
5093
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
36
package.json
Normal file
36
package.json
Normal file
@ -0,0 +1,36 @@
|
||||
{
|
||||
"name": "crawlerx",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"main": "crawler.js",
|
||||
"scripts": {
|
||||
"start": "node crawler.js https://example.com 200",
|
||||
"dev": "nodemon crawler.js https://example.com 200"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.12.2",
|
||||
"bcrypt": "^6.0.0",
|
||||
"cheerio": "^1.1.0",
|
||||
"chrome-launcher": "^1.2.1",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.2.2",
|
||||
"express": "^5.1.0",
|
||||
"got": "^14.4.7",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"lighthouse": "^12.8.2",
|
||||
"mongoose": "^8.18.1",
|
||||
"multer": "^2.0.2",
|
||||
"nodemailer": "^7.0.6",
|
||||
"normalize-url": "^8.0.2",
|
||||
"sitemapper": "^3.2.7",
|
||||
"slugify": "^1.6.6",
|
||||
"string-pixel-width": "^1.11.0",
|
||||
"string-similarity": "^4.0.4",
|
||||
"stripe": "^18.5.0",
|
||||
"text-readability": "^1.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.1.10",
|
||||
"playwright": "^1.55.0"
|
||||
}
|
||||
}
|
||||
2897
public/lighthouse-treemap/treemap-1758885492002.html
Normal file
2897
public/lighthouse-treemap/treemap-1758885492002.html
Normal file
File diff suppressed because one or more lines are too long
2897
public/lighthouse-treemap/treemap-1758885890915.html
Normal file
2897
public/lighthouse-treemap/treemap-1758885890915.html
Normal file
File diff suppressed because one or more lines are too long
2897
public/lighthouse-treemap/treemap-1758958684569.html
Normal file
2897
public/lighthouse-treemap/treemap-1758958684569.html
Normal file
File diff suppressed because one or more lines are too long
18
routes/auth.routes.js
Normal file
18
routes/auth.routes.js
Normal file
@ -0,0 +1,18 @@
|
||||
import express from "express";
|
||||
import { signup, login, changePassword, forgotPassword, resetPassword } from "../controllers/auth.controller.js";
|
||||
import { authMiddleware } from "../middlewares/auth.middleware.js";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.post("/signup", signup);
|
||||
router.post("/login", login);
|
||||
router.post("/change-password", authMiddleware, changePassword);
|
||||
router.post("/forgot-password", forgotPassword);
|
||||
router.post("/reset-password", resetPassword);
|
||||
|
||||
// example protected route
|
||||
router.get("/profile", authMiddleware, (req, res) => {
|
||||
res.json({ user: req.user });
|
||||
});
|
||||
|
||||
export default router;
|
||||
54
routes/blog.routes.js
Normal file
54
routes/blog.routes.js
Normal file
@ -0,0 +1,54 @@
|
||||
import express from "express";
|
||||
import multer from "multer";
|
||||
import {
|
||||
createBlog,
|
||||
getAllBlogs,
|
||||
getBlogBySlug,
|
||||
likeBlog,
|
||||
} from "../controllers/blog.controller.js";
|
||||
|
||||
import {
|
||||
createCategory,
|
||||
getCategories,
|
||||
deleteCategory,
|
||||
} from "../controllers/category.controller.js";
|
||||
|
||||
import {
|
||||
addComment as addCommentController,
|
||||
getComments,
|
||||
deleteComment,
|
||||
} from "../controllers/comment.controller.js";
|
||||
|
||||
const router = express.Router();
|
||||
const upload = multer({ dest: "uploads/" });
|
||||
|
||||
// =======================
|
||||
// Blog Routes
|
||||
// =======================
|
||||
// Create a blog
|
||||
router.post("/", upload.single("image"), createBlog);
|
||||
|
||||
// Get all blogs
|
||||
router.get("/", getAllBlogs);
|
||||
|
||||
// Get blog by slug
|
||||
router.get("/:slug", getBlogBySlug);
|
||||
|
||||
// Like a blog
|
||||
router.post("/:id/like", likeBlog);
|
||||
|
||||
// =======================
|
||||
// Category Routes
|
||||
// =======================
|
||||
router.post("/category", createCategory); // Create Category (admin)
|
||||
router.get("/category", getCategories); // List Categories
|
||||
router.delete("/category/:id", deleteCategory); // Delete Category (admin)
|
||||
|
||||
// =======================
|
||||
// Comment Routes
|
||||
// =======================
|
||||
router.post("/:blogId/comments", addCommentController); // Add Comment
|
||||
router.get("/:blogId/comments", getComments); // Get Comments
|
||||
router.delete("/:blogId/comments/:commentId", deleteComment); // Delete Comment (admin)
|
||||
|
||||
export default router;
|
||||
6
routes/crawl.routes.js
Normal file
6
routes/crawl.routes.js
Normal file
@ -0,0 +1,6 @@
|
||||
import { Router } from "express";
|
||||
import { crawlHandler } from "../controllers/crawl.controller.js";
|
||||
|
||||
const router = Router();
|
||||
router.get("/", crawlHandler);
|
||||
export default router;
|
||||
8
routes/lighthouse.routes.js
Normal file
8
routes/lighthouse.routes.js
Normal file
@ -0,0 +1,8 @@
|
||||
import express from 'express';
|
||||
import { runAudit } from '../controllers/lighthouseController.js';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.post('/audit', runAudit);
|
||||
|
||||
export default router;
|
||||
12
routes/maisondetreats/cakeOrder.routes.js
Normal file
12
routes/maisondetreats/cakeOrder.routes.js
Normal file
@ -0,0 +1,12 @@
|
||||
import express from "express";
|
||||
import { createCakeOrder, getAllCakeOrders } from "../../controllers/maisondetreats/cakeOrder.controller.js";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// Create a new cake order
|
||||
router.post("/", createCakeOrder);
|
||||
|
||||
// Get all cake orders
|
||||
router.get("/", getAllCakeOrders);
|
||||
|
||||
export default router;
|
||||
12
routes/message.routes.js
Normal file
12
routes/message.routes.js
Normal file
@ -0,0 +1,12 @@
|
||||
import express from "express";
|
||||
import { sendMessage, getMessages } from "../controllers/message.controller.js";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// POST /api/messages → Save a message
|
||||
router.post("/", sendMessage);
|
||||
|
||||
// GET /api/messages → Get all messages (optional)
|
||||
router.get("/", getMessages);
|
||||
|
||||
export default router;
|
||||
14
routes/payment.route.js
Normal file
14
routes/payment.route.js
Normal file
@ -0,0 +1,14 @@
|
||||
import express from "express";
|
||||
import {
|
||||
createPaymentIntent,
|
||||
createCheckoutSession,
|
||||
handleWebhook
|
||||
} from "../controllers/payment.controller.js";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.post("/create-intent", createPaymentIntent);
|
||||
router.post("/create-checkout-session", createCheckoutSession);
|
||||
router.post("/webhook", express.raw({ type: "application/json" }), handleWebhook);
|
||||
|
||||
export default router;
|
||||
6
routes/sitemap.routes.js
Normal file
6
routes/sitemap.routes.js
Normal file
@ -0,0 +1,6 @@
|
||||
import { Router } from "express";
|
||||
import { sitemapHandler } from "../controllers/sitemap.controller.js";
|
||||
|
||||
const router = Router();
|
||||
router.get("/", sitemapHandler);
|
||||
export default router;
|
||||
237
server copy.js
Normal file
237
server copy.js
Normal file
@ -0,0 +1,237 @@
|
||||
// // server.js
|
||||
// import express from "express";
|
||||
// import { Queue } from "bullmq";
|
||||
// import { connection } from "./redis.js";
|
||||
// import crypto from "crypto";
|
||||
|
||||
// const app = express();
|
||||
// app.use(express.json());
|
||||
|
||||
// const crawlQueue = new Queue("crawl", { connection });
|
||||
|
||||
// // Start a new crawl
|
||||
// app.post("/crawl", async (req, res) => {
|
||||
// const { startUrl } = req.body;
|
||||
// if (!startUrl) return res.status(400).json({ error: "Missing startUrl" });
|
||||
|
||||
// const crawlId = crypto.randomUUID();
|
||||
// await crawlQueue.add("fetch", { crawlId, url: startUrl });
|
||||
|
||||
// res.json({ crawlId, message: "Crawl started" });
|
||||
// });
|
||||
|
||||
// // (Optional) Check progress
|
||||
// app.get("/status/:id", async (req, res) => {
|
||||
// // For now just reply with "in progress"
|
||||
// res.json({ crawlId: req.params.id, status: "in progress" });
|
||||
// });
|
||||
|
||||
// app.listen(3000, () => {
|
||||
// console.log("Crawler API running at http://localhost:3000");
|
||||
// });
|
||||
|
||||
|
||||
// // server.js
|
||||
// import express from "express";
|
||||
// import cors from "cors"; // ← optional but recommended
|
||||
// import { crawl } from "./crawler.js"; // ensure crawl is a NAMED export; if default, use: import crawl from "./crawler.js";
|
||||
|
||||
// const app = express();
|
||||
// const PORT = process.env.PORT || 3010;
|
||||
|
||||
// /* Parse JSON BEFORE any middleware that might read req.body */
|
||||
// app.use(express.json());
|
||||
|
||||
// /* CORS (adjust origins as needed) */
|
||||
// app.use(cors({
|
||||
// origin: [
|
||||
// "http://localhost:3000",
|
||||
// "https://your-frontend.example" // ← replace or remove
|
||||
// ],
|
||||
// }));
|
||||
|
||||
// /* Safe request logger */
|
||||
// app.use((req, res, next) => {
|
||||
// console.log(`[${new Date().toISOString()}] ${req.method} ${req.originalUrl}`);
|
||||
// if (req.query && Object.keys(req.query).length) console.log("Query:", req.query);
|
||||
// if (req.body && typeof req.body === "object" && Object.keys(req.body).length) console.log("Body:", req.body);
|
||||
// next();
|
||||
// });
|
||||
|
||||
// /* GET /crawl?url=https://site.com&max=50 */
|
||||
// app.get("/crawl", async (req, res) => {
|
||||
// try {
|
||||
// const { url, max } = req.query;
|
||||
// if (!url) return res.status(400).json({ error: "Missing url param" });
|
||||
|
||||
// // validate & normalize
|
||||
// const target = new URL(String(url)); // throws if invalid
|
||||
// const limit = Math.min(Math.max(parseInt(max ?? "50", 10), 1), 500);
|
||||
|
||||
// await crawl(target.toString(), limit);
|
||||
// res.json({ ok: true, message: `Crawl started`, url: target.toString(), limit });
|
||||
// } catch (err) {
|
||||
// console.error("Crawl error:", err);
|
||||
// res.status(500).json({ error: "Crawl failed", details: String(err?.message ?? err) });
|
||||
// }
|
||||
// });
|
||||
|
||||
// /* Global safety nets so crashes don’t become silent restart loops */
|
||||
// process.on("unhandledRejection", (err) => console.error("unhandledRejection:", err));
|
||||
// process.on("uncaughtException", (err) => console.error("uncaughtException:", err));
|
||||
|
||||
// /* Bind to all interfaces so remote calls work */
|
||||
// app.listen(PORT, "0.0.0.0", () => {
|
||||
// console.log(`🚀 Server running at http://localhost:${PORT}`);
|
||||
// });
|
||||
|
||||
// server.js
|
||||
import express from "express";
|
||||
import cors from "cors";
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import fsp from "node:fs/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { crawl } from "./crawler.js"; // crawl(target, limit, onProgress?, options?)
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 3010;
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
/* ------------ Middleware ------------ */
|
||||
app.use(express.json());
|
||||
app.use(cors({ origin: ["http://localhost:3000", "https://app.crawlerx.co"] }));
|
||||
app.use(express.static(path.join(__dirname, "public")));
|
||||
app.get("/", (_req, res) => {
|
||||
const viewer = path.join(__dirname, "public", "crawlerx_viewer.html");
|
||||
return fs.existsSync(viewer)
|
||||
? res.sendFile(viewer)
|
||||
: res.type("text/plain").send("CrawlerX backend is running.");
|
||||
});
|
||||
app.get("/healthz", (_req, res) => res.json({ ok: true, time: new Date().toISOString() }));
|
||||
|
||||
/* ------------ Helpers ------------ */
|
||||
const ts = () =>
|
||||
new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-"); // safe filename
|
||||
function attachJson(res, filename, obj) {
|
||||
const json = JSON.stringify(obj, null, 2);
|
||||
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||
res.setHeader("Content-Disposition", `attachment; filename="${filename}"`);
|
||||
return res.send(json);
|
||||
}
|
||||
function isAbs(p) {
|
||||
try { return path.isAbsolute(p); } catch { return false; }
|
||||
}
|
||||
|
||||
/* ------------ Crawl endpoint ------------ */
|
||||
/**
|
||||
* GET /crawl?url=https://site.com&max=50[&stream=1][&download=1][&nostore=1]
|
||||
* - stream=1 : SSE live progress (no download)
|
||||
* - download=1 : respond as a JSON download (attachment)
|
||||
* - nostore=1 : ask crawler not to write files (if supported by your crawler)
|
||||
*/
|
||||
app.get("/crawl", async (req, res) => {
|
||||
try {
|
||||
const { url, max, stream, download, nostore } = req.query;
|
||||
if (!url) return res.status(400).json({ error: "Missing url param" });
|
||||
|
||||
const target = new URL(String(url)); // validate
|
||||
const limit = Math.min(Math.max(parseInt(max ?? "50", 10), 1), 500);
|
||||
const wantsStream =
|
||||
String(stream) === "1" ||
|
||||
(req.get("accept") || "").includes("text/event-stream");
|
||||
|
||||
/* ---------- SSE mode ---------- */
|
||||
if (wantsStream) {
|
||||
if (String(download) === "1") {
|
||||
return res.status(400).json({ error: "download not supported with stream=1" });
|
||||
}
|
||||
res.setHeader("Content-Type", "text/event-stream");
|
||||
res.setHeader("Cache-Control", "no-cache, no-transform");
|
||||
res.setHeader("Connection", "keep-alive");
|
||||
res.flushHeaders?.();
|
||||
const heartbeat = setInterval(() => res.write(":\n\n"), 15000);
|
||||
const send = (obj, evt) => {
|
||||
if (evt) res.write(`event: ${evt}\n`);
|
||||
res.write(`data: ${JSON.stringify(obj)}\n\n`);
|
||||
};
|
||||
send({ ok: true, message: "Crawl started", url: target.toString(), limit }, "started");
|
||||
|
||||
let finished = false;
|
||||
req.on("close", () => { clearInterval(heartbeat); if (!finished) console.warn("SSE client disconnected."); });
|
||||
|
||||
const result = await crawl(
|
||||
target.toString(),
|
||||
limit,
|
||||
(tick) => send(tick),
|
||||
// If your crawler supports it, this avoids writing files during SSE runs:
|
||||
{ persistReports: false, collectPages: true }
|
||||
);
|
||||
|
||||
finished = true;
|
||||
clearInterval(heartbeat);
|
||||
send({ ok: true, done: true, result }, "done");
|
||||
return res.end();
|
||||
}
|
||||
|
||||
/* ---------- Non-streaming mode ---------- */
|
||||
// Ask crawler (if it supports options) to avoid writing files when nostore=1 or download requested.
|
||||
const preferMemory = String(nostore) === "1" || String(download) === "1";
|
||||
const result = await crawl(
|
||||
target.toString(),
|
||||
limit,
|
||||
undefined,
|
||||
preferMemory ? { persistReports: false, collectPages: true } : undefined
|
||||
);
|
||||
|
||||
// If caller wants a downloadable JSON file...
|
||||
if (String(download) === "1") {
|
||||
const filename = `crawl-${ts()}.json`;
|
||||
|
||||
// 1) Best case: crawler returned in-memory data (no disk IO).
|
||||
// Use whichever property your crawler exposes. We try common shapes.
|
||||
const inMemory =
|
||||
result?.jsonData ??
|
||||
result?.pages ??
|
||||
result?.report ??
|
||||
(Array.isArray(result) ? result : null);
|
||||
|
||||
if (inMemory) {
|
||||
return attachJson(res, filename, inMemory);
|
||||
}
|
||||
|
||||
// 2) Fallback: crawler saved a JSON report path that we can stream.
|
||||
const jsonPath = result?.reports?.json;
|
||||
if (jsonPath && fs.existsSync(isAbs(jsonPath) ? jsonPath : path.join(__dirname, jsonPath))) {
|
||||
const abs = isAbs(jsonPath) ? jsonPath : path.join(__dirname, jsonPath);
|
||||
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||
res.setHeader("Content-Disposition", `attachment; filename="${filename}"`);
|
||||
return fs.createReadStream(abs).pipe(res);
|
||||
}
|
||||
|
||||
// 3) Last resort: send the entire result itself as JSON.
|
||||
return attachJson(res, filename, result);
|
||||
}
|
||||
|
||||
// Default JSON (inline, not attachment)
|
||||
return res.json({
|
||||
ok: true,
|
||||
message: "Crawl completed",
|
||||
url: target.toString(),
|
||||
limit,
|
||||
result
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("Crawl error:", err);
|
||||
return res.status(500).json({ error: "Crawl failed", details: String(err?.message ?? err) });
|
||||
}
|
||||
});
|
||||
|
||||
/* ------------ Safety nets ------------ */
|
||||
process.on("unhandledRejection", (e) => console.error("unhandledRejection:", e));
|
||||
process.on("uncaughtException", (e) => console.error("uncaughtException:", e));
|
||||
|
||||
/* ------------ Start server ------------ */
|
||||
app.listen(PORT, "0.0.0.0", () => {
|
||||
console.log(`🚀 Server running at http://localhost:${PORT}`);
|
||||
});
|
||||
80
server.js
Normal file
80
server.js
Normal file
@ -0,0 +1,80 @@
|
||||
import express from "express";
|
||||
import cors from "cors";
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
import crawlRoutes from "./routes/crawl.routes.js";
|
||||
import sitemapRoutes from "./routes/sitemap.routes.js";
|
||||
import authRoutes from "./routes/auth.routes.js"; // Login & Signup endpoints
|
||||
import paymentRoutes from "./routes/payment.route.js";
|
||||
import lighthouseRoutes from "./routes/lighthouse.routes.js"; // <-- ADD THIS
|
||||
import messageRoutes from "./routes/message.routes.js";
|
||||
import cakeOrderRoutes from "./routes/maisondetreats/cakeOrder.routes.js";
|
||||
import blogRoutes from "./routes/blog.routes.js";
|
||||
import { connectDB } from "./config/db.js";
|
||||
import { mailer } from "./utils/mailer.js";
|
||||
|
||||
// ------------------ Load environment ------------------
|
||||
dotenv.config(); // Must be first
|
||||
|
||||
// ------------------ Connect database ------------------
|
||||
await connectDB();
|
||||
|
||||
// ------------------ Express setup ------------------
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 3010;
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
app.use(express.json());
|
||||
app.use(
|
||||
cors({
|
||||
origin: [
|
||||
"http://localhost:3000",
|
||||
"http://127.0.0.1:3000",
|
||||
"https://api.crawlerx.co",
|
||||
"https://app.crawlerx.co",
|
||||
],
|
||||
})
|
||||
);
|
||||
app.use(express.static(path.join(__dirname, "public")));
|
||||
|
||||
// ------------------ SMTP verification ------------------
|
||||
|
||||
console.log("SMTP Host:", process.env.SMTP_HOST);
|
||||
console.log("SMTP Port:", process.env.SMTP_PORT);
|
||||
// ------------------ Routes ------------------
|
||||
app.get("/", (_req, res) => {
|
||||
const viewer = path.join(__dirname, "public", "crawlerx_viewer.html");
|
||||
if (fs.existsSync(viewer)) {
|
||||
return res.sendFile(viewer);
|
||||
} else {
|
||||
return res
|
||||
.type("text/plain")
|
||||
.send("CrawlerX backend is running.");
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/healthz", (_req, res) => res.json({ ok: true, time: new Date().toISOString() }));
|
||||
|
||||
app.use("/crawl", crawlRoutes);
|
||||
app.use("/sitemap", sitemapRoutes);
|
||||
app.use("/api/auth", authRoutes); // Login & Signup endpoints
|
||||
app.use("/api/payment", paymentRoutes);
|
||||
app.use("/api/lighthouse", lighthouseRoutes);
|
||||
app.use("/api/blog", blogRoutes); // All blog/category/comment routes now prefixed with /api/blog
|
||||
app.use("/api/messages", messageRoutes);
|
||||
app.use("/api/cake-orders", cakeOrderRoutes);
|
||||
|
||||
// Serve uploaded files
|
||||
app.use('/uploads', express.static(path.join(process.cwd(), 'uploads')));
|
||||
|
||||
// ------------------ Safety nets ------------------
|
||||
process.on("unhandledRejection", (err) => console.error("Unhandled Rejection:", err));
|
||||
process.on("uncaughtException", (err) => console.error("Uncaught Exception:", err));
|
||||
|
||||
// ------------------ Start server ------------------
|
||||
app.listen(PORT, "0.0.0.0", () => {
|
||||
console.log(`🚀 Server running at http://localhost:${PORT}`);
|
||||
});
|
||||
BIN
uploads/2176fe75251a06cf07a08e32a845b537
Normal file
BIN
uploads/2176fe75251a06cf07a08e32a845b537
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
BIN
uploads/240f5f275adf05f94ee3cb81d3a4c829
Normal file
BIN
uploads/240f5f275adf05f94ee3cb81d3a4c829
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
BIN
uploads/3d2a70806f726d28bd1814bce39b0906
Normal file
BIN
uploads/3d2a70806f726d28bd1814bce39b0906
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
BIN
uploads/866ff6699ca2b462d73b20c4d150b2e0
Normal file
BIN
uploads/866ff6699ca2b462d73b20c4d150b2e0
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
BIN
uploads/8930208f2d9cc158282fc31953abc6a3
Normal file
BIN
uploads/8930208f2d9cc158282fc31953abc6a3
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
BIN
uploads/d6d36b979d636c9f38a712bd3c298563
Normal file
BIN
uploads/d6d36b979d636c9f38a712bd3c298563
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
105
utils/mailer.js
Normal file
105
utils/mailer.js
Normal file
@ -0,0 +1,105 @@
|
||||
import nodemailer from "nodemailer";
|
||||
|
||||
//
|
||||
// Create reusable transporter object
|
||||
//
|
||||
export const mailer = nodemailer.createTransport({
|
||||
host: "mail.crawlerx.co", // your Hestia mail host
|
||||
port: 587, // STARTTLS
|
||||
secure: false, // must be false for 587
|
||||
auth: {
|
||||
user: "info@crawlerx.co", // e.g. info@crawlerx.co
|
||||
pass: "CrawlerX@2025", // mailbox password
|
||||
},
|
||||
name: "mail.crawlerx.co", // explicitly set hostname
|
||||
tls: {
|
||||
rejectUnauthorized: false, // allow self-signed certs
|
||||
},
|
||||
logger: true, // optional: logs connection steps
|
||||
debug: true, // optional: debug SMTP connection
|
||||
});
|
||||
|
||||
|
||||
//
|
||||
// Send welcome / signup email
|
||||
//
|
||||
export async function sendSignupMail(toEmail) {
|
||||
try {
|
||||
await mailer.sendMail({
|
||||
from: `"CrawlerX" <${process.env.SMTP_USER}>`,
|
||||
to: toEmail,
|
||||
subject: "Welcome to CrawlerX",
|
||||
html: `
|
||||
<h2>Welcome!</h2>
|
||||
<p>Your signup was successful. You can now log in and start using the app.</p>
|
||||
`,
|
||||
});
|
||||
console.log(`✅ Signup email sent to ${toEmail}`);
|
||||
} catch (err) {
|
||||
console.error("❌ Error sending signup email:", err);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Send reset-password email with 4-digit code or token link
|
||||
//
|
||||
export async function sendResetPasswordMail(email, token) {
|
||||
try {
|
||||
const resetURL = `${process.env.FRONTEND_URL}/reset-password?email=${email}&token=${token}`;
|
||||
await mailer.sendMail({
|
||||
from: `"CrawlerX" <${process.env.SMTP_USER}>`,
|
||||
to: email,
|
||||
subject: "Reset your password",
|
||||
html: `
|
||||
<p>You requested a password reset.</p>
|
||||
<p>Click here to reset: <a href="${resetURL}">${resetURL}</a></p>
|
||||
<p>This link is valid for 1 hour.</p>
|
||||
`,
|
||||
});
|
||||
console.log(`✅ Reset password email sent to ${email}`);
|
||||
} catch (err) {
|
||||
console.error("❌ Error sending reset password email:", err);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export const sendCakeOrderMail = async (toEmail, orderData) => {
|
||||
try {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: "mail.crawlerx.co",
|
||||
port: 587,
|
||||
secure: false, // use TLS? false for port 587
|
||||
auth: {
|
||||
user: "info@crawlerx.co",
|
||||
pass: "CrawlerX@2025",
|
||||
},
|
||||
tls: {
|
||||
rejectUnauthorized: false, // <--- allow self-signed certificate
|
||||
},
|
||||
});
|
||||
|
||||
const orderItems = Object.entries(orderData)
|
||||
.map(([category, flavours]) => {
|
||||
const items = Object.entries(flavours)
|
||||
.map(([flavour, qty]) => `• ${flavour}: ${qty}`)
|
||||
.join("\n");
|
||||
return `${category}:\n${items}`;
|
||||
})
|
||||
.join("\n\n");
|
||||
|
||||
const mailOptions = {
|
||||
from: `"Maison de Treats" <info@crawlerx.co>`,
|
||||
to: toEmail,
|
||||
subject: "🎉 Your Cake Order Confirmation",
|
||||
text: `Thank you for your order! Here are the details:\n\n${orderItems}`,
|
||||
html: `<h2>Thank you for your order!</h2>
|
||||
<p>Here are your cake order details:</p>
|
||||
<pre>${orderItems}</pre>`,
|
||||
};
|
||||
|
||||
await transporter.sendMail(mailOptions);
|
||||
console.log("Cake order email sent to", toEmail);
|
||||
} catch (err) {
|
||||
console.error("Failed to send cake order email:", err);
|
||||
}
|
||||
};
|
||||
20
utils/sitemap.js
Normal file
20
utils/sitemap.js
Normal file
@ -0,0 +1,20 @@
|
||||
import Sitemapper from "sitemapper";
|
||||
import normalizeUrl from "normalize-url";
|
||||
|
||||
/** Return an array of normalized URLs from <origin>/sitemap.xml (or [] if none). */
|
||||
export async function getSitemapUrls(startUrl) {
|
||||
const origin = new URL(startUrl).origin;
|
||||
const sitemapUrl = `${origin}/sitemap.xml`;
|
||||
const sm = new Sitemapper({ url: sitemapUrl, timeout: 15000 });
|
||||
|
||||
try {
|
||||
const { sites } = await sm.fetch();
|
||||
const out = [];
|
||||
for (const u of sites || []) {
|
||||
try { out.push(normalizeUrl(u, { stripHash: true })); } catch {}
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
8
utils/stripe.js
Normal file
8
utils/stripe.js
Normal file
@ -0,0 +1,8 @@
|
||||
import Stripe from "stripe";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
export const stripe = new Stripe(process.env.STRIPE_SECRET_KEY, {
|
||||
apiVersion: "2024-06-20",
|
||||
});
|
||||
11
utils/urlHelpers.js
Normal file
11
utils/urlHelpers.js
Normal file
@ -0,0 +1,11 @@
|
||||
export function isInternal(base, candidate) {
|
||||
try {
|
||||
const baseUrl = new URL(base);
|
||||
const testUrl = new URL(candidate, base);
|
||||
const protocolOk = testUrl.protocol === "http:" || testUrl.protocol === "https:";
|
||||
const stripWWW = (h) => h.replace(/^www\./i, "");
|
||||
return protocolOk && stripWWW(baseUrl.hostname) === stripWWW(testUrl.hostname);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user