/** * Seed the local D1 database with rank tracking data from DataForSEO. * * Usage: * pnpm seed:rank-tracking --domain=example.com [--projectId=xxx] * * Requires: * - DATAFORSEO_API_KEY in .env.local or .env * - Local D1 database (run `pnpm db:migrate:local` first) * - At least one project in the database (start the dev server once) */ import process from "node:process"; import { getPlatformProxy } from "wrangler"; import { drizzle } from "drizzle-orm/d1"; import { eq } from "drizzle-orm"; import { DataforseoLabsApi, DataforseoLabsGoogleRankedKeywordsLiveRequestInfo, } from "dataforseo-client"; import * as schema from "../src/db/schema"; import type { DomainRankedKeywordItem } from "../src/server/lib/dataforseo"; import { loadLocalEnv, parseArgs } from "./cli-utils"; loadLocalEnv(); const args = parseArgs(process.argv.slice(2)); await main(); async function main() { const domain = normalizeDomain(args.domain); if (!domain) { exitWithUsage("Missing --domain argument."); } const apiKey = process.env.DATAFORSEO_API_KEY; if (!apiKey) { exitWithUsage("Missing DATAFORSEO_API_KEY. Set it in .env.local or .env."); } console.log(`Setting up local D1 connection...`); const { env, dispose } = await getPlatformProxy<{ DB: D1Database }>(); const db = drizzle(env.DB, { schema }); try { // Resolve project const projectId = args.projectId ?? (await findFirstProject(db)); if (!projectId) { exitWithUsage( "No projects found in local DB. Start the dev server and create a project first.", ); } const existingProject = await db.query.projects.findFirst({ where: eq(schema.projects.id, projectId), }); if (!existingProject) { exitWithUsage(`Project ${projectId} not found in local DB.`); } console.log(`Using project: ${existingProject.name} (${projectId})`); // Check for existing config const existingConfig = await db.query.rankTrackingConfigs.findFirst({ where: eq(schema.rankTrackingConfigs.domain, domain), }); if (existingConfig) { console.log( `Rank tracking config already exists for ${domain} — skipping. Delete it manually to re-seed.`, ); return; } // Fetch top 50 keywords from DataForSEO console.log(`Fetching top 50 keywords for ${domain} from DataForSEO...`); const rankedItems = await fetchRankedKeywords(apiKey, domain, 50); console.log(`Got ${rankedItems.length} ranked keywords.`); if (rankedItems.length === 0) { console.log("No keywords found for this domain. Nothing to seed."); return; } // Map to keyword/position/url const keywords = rankedItems .map(mapKeywordItem) .filter((k): k is NonNullable => k !== null); console.log(`Mapped ${keywords.length} valid keywords.`); // Generate IDs const configId = crypto.randomUUID(); const runId = crypto.randomUUID(); const now = new Date().toISOString(); // Insert config await db.insert(schema.rankTrackingConfigs).values({ id: configId, projectId, domain, locationCode: 2840, languageCode: "en", devices: "mobile", serpDepth: 20, scheduleInterval: "weekly", isActive: true, lastCheckedAt: now, }); console.log(`Created config for ${domain} (mobile, weekly).`); // Insert keywords (batch individual statements to respect D1 bind limits) const keywordRows = keywords.map((k) => ({ id: crypto.randomUUID(), configId, keyword: k.keyword, })); const keywordStmts = keywordRows.map((row) => db.insert(schema.rankTrackingKeywords).values(row).onConflictDoNothing(), ); if (keywordStmts.length > 0) { const [first, ...rest] = keywordStmts; await db.batch([first, ...rest]); } console.log(`Inserted ${keywordRows.length} keywords.`); // Insert completed run await db.insert(schema.rankCheckRuns).values({ id: runId, configId, projectId, status: "completed", keywordsTotal: keywordRows.length, keywordsChecked: keywordRows.length, completedAt: now, }); console.log(`Created completed run.`); // Insert snapshots (mobile device) const snapshotStmts = keywordRows.map((kw, i) => db .insert(schema.rankSnapshots) .values({ runId, trackingKeywordId: kw.id, keyword: kw.keyword, device: "mobile" as const, position: keywords[i].position, url: keywords[i].url, serpFeatures: null, }) .onConflictDoNothing(), ); if (snapshotStmts.length > 0) { const [first, ...rest] = snapshotStmts; await db.batch([first, ...rest]); } console.log(`Inserted ${keywordRows.length} snapshots.`); console.log( `\nDone! Seeded rank tracking for ${domain} with ${keywords.length} keywords.`, ); } finally { await dispose(); } } // --------------------------------------------------------------------------- // DataForSEO // --------------------------------------------------------------------------- async function fetchRankedKeywords( apiKey: string, domain: string, limit: number, ): Promise { const api = new DataforseoLabsApi("https://api.dataforseo.com", { fetch: (url: RequestInfo, init?: RequestInit) => { const headers = new Headers(init?.headers); headers.set("Authorization", `Basic ${apiKey}`); return fetch(url, { ...init, headers }); }, }); const req = new DataforseoLabsGoogleRankedKeywordsLiveRequestInfo({ target: domain, location_code: 2840, language_code: "en", limit, order_by: ["keyword_data.keyword_info.search_volume,desc"], }); const response = await api.googleRankedKeywordsLive([req]); if (!response || response.status_code !== 20000) { throw new Error( `DataForSEO error: ${response?.status_message ?? "unknown"}`, ); } const task = response.tasks?.[0]; if (!task || task.status_code !== 20000) { throw new Error( `DataForSEO task error: ${task?.status_message ?? "no task returned"}`, ); } return task.result?.[0]?.items ?? []; } // --------------------------------------------------------------------------- // Mapping (mirrors DomainService.mapKeywordItem) // --------------------------------------------------------------------------- function mapKeywordItem(item: DomainRankedKeywordItem) { const keywordData = item.keyword_data; const rankedSerpElement = item.ranked_serp_element; const serpItem = rankedSerpElement?.serp_item; const keyword = keywordData?.keyword ?? item.keyword; if (!keyword) return null; const position = serpItem?.rank_absolute ?? rankedSerpElement?.rank_absolute ?? null; const url = serpItem?.url ?? rankedSerpElement?.url ?? null; return { keyword: keyword.toLowerCase().trim(), position: position != null ? Math.round(position) : null, url, }; } // --------------------------------------------------------------------------- // DB helpers // --------------------------------------------------------------------------- type SeedDb = ReturnType>; async function findFirstProject(db: SeedDb): Promise { const row = await db.query.projects.findFirst(); return row?.id ?? null; } // --------------------------------------------------------------------------- // CLI helpers // --------------------------------------------------------------------------- function normalizeDomain(raw: string | undefined): string | undefined { if (!raw) return undefined; return raw .trim() .toLowerCase() .replace(/^https?:\/\//u, "") .replace(/\/.*$/u, "") .replace(/^www\./u, ""); } function exitWithUsage(message: string): never { console.error(message); console.error( "Usage: pnpm seed:rank-tracking --domain=example.com [--projectId=xxx]", ); process.exit(1); }