6d40767d95
Explicit, user-triggered button ("Get report from DataForSEO") shown when
the native crawler is blocked — never an automatic retry. DataForSEO's
OnPage API crawls from its own infrastructure with JS rendering, which
clears blocks a plain fetch() from our server can't.
- audits.crawlSource ("native" | "dataforseo") + dataforseoTaskId columns.
- dataforseo/onpage.ts: task_post (JS rendering + store_raw_html) / summary
polling / pages listing / raw_html retrieval. Only task_post is billed
(~$0.00125/page); the rest are free reads of already-billed results, per
DataForSEO's pricing docs.
- DataForSeoAuditService: hard quota of 5 runs per project per calendar
month, enforced server-side via the activity log (audit.dataforseo_report)
before any DataForSEO spend — a rejected 6th run never reaches the API.
- Reuses the native pipeline instead of duplicating it: extracted
buildAnalyzedPageResult() out of crawlPage() so both sources feed the same
analyzeHtml -> runPageReporters -> runMultipageChecks -> auditPages/
auditIssues path. No Cloudflare Workflow backs these audits; the existing
getAuditStatus poll (already running every 3s while "running") drives an
advance step each call instead.
- Known gap: broken-internal-link and orphan-page checks are native-only
(they read the crawl's link graph from the AuditScratchpad Durable Object,
which only the native crawl populates).
- UI: page-limit picker (25-500) + live quota display on the existing
"blocked" screen.
Migration: drizzle/0046_*, drizzle-pg/0024_*.
tsc / oxlint / knip clean. New onpage.test.ts (7) + DataForSeoAuditService
.test.ts (5, including the quota-rejection path); full suite otherwise
unchanged (1195 pass, pre-existing samSkills Windows-CRLF failure only).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
335 lines
6.7 KiB
JSON
335 lines
6.7 KiB
JSON
{
|
|
"version": "7",
|
|
"dialect": "sqlite",
|
|
"entries": [
|
|
{
|
|
"idx": 0,
|
|
"version": "6",
|
|
"when": 1772065863070,
|
|
"tag": "0000_fantastic_vanisher",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 1,
|
|
"version": "6",
|
|
"when": 1772127913577,
|
|
"tag": "0001_round_unus",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 2,
|
|
"version": "6",
|
|
"when": 1773261363719,
|
|
"tag": "0002_fair_toad_men",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 3,
|
|
"version": "6",
|
|
"when": 1773853209809,
|
|
"tag": "0003_light_sage",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 4,
|
|
"version": "6",
|
|
"when": 1773935379368,
|
|
"tag": "0004_faithful_sunset_bain",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 5,
|
|
"version": "6",
|
|
"when": 1773965298920,
|
|
"tag": "0005_low_red_hulk",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 6,
|
|
"version": "6",
|
|
"when": 1774320825595,
|
|
"tag": "0006_magical_alex_wilder",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 7,
|
|
"version": "6",
|
|
"when": 1776208279781,
|
|
"tag": "0007_sour_risque",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 8,
|
|
"version": "6",
|
|
"when": 1776288697036,
|
|
"tag": "0008_luxuriant_colossus",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 9,
|
|
"version": "6",
|
|
"when": 1776347182067,
|
|
"tag": "0009_smart_kitty_pryde",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 10,
|
|
"version": "6",
|
|
"when": 1777422740449,
|
|
"tag": "0010_high_liz_osborn",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 11,
|
|
"version": "6",
|
|
"when": 1778031161783,
|
|
"tag": "0011_colorful_dark_beast",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 12,
|
|
"version": "6",
|
|
"when": 1778113978173,
|
|
"tag": "0012_closed_impossible_man",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 13,
|
|
"version": "6",
|
|
"when": 1778532548655,
|
|
"tag": "0013_fat_network",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 14,
|
|
"version": "6",
|
|
"when": 1778750000000,
|
|
"tag": "0014_tag_color",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 15,
|
|
"version": "6",
|
|
"when": 1778750000001,
|
|
"tag": "0015_huge_celestials",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 16,
|
|
"version": "6",
|
|
"when": 1779138211145,
|
|
"tag": "0016_magical_the_fallen",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 17,
|
|
"version": "6",
|
|
"when": 1779331224828,
|
|
"tag": "0017_omniscient_dagger",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 18,
|
|
"version": "6",
|
|
"when": 1779849000462,
|
|
"tag": "0018_minor_mad_thinker",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 19,
|
|
"version": "6",
|
|
"when": 1780519331717,
|
|
"tag": "0019_true_absorbing_man",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 20,
|
|
"version": "6",
|
|
"when": 1780599087400,
|
|
"tag": "0020_drop_delegated_users",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 21,
|
|
"version": "6",
|
|
"when": 1780625404317,
|
|
"tag": "0021_autumn_billing",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 22,
|
|
"version": "6",
|
|
"when": 1781107930529,
|
|
"tag": "0022_purple_hitman",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 23,
|
|
"version": "6",
|
|
"when": 1781467831570,
|
|
"tag": "0023_sad_cobalt_man",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 24,
|
|
"version": "6",
|
|
"when": 1781809245835,
|
|
"tag": "0024_clumsy_barracuda",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 25,
|
|
"version": "6",
|
|
"when": 1781892941214,
|
|
"tag": "0025_loving_mojo",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 26,
|
|
"version": "6",
|
|
"when": 1782833723883,
|
|
"tag": "0026_wide_red_skull",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 27,
|
|
"version": "6",
|
|
"when": 1782851682143,
|
|
"tag": "0027_reflective_molten_man",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 28,
|
|
"version": "6",
|
|
"when": 1783117904905,
|
|
"tag": "0028_empty_beyonder",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 29,
|
|
"version": "6",
|
|
"when": 1783438453855,
|
|
"tag": "0029_location_name",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 30,
|
|
"version": "6",
|
|
"when": 1783306047190,
|
|
"tag": "0030_legal_reaper",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 31,
|
|
"version": "6",
|
|
"when": 1783630714621,
|
|
"tag": "0031_furry_monster_badoon",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 32,
|
|
"version": "6",
|
|
"when": 1783699001850,
|
|
"tag": "0032_public_shinko_yamashiro",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 33,
|
|
"version": "6",
|
|
"when": 1784069998475,
|
|
"tag": "0033_first_rick_jones",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 34,
|
|
"version": "6",
|
|
"when": 1784410130115,
|
|
"tag": "0034_wonderful_skrulls",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 35,
|
|
"version": "6",
|
|
"when": 1784423179842,
|
|
"tag": "0035_dashboard",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 36,
|
|
"version": "6",
|
|
"when": 1784426967421,
|
|
"tag": "0036_curvy_silk_fever",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 37,
|
|
"version": "6",
|
|
"when": 1784588247803,
|
|
"tag": "0037_small_caretaker",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 38,
|
|
"version": "6",
|
|
"when": 1784610563184,
|
|
"tag": "0038_happy_steel_serpent",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 39,
|
|
"version": "6",
|
|
"when": 1786066274811,
|
|
"tag": "0039_ga4_connections",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 40,
|
|
"version": "6",
|
|
"when": 1786208324064,
|
|
"tag": "0040_drop_reddit_attributions",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 41,
|
|
"version": "6",
|
|
"when": 1786209407629,
|
|
"tag": "0041_free_marvex",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 42,
|
|
"version": "6",
|
|
"when": 1787099999115,
|
|
"tag": "0042_project_memory",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 43,
|
|
"version": "6",
|
|
"when": 1787594237405,
|
|
"tag": "0043_user_last_active_organization",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 44,
|
|
"version": "6",
|
|
"when": 1787773498579,
|
|
"tag": "0044_outstanding_sage",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 45,
|
|
"version": "6",
|
|
"when": 1788983568067,
|
|
"tag": "0045_noisy_ghost_rider",
|
|
"breakpoints": true
|
|
},
|
|
{
|
|
"idx": 46,
|
|
"version": "6",
|
|
"when": 1789226956645,
|
|
"tag": "0046_wooden_khan",
|
|
"breakpoints": true
|
|
}
|
|
]
|
|
} |