fix(ai-search): use valid Claude model_name and fail fast on unknown ones (#323)

DataForSEO dropped the Claude Sonnet 4.0 family from its llm_responses
catalog, so model_name=claude-sonnet-4-0 was rejected with 'Invalid
Field: model_name' while still billing the failed task. Point Claude at
claude-sonnet-4-5 and validate every model_name against DataForSEO's
accepted catalog before dispatching the paid call.
This commit is contained in:
Ben Senescu 2026-06-30 21:50:42 -04:00 committed by Ben Senescu
parent 4e48f8344f
commit a49ec51e95
3 changed files with 53 additions and 2 deletions

View File

@ -123,10 +123,12 @@ async function runModel(
return reapplyHighlightBrand(shaped, args.highlightBrand);
}
// DataForSEO's Claude catalog caps at the 4.0 family (no Sonnet 4.5+ yet).
// Each value must be a member of ACCEPTED_LLM_MODEL_NAMES in dataforseo/ai.ts,
// which mirrors DataForSEO's llm_responses/models catalog. DataForSEO dropped
// the Claude Sonnet 4.0 family, so we target the 4.5 alias (latest dated 4.5).
const MODEL_NAMES: Record<PromptExplorerModel, string> = {
chat_gpt: "gpt-5",
claude: "claude-sonnet-4-0",
claude: "claude-sonnet-4-5",
gemini: "gemini-2.5-pro",
perplexity: "sonar-reasoning-pro",
};

View File

@ -310,6 +310,24 @@ export async function fetchLlmCrossAggregatedMetrics(
type LlmResponseModelSlug = "chat_gpt" | "claude" | "gemini" | "perplexity";
/**
* Accepted `model_name` values per slug, mirroring DataForSEO's
* `/ai_optimization/{model}/llm_responses/models` catalog (verified 2026-06-30).
* We validate against this before dispatching because DataForSEO BILLS a task
* that fails with `Invalid Field: 'model_name'` a stale or mistyped model name
* would otherwise pay for a guaranteed-rejected call. DataForSEO resolves a
* basic alias (e.g. `claude-sonnet-4-5`) to its latest dated version.
*/
const ACCEPTED_LLM_MODEL_NAMES: Record<
LlmResponseModelSlug,
ReadonlySet<string>
> = {
chat_gpt: new Set(["gpt-5"]),
claude: new Set(["claude-sonnet-4-5", "claude-sonnet-4-6"]),
gemini: new Set(["gemini-2.5-pro"]),
perplexity: new Set(["sonar-reasoning-pro", "sonar-pro", "sonar"]),
};
type LlmResponsesInput = {
userPrompt: string;
modelSlug: LlmResponseModelSlug;
@ -348,6 +366,15 @@ function buildPerplexityLlmResponseRequest(
export async function fetchLlmResponse(
input: LlmResponsesInput,
): Promise<DataforseoApiResponse<LlmResponseResult>> {
// Fail fast on an unknown model_name: DataForSEO charges for tasks that fail
// with `Invalid Field: 'model_name'`, so we must never dispatch one.
if (!ACCEPTED_LLM_MODEL_NAMES[input.modelSlug].has(input.modelName)) {
throw new AppError(
"VALIDATION_ERROR",
`Unsupported DataForSEO model_name "${input.modelName}" for ${input.modelSlug}`,
);
}
// DataForSEO's Gemini endpoint rejects `web_search_country_iso_code` with a
// 40501 "Invalid Field" error. The other three models accept it.
const supportsCountry = input.modelSlug !== "gemini";

View File

@ -356,3 +356,25 @@ describe("DataForSEO SDK-backed endpoints", () => {
]);
});
});
describe("fetchLlmResponse model_name validation", () => {
beforeEach(() => {
vi.restoreAllMocks();
});
it("rejects an unknown model_name before dispatching a paid LLM task", async () => {
const fetchMock = vi.fn<typeof fetch>();
vi.stubGlobal("fetch", fetchMock);
await expect(
fetchLlmResponse({
userPrompt: "What is OpenSEO?",
modelSlug: "claude",
// DataForSEO dropped this from its catalog; it must never be dispatched.
modelName: "claude-sonnet-4-0",
}),
).rejects.toThrow(/Unsupported DataForSEO model_name/);
expect(fetchMock).not.toHaveBeenCalled();
});
});