diff --git a/js/llm.ts b/js/llm.ts index baed1770..d973fb38 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -518,3 +518,27 @@ export const Translation = buildLLMClassifier<{ language: string; input: string; }>("Translation", "translation"); + +/** + * Test whether a voice agent completed what the user wanted from the call. + */ +export const VoiceTaskCompletion = buildLLMClassifier<{}>( + "VoiceTaskCompletion", + "voice_task_completion", +); + +/** + * Test whether speech-to-text kept the meaning of what the user said. + */ +export const VoiceTranscriptionAccuracy = buildLLMClassifier<{}>( + "VoiceTranscriptionAccuracy", + "voice_transcription_accuracy", +); + +/** + * Test whether the user never had to check that the voice agent was still there. + */ +export const VoiceDeadAir = buildLLMClassifier<{}>( + "VoiceDeadAir", + "voice_dead_air", +); diff --git a/js/manifest.ts b/js/manifest.ts index c21d3703..f44a1610 100644 --- a/js/manifest.ts +++ b/js/manifest.ts @@ -9,6 +9,9 @@ import { Sql, Summary, Translation, + VoiceDeadAir, + VoiceTaskCompletion, + VoiceTranscriptionAccuracy, } from "./llm"; import { NumericDiff } from "./number"; import { EmbeddingSimilarity, Levenshtein } from "./string"; @@ -101,6 +104,24 @@ export const Evaluators: { template: templates.translation, requiresExtraParams: true, }, + { + method: VoiceTaskCompletion, + description: + "Test whether a voice agent completed what the user wanted from the call, from the call recording and trajectory.", + template: templates.voice_task_completion, + }, + { + method: VoiceTranscriptionAccuracy, + description: + "Test whether speech-to-text kept the meaning of what the user said, from the call recording and trajectory.", + template: templates.voice_transcription_accuracy, + }, + { + method: VoiceDeadAir, + description: + "Test whether the user never had to check that the voice agent was still there, from the call recording and trajectory.", + template: templates.voice_dead_air, + }, ], }, { diff --git a/js/templates.ts b/js/templates.ts index 69f4637c..1a2ff8f3 100644 --- a/js/templates.ts +++ b/js/templates.ts @@ -10,6 +10,9 @@ import security from "../templates/security.yaml"; import sql from "../templates/sql.yaml"; import summary from "../templates/summary.yaml"; import translation from "../templates/translation.yaml"; +import voice_dead_air from "../templates/voice_dead_air.yaml"; +import voice_task_completion from "../templates/voice_task_completion.yaml"; +import voice_transcription_accuracy from "../templates/voice_transcription_accuracy.yaml"; export const modelGradedSpecSchema = z.object({ prompt: z.string(), @@ -32,6 +35,9 @@ const templateStrings = { sql, summary, translation, + voice_dead_air, + voice_task_completion, + voice_transcription_accuracy, } as const; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions diff --git a/templates/voice_dead_air.yaml b/templates/voice_dead_air.yaml new file mode 100644 index 00000000..ffc59b0a --- /dev/null +++ b/templates/voice_dead_air.yaml @@ -0,0 +1,10 @@ +prompt: |- + This is a recording of a call between a user and a voice agent, followed by its transcript. `User` lines are what the agent's speech-to-text produced, `Assistant` lines are what the agent said, and `Tool` lines are its tool calls and results. The agent is the voice that speaks the `Assistant` lines. Use the recording for what was actually said and the transcript for what the agent received. + {{recording}} + [Transcript]: {{trajectory}} + + Did the user have to check that the agent was still there, for example "Hello? Are you still there?", after the agent stayed silent when it was its turn to speak? Count it even if the agent recovered. Do not count a greeting when the call connects, a request to repeat, or words spoken while the agent is talking. +choice_scores: + "Y": 0.0 + "N": 1.0 +model: gemini-4-argon diff --git a/templates/voice_task_completion.yaml b/templates/voice_task_completion.yaml new file mode 100644 index 00000000..01b39051 --- /dev/null +++ b/templates/voice_task_completion.yaml @@ -0,0 +1,10 @@ +prompt: |- + This is a recording of a call between a user and a voice agent, followed by its transcript. `User` lines are what the agent's speech-to-text produced, `Assistant` lines are what the agent said, `Tool` lines are its tool calls and results, and `System` lines are its instructions. The agent is the voice that speaks the `Assistant` lines. Use the recording for what was actually said and the transcript for what the agent received. + {{recording}} + [Transcript]: {{trajectory_with_system}} + + Did the agent complete what the user wanted from the call? Take the request from the recording, since the transcript may have misheard it. For an action, such as a booking, change or transfer, require a tool result showing it succeeded; the agent saying so is not proof. For a question, require an answer consistent with any tool results. A clear refusal or handoff counts when the request could not or should not be done. Answer N if the user gave up or the call ended first. +choice_scores: + "Y": 1.0 + "N": 0.0 +model: gemini-4-argon diff --git a/templates/voice_transcription_accuracy.yaml b/templates/voice_transcription_accuracy.yaml new file mode 100644 index 00000000..c49b8003 --- /dev/null +++ b/templates/voice_transcription_accuracy.yaml @@ -0,0 +1,10 @@ +prompt: |- + This is a recording of a call between a user and a voice agent, followed by its transcript. `User` lines are what the agent's speech-to-text produced, `Assistant` lines are what the agent said, and `Tool` lines are its tool calls and results. The agent is the voice that speaks the `Assistant` lines. Use the recording for what was actually said and the transcript for what the agent received. + {{recording}} + [Transcript]: {{trajectory}} + + Did speech-to-text change the meaning of what the user said in any `User` line: wrong words, words they didn't say, or speech that wasn't the user, such as background voices or a recorded announcement? Count it even if the agent recovered. Judge meaning only: ignore punctuation, fillers, number formats, and turns cut short or split across lines. +choice_scores: + "Y": 0.0 + "N": 1.0 +model: gemini-4-argon