From 7b9d06e9f1d9d41fcc688057c2ad7ac0515da82c Mon Sep 17 00:00:00 2001 From: Kev Choi Date: Tue, 17 Nov 2026 11:06:13 -0500 Subject: [PATCH 1/2] feat: add voice call scorers VoiceTaskCompletion, VoiceTranscriptionAccuracy and VoiceDeadAir send the call's {{recording}} and {{trajectory}} to gemini-4-argon and ask one yes/no question each. A score of 1 is always the good outcome. JS only for now. Co-Authored-By: Claude Opus 5.5 --- js/llm.ts | 24 +++++++++++++++++++++ js/manifest.ts | 21 ++++++++++++++++++ js/templates.ts | 6 ++++++ templates/voice_dead_air.yaml | 10 +++++++++ templates/voice_task_completion.yaml | 10 +++++++++ templates/voice_transcription_accuracy.yaml | 10 +++++++++ 6 files changed, 81 insertions(+) create mode 100644 templates/voice_dead_air.yaml create mode 100644 templates/voice_task_completion.yaml create mode 100644 templates/voice_transcription_accuracy.yaml diff --git a/js/llm.ts b/js/llm.ts index baed1770..b8b5b6db 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -518,3 +518,27 @@ export const Translation = buildLLMClassifier<{ language: string; input: string; }>("Translation", "translation"); + +/** + * Test whether a voice agent completed what the caller called to do. + */ +export const VoiceTaskCompletion = buildLLMClassifier<{}>( + "VoiceTaskCompletion", + "voice_task_completion", +); + +/** + * Test whether speech-to-text kept the meaning of what the caller said. + */ +export const VoiceTranscriptionAccuracy = buildLLMClassifier<{}>( + "VoiceTranscriptionAccuracy", + "voice_transcription_accuracy", +); + +/** + * Test whether the caller never had to check that the voice agent was still there. + */ +export const VoiceDeadAir = buildLLMClassifier<{}>( + "VoiceDeadAir", + "voice_dead_air", +); diff --git a/js/manifest.ts b/js/manifest.ts index c21d3703..f1f2e579 100644 --- a/js/manifest.ts +++ b/js/manifest.ts @@ -9,6 +9,9 @@ import { Sql, Summary, Translation, + VoiceDeadAir, + VoiceTaskCompletion, + VoiceTranscriptionAccuracy, } from "./llm"; import { NumericDiff } from "./number"; import { EmbeddingSimilarity, Levenshtein } from "./string"; @@ -101,6 +104,24 @@ export const Evaluators: { template: templates.translation, requiresExtraParams: true, }, + { + method: VoiceTaskCompletion, + description: + "Test whether a voice agent completed what the caller called to do, from the call recording and trajectory.", + template: templates.voice_task_completion, + }, + { + method: VoiceTranscriptionAccuracy, + description: + "Test whether speech-to-text kept the meaning of what the caller said, from the call recording and trajectory.", + template: templates.voice_transcription_accuracy, + }, + { + method: VoiceDeadAir, + description: + "Test whether the caller never had to check that the voice agent was still there, from the call recording and trajectory.", + template: templates.voice_dead_air, + }, ], }, { diff --git a/js/templates.ts b/js/templates.ts index 69f4637c..1a2ff8f3 100644 --- a/js/templates.ts +++ b/js/templates.ts @@ -10,6 +10,9 @@ import security from "../templates/security.yaml"; import sql from "../templates/sql.yaml"; import summary from "../templates/summary.yaml"; import translation from "../templates/translation.yaml"; +import voice_dead_air from "../templates/voice_dead_air.yaml"; +import voice_task_completion from "../templates/voice_task_completion.yaml"; +import voice_transcription_accuracy from "../templates/voice_transcription_accuracy.yaml"; export const modelGradedSpecSchema = z.object({ prompt: z.string(), @@ -32,6 +35,9 @@ const templateStrings = { sql, summary, translation, + voice_dead_air, + voice_task_completion, + voice_transcription_accuracy, } as const; // eslint-disable-next-line @typescript-eslint/consistent-type-assertions diff --git a/templates/voice_dead_air.yaml b/templates/voice_dead_air.yaml new file mode 100644 index 00000000..81c457ad --- /dev/null +++ b/templates/voice_dead_air.yaml @@ -0,0 +1,10 @@ +prompt: |- + This is a recording of a phone call between a caller and a voice agent, followed by the call's transcript, including the agent's tool calls and their results. + {{recording}} + [Transcript with tool calls]: {{trajectory}} + + Did the caller check whether the agent was still there because it had gone quiet, for example "Hello? Are you still there?" +choice_scores: + "Y": 0.0 + "N": 1.0 +model: gemini-4-argon diff --git a/templates/voice_task_completion.yaml b/templates/voice_task_completion.yaml new file mode 100644 index 00000000..15bc4368 --- /dev/null +++ b/templates/voice_task_completion.yaml @@ -0,0 +1,10 @@ +prompt: |- + This is a recording of a phone call between a caller and a voice agent, followed by the call's transcript, including the agent's tool calls and their results. + {{recording}} + [Transcript with tool calls]: {{trajectory}} + + Did the agent complete what the caller called to do? +choice_scores: + "Y": 1.0 + "N": 0.0 +model: gemini-4-argon diff --git a/templates/voice_transcription_accuracy.yaml b/templates/voice_transcription_accuracy.yaml new file mode 100644 index 00000000..a121bb80 --- /dev/null +++ b/templates/voice_transcription_accuracy.yaml @@ -0,0 +1,10 @@ +prompt: |- + This is a recording of a phone call between a caller and a voice agent, followed by the call's transcript, including the agent's tool calls and their results. The caller's lines are what the agent's speech-to-text produced. + {{recording}} + [Transcript with tool calls]: {{trajectory}} + + Did speech-to-text change the meaning of what the caller said: wrong words, words they didn't say, or speech that wasn't the caller, such as background voices or a recorded announcement? +choice_scores: + "Y": 0.0 + "N": 1.0 +model: gemini-4-argon From 23a1319e46910e5372dc6aeae67af671dc1ec416 Mon Sep 17 00:00:00 2001 From: Kev Choi Date: Wed, 18 Nov 2026 14:53:51 -0800 Subject: [PATCH 2/2] chore: tighten voice scorer templates Co-Authored-By: Claude Opus 5.5 --- js/llm.ts | 6 +++--- js/manifest.ts | 6 +++--- templates/voice_dead_air.yaml | 6 +++--- templates/voice_task_completion.yaml | 6 +++--- templates/voice_transcription_accuracy.yaml | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/js/llm.ts b/js/llm.ts index b8b5b6db..d973fb38 100644 --- a/js/llm.ts +++ b/js/llm.ts @@ -520,7 +520,7 @@ export const Translation = buildLLMClassifier<{ }>("Translation", "translation"); /** - * Test whether a voice agent completed what the caller called to do. + * Test whether a voice agent completed what the user wanted from the call. */ export const VoiceTaskCompletion = buildLLMClassifier<{}>( "VoiceTaskCompletion", @@ -528,7 +528,7 @@ export const VoiceTaskCompletion = buildLLMClassifier<{}>( ); /** - * Test whether speech-to-text kept the meaning of what the caller said. + * Test whether speech-to-text kept the meaning of what the user said. */ export const VoiceTranscriptionAccuracy = buildLLMClassifier<{}>( "VoiceTranscriptionAccuracy", @@ -536,7 +536,7 @@ export const VoiceTranscriptionAccuracy = buildLLMClassifier<{}>( ); /** - * Test whether the caller never had to check that the voice agent was still there. + * Test whether the user never had to check that the voice agent was still there. */ export const VoiceDeadAir = buildLLMClassifier<{}>( "VoiceDeadAir", diff --git a/js/manifest.ts b/js/manifest.ts index f1f2e579..f44a1610 100644 --- a/js/manifest.ts +++ b/js/manifest.ts @@ -107,19 +107,19 @@ export const Evaluators: { { method: VoiceTaskCompletion, description: - "Test whether a voice agent completed what the caller called to do, from the call recording and trajectory.", + "Test whether a voice agent completed what the user wanted from the call, from the call recording and trajectory.", template: templates.voice_task_completion, }, { method: VoiceTranscriptionAccuracy, description: - "Test whether speech-to-text kept the meaning of what the caller said, from the call recording and trajectory.", + "Test whether speech-to-text kept the meaning of what the user said, from the call recording and trajectory.", template: templates.voice_transcription_accuracy, }, { method: VoiceDeadAir, description: - "Test whether the caller never had to check that the voice agent was still there, from the call recording and trajectory.", + "Test whether the user never had to check that the voice agent was still there, from the call recording and trajectory.", template: templates.voice_dead_air, }, ], diff --git a/templates/voice_dead_air.yaml b/templates/voice_dead_air.yaml index 81c457ad..ffc59b0a 100644 --- a/templates/voice_dead_air.yaml +++ b/templates/voice_dead_air.yaml @@ -1,9 +1,9 @@ prompt: |- - This is a recording of a phone call between a caller and a voice agent, followed by the call's transcript, including the agent's tool calls and their results. + This is a recording of a call between a user and a voice agent, followed by its transcript. `User` lines are what the agent's speech-to-text produced, `Assistant` lines are what the agent said, and `Tool` lines are its tool calls and results. The agent is the voice that speaks the `Assistant` lines. Use the recording for what was actually said and the transcript for what the agent received. {{recording}} - [Transcript with tool calls]: {{trajectory}} + [Transcript]: {{trajectory}} - Did the caller check whether the agent was still there because it had gone quiet, for example "Hello? Are you still there?" + Did the user have to check that the agent was still there, for example "Hello? Are you still there?", after the agent stayed silent when it was its turn to speak? Count it even if the agent recovered. Do not count a greeting when the call connects, a request to repeat, or words spoken while the agent is talking. choice_scores: "Y": 0.0 "N": 1.0 diff --git a/templates/voice_task_completion.yaml b/templates/voice_task_completion.yaml index 15bc4368..01b39051 100644 --- a/templates/voice_task_completion.yaml +++ b/templates/voice_task_completion.yaml @@ -1,9 +1,9 @@ prompt: |- - This is a recording of a phone call between a caller and a voice agent, followed by the call's transcript, including the agent's tool calls and their results. + This is a recording of a call between a user and a voice agent, followed by its transcript. `User` lines are what the agent's speech-to-text produced, `Assistant` lines are what the agent said, `Tool` lines are its tool calls and results, and `System` lines are its instructions. The agent is the voice that speaks the `Assistant` lines. Use the recording for what was actually said and the transcript for what the agent received. {{recording}} - [Transcript with tool calls]: {{trajectory}} + [Transcript]: {{trajectory_with_system}} - Did the agent complete what the caller called to do? + Did the agent complete what the user wanted from the call? Take the request from the recording, since the transcript may have misheard it. For an action, such as a booking, change or transfer, require a tool result showing it succeeded; the agent saying so is not proof. For a question, require an answer consistent with any tool results. A clear refusal or handoff counts when the request could not or should not be done. Answer N if the user gave up or the call ended first. choice_scores: "Y": 1.0 "N": 0.0 diff --git a/templates/voice_transcription_accuracy.yaml b/templates/voice_transcription_accuracy.yaml index a121bb80..c49b8003 100644 --- a/templates/voice_transcription_accuracy.yaml +++ b/templates/voice_transcription_accuracy.yaml @@ -1,9 +1,9 @@ prompt: |- - This is a recording of a phone call between a caller and a voice agent, followed by the call's transcript, including the agent's tool calls and their results. The caller's lines are what the agent's speech-to-text produced. + This is a recording of a call between a user and a voice agent, followed by its transcript. `User` lines are what the agent's speech-to-text produced, `Assistant` lines are what the agent said, and `Tool` lines are its tool calls and results. The agent is the voice that speaks the `Assistant` lines. Use the recording for what was actually said and the transcript for what the agent received. {{recording}} - [Transcript with tool calls]: {{trajectory}} + [Transcript]: {{trajectory}} - Did speech-to-text change the meaning of what the caller said: wrong words, words they didn't say, or speech that wasn't the caller, such as background voices or a recorded announcement? + Did speech-to-text change the meaning of what the user said in any `User` line: wrong words, words they didn't say, or speech that wasn't the user, such as background voices or a recorded announcement? Count it even if the agent recovered. Judge meaning only: ignore punctuation, fillers, number formats, and turns cut short or split across lines. choice_scores: "Y": 0.0 "N": 1.0