← back to Butlr
ai-agent: fix 3 bugs exposed by AAA call PSmRTAJ5
d68c3bfa06c0d40dcb2024c27b5652d4ad08053b · 2026-05-13 07:31:26 -0700 · SteveStudio2
The AAA Roadside call (Steve testing membership verification) exposed
three prompt-engineering bugs:
1. AI greeted rep "Carol" as "Sarah" because Sarah was in the worked
example and the model copied it literally. Fix: explicit "DO NOT
memorize names from examples below. Extract from each input. The
name CHANGES per call" instruction + 4 worked examples with 4
different names (Sarah/Carol/Marcus/no-name).
2. AI pressed digits not present in the menu — pressed 4 when "for
roadside press 1, for travel press 2" was offered. Fix: "ONLY
press a digit that was EXPLICITLY mentioned. The digit MUST
literally appear in the input." Plus new yes/no question rule
(the original Kentucky prompt got pressed-0 when it should have
been "No" spoken aloud).
3. AI hung up claiming "goal achieved" on empty input right after
greeting Carol — never actually heard the answer. Fix: GOAL
ACHIEVED now requires "the rep EXPLICITLY answered the goal
question". Empty input now ALWAYS routes to wait, never hangup.
Also added rule 7: rep follow-up handler so when the rep asks for
phone-on-account or name-on-account, we answer concisely with the
real data.
Files touched
A lib/call-overrides.jsM routes/listen.jsM routes/twilio-webhooks.jsM views/public/listen.ejs
Diff
commit d68c3bfa06c0d40dcb2024c27b5652d4ad08053b
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Wed May 13 07:31:26 2026 -0700
ai-agent: fix 3 bugs exposed by AAA call PSmRTAJ5
The AAA Roadside call (Steve testing membership verification) exposed
three prompt-engineering bugs:
1. AI greeted rep "Carol" as "Sarah" because Sarah was in the worked
example and the model copied it literally. Fix: explicit "DO NOT
memorize names from examples below. Extract from each input. The
name CHANGES per call" instruction + 4 worked examples with 4
different names (Sarah/Carol/Marcus/no-name).
2. AI pressed digits not present in the menu — pressed 4 when "for
roadside press 1, for travel press 2" was offered. Fix: "ONLY
press a digit that was EXPLICITLY mentioned. The digit MUST
literally appear in the input." Plus new yes/no question rule
(the original Kentucky prompt got pressed-0 when it should have
been "No" spoken aloud).
3. AI hung up claiming "goal achieved" on empty input right after
greeting Carol — never actually heard the answer. Fix: GOAL
ACHIEVED now requires "the rep EXPLICITLY answered the goal
question". Empty input now ALWAYS routes to wait, never hangup.
Also added rule 7: rep follow-up handler so when the rep asks for
phone-on-account or name-on-account, we answer concisely with the
real data.
---
lib/call-overrides.js | 27 ++++++++++
routes/listen.js | 29 +++++++++++
routes/twilio-webhooks.js | 123 +++++++++++++++++++++++++++++++++++-----------
views/public/listen.ejs | 84 +++++++++++++++++++++++++++----
4 files changed, 226 insertions(+), 37 deletions(-)
diff --git a/lib/call-overrides.js b/lib/call-overrides.js
new file mode 100644
index 0000000..2b0c483
--- /dev/null
+++ b/lib/call-overrides.js
@@ -0,0 +1,27 @@
+// Live human-in-the-loop overrides for in-progress calls.
+//
+// The listen page (gated to call owner) can POST one of:
+// /api/calls/:id/say — speak this text (in caller's voice) on the next gather turn
+// /api/calls/:id/hangup — say goodbye and hang up on the next gather turn
+//
+// The gather handler consumes any pending override BEFORE asking the LLM.
+// Per-call queue; consumed once, then cleared.
+
+const OVERRIDES = new Map(); // callId → { sayText?: string, hangup?: boolean }
+
+function set(callId, patch) {
+ const cur = OVERRIDES.get(callId) || {};
+ OVERRIDES.set(callId, { ...cur, ...patch });
+}
+
+function consume(callId) {
+ const cur = OVERRIDES.get(callId);
+ if (cur) OVERRIDES.delete(callId);
+ return cur || {};
+}
+
+function peek(callId) {
+ return OVERRIDES.get(callId) || {};
+}
+
+module.exports = { set, consume, peek };
diff --git a/routes/listen.js b/routes/listen.js
index ada8067..f267c2a 100644
--- a/routes/listen.js
+++ b/routes/listen.js
@@ -10,6 +10,7 @@ const path = require('path');
const fs = require('fs');
const data = require('../lib/data');
const transcribe = require('../lib/transcribe');
+const callOverrides = require('../lib/call-overrides');
const router = express.Router();
@@ -76,4 +77,32 @@ router.get('/api/calls/:call_id/has-transcript', (req, res) => {
res.json({ ok: true, exists: true, size_bytes: fs.statSync(fp).size });
});
+// Live takeover — speak as the call owner on the next gather turn.
+// The text gets synthesized through ElevenLabs (cloned voice if VOICE_ID
+// is set to the user's IVC) and played on the call. Best UX is to type
+// quickly while the rep is still talking; the speak lands within ~3-20s
+// depending on where in the gather cycle we are.
+router.post('/api/calls/:call_id/say', express.json(), (req, res) => {
+ const callId = req.params.call_id;
+ if (!/^[A-Za-z0-9_-]{6,16}$/.test(callId)) return res.status(404).json({ ok: false });
+ const call = data.getCall(callId, req.user && req.user.id, false);
+ if (!call) return res.status(404).json({ ok: false });
+ const text = String((req.body && req.body.text) || '').trim().slice(0, 500);
+ if (!text) return res.status(400).json({ ok: false, error: 'no_text' });
+ callOverrides.set(callId, { sayText: text });
+ console.log(`[takeover] say queued for ${callId}: "${text.slice(0, 80)}"`);
+ res.json({ ok: true });
+});
+
+// Live takeover — politely end the call on the next gather turn.
+router.post('/api/calls/:call_id/hangup', (req, res) => {
+ const callId = req.params.call_id;
+ if (!/^[A-Za-z0-9_-]{6,16}$/.test(callId)) return res.status(404).json({ ok: false });
+ const call = data.getCall(callId, req.user && req.user.id, false);
+ if (!call) return res.status(404).json({ ok: false });
+ callOverrides.set(callId, { hangup: true });
+ console.log(`[takeover] hangup queued for ${callId}`);
+ res.json({ ok: true });
+});
+
module.exports = router;
diff --git a/routes/twilio-webhooks.js b/routes/twilio-webhooks.js
index 63d934b..3d2def5 100644
--- a/routes/twilio-webhooks.js
+++ b/routes/twilio-webhooks.js
@@ -22,6 +22,7 @@ const { buildAnnouncement } = require('../lib/twilio');
const elevenlabs = require('../lib/elevenlabs');
const data = require('../lib/data');
const transcribe = require('../lib/transcribe');
+const callOverrides = require('../lib/call-overrides');
// IVR DTMF lookup — given a call's business phone, return the configured
// digit sequence (delay → press) for navigating to a human. Pulled from
@@ -159,6 +160,32 @@ router.post('/gather/:call_id', express.urlencoded({ extended: true }), async (r
if (incoming) turns.push({ role: 'user', content: incoming });
console.log(`[ai-agent] call=${callId} heard: "${incoming.slice(0,80)}" turn=${turns.length}`);
+ // ── Live takeover check — skip LLM entirely if owner queued an action ────
+ // Owner can POST to /api/calls/:id/say or /hangup from the listen page;
+ // we consume that here and short-circuit the LLM round-trip.
+ const override = callOverrides.consume(callId);
+ if (override.hangup) {
+ console.log(`[takeover] call=${callId} hangup`);
+ const goodbye = await playOrSay('Thank you. Goodbye.', pub);
+ data.updateStatus(callId, 'done');
+ return res.type('text/xml').send(`<?xml version="1.0" encoding="UTF-8"?>\n<Response>${goodbye}<Hangup/></Response>`);
+ }
+ if (override.sayText) {
+ console.log(`[takeover] call=${callId} say: "${override.sayText.slice(0,80)}"`);
+ turns.push({ role: 'assistant', content: `[OWNER SPOKE]: ${override.sayText}` });
+ const speak = await playOrSay(override.sayText, pub);
+ const continueGather = `<Gather input="speech dtmf"
+ action="${pub}/twilio/gather/${callId}"
+ method="POST"
+ speechTimeout="3"
+ timeout="20"
+ actionOnEmptyResult="true"
+ speechModel="phone_call">
+ <Pause length="20"/>
+ </Gather>`;
+ return res.type('text/xml').send(`<?xml version="1.0" encoding="UTF-8"?>\n<Response>${speak}${continueGather}</Response>`);
+ }
+
// Build the LLM prompt
// Action-first design: model must emit a JSON {action:...} object whenever
// the input contains anything that resembles an IVR menu, hold music,
@@ -167,41 +194,81 @@ router.post('/gather/:call_id', express.urlencoded({ extended: true }), async (r
// call exposed that qwen2.5:7b would deflect with "One moment please…"
// instead of pressing a digit; this prompt corrects that with worked
// examples for every action type.
- const systemPrompt = `You are Butlr, an automated phone assistant calling ${call.business_name} for ${call.callback_name || 'the customer'}.
+ const goalShort = call.goal.length > 80 ? call.goal.slice(0, 80) + '…' : call.goal;
+ const customerName = call.callback_name || 'a customer';
+ const systemPrompt = `You are Butlr, an automated phone assistant calling ${call.business_name} for ${customerName}.
GOAL: ${call.goal}
═══ DECISION TABLE — pick ONE action per turn ═══
-You will hear text from the other end of the line. Identify what KIND of input it is, then emit the matching response. JSON actions MUST be on their own line with no other text. Examples below — match the PATTERN, substitute the right digit/reason:
-
-1. IVR MENU heard (any list of "press N for X" / "for X press N" / "say or press N") → emit JSON, no prose:
- Input: "For sales, press 2. For service, press 3. For store hours and address, press 5."
- Output: {"action":"press","digit":"5"}
-
- Input: "Para español, oprima 9. For English, press 1."
- Output: {"action":"press","digit":"1"}
-
-2. HOLD MUSIC or "please continue holding" / "your call is important" / silence → wait:
- Input: "Please continue to hold. Your call is important to us."
- Output: {"action":"wait"}
-
-3. VERIFICATION request that ONLY the customer can answer (full SSN, verbal password, full DOB) → escalate:
- Input: "I'll need your full social security number and date of birth to continue."
- Output: {"action":"escalate","reason":"rep asked for full SSN + DOB"}
-
-4. GOAL ACHIEVED or call should end (rep gave the info, transferred, said goodbye) → hangup:
- Input: "Our address is 1234 Pico Boulevard and we're open ten to six."
- Output: {"action":"hangup","reason":"got address and hours"}
-
-5. HUMAN REP greets you (only after pressing through IVR) → speak ONE short sentence introducing yourself + goal, no JSON:
- Input: "Thanks for calling MacEnthusiasts, this is Sarah, how can I help?"
- Output: Hi Sarah, I'm calling for ${call.callback_name || 'a customer'} — quick question about ${call.goal.slice(0, 40)}…
+You will hear text from the other end of the line. Identify what KIND of input it is, then emit the matching response. JSON actions MUST be on their own line with no other text.
+
+1. IVR MENU heard (any list of "press N for X" / "for X press N" / "say or press N") → emit JSON.
+ CRITICAL: ONLY press a digit that was EXPLICITLY mentioned in the menu. NEVER invent a digit.
+ If the menu has options but none directly serves the goal, press 0 (operator).
+ The digit you output MUST literally appear in the input you just heard.
+ Examples:
+ Input: "For sales, press 2. For service, press 3. For store hours, press 5."
+ Output: {"action":"press","digit":"5"}
+ Input: "Para español, oprima 9. For English, press 1."
+ Output: {"action":"press","digit":"1"}
+ Input: "For roadside, press 1. For membership, press 3." (goal: check membership)
+ Output: {"action":"press","digit":"3"}
+
+2. YES/NO QUESTION from IVR ("are you calling from X? Say yes or no") → speak the word, NOT JSON.
+ Input: "Are you calling from Kentucky? Say yes or no."
+ Output: No
+ Input: "Is this a roadside emergency? Yes or no?"
+ Output: No
+
+3. HOLD MUSIC / "please continue holding" / silence / empty input → wait:
+ Input: "Please continue to hold. Your call is important to us."
+ Output: {"action":"wait"}
+ Input: "" (empty — they didn't speak, or our speech got dropped)
+ Output: {"action":"wait"}
+
+4. VERIFICATION request only the customer can answer (full SSN, password, full DOB) → escalate:
+ Input: "I'll need your full social security number and date of birth."
+ Output: {"action":"escalate","reason":"rep asked for full SSN + DOB"}
+
+5. GOAL ACHIEVED — ONLY when the rep EXPLICITLY answered the goal question.
+ "Goal achieved" means the rep gave you the actual answer, not just greeted you.
+ NEVER hang up on empty input. NEVER hang up immediately after your own introduction.
+ Goal "verify membership" — achieved means: "Yes that membership is active" or "No it expired".
+ Goal "get store hours" — achieved means: "We're open 10 to 6 Monday-Saturday".
+ Examples:
+ Input: "Yes, membership is active and renews March 2027."
+ Output: {"action":"hangup","reason":"confirmed active through 2027"}
+ Input: "We're open 10 to 6 Monday through Saturday."
+ Output: {"action":"hangup","reason":"got store hours M-Sat 10-6"}
+
+6. HUMAN REP greets you (after pressing through IVR) → speak ONE short sentence using THE ACTUAL NAME they JUST SAID.
+ Extract the rep's name from the input. Common patterns: "this is NAME", "my name is NAME", "I'm NAME".
+ If no name was said, say "Hi" with no name. DO NOT memorize names from examples below.
+ Examples (notice the name CHANGES — extract from each input, never copy from another):
+ Input: "Thanks for calling MacEnthusiasts, this is Sarah, how can I help?"
+ Output: Hi Sarah, I'm calling for ${customerName} — quick question about ${goalShort}
+ Input: "Hi this is Carol, are you safe and where are you located?"
+ Output: Hi Carol, I'm calling for ${customerName} — quick question about ${goalShort}
+ Input: "Triple A this is Marcus how can I help?"
+ Output: Hi Marcus, I'm calling for ${customerName} — quick question about ${goalShort}
+ Input: "Customer service, how can I help?" (no name given)
+ Output: Hi, I'm calling for ${customerName} — quick question about ${goalShort}
+
+7. REP FOLLOW-UP after your intro (clarifying question, asks for lookup info) → answer concisely.
+ Input: "Sure, what's the phone or membership number on the account?"
+ Output: The phone on file is ${call.callback_phone || 'unknown'}.
+ Input: "What's the name on the account?"
+ Output: It's ${customerName}.
═══ HARD RULES ═══
-- NEVER reply with "One moment please" or "I'm checking on that" — those are useless deflections.
-- If unsure which digit, press 0 (usually goes to operator): {"action":"press","digit":"0"}
-- If the IVR repeats itself, you MUST press a digit on the SECOND repeat, not wait again.
+- NEVER greet a rep by a name they didn't just say. If no name was given, just "Hi".
+- NEVER press a digit that wasn't explicitly offered in the menu.
+- NEVER claim "goal achieved" without an explicit answer from the rep.
+- NEVER reply with "One moment please" or "I'm checking on that".
+- Empty input → wait. Don't hangup. Don't speak filler.
+- If the IVR repeats itself, press 0 on the SECOND repeat.
- Keep ANY prose reply under 20 words.
- Account info if asked: last4 ${(call.account_number||'').slice(-4) || 'n/a'}, SSN ${call.last4_ssn || 'n/a'}, ZIP ${call.billing_zip || 'n/a'}, DOB ${call.date_of_birth || 'n/a'}.
diff --git a/views/public/listen.ejs b/views/public/listen.ejs
index 3bcb551..1334c56 100644
--- a/views/public/listen.ejs
+++ b/views/public/listen.ejs
@@ -44,15 +44,42 @@
</div>
</div>
- <section style="margin-top: 18px; padding: 16px; border: 1px solid var(--line); border-radius: 10px; background: var(--panel);">
- <header style="display:flex; align-items:center; gap:10px; margin-bottom:10px;">
- <strong style="font-size:15px;">🎩 Live AI-agent transcript</strong>
- <span id="transcript-count" class="muted" style="font-size:12px; margin-left:auto;">— turns</span>
- </header>
- <div id="transcript-feed" style="max-height: 480px; overflow-y: auto; font-size: 14px; line-height: 1.55;">
- <p class="muted" style="font-style: italic;">Waiting for AI-agent turns…</p>
- </div>
- </section>
+ <!-- Two-column layout: live transcript LEFT, takeover panel RIGHT -->
+ <div style="display:grid; grid-template-columns: 1fr 1fr; gap:16px; margin-top:18px;">
+
+ <section style="padding: 16px; border: 1px solid var(--line); border-radius: 10px; background: var(--panel);">
+ <header style="display:flex; align-items:center; gap:10px; margin-bottom:10px;">
+ <strong style="font-size:15px;">🎩 Live AI-agent transcript</strong>
+ <span id="transcript-count" class="muted" style="font-size:12px; margin-left:auto;">— turns</span>
+ </header>
+ <div id="transcript-feed" style="max-height: 480px; overflow-y: auto; font-size: 14px; line-height: 1.55;">
+ <p class="muted" style="font-style: italic;">Waiting for AI-agent turns…</p>
+ </div>
+ </section>
+
+ <section style="padding: 16px; border: 1px solid var(--line); border-radius: 10px; background: var(--panel);">
+ <header style="display:flex; align-items:center; gap:10px; margin-bottom:10px;">
+ <strong style="font-size:15px;">🎙️ Take over — speak / hang up</strong>
+ <span class="muted" style="font-size:12px; margin-left:auto;">applies on next AI turn</span>
+ </header>
+ <textarea id="speak-text" rows="4" placeholder="Type what to say in your cloned voice (lands in ~3-20s on next gather)…" style="width:100%; box-sizing:border-box; padding:10px; border:1px solid var(--line); border-radius:6px; font-size:14px; font-family:inherit; resize:vertical;"></textarea>
+ <div style="display:flex; gap:8px; margin-top:10px; align-items:center;">
+ <button id="speak-send" style="padding:10px 16px; font-size:14px; flex:1;">🎙️ Speak as me</button>
+ <button id="hangup-now" style="padding:10px 16px; font-size:14px; background:#fee2e2; border:1px solid #d33; color:#900; cursor:pointer; border-radius:6px;">📴 Hang up</button>
+ </div>
+ <div id="takeover-status" class="muted" style="font-size:12px; margin-top:8px; min-height:16px;"></div>
+ <details style="margin-top:10px;">
+ <summary style="cursor:pointer; color:var(--ink-dim); font-size:12px;">quick phrases</summary>
+ <div style="display:flex; flex-wrap:wrap; gap:6px; margin-top:8px;">
+ <button class="quick-phrase" data-text="Hi, this is Steve Abrams calling on behalf of myself. Quick question please." style="padding:6px 10px; font-size:12px;">👋 Intro</button>
+ <button class="quick-phrase" data-text="Could you transfer me to a representative please?" style="padding:6px 10px; font-size:12px;">🔁 Transfer me</button>
+ <button class="quick-phrase" data-text="Could you repeat that please?" style="padding:6px 10px; font-size:12px;">🔂 Repeat</button>
+ <button class="quick-phrase" data-text="Thank you for your help. Have a great day. Goodbye." style="padding:6px 10px; font-size:12px;">👋 Polite bye</button>
+ </div>
+ </details>
+ </section>
+
+ </div>
<details style="margin-top: 16px;">
<summary style="cursor: pointer; color: var(--ink-dim); font-size: 13px;">technical notes</summary>
@@ -150,6 +177,45 @@ transcript: data/transcripts/<%= call.id %>.json (post-call)</pre>
} catch {}
}, 5000);
+ // ── Live takeover (speak / hangup) ────────────────────────────────
+ const takeoverStatus = document.getElementById('takeover-status');
+ function setTakeoverStatus(text, isErr) {
+ takeoverStatus.textContent = text || '';
+ takeoverStatus.style.color = isErr ? '#d33' : 'var(--ink-dim)';
+ if (text) setTimeout(() => { if (takeoverStatus.textContent === text) takeoverStatus.textContent = ''; }, 8000);
+ }
+ document.getElementById('speak-send').addEventListener('click', async () => {
+ const ta = document.getElementById('speak-text');
+ const text = ta.value.trim();
+ if (!text) { setTakeoverStatus('Type something first.', true); return; }
+ setTakeoverStatus('Sending to Butlr…');
+ try {
+ const r = await fetch('/api/calls/' + callId + '/say', {
+ method: 'POST', headers: {'Content-Type':'application/json'},
+ body: JSON.stringify({ text })
+ });
+ const j = await r.json();
+ if (j.ok) { setTakeoverStatus('✅ queued — will play on the next AI turn'); ta.value = ''; }
+ else setTakeoverStatus('❌ ' + (j.error || 'failed'), true);
+ } catch (e) { setTakeoverStatus('❌ network error: ' + e.message, true); }
+ });
+ document.getElementById('hangup-now').addEventListener('click', async () => {
+ if (!confirm('Hang up the call now?')) return;
+ setTakeoverStatus('Hanging up…');
+ try {
+ const r = await fetch('/api/calls/' + callId + '/hangup', { method:'POST' });
+ const j = await r.json();
+ setTakeoverStatus(j.ok ? '📴 hangup queued' : ('❌ ' + (j.error||'failed')), !j.ok);
+ } catch (e) { setTakeoverStatus('❌ network error: ' + e.message, true); }
+ });
+ document.querySelectorAll('.quick-phrase').forEach(btn => {
+ btn.addEventListener('click', () => {
+ const ta = document.getElementById('speak-text');
+ ta.value = btn.getAttribute('data-text') || '';
+ ta.focus();
+ });
+ });
+
// Live AI-agent transcript — polls /api/calls/:id/agent-log every 3s
const feedEl = document.getElementById('transcript-feed');
const countEl2 = document.getElementById('transcript-count');
← 6fe29bf predeploy-guard: abort deploy if any call is in flight
·
back to Butlr
·
twilio: Stream track=both_tracks so listener hears Butlr's v 83398b4 →