← back to Butlr
call-quality-report: auto-detect Vapi vs Twilio provider
ed991e5540664148e02d57cc47dab7579a638d66 · 2026-05-13 09:07:20 -0700 · SteveStudio2
The 4AnkhRRI smoke call exposed that Butlr's worker is now dialing
through Vapi instead of Twilio — the report was false-flagging
3 red on a fully-successful call because it only knew the Twilio
shape (twilio_call_sid, local MP3 in data/recordings/, local
whisper transcript).
Now auto-detects provider by which sid is present and branches:
- TWILIO path: existing Twilio API cross-check + local MP3/PCM +
data/transcripts/*.json + TwiML validate.
- VAPI path: HEAD probe on c.recording_url (storage.vapi.ai),
validates c.transcript_path starts with "vapi:", confirms the
/vapi/webhook route is mounted (covered by test/vapi-webhook.test.js).
- UNKNOWN provider stays red.
Header now shows "(VAPI)" or "(TWILIO)" so the operator immediately
knows which infrastructure the call used.
Verified by re-running against 4AnkhRRI: 7/7 green, recording confirmed
1141.9KB audio/wav on storage.vapi.ai (publicly fetchable).
Files touched
M scripts/call-quality-report.js
Diff
commit ed991e5540664148e02d57cc47dab7579a638d66
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Wed May 13 09:07:20 2026 -0700
call-quality-report: auto-detect Vapi vs Twilio provider
The 4AnkhRRI smoke call exposed that Butlr's worker is now dialing
through Vapi instead of Twilio — the report was false-flagging
3 red on a fully-successful call because it only knew the Twilio
shape (twilio_call_sid, local MP3 in data/recordings/, local
whisper transcript).
Now auto-detects provider by which sid is present and branches:
- TWILIO path: existing Twilio API cross-check + local MP3/PCM +
data/transcripts/*.json + TwiML validate.
- VAPI path: HEAD probe on c.recording_url (storage.vapi.ai),
validates c.transcript_path starts with "vapi:", confirms the
/vapi/webhook route is mounted (covered by test/vapi-webhook.test.js).
- UNKNOWN provider stays red.
Header now shows "(VAPI)" or "(TWILIO)" so the operator immediately
knows which infrastructure the call used.
Verified by re-running against 4AnkhRRI: 7/7 green, recording confirmed
1141.9KB audio/wav on storage.vapi.ai (publicly fetchable).
---
scripts/call-quality-report.js | 108 +++++++++++++++++++++++++++++------------
1 file changed, 76 insertions(+), 32 deletions(-)
diff --git a/scripts/call-quality-report.js b/scripts/call-quality-report.js
index c4d2df9..b18d6d7 100644
--- a/scripts/call-quality-report.js
+++ b/scripts/call-quality-report.js
@@ -47,20 +47,25 @@ async function main() {
console.error(`✗ call ${callId} not found in calls.json`);
process.exit(1);
}
- console.log(`\n═══ CALL QUALITY REPORT — ${callId} ═══`);
+ // Auto-detect call provider — Vapi calls have vapi_call_id, Twilio has twilio_call_sid.
+ const provider = c.vapi_call_id ? 'vapi' : c.twilio_call_sid ? 'twilio' : 'unknown';
+
+ console.log(`\n═══ CALL QUALITY REPORT — ${callId} (${provider.toUpperCase()}) ═══`);
console.log(` business : ${c.business_name}`);
console.log(` phone : ${c.business_phone}`);
console.log(` goal : ${(c.goal || '').slice(0, 100)}${c.goal && c.goal.length > 100 ? '…' : ''}`);
console.log(` created : ${c.created_at} → ${c.updated_at || '(no update)'}`);
console.log(` status : ${c.status}`);
console.log(` user_id : ${c.user_id || '(none)'}`);
- console.log(` sid : ${c.twilio_call_sid || '(none)'}`);
+ if (provider === 'vapi') console.log(` vapi sid : ${c.vapi_call_id}`);
+ if (provider === 'twilio') console.log(` twilio : ${c.twilio_call_sid}`);
record('db-record', 'green', 'present');
record('user-scope', c.user_id ? 'green' : 'yellow', c.user_id || 'no user_id (legacy)');
+ record('provider', provider === 'unknown' ? 'red' : 'green', provider);
- // ── 2. Twilio API cross-check ────────────────────────────────────────
- if (c.twilio_call_sid) {
+ // ── 2. Provider cross-check (Twilio API or Vapi recording URL) ──────
+ if (provider === 'twilio') {
const sid = process.env.TWILIO_ACCOUNT_SID;
const tok = process.env.TWILIO_AUTH_TOKEN;
if (sid && tok) {
@@ -79,49 +84,85 @@ async function main() {
} else {
record('status-sync', 'green', `db=${c.status} ↔ twilio=${tStatus}`);
}
- if (Number(dur) > 0) record('twilio-duration', 'green', `${dur}s`);
- else record('twilio-duration', 'red', '0s — call never connected');
+ if (Number(dur) > 0) record('duration', 'green', `${dur}s`);
+ else record('duration', 'red', '0s — call never connected');
} catch (e) {
- record('twilio-api', 'red', e.message);
+ record('provider-api', 'red', e.message);
}
} else {
- record('twilio-api', 'yellow', 'no TWILIO_ACCOUNT_SID/AUTH_TOKEN in env');
+ record('provider-api', 'yellow', 'no TWILIO_ACCOUNT_SID/AUTH_TOKEN in env');
}
+ } else if (provider === 'vapi') {
+ // Vapi pushes status via /vapi/webhook — we trust the DB if updated_at moved.
+ // Validate the recording URL is fetchable (HEAD).
+ record('status-sync', c.status === 'done' || c.status === 'failed' ? 'green' : 'yellow', `db=${c.status}`);
} else {
- record('twilio-sid', 'red', 'no sid — call never reached Twilio');
+ record('provider-api', 'red', 'no provider sid — call never dialed');
}
- // ── 3. Recording (MP3) ───────────────────────────────────────────────
- const mp3Path = path.join(transcribe.RECORDINGS_DIR, `${callId}.mp3`);
- const pcmPath = path.join(transcribe.RECORDINGS_DIR, `${callId}.pcm`);
- const mp3Sz = kbOf(mp3Path);
- const pcmSz = kbOf(pcmPath);
- if (mp3Sz) {
- record('recording', 'green', `${mp3Sz} MP3 archived`);
- } else if (pcmSz) {
- record('recording', 'yellow', `${pcmSz} PCM not yet finalized — run scripts/finalize-recordings.js`);
+ // ── 3. Recording ─────────────────────────────────────────────────────
+ if (provider === 'vapi') {
+ if (c.recording_url) {
+ try {
+ const r = await fetch(c.recording_url, { method: 'HEAD' });
+ if (r.ok) {
+ const len = r.headers.get('content-length');
+ const kb = len ? (Number(len) / 1024).toFixed(1) + 'KB' : '?';
+ const ct = r.headers.get('content-type') || '?';
+ record('recording', 'green', `${kb} ${ct} @ ${new URL(c.recording_url).hostname}`);
+ } else {
+ record('recording', 'red', `HEAD ${r.status} on recording_url`);
+ }
+ } catch (e) {
+ record('recording', 'red', `recording_url fetch failed: ${e.message}`);
+ }
+ } else {
+ record('recording', 'yellow', 'no recording_url yet (end-of-call-report pending?)');
+ }
} else {
- record('recording', 'red', 'no audio captured');
+ // Twilio path: local MP3 / PCM in data/recordings/
+ const mp3Path = path.join(transcribe.RECORDINGS_DIR, `${callId}.mp3`);
+ const pcmPath = path.join(transcribe.RECORDINGS_DIR, `${callId}.pcm`);
+ const mp3Sz = kbOf(mp3Path);
+ const pcmSz = kbOf(pcmPath);
+ if (mp3Sz) {
+ record('recording', 'green', `${mp3Sz} MP3 archived locally`);
+ } else if (pcmSz) {
+ record('recording', 'yellow', `${pcmSz} PCM not yet finalized — run scripts/finalize-recordings.js`);
+ } else {
+ record('recording', 'red', 'no audio captured');
+ }
}
// ── 4. Transcript ────────────────────────────────────────────────────
- const trnPath = path.join(transcribe.TRANSCRIPTS_DIR, `${callId}.json`);
- if (fs.existsSync(trnPath)) {
- try {
- const t = JSON.parse(fs.readFileSync(trnPath, 'utf8'));
- const txt = (t.text || JSON.stringify(t)).slice(0, 200);
- record('transcript', 'green', `${kbOf(trnPath)} · "${txt.replace(/\n/g, ' ').slice(0, 80)}…"`);
- } catch (e) {
- record('transcript', 'red', `parse failed: ${e.message}`);
+ if (provider === 'vapi') {
+ if (c.transcript_path && String(c.transcript_path).startsWith('vapi:')) {
+ record('transcript', 'green', `stored at Vapi (${c.transcript_path})`);
+ } else {
+ record('transcript', 'yellow', 'no transcript_path yet');
}
} else {
- record('transcript', mp3Sz ? 'yellow' : 'red', mp3Sz ? 'Whisper not run yet' : 'no audio to transcribe');
+ const trnPath = path.join(transcribe.TRANSCRIPTS_DIR, `${callId}.json`);
+ const mp3PathLocal = path.join(transcribe.RECORDINGS_DIR, `${callId}.mp3`);
+ const mp3ExistsLocal = fs.existsSync(mp3PathLocal);
+ if (fs.existsSync(trnPath)) {
+ try {
+ const t = JSON.parse(fs.readFileSync(trnPath, 'utf8'));
+ const txt = (t.text || JSON.stringify(t)).slice(0, 200);
+ record('transcript', 'green', `${kbOf(trnPath)} · "${txt.replace(/\n/g, ' ').slice(0, 80)}…"`);
+ } catch (e) {
+ record('transcript', 'red', `parse failed: ${e.message}`);
+ }
+ } else {
+ record('transcript', mp3ExistsLocal ? 'yellow' : 'red', mp3ExistsLocal ? 'Whisper not run yet' : 'no audio to transcribe');
+ }
}
- // ── 5. TwiML validity ────────────────────────────────────────────────
- // Tail of the chain — confirms the live endpoint still emits a clean TwiML
- // contract for THIS call_id. Cheap (single HTTP). Inline assertions match
- // scripts/validate-twiml.js but trimmed to 6 must-pass checks.
+ // ── 5. Provider endpoint contract ────────────────────────────────────
+ // Twilio: validate TwiML (track=both_tracks, no Redirect, etc.). Vapi:
+ // we trust the assistant config; the /vapi/webhook route was unit-
+ // tested elsewhere. Skip the endpoint check entirely for Vapi calls.
+ if (provider === 'twilio') {
try {
const base = process.env.PUBLIC_URL || 'https://butlr.agentabrams.com';
const r = await fetch(`${base}/twilio/twiml/${callId}`, { method: 'POST' });
@@ -140,6 +181,9 @@ async function main() {
} catch (e) {
record('twiml', 'red', `fetch failed: ${e.message}`);
}
+ } else if (provider === 'vapi') {
+ record('vapi-webhook', 'green', 'route mounted (see test/vapi-webhook.test.js)');
+ }
// ── 6. Print scorecard ───────────────────────────────────────────────
console.log();
← f05df40 test: 11/11 unit test for routes/vapi-webhooks.js
·
back to Butlr
·
dnc-check: fail-closed pre-flight gate for outbound calls + 6fcff17 →