← back to Butlr
live-audio bridge + recording + Whisper transcript on finish
9846567e65e356323d21397015856a9d43547f0e · 2026-05-12 12:59:14 -0700 · Steve
End-to-end pipeline for Phase 2 going live:
lib/elevenlabs.js — TTS for the announce preamble, sha256 cached,
graceful fallback to Twilio <Say> on 401/network err
lib/listen-bridge.js — single ws server, two lanes:
/stream/<id> = Twilio Media Streams (µ-law → PCM16)
/listen-ws/<id> = browser playback (raw PCM frames)
lib/transcribe.js — Twilio Recording download (basic-auth) + Whisper
verbose_json, stored at data/transcripts/<id>.json
lib/twilio.js — LIVE dialCall() now real, POSTs to Twilio Calls API
with Url=/twilio/twiml/<id>, status/AMD callbacks,
AsyncAmd for human-vs-voicemail branch
routes/twilio-webhooks.js — /twiml /status /recording /amd /audio/announce
TwiML: <Stream> + announce + <Record/> with
recordingStatusCallback → kicks Whisper
routes/listen.js — /listen/<id> browser page + /calls/<id>/{transcript,recording}
views/public/listen.ejs — Web Audio API playback page with consent banner
server.js — http.createServer, attachListenBridge(server),
CSP allows ws:/wss: + media-src blob:
lib/data.js — setTwilioSid/setRecordingMeta/setTranscriptMeta
All gated by feedback_holdforme_always_announce.md — every outbound call
opens with two-party-consent announcement, no skip flag, ever.
Files touched
M lib/data.jsA lib/elevenlabs.jsA lib/listen-bridge.jsA lib/transcribe.jsM lib/twilio.jsM package-lock.jsonM package.jsonA routes/listen.jsA routes/twilio-webhooks.jsM server.jsA views/public/listen.ejs
Diff
commit 9846567e65e356323d21397015856a9d43547f0e
Author: Steve <steve@designerwallcoverings.com>
Date: Tue May 12 12:59:14 2026 -0700
live-audio bridge + recording + Whisper transcript on finish
End-to-end pipeline for Phase 2 going live:
lib/elevenlabs.js — TTS for the announce preamble, sha256 cached,
graceful fallback to Twilio <Say> on 401/network err
lib/listen-bridge.js — single ws server, two lanes:
/stream/<id> = Twilio Media Streams (µ-law → PCM16)
/listen-ws/<id> = browser playback (raw PCM frames)
lib/transcribe.js — Twilio Recording download (basic-auth) + Whisper
verbose_json, stored at data/transcripts/<id>.json
lib/twilio.js — LIVE dialCall() now real, POSTs to Twilio Calls API
with Url=/twilio/twiml/<id>, status/AMD callbacks,
AsyncAmd for human-vs-voicemail branch
routes/twilio-webhooks.js — /twiml /status /recording /amd /audio/announce
TwiML: <Stream> + announce + <Record/> with
recordingStatusCallback → kicks Whisper
routes/listen.js — /listen/<id> browser page + /calls/<id>/{transcript,recording}
views/public/listen.ejs — Web Audio API playback page with consent banner
server.js — http.createServer, attachListenBridge(server),
CSP allows ws:/wss: + media-src blob:
lib/data.js — setTwilioSid/setRecordingMeta/setTranscriptMeta
All gated by feedback_holdforme_always_announce.md — every outbound call
opens with two-party-consent announcement, no skip flag, ever.
---
lib/data.js | 13 ++++
lib/elevenlabs.js | 74 ++++++++++++++++++++++
lib/listen-bridge.js | 98 +++++++++++++++++++++++++++++
lib/transcribe.js | 96 +++++++++++++++++++++++++++++
lib/twilio.js | 76 +++++++++++++----------
package-lock.json | 24 +++++++-
package.json | 7 ++-
routes/listen.js | 48 +++++++++++++++
routes/twilio-webhooks.js | 154 ++++++++++++++++++++++++++++++++++++++++++++++
server.js | 21 ++++++-
views/public/listen.ejs | 97 +++++++++++++++++++++++++++++
11 files changed, 671 insertions(+), 37 deletions(-)
diff --git a/lib/data.js b/lib/data.js
index ae94080..4b853a9 100644
--- a/lib/data.js
+++ b/lib/data.js
@@ -175,6 +175,18 @@ function updateStatus(id, status) {
return all[idx];
}
+function patchRow(id, patch) {
+ const all = readAll();
+ const idx = all.findIndex(c => c.id === id);
+ if (idx < 0) return null;
+ all[idx] = { ...all[idx], ...patch, updated_at: new Date().toISOString() };
+ writeAll(all);
+ return all[idx];
+}
+function setTwilioSid(id, sid) { return patchRow(id, { twilio_call_sid: sid }); }
+function setRecordingMeta(id, meta) { return patchRow(id, meta); }
+function setTranscriptMeta(id, meta) { return patchRow(id, meta); }
+
// ── Preset business directory ─────────────────────────────────────────
let _businesses = null;
function loadBusinesses() {
@@ -195,5 +207,6 @@ module.exports = {
CATEGORIES, categoryById,
validatePhone, sanitizeRow,
addCall, listCalls, getCall, updateStatus,
+ patchRow, setTwilioSid, setRecordingMeta, setTranscriptMeta,
loadBusinesses, businessBySlug, businessesForCategory,
};
diff --git a/lib/elevenlabs.js b/lib/elevenlabs.js
new file mode 100644
index 0000000..49af280
--- /dev/null
+++ b/lib/elevenlabs.js
@@ -0,0 +1,74 @@
+// ElevenLabs TTS synthesis for the always-announce preamble.
+//
+// Standing rules in play:
+// • feedback_always_elevenlabs_voice.md — never piper/say/coqui as primary
+// • feedback_holdforme_always_announce.md — every outbound call announces consent
+//
+// Cache strategy: hash(text + voiceId + model) → mp3 file on disk.
+// If the EL key is missing/invalid (401), synth returns null and the caller
+// falls back to Twilio `<Say voice="Polly.Joanna">…</Say>` so the announcement
+// still plays. The rule is non-negotiable — only the voice quality is.
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const API_KEY = process.env.ELEVENLABS_API_KEY || '';
+const DEFAULT_VOICE = process.env.ELEVENLABS_VOICE_ID || '21m00Tcm4TlvDq8ikWAM'; // Rachel — neutral pro
+const DEFAULT_MODEL = process.env.ELEVENLABS_MODEL || 'eleven_turbo_v2_5';
+
+const CACHE_DIR = path.join(__dirname, '..', 'data', 'audio-cache');
+fs.mkdirSync(CACHE_DIR, { recursive: true });
+
+function cachePathFor(text, voiceId, modelId) {
+ const h = crypto.createHash('sha256').update(`${voiceId}|${modelId}|${text}`).digest('hex').slice(0, 24);
+ return path.join(CACHE_DIR, `${h}.mp3`);
+}
+
+async function synthesize(text, opts = {}) {
+ const voiceId = opts.voiceId || DEFAULT_VOICE;
+ const modelId = opts.modelId || DEFAULT_MODEL;
+ const cachePath = cachePathFor(text, voiceId, modelId);
+
+ if (fs.existsSync(cachePath)) {
+ return { ok: true, path: cachePath, cached: true };
+ }
+ if (!API_KEY) {
+ return { ok: false, error: 'no_api_key', fallback: 'twilio_say' };
+ }
+
+ try {
+ const url = `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voiceId)}`;
+ const resp = await fetch(url, {
+ method: 'POST',
+ headers: {
+ 'xi-api-key': API_KEY,
+ 'Content-Type': 'application/json',
+ 'Accept': 'audio/mpeg',
+ },
+ body: JSON.stringify({
+ text,
+ model_id: modelId,
+ voice_settings: { stability: 0.5, similarity_boost: 0.75 },
+ }),
+ });
+ if (!resp.ok) {
+ const body = await resp.text().catch(() => '');
+ return { ok: false, error: `el_http_${resp.status}`, body: body.slice(0, 200), fallback: 'twilio_say' };
+ }
+ const buf = Buffer.from(await resp.arrayBuffer());
+ fs.writeFileSync(cachePath, buf);
+ return { ok: true, path: cachePath, cached: false, bytes: buf.length };
+ } catch (e) {
+ return { ok: false, error: 'el_exception', detail: e.message, fallback: 'twilio_say' };
+ }
+}
+
+// Pre-warm the cache for a call's announcement so the TwiML endpoint can
+// serve it immediately. Returns the cache path or null on failure.
+async function prewarmAnnouncement(text, opts = {}) {
+ const r = await synthesize(text, opts);
+ return r.ok ? r.path : null;
+}
+
+module.exports = { synthesize, prewarmAnnouncement, cachePathFor, DEFAULT_VOICE, DEFAULT_MODEL };
diff --git a/lib/listen-bridge.js b/lib/listen-bridge.js
new file mode 100644
index 0000000..7dc7e41
--- /dev/null
+++ b/lib/listen-bridge.js
@@ -0,0 +1,98 @@
+// Live-audio listen bridge.
+//
+// Twilio Voice → <Start><Stream url="wss://.../stream/<call_id>"/> sends
+// inbound µ-law audio frames at 8kHz mono in 20ms chunks as base64 inside
+// JSON envelopes: { event: "media", media: { payload: "<base64>" } }
+//
+// We accept those, decode µ-law → PCM16 LE, and broadcast the raw PCM bytes
+// to every browser WebSocket listening on /listen/<call_id>. The browser
+// uses AudioContext to play it back.
+//
+// Two WebSocket "lanes" on the same upgrade handler — Twilio Streams on
+// /stream/:call_id, browser listeners on /listen-ws/:call_id.
+
+const { WebSocketServer } = require('ws');
+const url = require('url');
+
+// µ-law → linear PCM16 LE decoder (G.711 standard).
+// Bit-fiddly but small. Adapted from RFC 5391.
+const MULAW_BIAS = 33;
+const MULAW_CLIP = 32635;
+function mulawDecodeOne(mulawByte) {
+ mulawByte = ~mulawByte & 0xff;
+ const sign = (mulawByte & 0x80) ? -1 : 1;
+ const exponent = (mulawByte >> 4) & 0x07;
+ const mantissa = mulawByte & 0x0f;
+ let sample = ((mantissa << 3) + MULAW_BIAS) << exponent;
+ sample -= MULAW_BIAS;
+ return sign * sample;
+}
+function mulawDecodeBuffer(mulawBuf) {
+ const out = Buffer.alloc(mulawBuf.length * 2);
+ for (let i = 0; i < mulawBuf.length; i++) {
+ const s = mulawDecodeOne(mulawBuf[i]);
+ out.writeInt16LE(s, i * 2);
+ }
+ return out;
+}
+
+// callId → Set<browserWs>
+const listeners = new Map();
+
+function getListeners(callId) {
+ let set = listeners.get(callId);
+ if (!set) { set = new Set(); listeners.set(callId, set); }
+ return set;
+}
+
+function broadcastPcm(callId, pcmBuf) {
+ const set = listeners.get(callId);
+ if (!set || set.size === 0) return;
+ for (const ws of set) {
+ if (ws.readyState === ws.OPEN) {
+ try { ws.send(pcmBuf, { binary: true }); } catch (_) { /* swallow */ }
+ }
+ }
+}
+
+function attachListenBridge(httpServer) {
+ const wss = new WebSocketServer({ noServer: true });
+
+ httpServer.on('upgrade', (req, socket, head) => {
+ const parsed = url.parse(req.url);
+ const m = (parsed.pathname || '').match(/^\/(stream|listen-ws)\/([A-Za-z0-9_-]{6,16})$/);
+ if (!m) { socket.destroy(); return; }
+ const lane = m[1];
+ const callId = m[2];
+
+ wss.handleUpgrade(req, socket, head, (ws) => {
+ ws._lane = lane;
+ ws._callId = callId;
+
+ if (lane === 'listen-ws') {
+ getListeners(callId).add(ws);
+ ws.on('close', () => {
+ const set = listeners.get(callId);
+ if (set) { set.delete(ws); if (set.size === 0) listeners.delete(callId); }
+ });
+ ws.on('error', () => {});
+ return;
+ }
+
+ // lane === 'stream' (Twilio Media Streams)
+ ws.on('message', (data, isBinary) => {
+ if (isBinary) return; // Twilio sends JSON
+ let msg;
+ try { msg = JSON.parse(data.toString('utf8')); } catch { return; }
+ if (msg.event !== 'media' || !msg.media || !msg.media.payload) return;
+ const mulaw = Buffer.from(msg.media.payload, 'base64');
+ const pcm = mulawDecodeBuffer(mulaw);
+ broadcastPcm(callId, pcm);
+ });
+ ws.on('close', () => {});
+ ws.on('error', () => {});
+ });
+ });
+}
+
+module.exports = { attachListenBridge, getListeners, mulawDecodeBuffer };
diff --git a/lib/transcribe.js b/lib/transcribe.js
new file mode 100644
index 0000000..5465320
--- /dev/null
+++ b/lib/transcribe.js
@@ -0,0 +1,96 @@
+// Whisper transcription via OpenAI API.
+//
+// Standing rule (Steve, 2026-05-12): "record all conversations and create
+// transcript when finished." Recording happens via Twilio Recording API
+// (recording_url comes back on call.completed). This module fetches that
+// MP3, ships it to Whisper, and writes the JSON transcript to disk.
+//
+// Whisper outputs an SRT-style JSON with timestamps + speaker turns. We
+// store the raw response so we can re-render the UI later without re-paying.
+
+const fs = require('fs');
+const path = require('path');
+
+const API_KEY = process.env.OPENAI_API_KEY || '';
+const MODEL = process.env.WHISPER_MODEL || 'whisper-1';
+
+const TRANSCRIPTS_DIR = path.join(__dirname, '..', 'data', 'transcripts');
+const RECORDINGS_DIR = path.join(__dirname, '..', 'data', 'recordings');
+fs.mkdirSync(TRANSCRIPTS_DIR, { recursive: true });
+fs.mkdirSync(RECORDINGS_DIR, { recursive: true });
+
+// Download a Twilio recording (basic auth required) to local disk.
+async function downloadRecording(recordingUrl, callId, { twilioSid, twilioToken } = {}) {
+ const sid = twilioSid || process.env.TWILIO_ACCOUNT_SID;
+ const tok = twilioToken || process.env.TWILIO_AUTH_TOKEN;
+ if (!sid || !tok) return { ok: false, error: 'no_twilio_creds' };
+
+ // Twilio recording URL format: append .mp3 to force MP3 download
+ const url = recordingUrl.endsWith('.mp3') ? recordingUrl : recordingUrl + '.mp3';
+ const auth = 'Basic ' + Buffer.from(`${sid}:${tok}`).toString('base64');
+ try {
+ const resp = await fetch(url, { headers: { 'Authorization': auth } });
+ if (!resp.ok) return { ok: false, error: `twilio_http_${resp.status}` };
+ const buf = Buffer.from(await resp.arrayBuffer());
+ const dest = path.join(RECORDINGS_DIR, `${callId}.mp3`);
+ fs.writeFileSync(dest, buf);
+ return { ok: true, path: dest, bytes: buf.length };
+ } catch (e) {
+ return { ok: false, error: 'download_exception', detail: e.message };
+ }
+}
+
+// Run an MP3 through Whisper. Returns the JSON response with `text` +
+// segments + words (if requested). Saves to data/transcripts/<callId>.json.
+async function transcribe(callId, mp3Path) {
+ if (!API_KEY) return { ok: false, error: 'no_openai_key' };
+ if (!fs.existsSync(mp3Path)) return { ok: false, error: 'mp3_not_found' };
+
+ try {
+ const audio = fs.readFileSync(mp3Path);
+ // node 20+ has FormData + Blob
+ const form = new FormData();
+ form.append('file', new Blob([audio], { type: 'audio/mpeg' }), `${callId}.mp3`);
+ form.append('model', MODEL);
+ form.append('response_format', 'verbose_json');
+ form.append('timestamp_granularities[]', 'segment');
+
+ const resp = await fetch('https://api.openai.com/v1/audio/transcriptions', {
+ method: 'POST',
+ headers: { 'Authorization': `Bearer ${API_KEY}` },
+ body: form,
+ });
+ if (!resp.ok) {
+ const body = await resp.text().catch(() => '');
+ return { ok: false, error: `whisper_http_${resp.status}`, body: body.slice(0, 400) };
+ }
+ const json = await resp.json();
+ const dest = path.join(TRANSCRIPTS_DIR, `${callId}.json`);
+ fs.writeFileSync(dest, JSON.stringify(json, null, 2));
+ return { ok: true, path: dest, text: json.text, segments: (json.segments || []).length };
+ } catch (e) {
+ return { ok: false, error: 'whisper_exception', detail: e.message };
+ }
+}
+
+// One-shot: download + transcribe. Called from the Twilio status callback
+// when call.completed fires with a RecordingUrl.
+async function recordAndTranscribe(callId, recordingUrl) {
+ const dl = await downloadRecording(recordingUrl, callId);
+ if (!dl.ok) return { ok: false, step: 'download', ...dl };
+ const tx = await transcribe(callId, dl.path);
+ if (!tx.ok) return { ok: false, step: 'transcribe', ...tx, mp3Path: dl.path };
+ return { ok: true, mp3Path: dl.path, transcriptPath: tx.path, text: tx.text };
+}
+
+function readTranscript(callId) {
+ const p = path.join(TRANSCRIPTS_DIR, `${callId}.json`);
+ if (!fs.existsSync(p)) return null;
+ try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return null; }
+}
+
+function recordingExists(callId) {
+ return fs.existsSync(path.join(RECORDINGS_DIR, `${callId}.mp3`));
+}
+
+module.exports = { downloadRecording, transcribe, recordAndTranscribe, readTranscript, recordingExists, TRANSCRIPTS_DIR, RECORDINGS_DIR };
diff --git a/lib/twilio.js b/lib/twilio.js
index 099c087..6e592ec 100644
--- a/lib/twilio.js
+++ b/lib/twilio.js
@@ -53,8 +53,6 @@ function announceTwiml(call) {
// Use neutral voice; <Pause/> gives the called party a beat to object.
return `<Say voice="Polly.Joanna">${text}</Say><Pause length="2"/>`;
}
-module.exports.buildAnnouncement = buildAnnouncement;
-module.exports.announceTwiml = announceTwiml;
// Dry-run state machine — simulates a real call advancing through stages.
// queued → dialing → on_hold → connected → done
@@ -115,34 +113,50 @@ async function dialCall(call) {
data.updateStatus(call.id, 'dialing');
return { ok: true, dry_run: true };
}
- // LIVE — would call Twilio REST API here
- // const accountSid = process.env.TWILIO_ACCOUNT_SID;
- // const authToken = process.env.TWILIO_AUTH_TOKEN;
- // const fromNum = process.env.TWILIO_PHONE_NUMBER;
- // const publicUrl = process.env.PUBLIC_URL;
- // const body = new URLSearchParams({
- // To: call.business_phone,
- // From: fromNum,
- // Url: `${publicUrl}/twilio/twiml/hold/${call.id}`,
- // StatusCallback: `${publicUrl}/twilio/status/${call.id}`,
- // MachineDetection: 'DetectMessageEnd',
- // AmdStatusCallback: `${publicUrl}/twilio/amd/${call.id}`,
- // AmdStatusCallbackMethod: 'POST',
- // });
- // const resp = await fetch(`https://api.twilio.com/2010-04-01/Accounts/${accountSid}/Calls.json`, {
- // method: 'POST',
- // headers: {
- // 'Authorization': 'Basic ' + Buffer.from(`${accountSid}:${authToken}`).toString('base64'),
- // 'Content-Type': 'application/x-www-form-urlencoded',
- // },
- // body,
- // });
- // const json = await resp.json();
- // if (!resp.ok) { log('LIVE dial failed:', json); return { ok: false, error: json }; }
- // data.updateStatus(call.id, 'dialing');
- // return { ok: true, sid: json.sid };
- log('LIVE mode requested but not yet implemented — leaving call queued');
- return { ok: false, error: 'live_mode_not_implemented' };
+ // LIVE — actually dial via Twilio REST API. TwiML lives at
+ // /twilio/twiml/:call_id and always opens with the consent announcement.
+ const accountSid = process.env.TWILIO_ACCOUNT_SID;
+ const authToken = process.env.TWILIO_AUTH_TOKEN;
+ const fromNum = process.env.TWILIO_PHONE_NUMBER;
+ const publicUrl = process.env.PUBLIC_URL;
+ if (!accountSid || !authToken || !fromNum || !publicUrl) {
+ log('LIVE mode requested but Twilio creds missing — leaving call queued');
+ return { ok: false, error: 'twilio_creds_missing' };
+ }
+ const body = new URLSearchParams({
+ To: call.business_phone,
+ From: fromNum,
+ Url: `${publicUrl}/twilio/twiml/${call.id}`,
+ StatusCallback: `${publicUrl}/twilio/status/${call.id}`,
+ StatusCallbackMethod: 'POST',
+ StatusCallbackEvent: 'initiated ringing answered completed',
+ MachineDetection: 'DetectMessageEnd',
+ AsyncAmd: 'true',
+ AsyncAmdStatusCallback: `${publicUrl}/twilio/amd/${call.id}`,
+ AsyncAmdStatusCallbackMethod: 'POST',
+ });
+ try {
+ const resp = await fetch(`https://api.twilio.com/2010-04-01/Accounts/${accountSid}/Calls.json`, {
+ method: 'POST',
+ headers: {
+ 'Authorization': 'Basic ' + Buffer.from(`${accountSid}:${authToken}`).toString('base64'),
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ },
+ body,
+ });
+ const json = await resp.json().catch(() => ({}));
+ if (!resp.ok) {
+ log('LIVE dial failed:', resp.status, json && json.message);
+ return { ok: false, error: 'twilio_dial_failed', status: resp.status, detail: json };
+ }
+ data.updateStatus(call.id, 'dialing');
+ if (data.setTwilioSid) data.setTwilioSid(call.id, json.sid);
+ log(`LIVE dial placed · sid=${json.sid} · to=${call.business_phone}`);
+ return { ok: true, sid: json.sid };
+ } catch (e) {
+ log('LIVE dial exception:', e.message);
+ return { ok: false, error: 'twilio_exception', detail: e.message };
+ }
}
async function advanceDryRun(call) {
@@ -189,4 +203,4 @@ function start({ intervalMs = 5000 } = {}) {
setTimeout(tick, 500);
}
-module.exports = { start, tick, dialCall, sendSms, DRY_RUN };
+module.exports = { start, tick, dialCall, sendSms, DRY_RUN, buildAnnouncement, announceTwiml };
diff --git a/package-lock.json b/package-lock.json
index c1bb051..0be2bae 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -13,7 +13,8 @@
"express": "^4.22.2",
"express-rate-limit": "^7.4.0",
"helmet": "^7.1.0",
- "morgan": "^1.10.0"
+ "morgan": "^1.10.0",
+ "ws": "^8.20.1"
},
"engines": {
"node": ">=20"
@@ -1001,6 +1002,27 @@
"engines": {
"node": ">= 0.8"
}
+ },
+ "node_modules/ws": {
+ "version": "8.20.1",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.1.tgz",
+ "integrity": "sha512-It4dO0K5v//JtTXuPkfEOaI3uUN87iYPnqo/ZzqCoG3g8uhA66QUMs/SrM0YK7/NAu+r4LMh/9dq2A7k+rHs+w==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=10.0.0"
+ },
+ "peerDependencies": {
+ "bufferutil": "^4.0.1",
+ "utf-8-validate": ">=5.0.2"
+ },
+ "peerDependenciesMeta": {
+ "bufferutil": {
+ "optional": true
+ },
+ "utf-8-validate": {
+ "optional": true
+ }
+ }
}
}
}
diff --git a/package.json b/package.json
index ccb811a..e9aad0f 100644
--- a/package.json
+++ b/package.json
@@ -14,7 +14,10 @@
"express": "^4.22.2",
"express-rate-limit": "^7.4.0",
"helmet": "^7.1.0",
- "morgan": "^1.10.0"
+ "morgan": "^1.10.0",
+ "ws": "^8.20.1"
},
- "engines": { "node": ">=20" }
+ "engines": {
+ "node": ">=20"
+ }
}
diff --git a/routes/listen.js b/routes/listen.js
new file mode 100644
index 0000000..e6719cb
--- /dev/null
+++ b/routes/listen.js
@@ -0,0 +1,48 @@
+// Listen-in routes:
+// GET /listen/:call_id — browser page that streams the live audio
+// GET /calls/:call_id/transcript — transcript JSON (post-call)
+// GET /calls/:call_id/recording — MP3 recording (post-call)
+
+const express = require('express');
+const path = require('path');
+const fs = require('fs');
+const data = require('../lib/data');
+const transcribe = require('../lib/transcribe');
+
+const router = express.Router();
+
+router.get('/listen/:call_id', (req, res) => {
+ const callId = req.params.call_id;
+ if (!/^[A-Za-z0-9_-]{6,16}$/.test(callId)) {
+ return res.status(404).render('public/404', { title: 'Not found — HoldForMe' });
+ }
+ const call = data.getCall(callId, false);
+ if (!call) {
+ return res.status(404).render('public/404', { title: 'Not found — HoldForMe' });
+ }
+ res.render('public/listen', {
+ title: `Listen in — ${call.business_name} — HoldForMe`,
+ meta_desc: 'Live audio listen-in.',
+ call,
+ });
+});
+
+router.get('/calls/:call_id/transcript', (req, res) => {
+ const callId = req.params.call_id;
+ if (!/^[A-Za-z0-9_-]{6,16}$/.test(callId)) return res.status(404).json({ ok: false });
+ const t = transcribe.readTranscript(callId);
+ if (!t) return res.status(404).json({ ok: false, error: 'no_transcript' });
+ res.json({ ok: true, transcript: t });
+});
+
+router.get('/calls/:call_id/recording', (req, res) => {
+ const callId = req.params.call_id;
+ if (!/^[A-Za-z0-9_-]{6,16}$/.test(callId)) return res.status(404).end();
+ const fp = path.join(transcribe.RECORDINGS_DIR, `${callId}.mp3`);
+ if (!fs.existsSync(fp)) return res.status(404).end();
+ res.type('audio/mpeg');
+ res.setHeader('Content-Disposition', `inline; filename="${callId}.mp3"`);
+ fs.createReadStream(fp).pipe(res);
+});
+
+module.exports = router;
diff --git a/routes/twilio-webhooks.js b/routes/twilio-webhooks.js
new file mode 100644
index 0000000..1f20cc4
--- /dev/null
+++ b/routes/twilio-webhooks.js
@@ -0,0 +1,154 @@
+// Twilio webhook routes:
+// GET /twilio/twiml/:call_id — the call's TwiML script
+// POST /twilio/status/:call_id — status callback (in-progress, completed)
+// POST /twilio/recording/:call_id — recording-complete callback (kicks Whisper)
+// POST /twilio/amd/:call_id — answering machine detection
+// GET /twilio/audio/announce/:hash.mp3 — pre-rendered ElevenLabs announce file
+//
+// The TwiML script ALWAYS:
+// 1. <Start><Stream url="wss://.../stream/<call_id>"/></Start> — start listen-in
+// 2. <Play>EL-announce.mp3</Play> (or <Say> fallback) — two-party-consent preamble
+// 3. <Pause length="2"/> — beat for objection
+// 4. <Record recordingStatusCallback=…/> — record entire call
+// OR <Dial record="record-from-answer-dual"/> — when bridging
+// 5. <Pause length="6000"/> — keep call open to hold
+//
+// Recording consent is in the announce; we never start recording without it.
+
+const express = require('express');
+const path = require('path');
+const fs = require('fs');
+const { buildAnnouncement } = require('../lib/twilio');
+const elevenlabs = require('../lib/elevenlabs');
+const data = require('../lib/data');
+const transcribe = require('../lib/transcribe');
+
+const router = express.Router();
+
+function publicUrl(req) {
+ return process.env.PUBLIC_URL || `${req.protocol}://${req.get('host')}`;
+}
+function wsUrl(req) {
+ const pub = publicUrl(req);
+ return pub.replace(/^http/, 'ws');
+}
+
+// ── GET /twilio/twiml/:call_id ────────────────────────────────────────
+router.get('/twiml/:call_id', async (req, res) => {
+ const callId = req.params.call_id;
+ if (!/^[A-Za-z0-9_-]{6,16}$/.test(callId)) return res.status(404).type('text/xml').send('<Response><Hangup/></Response>');
+
+ const call = data.getCall(callId, true);
+ if (!call) return res.status(404).type('text/xml').send('<Response><Hangup/></Response>');
+
+ const text = buildAnnouncement(call);
+ const pub = publicUrl(req);
+ const ws = wsUrl(req);
+
+ // Pre-warm ElevenLabs announce; fall back to <Say> if EL unavailable.
+ let announceXml;
+ try {
+ const synth = await elevenlabs.synthesize(text);
+ if (synth.ok) {
+ const fname = path.basename(synth.path);
+ announceXml = `<Play>${pub}/twilio/audio/announce/${fname}</Play>`;
+ } else {
+ announceXml = `<Say voice="Polly.Joanna">${escapeXml(text)}</Say>`;
+ }
+ } catch {
+ announceXml = `<Say voice="Polly.Joanna">${escapeXml(text)}</Say>`;
+ }
+
+ // Full TwiML — stream + announce + pause + record + extended pause.
+ // recordingStatusCallback fires when the recording is ready for Whisper.
+ const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+ <Start>
+ <Stream url="${ws}/stream/${callId}"/>
+ </Start>
+ ${announceXml}
+ <Pause length="2"/>
+ <Record
+ timeout="3600"
+ maxLength="3600"
+ recordingStatusCallback="${pub}/twilio/recording/${callId}"
+ recordingStatusCallbackMethod="POST"
+ recordingStatusCallbackEvent="completed"
+ playBeep="false"
+ trim="do-not-trim"
+ />
+ <Pause length="3600"/>
+</Response>`;
+ res.type('text/xml').send(xml);
+});
+
+// ── POST /twilio/status/:call_id ──────────────────────────────────────
+// Maps Twilio call lifecycle to HoldForMe internal status.
+router.post('/status/:call_id', (req, res) => {
+ const callId = req.params.call_id;
+ const cs = String(req.body.CallStatus || '').toLowerCase();
+ // initiated, ringing, in-progress, completed, busy, failed, no-answer, canceled
+ let internal = null;
+ if (cs === 'initiated' || cs === 'ringing') internal = 'dialing';
+ else if (cs === 'in-progress') internal = 'connected';
+ else if (cs === 'completed') internal = 'done';
+ else if (['busy','failed','no-answer','canceled'].includes(cs)) internal = 'failed';
+ if (internal) data.updateStatus(callId, internal);
+ res.type('text/xml').send('<Response/>');
+});
+
+// ── POST /twilio/recording/:call_id ───────────────────────────────────
+// Twilio fires this when the recording is ready. We then download it and
+// run Whisper. Both steps are best-effort; we log failures, never crash.
+router.post('/recording/:call_id', async (req, res) => {
+ const callId = req.params.call_id;
+ const recordingUrl = req.body.RecordingUrl;
+ const recordingSid = req.body.RecordingSid;
+ // ack Twilio immediately
+ res.type('text/xml').send('<Response/>');
+
+ if (!recordingUrl) {
+ console.error('[twilio-webhook] /recording: no RecordingUrl', { callId });
+ return;
+ }
+ try {
+ data.setRecordingMeta && data.setRecordingMeta(callId, { recording_url: recordingUrl, recording_sid: recordingSid });
+ const r = await transcribe.recordAndTranscribe(callId, recordingUrl);
+ if (r.ok) {
+ data.setTranscriptMeta && data.setTranscriptMeta(callId, { transcript_path: r.transcriptPath, recording_path: r.mp3Path });
+ console.log('[twilio-webhook] transcript ready', { callId, segments: r.text ? r.text.length : 0 });
+ } else {
+ console.error('[twilio-webhook] transcribe failed', { callId, ...r });
+ }
+ } catch (e) {
+ console.error('[twilio-webhook] recording handler exception', e.message);
+ }
+});
+
+// ── POST /twilio/amd/:call_id ─────────────────────────────────────────
+// Answering-machine detection. AnsweredBy = human | machine_end_beep | etc.
+router.post('/amd/:call_id', (req, res) => {
+ const callId = req.params.call_id;
+ const answeredBy = req.body.AnsweredBy;
+ // For now, just log. Future: branch TwiML based on human vs voicemail.
+ console.log('[twilio-webhook] AMD result', { callId, answeredBy });
+ res.type('text/xml').send('<Response/>');
+});
+
+// ── GET /twilio/audio/announce/:hash.mp3 ──────────────────────────────
+// Serve the cached ElevenLabs MP3. Hash filename is opaque, no enumeration.
+router.get('/audio/announce/:hash', (req, res) => {
+ const hash = req.params.hash;
+ if (!/^[a-f0-9]{6,64}\.mp3$/.test(hash)) return res.status(404).end();
+ const fp = path.join(__dirname, '..', 'data', 'audio-cache', hash);
+ if (!fs.existsSync(fp)) return res.status(404).end();
+ res.type('audio/mpeg');
+ res.setHeader('Cache-Control', 'public, max-age=86400');
+ fs.createReadStream(fp).pipe(res);
+});
+
+function escapeXml(s) {
+ return String(s || '').replace(/[<>&"']/g, c => ({'<':'<','>':'>','&':'&','"':'"',"'":'''}[c]));
+}
+
+module.exports = router;
diff --git a/server.js b/server.js
index 8eeabc5..ad611ce 100644
--- a/server.js
+++ b/server.js
@@ -1,12 +1,16 @@
require('dotenv').config();
const express = require('express');
+const http = require('http');
const path = require('path');
const morgan = require('morgan');
const helmet = require('helmet');
const rateLimit = require('express-rate-limit');
const publicRoutes = require('./routes/public');
+const listenRoutes = require('./routes/listen');
+const twilioWebhooks = require('./routes/twilio-webhooks');
+const { attachListenBridge } = require('./lib/listen-bridge');
const app = express();
const PORT = parseInt(process.env.PORT || '9932', 10);
@@ -19,6 +23,8 @@ app.set('views', path.join(__dirname, 'views'));
app.disable('x-powered-by');
// Security baseline (per dw-site-build standing rule, 2026-05-12)
+// connectSrc now allows ws:/wss: same-origin for the listen-in bridge.
+// mediaSrc allows blob:/self for AudioContext playback.
app.use(helmet({
contentSecurityPolicy: {
directives: {
@@ -27,7 +33,8 @@ app.use(helmet({
styleSrc: ["'self'", "'unsafe-inline'", 'https://fonts.googleapis.com'],
imgSrc: ["'self'", 'data:', 'https:'],
fontSrc: ["'self'", 'https://fonts.gstatic.com', 'data:'],
- connectSrc: ["'self'"],
+ connectSrc: ["'self'", 'ws:', 'wss:'],
+ mediaSrc: ["'self'", 'blob:'],
frameAncestors: ["'none'"],
objectSrc: ["'none'"],
baseUri: ["'self'"],
@@ -63,6 +70,10 @@ app.use((req, res, next) => {
next();
});
+// Twilio webhooks must NOT pass through morgan body-noise & must accept Twilio's
+// own urlencoded callbacks — mount before publicRoutes so /twilio/* short-circuits.
+app.use('/twilio', twilioWebhooks);
+app.use('/', listenRoutes);
app.use('/', publicRoutes);
app.use((req, res) => {
@@ -77,8 +88,12 @@ app.use((err, req, res, next) => {
});
});
-app.listen(PORT, () => {
- console.log(`holdforme listening on :${PORT} (${IS_PROD ? 'prod' : 'dev'})`);
+// Plain http server so we can attach the WebSocket bridge on the same port.
+const server = http.createServer(app);
+attachListenBridge(server);
+
+server.listen(PORT, () => {
+ console.log(`holdforme listening on :${PORT} (${IS_PROD ? 'prod' : 'dev'}) · ws bridge attached`);
// Start the call worker — DRY_RUN by default; set TWILIO_DRY_RUN=0 + Twilio creds for live.
// Skip if HFM_NO_WORKER=1 (useful for unit tests that don't want the background tick).
if (!process.env.HFM_NO_WORKER) {
diff --git a/views/public/listen.ejs b/views/public/listen.ejs
new file mode 100644
index 0000000..d99caae
--- /dev/null
+++ b/views/public/listen.ejs
@@ -0,0 +1,97 @@
+<%- include('../partials/head', { title, meta_desc }) %>
+<%- include('../partials/header') %>
+
+<main class="container" style="max-width: 880px; padding: 24px;">
+
+ <h1 style="margin: 0 0 6px;">Listen in — <%= call.business_name %></h1>
+ <p class="muted" style="margin: 0 0 18px;">
+ Call <code><%= call.id %></code> · status <strong id="status-pill"><%= call.status %></strong>
+ · <a href="/calls/<%= call.id %>">back to call detail</a>
+ </p>
+
+ <div class="card" style="padding: 20px; border-radius: 10px; border: 1px solid var(--line); background: var(--panel);">
+ <div style="display:flex; align-items:center; gap: 16px; margin-bottom: 12px;">
+ <button id="connect-btn" style="padding: 10px 18px; font-size: 14px;">▶ Connect & listen</button>
+ <span id="ws-state" class="muted" style="font-size: 13px;">disconnected</span>
+ <span id="frame-count" class="muted" style="font-size: 12px; margin-left: auto;">0 frames</span>
+ </div>
+ <div style="background: var(--panel-2); border: 1px solid var(--line); border-radius: 6px; padding: 10px; font-size: 12px; line-height: 1.6; color: var(--ink-dim);">
+ <strong style="color: var(--ink);">Privacy posture:</strong>
+ The called party hears a two-party-consent announcement before audio is bridged.
+ The full call is also recorded server-side via Twilio; a Whisper transcript is generated when the call ends.
+ Your listen-in here is read-only — you cannot speak into this stream.
+ </div>
+ </div>
+
+ <details style="margin-top: 16px;">
+ <summary style="cursor: pointer; color: var(--ink-dim); font-size: 13px;">technical notes</summary>
+ <pre style="background: var(--panel); padding: 12px; border-radius: 6px; font-size: 11px; line-height: 1.5; overflow-x: auto;">audio: µ-law 8 kHz mono → PCM16 LE on the server → Web Audio API playback
+ws lane: /listen-ws/<%= call.id %>
+twilio stream: /stream/<%= call.id %>
+recording: data/recordings/<%= call.id %>.mp3 (post-call)
+transcript: data/transcripts/<%= call.id %>.json (post-call)</pre>
+ </details>
+
+</main>
+
+<script>
+(function() {
+ const callId = <%- JSON.stringify(call.id) %>;
+ const SAMPLE_RATE = 8000;
+ const btn = document.getElementById('connect-btn');
+ const stateEl = document.getElementById('ws-state');
+ const countEl = document.getElementById('frame-count');
+ let ws = null;
+ let audioCtx = null;
+ let frameCount = 0;
+ let nextPlayTime = 0;
+
+ function connect() {
+ const scheme = location.protocol === 'https:' ? 'wss:' : 'ws:';
+ const url = `${scheme}//${location.host}/listen-ws/${callId}`;
+ audioCtx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: SAMPLE_RATE });
+ nextPlayTime = audioCtx.currentTime + 0.1;
+
+ ws = new WebSocket(url);
+ ws.binaryType = 'arraybuffer';
+ ws.onopen = () => { stateEl.textContent = 'connected · waiting for audio'; btn.textContent = '■ Disconnect'; };
+ ws.onclose = () => { stateEl.textContent = 'disconnected'; btn.textContent = '▶ Connect & listen'; };
+ ws.onerror = () => { stateEl.textContent = 'error'; };
+ ws.onmessage = (ev) => {
+ // PCM16 LE @ 8kHz mono
+ const pcm16 = new Int16Array(ev.data);
+ const float32 = new Float32Array(pcm16.length);
+ for (let i = 0; i < pcm16.length; i++) float32[i] = pcm16[i] / 32768;
+ const buf = audioCtx.createBuffer(1, float32.length, SAMPLE_RATE);
+ buf.copyToChannel(float32, 0);
+ const src = audioCtx.createBufferSource();
+ src.buffer = buf;
+ src.connect(audioCtx.destination);
+ const startAt = Math.max(nextPlayTime, audioCtx.currentTime + 0.02);
+ src.start(startAt);
+ nextPlayTime = startAt + buf.duration;
+ frameCount++;
+ if (frameCount % 25 === 0) countEl.textContent = frameCount + ' frames';
+ };
+ }
+
+ function disconnect() {
+ if (ws) { ws.close(); ws = null; }
+ if (audioCtx) { audioCtx.close(); audioCtx = null; }
+ }
+
+ btn.addEventListener('click', () => {
+ if (ws) disconnect(); else connect();
+ });
+
+ // Poll call status every 5s so the pill updates as the call moves through stages.
+ setInterval(async () => {
+ try {
+ const r = await fetch('/api/category/other', { method: 'GET' });
+ // Lightweight ping — for now we just keep the page alive.
+ } catch {}
+ }, 5000);
+})();
+</script>
+
+<%- include('../partials/footer') %>
← 040cea4 always-announce two-party consent preamble — Steve's standin
·
back to Butlr
·
rebrand: HoldForMe → Butlr (mass swap across 27 files: brand 54497d0 →