← back to Voice Chat
initial scaffold: voice-chat — OpenAI Realtime WebRTC, sub-500ms turn
dcc17073ae0e454316e9109e8c8e10a130f73645 · 2026-05-10 22:50:32 -0700 · Steve Abrams
Files touched
A .gitignoreA ecosystem.config.cjsA package.jsonA public/index.htmlA public/voice-chat.jsA server.js
Diff
commit dcc17073ae0e454316e9109e8c8e10a130f73645
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sun May 10 22:50:32 2026 -0700
initial scaffold: voice-chat — OpenAI Realtime WebRTC, sub-500ms turn
---
.gitignore | 5 ++
ecosystem.config.cjs | 12 ++++
package.json | 13 ++++
public/index.html | 96 ++++++++++++++++++++++++++++++
public/voice-chat.js | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++
server.js | 145 +++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 434 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..e375bd2
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+node_modules/
+.env
+.env.*
+*.log
+.DS_Store
diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs
new file mode 100644
index 0000000..4235b68
--- /dev/null
+++ b/ecosystem.config.cjs
@@ -0,0 +1,12 @@
+module.exports = {
+ apps: [{
+ name: "voice-chat",
+ script: "server.js",
+ cwd: __dirname,
+ env: {
+ PORT: 9870,
+ },
+ autorestart: true,
+ max_memory_restart: "300M",
+ }],
+};
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..827fd54
--- /dev/null
+++ b/package.json
@@ -0,0 +1,13 @@
+{
+ "name": "voice-chat",
+ "version": "0.1.0",
+ "description": "Voice-native chat using OpenAI Realtime API — sub-500ms turn time. Browser captures mic, server mints ephemeral session, client connects directly to OpenAI for the audio stream.",
+ "type": "module",
+ "main": "server.js",
+ "scripts": {
+ "start": "node server.js"
+ },
+ "dependencies": {
+ "express": "^4.21.0"
+ }
+}
diff --git a/public/index.html b/public/index.html
new file mode 100644
index 0000000..d3f6573
--- /dev/null
+++ b/public/index.html
@@ -0,0 +1,96 @@
+<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
+ <title>voice-chat</title>
+ <style>
+ :root {
+ --bg: #0a0a0c;
+ --fg: #f4f4f5;
+ --muted: #9ca3af;
+ --accent: #ef4444;
+ --green: #22c55e;
+ --border: #1f2937;
+ }
+ *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+ html, body {
+ background: var(--bg);
+ color: var(--fg);
+ font-family: -apple-system, BlinkMacSystemFont, "Inter", "Segoe UI", sans-serif;
+ height: 100vh;
+ overflow: hidden;
+ }
+ body {
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: center;
+ gap: 32px;
+ padding: 24px;
+ }
+ #mic {
+ width: 220px;
+ height: 220px;
+ border-radius: 50%;
+ background: var(--accent);
+ border: none;
+ color: white;
+ font-size: 18px;
+ font-weight: 600;
+ letter-spacing: 0.5px;
+ cursor: pointer;
+ box-shadow: 0 12px 48px rgba(239, 68, 68, 0.4);
+ transition: transform 0.15s ease, box-shadow 0.15s ease, background 0.2s;
+ user-select: none;
+ }
+ #mic:hover { transform: scale(1.03); }
+ #mic:active { transform: scale(0.97); }
+ #mic.connected {
+ background: var(--green);
+ box-shadow: 0 12px 48px rgba(34, 197, 94, 0.35);
+ animation: pulse 1.8s ease-in-out infinite;
+ }
+ #mic.error { background: #6b7280; }
+ @keyframes pulse {
+ 0%, 100% { box-shadow: 0 12px 48px rgba(34, 197, 94, 0.35); }
+ 50% { box-shadow: 0 12px 64px rgba(34, 197, 94, 0.7); }
+ }
+ #status {
+ color: var(--muted);
+ font-size: 14px;
+ letter-spacing: 0.5px;
+ text-align: center;
+ min-height: 18px;
+ }
+ #log {
+ width: min(720px, 95vw);
+ max-height: 240px;
+ overflow-y: auto;
+ border: 1px solid var(--border);
+ border-radius: 12px;
+ padding: 16px;
+ font-size: 13px;
+ line-height: 1.55;
+ background: rgba(255,255,255,0.02);
+ }
+ #log .me { color: #facc15; }
+ #log .ai { color: #e0e7ff; }
+ #log .sys { color: var(--muted); font-style: italic; }
+ #log .row { margin-bottom: 4px; }
+ #latency {
+ color: var(--muted);
+ font-size: 12px;
+ font-variant-numeric: tabular-nums;
+ }
+ </style>
+</head>
+<body>
+ <button id="mic">START</button>
+ <div id="status">tap to begin</div>
+ <div id="log"></div>
+ <div id="latency"></div>
+
+ <script type="module" src="voice-chat.js"></script>
+</body>
+</html>
diff --git a/public/voice-chat.js b/public/voice-chat.js
new file mode 100644
index 0000000..f961fea
--- /dev/null
+++ b/public/voice-chat.js
@@ -0,0 +1,163 @@
+// voice-chat client — opens a WebRTC peer connection directly to OpenAI
+// using an ephemeral key minted by our server. No audio passes through us.
+
+const $mic = document.getElementById("mic");
+const $status = document.getElementById("status");
+const $log = document.getElementById("log");
+const $latency = document.getElementById("latency");
+
+let pc = null; // RTCPeerConnection
+let dc = null; // RTCDataChannel for control messages
+let audioEl = null; // remote audio playback
+let micStream = null;
+let lastUserSpokeAt = 0;
+
+function log(role, text) {
+ const row = document.createElement("div");
+ row.className = "row " + role;
+ const prefix = role === "me" ? "you · " : role === "ai" ? "ai · " : "· ";
+ row.textContent = prefix + text;
+ $log.appendChild(row);
+ $log.scrollTop = $log.scrollHeight;
+}
+
+function setStatus(msg) { $status.textContent = msg; }
+
+async function start() {
+ $mic.disabled = true;
+ setStatus("requesting mic…");
+
+ try {
+ micStream = await navigator.mediaDevices.getUserMedia({
+ audio: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+ channelCount: 1,
+ sampleRate: 24000,
+ },
+ });
+ } catch (e) {
+ setStatus("mic denied — check browser permissions");
+ $mic.classList.add("error");
+ $mic.textContent = "MIC DENIED";
+ $mic.disabled = false;
+ return;
+ }
+
+ setStatus("minting session…");
+ let session;
+ try {
+ const r = await fetch("/api/session", { method: "POST" });
+ if (!r.ok) throw new Error("session " + r.status);
+ session = await r.json();
+ } catch (e) {
+ setStatus("session error: " + e.message);
+ $mic.classList.add("error");
+ $mic.disabled = false;
+ return;
+ }
+ const EPHEMERAL_KEY = session.client_secret.value;
+ const MODEL = session.model;
+
+ setStatus("connecting to openai…");
+
+ // RTCPeerConnection
+ pc = new RTCPeerConnection();
+
+ // Remote audio: AI voice
+ audioEl = document.createElement("audio");
+ audioEl.autoplay = true;
+ pc.ontrack = (e) => {
+ audioEl.srcObject = e.streams[0];
+ };
+
+ // Local mic → peer
+ for (const track of micStream.getTracks()) {
+ pc.addTrack(track, micStream);
+ }
+
+ // Data channel for control + transcripts
+ dc = pc.createDataChannel("oai-events");
+ dc.onopen = () => {
+ setStatus("connected — speak naturally");
+ $mic.classList.add("connected");
+ $mic.textContent = "LISTENING";
+ log("sys", "session open");
+ };
+ dc.onmessage = (e) => {
+ let m;
+ try { m = JSON.parse(e.data); } catch { return; }
+ switch (m.type) {
+ case "response.audio_transcript.done":
+ log("ai", m.transcript);
+ if (lastUserSpokeAt) {
+ $latency.textContent = `last turn: ${Date.now() - lastUserSpokeAt} ms (mouth-to-mouth)`;
+ }
+ break;
+ case "conversation.item.input_audio_transcription.completed":
+ log("me", m.transcript);
+ lastUserSpokeAt = Date.now();
+ break;
+ case "input_audio_buffer.speech_started":
+ setStatus("you're speaking…");
+ break;
+ case "input_audio_buffer.speech_stopped":
+ setStatus("thinking…");
+ lastUserSpokeAt = Date.now();
+ break;
+ case "response.done":
+ setStatus("connected — speak naturally");
+ break;
+ case "error":
+ log("sys", "ERROR: " + (m.error?.message || JSON.stringify(m.error)));
+ break;
+ }
+ };
+
+ // SDP offer/answer
+ const offer = await pc.createOffer();
+ await pc.setLocalDescription(offer);
+
+ const sdpResp = await fetch(`https://api.openai.com/v1/realtime?model=${MODEL}`, {
+ method: "POST",
+ body: offer.sdp,
+ headers: {
+ Authorization: `Bearer ${EPHEMERAL_KEY}`,
+ "Content-Type": "application/sdp",
+ },
+ });
+ if (!sdpResp.ok) {
+ const err = await sdpResp.text();
+ setStatus("openai webrtc error: " + sdpResp.status);
+ log("sys", err.slice(0, 300));
+ return;
+ }
+ const answer = { type: "answer", sdp: await sdpResp.text() };
+ await pc.setRemoteDescription(answer);
+
+ $mic.disabled = false;
+ $mic.textContent = "STOP";
+}
+
+async function stop() {
+ setStatus("disconnecting…");
+ try { dc && dc.close(); } catch {}
+ try { pc && pc.close(); } catch {}
+ if (micStream) micStream.getTracks().forEach(t => t.stop());
+ pc = null; dc = null;
+ $mic.classList.remove("connected", "error");
+ $mic.textContent = "START";
+ setStatus("disconnected");
+}
+
+let running = false;
+$mic.addEventListener("click", async () => {
+ if (running) {
+ running = false;
+ await stop();
+ } else {
+ running = true;
+ await start();
+ }
+});
diff --git a/server.js b/server.js
new file mode 100644
index 0000000..c221fd5
--- /dev/null
+++ b/server.js
@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+/**
+ * voice-chat — minimal voice-native chat server.
+ *
+ * Architecture:
+ * Browser ──(WebRTC + ephemeral key)──> OpenAI Realtime API
+ * │
+ * │ (1) GET / → serves public/index.html
+ * │ (2) POST /api/session → server mints ephemeral session via OpenAI,
+ * │ returns client_secret to browser. Browser
+ * │ uses that token to open a direct WebRTC
+ * │ connection to OpenAI — no audio passes
+ * │ through this server. Latency ≈ 300-500ms.
+ * └─→ Express
+ *
+ * Why ephemeral keys: the long-lived OPENAI_API_KEY never leaves this server.
+ * Each /api/session call mints a short-lived token (~1 min TTL) the browser
+ * can use for one direct WebRTC connection. OpenAI's recommended pattern
+ * for browser clients.
+ *
+ * Env vars (in .env):
+ * OPENAI_API_KEY — required, must have Realtime API access
+ * VOICE — optional, default "alloy" (alloy|echo|fable|onyx|nova|shimmer|ash|ballad|coral|sage|verse)
+ * MODEL — optional, default "gpt-4o-realtime-preview-2024-12-17"
+ * PORT — optional, default 9870
+ * SYSTEM_INSTRUCTIONS — optional, default short Steve-flavored prompt
+ * BASIC_AUTH — optional, "user:pw" for nginx-style protection
+ */
+
+import express from "express";
+import { promises as fs } from "node:fs";
+import { homedir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// ─── env loading (read .env file if present, no dotenv dep) ──────────────
+async function loadEnvFile() {
+ for (const path of [
+ join(__dirname, ".env"),
+ join(homedir(), "Projects/secrets-manager/.env"),
+ ]) {
+ try {
+ const txt = await fs.readFile(path, "utf-8");
+ for (const line of txt.split("\n")) {
+ const m = line.match(/^\s*([A-Z_][A-Z0-9_]*)\s*=\s*(.*?)\s*$/);
+ if (!m) continue;
+ const k = m[1];
+ if (process.env[k]) continue; // existing env wins
+ let v = m[2];
+ if (v.startsWith('"') && v.endsWith('"')) v = v.slice(1, -1);
+ if (v.startsWith("'") && v.endsWith("'")) v = v.slice(1, -1);
+ process.env[k] = v;
+ }
+ } catch {}
+ }
+}
+await loadEnvFile();
+
+const PORT = Number(process.env.PORT || 9870);
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+const MODEL = process.env.MODEL || "gpt-4o-realtime-preview-2024-12-17";
+const VOICE = process.env.VOICE || "alloy";
+const BASIC_AUTH = process.env.BASIC_AUTH || "";
+const SYSTEM_INSTRUCTIONS =
+ process.env.SYSTEM_INSTRUCTIONS ||
+ "You are a brisk, candid assistant for Steve. Keep replies short — one or two sentences unless asked to elaborate. Skip filler ('great question', 'sure'). If unsure, say so. Talk like a colleague, not a brochure.";
+
+if (!OPENAI_API_KEY) {
+ console.error("voice-chat: OPENAI_API_KEY missing — set it in ~/Projects/voice-chat/.env or via /secrets.");
+ process.exit(1);
+}
+
+// ─── express ─────────────────────────────────────────────────────────────
+
+const app = express();
+app.use(express.json());
+
+// Optional basic-auth gate
+if (BASIC_AUTH) {
+ const expected = "Basic " + Buffer.from(BASIC_AUTH).toString("base64");
+ app.use((req, res, next) => {
+ if (req.path === "/api/health") return next();
+ const got = req.headers.authorization || "";
+ if (got !== expected) {
+ res.set("WWW-Authenticate", 'Basic realm="voice-chat"');
+ return res.status(401).send("auth required");
+ }
+ next();
+ });
+}
+
+// Health
+app.get("/api/health", (req, res) => {
+ res.json({ ok: true, model: MODEL, voice: VOICE });
+});
+
+// Mint ephemeral session
+app.post("/api/session", async (req, res) => {
+ try {
+ const body = {
+ model: MODEL,
+ voice: VOICE,
+ instructions: SYSTEM_INSTRUCTIONS,
+ modalities: ["audio", "text"],
+ turn_detection: {
+ type: "server_vad",
+ threshold: 0.5,
+ silence_duration_ms: 500,
+ },
+ };
+ const r = await fetch("https://api.openai.com/v1/realtime/sessions", {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${OPENAI_API_KEY}`,
+ "Content-Type": "application/json",
+ "OpenAI-Beta": "realtime=v1",
+ },
+ body: JSON.stringify(body),
+ });
+ const data = await r.json();
+ if (!r.ok) {
+ console.error("OpenAI session mint failed:", r.status, data);
+ return res.status(r.status).json({ error: data });
+ }
+ // Return only what the browser needs.
+ res.json({
+ client_secret: data.client_secret,
+ model: MODEL,
+ voice: VOICE,
+ });
+ } catch (e) {
+ console.error("session error:", e);
+ res.status(500).json({ error: e.message });
+ }
+});
+
+app.use(express.static(join(__dirname, "public")));
+
+app.listen(PORT, () => {
+ console.log(`voice-chat ready on http://localhost:${PORT}`);
+ console.log(` model=${MODEL} voice=${VOICE}`);
+ if (BASIC_AUTH) console.log(" basic-auth ON");
+});
(oldest)
·
back to Voice Chat
·
port: 9870 collided with Shop by Color → 9893 182cab1 →