← back to Exo

fix audio buffering

c94ffa0e2bc02f541f3adf836db21c8e4986d94f · 2024-08-18 20:53:16 +0100 · Alex Cheema

Files touched

• M examples/astra/astra/ContentView.swift

Diff

commit c94ffa0e2bc02f541f3adf836db21c8e4986d94f
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Sun Aug 18 20:53:16 2024 +0100

    fix audio buffering
---
 examples/astra/astra/ContentView.swift | 42 +++++++++++++++++++++++++++-------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/examples/astra/astra/ContentView.swift b/examples/astra/astra/ContentView.swift
index 5e3d055c..b75e9601 100644
--- a/examples/astra/astra/ContentView.swift
+++ b/examples/astra/astra/ContentView.swift
@@ -10,7 +10,7 @@ struct ContentView: View {
     @State private var bufferSeconds: Double = 0.5 // or whatever the actual buffer size is
     @State private var modelState: ModelState = .unloaded
 
-    @AppStorage("selectedModel") private var selectedModel: String = "large"
+    @AppStorage("selectedModel") private var selectedModel: String = "large-v3"
     @AppStorage("selectedLanguage") private var selectedLanguage: String = "english"
     @AppStorage("selectedTask") private var selectedTask: String = "transcribe"
 
@@ -18,10 +18,13 @@ struct ContentView: View {
     @State private var currentMemo = ""
     @State private var lastVoiceActivityTime = Date()
     @State private var silenceTimer: Timer?
-    @State private var voiceActivityThreshold: Float = 0.3 // Start with a lower value
+    @State private var voiceActivityThreshold: Float = 0.1 // Lower this value
     @State private var silenceTimeThreshold = 1.0
     @State private var debugText = ""
     @State private var apiEndpoint = "http://192.168.212.74:8000/v1/chat/completions"
+    @State private var audioBuffer: [Float] = []
+    @State private var bufferDuration: Double = 0.5 // 0.5 seconds buffer
+    @State private var isInitialTranscription = true
 
     var body: some View {
         VStack {
@@ -37,7 +40,7 @@ struct ContentView: View {
             }
 
             Picker("Model", selection: $selectedModel) {
-                Text("large").tag("large")
+                Text("large-v3").tag("large-v3")
                 Text("base").tag("base")
                 Text("small").tag("small")
             }
@@ -71,12 +74,26 @@ struct ContentView: View {
                 whisperKit = try await WhisperKit(verbose: true)
                 print("WhisperKit initialized successfully")
                 startListening()
+                startAudioBuffering() // Add this line
             } catch {
                 print("Error initializing WhisperKit: \(error)")
             }
         }
     }
 
+    // Add this new function
+    private func startAudioBuffering() {
+        Task {
+            while true {
+                if let samples = whisperKit?.audioProcessor.audioSamples {
+                    let bufferSize = Int(Double(WhisperKit.sampleRate) * bufferDuration)
+                    audioBuffer = Array(samples.suffix(bufferSize))
+                }
+                try await Task.sleep(nanoseconds: 100_000_000) // Update every 0.1 seconds
+            }
+        }
+    }
+
     private func loadModel(_ model: String) {
         Task {
             let success = try await loadModel(selectedModel)
@@ -170,6 +187,7 @@ struct ContentView: View {
     private func startNewMemo() {
         isRecordingMemo = true
         currentMemo = ""
+        isInitialTranscription = true
         silenceTimer?.invalidate()
         silenceTimer = Timer.scheduledTimer(withTimeInterval: 0.5, repeats: true) { _ in
             checkSilence()
@@ -183,15 +201,22 @@ struct ContentView: View {
             while isRecordingMemo {
                 if let samples = whisperKit?.audioProcessor.audioSamples, samples.count > WhisperKit.sampleRate {
                     do {
-                        let result = try await whisperKit?.transcribe(audioArray: Array(samples))
+                        let samplesToTranscribe: [Float]
+                        if isInitialTranscription {
+                            samplesToTranscribe = audioBuffer + samples
+                            isInitialTranscription = false
+                        } else {
+                            samplesToTranscribe = Array(samples)
+                        }
+                        
+                        let result = try await whisperKit?.transcribe(audioArray: samplesToTranscribe)
                         await MainActor.run {
                             let newText = result?.first?.text ?? ""
                             if !newText.isEmpty {
-                                currentMemo += newText
-                                currentText += newText
+                                currentMemo = newText
+                                currentText = newText
                             }
                         }
-                        whisperKit?.audioProcessor.purgeAudioSamples(keepingLast: 0)
                     } catch {
                         print("Transcription error: \(error)")
                     }
@@ -233,7 +258,8 @@ struct ContentView: View {
         let payload: [String: Any] = [
             "model": "llama-3.1-8b",
             "messages": [["role": "user", "content": memo]],
-            "temperature": 0.7
+            "temperature": 0.7,
+            "stream": true
         ]
 
         guard let jsonData = try? JSONSerialization.data(withJSONObject: payload) else {

← 23c713c0 better readme for astra  ·  back to Exo  ·  fix streaming, change default model to llava 85035438 →