← back to Exo
astra better ui with camera vlm
1a419f1f004da19bbe66a31ca1c2edc6841e9ae9 · 2024-08-19 18:18:05 +0100 · Alex Cheema
Files touched
M examples/astra/astra.xcodeproj/project.pbxprojM examples/astra/astra/ContentView.swiftM examples/astra/astra/astra.entitlements
Diff
commit 1a419f1f004da19bbe66a31ca1c2edc6841e9ae9
Author: Alex Cheema <alexcheema123@gmail.com>
Date: Mon Aug 19 18:18:05 2024 +0100
astra better ui with camera vlm
---
examples/astra/astra.xcodeproj/project.pbxproj | 2 +
examples/astra/astra/ContentView.swift | 402 +++++++++++++++++++++++--
examples/astra/astra/astra.entitlements | 2 +
3 files changed, 385 insertions(+), 21 deletions(-)
diff --git a/examples/astra/astra.xcodeproj/project.pbxproj b/examples/astra/astra.xcodeproj/project.pbxproj
index 3cd74239..bcbf09ed 100644
--- a/examples/astra/astra.xcodeproj/project.pbxproj
+++ b/examples/astra/astra.xcodeproj/project.pbxproj
@@ -432,6 +432,7 @@
ENABLE_HARDENED_RUNTIME = YES;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
+ INFOPLIST_KEY_NSCameraUsageDescription = "Capture from camera to send to vision model";
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Uses your microphone for transcribing audio";
"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphoneos*]" = YES;
"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphonesimulator*]" = YES;
@@ -471,6 +472,7 @@
ENABLE_HARDENED_RUNTIME = YES;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
+ INFOPLIST_KEY_NSCameraUsageDescription = "Capture from camera to send to vision model";
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Uses your microphone for transcribing audio";
"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphoneos*]" = YES;
"INFOPLIST_KEY_UIApplicationSceneManifest_Generation[sdk=iphonesimulator*]" = YES;
diff --git a/examples/astra/astra/ContentView.swift b/examples/astra/astra/ContentView.swift
index 42c845b0..0dfb40de 100644
--- a/examples/astra/astra/ContentView.swift
+++ b/examples/astra/astra/ContentView.swift
@@ -3,6 +3,131 @@ import WhisperKit
import AVFoundation
import Foundation
import Combine
+import Vision
+import AVFAudio
+
+actor CameraActor {
+ let captureSession = AVCaptureSession()
+ private let photoOutput = AVCapturePhotoOutput()
+ private var isConfigured = false
+ private var currentPhotoCaptureDelegate: PhotoCaptureDelegate?
+
+ func configure() throws {
+ guard !isConfigured else {
+ print("Camera already configured")
+ return
+ }
+
+ print("Starting camera configuration")
+
+ guard let camera = AVCaptureDevice.default(for: .video) else {
+ print("No camera device available")
+ throw CameraError.cameraUnavailable
+ }
+
+ do {
+ let input = try AVCaptureDeviceInput(device: camera)
+ print("Camera input created successfully")
+
+ guard captureSession.canAddInput(input) else {
+ print("Cannot add camera input to session")
+ throw CameraError.cannotAddInputOutput
+ }
+
+ guard captureSession.canAddOutput(photoOutput) else {
+ print("Cannot add photo output to session")
+ throw CameraError.cannotAddInputOutput
+ }
+
+ captureSession.beginConfiguration()
+ captureSession.addInput(input)
+ captureSession.addOutput(photoOutput)
+ captureSession.commitConfiguration()
+
+ print("Camera session configured successfully")
+
+ Task.detached { [weak self] in
+ self?.captureSession.startRunning()
+ print("Camera session started running")
+ }
+
+ isConfigured = true
+ print("Camera fully configured and ready")
+ } catch {
+ print("Error during camera configuration: \(error)")
+ throw error
+ }
+ }
+
+ func capturePhoto() async throws -> String {
+ guard isConfigured else {
+ throw CameraError.notConfigured
+ }
+
+ return try await withCheckedThrowingContinuation { continuation in
+ let photoSettings = AVCapturePhotoSettings()
+
+ let delegate = PhotoCaptureDelegate { result in
+ self.currentPhotoCaptureDelegate = nil
+ continuation.resume(with: result)
+ }
+
+ self.currentPhotoCaptureDelegate = delegate
+
+ Task { @MainActor in
+ self.photoOutput.capturePhoto(with: photoSettings, delegate: delegate)
+ }
+ }
+ }
+}
+
+class PhotoCaptureDelegate: NSObject, AVCapturePhotoCaptureDelegate {
+ private let completionHandler: (Result<String, Error>) -> Void
+
+ init(completionHandler: @escaping (Result<String, Error>) -> Void) {
+ self.completionHandler = completionHandler
+ }
+
+ func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
+ if let error = error {
+ completionHandler(.failure(error))
+ return
+ }
+
+ guard let imageData = photo.fileDataRepresentation() else {
+ completionHandler(.failure(CameraError.imageProcessingFailed))
+ return
+ }
+
+ let base64String = imageData.base64EncodedString()
+ completionHandler(.success(base64String))
+ }
+}
+
+enum CameraError: Error {
+ case cameraUnavailable
+ case cannotAddInputOutput
+ case notConfigured
+ case imageProcessingFailed
+}
+
+struct CameraPreview: UIViewRepresentable {
+ let cameraActor: CameraActor
+
+ func makeUIView(context: Context) -> UIView {
+ let view = UIView(frame: .zero)
+ let previewLayer = AVCaptureVideoPreviewLayer(session: cameraActor.captureSession)
+ previewLayer.videoGravity = .resizeAspectFill
+ view.layer.addSublayer(previewLayer)
+ return view
+ }
+
+ func updateUIView(_ uiView: UIView, context: Context) {
+ if let previewLayer = uiView.layer.sublayers?.first as? AVCaptureVideoPreviewLayer {
+ previewLayer.frame = uiView.bounds
+ }
+ }
+}
struct ContentView: View {
@State private var whisperKit: WhisperKit?
@@ -19,7 +144,7 @@ struct ContentView: View {
@State private var currentMemo = ""
@State private var lastVoiceActivityTime = Date()
@State private var silenceTimer: Timer?
- @State private var voiceActivityThreshold: Float = 0.33
+ @State private var voiceActivityThreshold: Float = 0.40
@State private var silenceTimeThreshold = 1.0
@State private var debugText = ""
@State private var apiEndpoint = "http://192.168.212.74:8000/v1/chat/completions"
@@ -29,6 +154,24 @@ struct ContentView: View {
@State private var streamingResponse = ""
@State private var cancellables = Set<AnyCancellable>()
+ @State private var cameraActor: CameraActor?
+ @State private var showLiveCamera = false
+ @State private var capturedImageBase64: String?
+ @State private var errorMessage: String?
+ @State private var isCameraReady = false
+
+ @State private var speechSynthesizer = AVSpeechSynthesizer()
+ @State private var speechBuffer = ""
+ @State private var wordCount = 0
+ let maxWords = 12
+ @State private var originalSilenceThreshold: Float = 0.40
+ @State private var isTTSActive: Bool = false
+ @State private var canRecordAudio: Bool = true
+ @State private var ttsFinishTime: Date?
+
+ @State private var isRequestInProgress = false
+ @State private var isFirst3WordsOfResponse = true
+
var body: some View {
VStack {
Text(currentText)
@@ -62,6 +205,18 @@ struct ContentView: View {
.font(.caption)
.foregroundColor(.gray)
+ Text("TTS Active: \(isTTSActive ? "Yes" : "No")")
+ .font(.caption)
+ .foregroundColor(isTTSActive ? .green : .red)
+
+ Text("Current Silence Threshold: \(voiceActivityThreshold, specifier: "%.2f")")
+ .font(.caption)
+ .foregroundColor(.blue)
+
+ Text("Original Silence Threshold: \(originalSilenceThreshold, specifier: "%.2f")")
+ .font(.caption)
+ .foregroundColor(.orange)
+
Slider(value: $voiceActivityThreshold, in: 0.01...1.0) {
Text("Voice Activity Threshold: \(voiceActivityThreshold, specifier: "%.2f")")
}
@@ -76,9 +231,52 @@ struct ContentView: View {
}
.frame(height: 200)
.border(Color.gray, width: 1)
+
+ Toggle("Show Live Camera", isOn: $showLiveCamera)
+ .padding()
+ .onChange(of: showLiveCamera) { newValue in
+ if newValue {
+ Task {
+ await setupCamera()
+ }
+ } else {
+ cameraActor = nil
+ isCameraReady = false
+ print("Camera disabled")
+ }
+ }
+
+ if showLiveCamera {
+ if isCameraReady, let actor = cameraActor {
+ CameraPreview(cameraActor: actor)
+ .frame(height: 200)
+ .cornerRadius(10)
+ .padding()
+
+ Button("Capture Photo") {
+ Task {
+ await capturePhoto()
+ }
+ }
+ .padding()
+ } else {
+ ProgressView("Initializing camera...")
+ .padding()
+ }
+ }
+
+ Text("Camera Ready: \(isCameraReady ? "Yes" : "No")")
+ .padding()
+
+ if let errorMessage = errorMessage {
+ Text("Error: \(errorMessage)")
+ .foregroundColor(.red)
+ .padding()
+ }
}
.onAppear {
setupWhisperKit()
+ startTTSMonitoring()
}
}
@@ -88,14 +286,48 @@ struct ContentView: View {
whisperKit = try await WhisperKit(verbose: true)
print("WhisperKit initialized successfully")
startListening()
- startAudioBuffering() // Add this line
+ startAudioBuffering()
} catch {
print("Error initializing WhisperKit: \(error)")
}
}
}
- // Add this new function
+ private func startTTSMonitoring() {
+ Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { _ in
+ let newTTSActive = speechSynthesizer.isSpeaking
+ if newTTSActive != isTTSActive {
+ isTTSActive = newTTSActive
+ canRecordAudio = !newTTSActive
+ if isTTSActive {
+ voiceActivityThreshold = 1.0 // Set to max to prevent recording
+ whisperKit?.audioProcessor.purgeAudioSamples(keepingLast: 0) // Flush audio buffer
+ print("TTS Started - Audio recording paused")
+ } else {
+ ttsFinishTime = Date()
+ print("TTS Finished - Waiting 0.5 seconds before resuming audio recording")
+ }
+ updateDebugText()
+ }
+
+ if !isTTSActive, let finishTime = ttsFinishTime, Date().timeIntervalSince(finishTime) >= 0.5 {
+ whisperKit?.audioProcessor.purgeAudioSamples(keepingLast: 0) // Flush audio buffer
+ voiceActivityThreshold = originalSilenceThreshold
+ canRecordAudio = true
+ ttsFinishTime = nil
+ print("Audio recording resumed after TTS delay")
+ updateDebugText()
+ }
+ }
+ }
+
+ private func updateDebugText() {
+ debugText += "\nTTS Active: \(isTTSActive)"
+ debugText += "\nCurrent Silence Threshold: \(voiceActivityThreshold)"
+ debugText += "\nOriginal Silence Threshold: \(originalSilenceThreshold)"
+ debugText += "\n---"
+ }
+
private func startAudioBuffering() {
Task {
while true {
@@ -138,7 +370,7 @@ struct ContentView: View {
}
private func checkVoiceActivity() {
- guard let audioProcessor = whisperKit?.audioProcessor else { return }
+ guard canRecordAudio, let audioProcessor = whisperKit?.audioProcessor else { return }
let voiceDetected = AudioProcessor.isVoiceDetected(
in: audioProcessor.relativeEnergy,
@@ -146,7 +378,6 @@ struct ContentView: View {
silenceThreshold: Float(voiceActivityThreshold)
)
- // Debug logging
let energyValuesToConsider = Int(Float(bufferSeconds) / 0.1)
let nextBufferEnergies = audioProcessor.relativeEnergy.suffix(energyValuesToConsider)
let numberOfValuesToCheck = max(10, nextBufferEnergies.count - 10)
@@ -190,7 +421,6 @@ struct ContentView: View {
saveMemoToFile(currentMemo)
currentMemo = ""
}
- // Flush the transcribed text and reset audio samples
currentText = ""
whisperKit?.audioProcessor.purgeAudioSamples(keepingLast: 0)
print("Ended memo")
@@ -213,7 +443,7 @@ struct ContentView: View {
private func transcribeInRealTime() {
Task {
while isRecordingMemo {
- if let samples = whisperKit?.audioProcessor.audioSamples, samples.count > WhisperKit.sampleRate {
+ if canRecordAudio, let samples = whisperKit?.audioProcessor.audioSamples, samples.count > WhisperKit.sampleRate {
do {
let samplesToTranscribe: [Float]
if isInitialTranscription {
@@ -222,7 +452,7 @@ struct ContentView: View {
} else {
samplesToTranscribe = Array(samples)
}
-
+
let result = try await whisperKit?.transcribe(audioArray: samplesToTranscribe)
await MainActor.run {
let newText = result?.first?.text ?? ""
@@ -259,12 +489,85 @@ struct ContentView: View {
print("Error saving memo: \(error)")
}
- // After saving to file, send HTTP request
- sendMemoToAPI(memo)
+ Task {
+ if !isCameraReady {
+ print("Camera not ready, initializing...")
+ await setupCamera()
+ }
+
+ if let imageBase64 = await capturePhotoBase64() {
+ sendMemoToAPI(memo, imageBase64: imageBase64)
+ } else {
+ sendMemoToAPI(memo)
+ }
+ }
+ }
+
+ private func setupCamera() async {
+ print("Setting up camera...")
+ do {
+ let actor = CameraActor()
+ print("CameraActor instance created")
+ try await actor.configure()
+ print("Camera configured successfully")
+ await MainActor.run {
+ self.cameraActor = actor
+ self.errorMessage = nil
+ self.isCameraReady = true
+ print("Camera setup complete, UI updated")
+ }
+ } catch {
+ print("Camera setup failed: \(error)")
+ await MainActor.run {
+ self.errorMessage = "Failed to initialize camera: \(error.localizedDescription)"
+ self.isCameraReady = false
+ print("Camera setup failure reflected in UI")
+ }
+ }
+ }
+
+ private func capturePhotoBase64() async -> String? {
+ print("Attempting to capture photo...")
+ if !isCameraReady {
+ print("Camera not ready, attempting to initialize...")
+ await setupCamera()
+ }
+
+ guard let actor = cameraActor, isCameraReady else {
+ print("Camera not initialized or not ready, cannot capture photo")
+ await MainActor.run {
+ self.errorMessage = "Camera not initialized or not ready"
+ }
+ return nil
+ }
+
+ do {
+ let base64String = try await actor.capturePhoto()
+ print("Photo captured successfully")
+ await MainActor.run {
+ self.errorMessage = nil
+ }
+ return base64String
+ } catch {
+ print("Error capturing photo: \(error)")
+ await MainActor.run {
+ self.errorMessage = "Failed to capture photo: \(error.localizedDescription)"
+ }
+ return nil
+ }
}
- private func sendMemoToAPI(_ memo: String) {
+ private func sendMemoToAPI(_ memo: String, imageBase64: String? = nil) {
Task {
+ guard !isRequestInProgress else {
+ print("A request is already in progress. Skipping this one.")
+ return
+ }
+
+ isRequestInProgress = true
+ isFirst3WordsOfResponse = true // Reset for new request
+ defer { isRequestInProgress = false }
+
do {
print("Starting API request for memo: \(memo.prefix(50))...")
@@ -273,21 +576,25 @@ struct ContentView: View {
return
}
- let payload: [String: Any] = [
+ var payload: [String: Any] = [
"model": "llava-1.5-7b-hf",
"messages": [
- ["role": "system", "content": ["type": "text", "text": "You are a helpful chat assistant being used with Whisper voice transcription. Please assist the user with their queries."]],
- ["role": "user", "content": ["type": "text", "text": memo]]
+ ["role": "user", "content": [
+ ["type": "text", "text": "You are a helpful conversational assistant chatting with a Gen Z user using their iPhone for voice transcription and sending images to you with their iPhone camera. Be conversational and concise, with a laid back attitude and be cheerful with humour. User said: " + memo],
+ ]]
],
"temperature": 0.7,
"stream": true
]
- // let payload: [String: Any] = [
- // "model": "llama-3.1-8b",
- // "messages": [["role": "system", "content": "You are a helpful chat assistant being used with Whisper voice transcription. Please assist the user with their queries."], ["role": "user", "content": memo]],
- // "temperature": 0.7,
- // "stream": true
- // ]
+
+ if let imageBase64 = imageBase64 {
+ if var userMessage = (payload["messages"] as? [[String: Any]])?.last,
+ var content = userMessage["content"] as? [[String: Any]] {
+ content.append(["type": "image_url", "image_url": ["url": "data:image/jpeg;base64,\(imageBase64)"]])
+ userMessage["content"] = content
+ payload["messages"] = [userMessage]
+ }
+ }
guard let jsonData = try? JSONSerialization.data(withJSONObject: payload) else {
print("Failed to serialize JSON payload")
@@ -301,7 +608,6 @@ struct ContentView: View {
print("Sending request to \(url.absoluteString)")
- // Reset the streaming response
await MainActor.run {
self.streamingResponse = ""
}
@@ -348,10 +654,38 @@ struct ContentView: View {
print("Extracted content: \(content)")
await MainActor.run {
self.streamingResponse += content
+ bufferContent(content)
}
}
}
+ private func bufferContent(_ content: String) {
+ speechBuffer += content
+ let words = speechBuffer.split(separator: " ")
+ wordCount = words.count
+
+ if isFirst3WordsOfResponse && wordCount >= 3 {
+ isFirst3WordsOfResponse = false
+ speakBufferedContent()
+ } else if content.contains(".") || content.contains("!") || content.contains("?") || wordCount >= maxWords {
+ speakBufferedContent()
+ }
+ }
+
+ private func speakBufferedContent() {
+ guard !speechBuffer.isEmpty else { return }
+ speakContent(speechBuffer)
+ speechBuffer = ""
+ wordCount = 0
+ }
+
+ private func speakContent(_ content: String) {
+ let utterance = AVSpeechUtterance(string: content)
+ utterance.voice = AVSpeechSynthesisVoice(language: "en-US")
+ utterance.rate = 0.5
+ speechSynthesizer.speak(utterance)
+ }
+
private func loadModel(_ model: String) async throws -> Bool {
guard let whisperKit = whisperKit else {
print("WhisperKit instance not initialized")
@@ -372,4 +706,30 @@ struct ContentView: View {
return false
}
}
+
+ private func capturePhoto() async {
+ print("Attempting to capture photo...")
+ print("Camera ready: \(isCameraReady), CameraActor exists: \(cameraActor != nil)")
+ guard let actor = cameraActor, isCameraReady else {
+ print("Camera not initialized or not ready, cannot capture photo")
+ await MainActor.run {
+ self.errorMessage = "Camera not initialized or not ready"
+ }
+ return
+ }
+
+ do {
+ let base64String = try await actor.capturePhoto()
+ print("Photo captured successfully")
+ await MainActor.run {
+ self.capturedImageBase64 = base64String
+ self.errorMessage = nil
+ }
+ } catch {
+ print("Error capturing photo: \(error)")
+ await MainActor.run {
+ self.errorMessage = "Failed to capture photo: \(error.localizedDescription)"
+ }
+ }
+ }
}
diff --git a/examples/astra/astra/astra.entitlements b/examples/astra/astra/astra.entitlements
index dc74f055..ee23443d 100644
--- a/examples/astra/astra/astra.entitlements
+++ b/examples/astra/astra/astra.entitlements
@@ -16,5 +16,7 @@
<true/>
<key>com.apple.security.network.server</key>
<true/>
+ <key>com.apple.security.device.camera</key>
+ <true/>
</dict>
</plist>
← 85035438 fix streaming, change default model to llava
·
back to Exo
·
astra: live camera with overlay debug info / ui e87e7260 →