← back to Exo
astra: live camera with overlay debug info / ui
e87e7260f8349b2b73395268db59198f14daa9c1 · 2024-08-20 12:50:05 +0100 · Alex Cheema
Files touched
M examples/astra/astra/ContentView.swift
Diff
commit e87e7260f8349b2b73395268db59198f14daa9c1
Author: Alex Cheema <alexcheema123@gmail.com>
Date: Tue Aug 20 12:50:05 2024 +0100
astra: live camera with overlay debug info / ui
---
examples/astra/astra/ContentView.swift | 170 ++++++++++++++++-----------------
1 file changed, 82 insertions(+), 88 deletions(-)
diff --git a/examples/astra/astra/ContentView.swift b/examples/astra/astra/ContentView.swift
index 0dfb40de..3ae6f1ce 100644
--- a/examples/astra/astra/ContentView.swift
+++ b/examples/astra/astra/ContentView.swift
@@ -111,20 +111,21 @@ enum CameraError: Error {
case imageProcessingFailed
}
-struct CameraPreview: UIViewRepresentable {
+struct CameraPreview: UIViewControllerRepresentable {
let cameraActor: CameraActor
- func makeUIView(context: Context) -> UIView {
- let view = UIView(frame: .zero)
+ func makeUIViewController(context: Context) -> UIViewController {
+ let viewController = UIViewController()
let previewLayer = AVCaptureVideoPreviewLayer(session: cameraActor.captureSession)
previewLayer.videoGravity = .resizeAspectFill
- view.layer.addSublayer(previewLayer)
- return view
+ viewController.view.layer.addSublayer(previewLayer)
+ previewLayer.frame = viewController.view.bounds
+ return viewController
}
- func updateUIView(_ uiView: UIView, context: Context) {
- if let previewLayer = uiView.layer.sublayers?.first as? AVCaptureVideoPreviewLayer {
- previewLayer.frame = uiView.bounds
+ func updateUIViewController(_ uiViewController: UIViewController, context: Context) {
+ if let previewLayer = uiViewController.view.layer.sublayers?.first as? AVCaptureVideoPreviewLayer {
+ previewLayer.frame = uiViewController.view.bounds
}
}
}
@@ -173,106 +174,99 @@ struct ContentView: View {
@State private var isFirst3WordsOfResponse = true
var body: some View {
- VStack {
- Text(currentText)
- .padding()
+ ZStack {
+ if showLiveCamera, isCameraReady, let actor = cameraActor {
+ CameraPreview(cameraActor: actor)
+ .edgesIgnoringSafeArea(.all)
+ }
- Text(isListening ? "Listening..." : "Not listening")
- .foregroundColor(isListening ? .green : .red)
+ ScrollView {
+ VStack {
+ Text(currentText)
+ .padding()
- if isRecordingMemo {
- Text("Recording memo...")
- .foregroundColor(.blue)
- }
+ Text(isListening ? "Listening..." : "Not listening")
+ .foregroundColor(isListening ? .green : .red)
- Picker("Model", selection: $selectedModel) {
- Text("large-v3").tag("large-v3")
- Text("base").tag("base")
- Text("small").tag("small")
- }
- .pickerStyle(SegmentedPickerStyle())
- .padding()
+ if isRecordingMemo {
+ Text("Recording...")
+ .foregroundColor(.blue)
+ }
- Button("Load Model") {
- loadModel(selectedModel)
- }
- .disabled(modelState == .loaded)
- .padding()
+ Picker("Model", selection: $selectedModel) {
+ Text("large-v3").tag("large-v3")
+ Text("base").tag("base")
+ Text("small").tag("small")
+ }
+ .pickerStyle(SegmentedPickerStyle())
+ .padding()
- Text("Model State: \(modelState.description)")
+ Button("Load Model") {
+ loadModel(selectedModel)
+ }
+ .disabled(modelState == .loaded)
+ .padding()
- Text(debugText)
- .font(.caption)
- .foregroundColor(.gray)
+ Text("Model State: \(modelState.description)")
- Text("TTS Active: \(isTTSActive ? "Yes" : "No")")
- .font(.caption)
- .foregroundColor(isTTSActive ? .green : .red)
+ Text(debugText)
+ .font(.caption)
+ .foregroundColor(.gray)
- Text("Current Silence Threshold: \(voiceActivityThreshold, specifier: "%.2f")")
- .font(.caption)
- .foregroundColor(.blue)
+ Text("TTS Active: \(isTTSActive ? "Yes" : "No")")
+ .font(.caption)
+ .foregroundColor(isTTSActive ? .green : .red)
- Text("Original Silence Threshold: \(originalSilenceThreshold, specifier: "%.2f")")
- .font(.caption)
- .foregroundColor(.orange)
+ Text("Current Silence Threshold: \(voiceActivityThreshold, specifier: "%.2f")")
+ .font(.caption)
+ .foregroundColor(.blue)
- Slider(value: $voiceActivityThreshold, in: 0.01...1.0) {
- Text("Voice Activity Threshold: \(voiceActivityThreshold, specifier: "%.2f")")
- }
+ Text("Original Silence Threshold: \(originalSilenceThreshold, specifier: "%.2f")")
+ .font(.caption)
+ .foregroundColor(.orange)
- Text("API Response:")
- .font(.headline)
- .padding(.top)
+ Slider(value: $voiceActivityThreshold, in: 0.01...1.0) {
+ Text("Voice Activity Threshold: \(voiceActivityThreshold, specifier: "%.2f")")
+ }
- ScrollView {
- Text(streamingResponse)
- .padding()
- }
- .frame(height: 200)
- .border(Color.gray, width: 1)
-
- Toggle("Show Live Camera", isOn: $showLiveCamera)
- .padding()
- .onChange(of: showLiveCamera) { newValue in
- if newValue {
- Task {
- await setupCamera()
- }
- } else {
- cameraActor = nil
- isCameraReady = false
- print("Camera disabled")
+ Text("API Response:")
+ .font(.headline)
+ .padding(.top)
+
+ ScrollView {
+ Text(streamingResponse)
+ .padding()
}
- }
+ .frame(height: 200)
+ .border(Color.gray, width: 1)
- if showLiveCamera {
- if isCameraReady, let actor = cameraActor {
- CameraPreview(cameraActor: actor)
- .frame(height: 200)
- .cornerRadius(10)
+ Toggle("Show Live Camera", isOn: $showLiveCamera)
.padding()
+ .onChange(of: showLiveCamera) { newValue in
+ if newValue {
+ Task {
+ await setupCamera()
+ }
+ } else {
+ cameraActor = nil
+ isCameraReady = false
+ print("Camera disabled")
+ }
+ }
+
+ if !showLiveCamera {
+ Text("Camera Ready: \(isCameraReady ? "Yes" : "No")")
+ .padding()
- Button("Capture Photo") {
- Task {
- await capturePhoto()
+ if let errorMessage = errorMessage {
+ Text("Error: \(errorMessage)")
+ .foregroundColor(.red)
+ .padding()
}
}
- .padding()
- } else {
- ProgressView("Initializing camera...")
- .padding()
}
}
-
- Text("Camera Ready: \(isCameraReady ? "Yes" : "No")")
- .padding()
-
- if let errorMessage = errorMessage {
- Text("Error: \(errorMessage)")
- .foregroundColor(.red)
- .padding()
- }
+ .opacity(showLiveCamera ? 0.7 : 1)
}
.onAppear {
setupWhisperKit()
← 1a419f1f astra better ui with camera vlm
·
back to Exo
·
astra clarify readme, it's an example app 430d4c0c →