Implement speech synthesis

fullmoon.xcodeproj/project.pbxproj

@@ -16,7 +16,7 @@
 /* End PBXFileReference section */
 
 /* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */
-		860E9CF12CB0649400C5BB52 /* Exceptions for "fullmoon" folder in "fullmoon" target */ = {
+		8C48AC642CD5C74E00E5CCE9 /* Exceptions for "fullmoon" folder in "fullmoon" target */ = {
 			isa = PBXFileSystemSynchronizedBuildFileExceptionSet;
 			membershipExceptions = (
 				Info.plist,
@@ -29,7 +29,7 @@
 		860E9CD02CB055B000C5BB52 /* fullmoon */ = {
 			isa = PBXFileSystemSynchronizedRootGroup;
 			exceptions = (
-				860E9CF12CB0649400C5BB52 /* Exceptions for "fullmoon" folder in "fullmoon" target */,
+				8C48AC642CD5C74E00E5CCE9 /* Exceptions for "fullmoon" folder in "fullmoon" target */,
 			);
 			path = fullmoon;
 			sourceTree = "<group>";

fullmoon/ContentView.swift

@@ -12,6 +12,7 @@ import MarkdownUI
 struct ContentView: View {
     private var idiom : UIUserInterfaceIdiom { UIDevice.current.userInterfaceIdiom }
     @EnvironmentObject var appManager: AppManager
+    @EnvironmentObject var speechSynthesizer: SpeechSynthesizer
     @Environment(\.modelContext) var modelContext
     @Environment(LLMEvaluator.self) var llm
     @State var showOnboarding = false
@@ -31,7 +32,7 @@ struct ContentView: View {
                         ScrollView(.vertical) {
                             VStack(alignment: .leading, spacing: 0) {
                                 ForEach(currentThread.sortedMessages) { message in
-                                    HStack {
+                                    HStack(alignment: .lastTextBaseline) {
                                         if message.role == .user {
                                             Spacer()
                                         }
@@ -49,6 +50,14 @@ struct ContentView: View {
 
                                         if message.role == .assistant {
                                             Spacer()
+
+                                            if !speechSynthesizer.auto {
+                                                Button {
+                                                    speechSynthesizer.speak(message.content)
+                                                } label: {
+                                                    Image(systemName: "speaker.wave.2")
+                                                }
+                                            }
                                         }
                                     }
                                     .padding()
@@ -74,6 +83,10 @@ struct ContentView: View {
                         .onChange(of: llm.output) { _, _ in
                             scrollView.scrollTo(bottomID)
                             appManager.playHaptic()
+
+                            if !llm.running && speechSynthesizer.auto {
+                                speechSynthesizer.speak(llm.output)
+                            }
                         }
                     }
                     .defaultScrollAnchor(.bottom)

fullmoon/Models/SpeechSynthesizer.swift

@@ -0,0 +1,32 @@
+//
+//  SpeechSynthesizer.swift
+//  fullmoon
+//
+//  Created by Kuba Szulaczkowski on 11/10/24.
+//
+
+import AVFoundation
+import SwiftUI
+
+final class SpeechSynthesizer: ObservableObject {
+    static let shared = SpeechSynthesizer()
+    private init() {}
+
+    @AppStorage("autoSpeak") var auto = true
+    @AppStorage("speechRate") var rate = Double(AVSpeechUtteranceDefaultSpeechRate)
+    @AppStorage("speechPitch") var pitch = 1.0
+    @AppStorage("speechVolume") var volume = 1.0
+    @AppStorage("voice") var voice = AVSpeechSynthesisVoice.speechVoices().filter({ $0.language.hasPrefix("en") }).first?.identifier ?? "com.apple.voice.compact.en-GB.Daniel"
+
+    private let synthesizer = AVSpeechSynthesizer()
+
+    func speak(_ textToSpeak: String) {
+        let utterance = AVSpeechUtterance(string: textToSpeak)
+        utterance.rate = Float(rate)
+        utterance.pitchMultiplier = Float(pitch)
+        utterance.volume = Float(volume)
+        utterance.voice = AVSpeechSynthesisVoice.speechVoices().first(where: { $0.identifier == voice }) ?? AVSpeechSynthesisVoice.speechVoices().filter({ $0.language.hasPrefix("en") }).first
+
+        synthesizer.speak(utterance)
+    }
+}

fullmoon/Views/Settings/SettingsView.swift

@@ -25,6 +25,10 @@ struct SettingsView: View {
                         Label("chats", systemImage: "message")
                     }
 
+                    NavigationLink(destination: SpeechSettingsView()) {
+                        Label("speech", systemImage: "speaker.wave.2.bubble")
+                    }
+
                     NavigationLink(destination: ModelsSettingsView()) {
                         Label("models", systemImage: "arrow.down.circle")
                             .badge(appManager.modelDisplayName(appManager.currentModelName ?? ""))

fullmoon/Views/Settings/SpeechSettingsView.swift

@@ -0,0 +1,73 @@
+//
+//  SpeechSettingsView.swift
+//  fullmoon
+//
+//  Created by Kuba Szulaczkowski on 11/1/24.
+//
+
+import AVFoundation
+import SwiftUI
+
+struct SpeechSettingsView: View {
+    @EnvironmentObject var speechSynthesizer: SpeechSynthesizer    
+    @State private var textToSpeak: String = "Hi, I'm a helpful assistant!"
+
+    var body: some View {
+        List {
+            Section {
+                Toggle("auto speak responses", isOn: $speechSynthesizer.auto)
+            }
+
+            Section {
+                Picker(selection: $speechSynthesizer.voice) {
+                    ForEach(AVSpeechSynthesisVoice.speechVoices().filter({ $0.language.hasPrefix("en") }), id: \.identifier) { voice in
+                        Text("\(voice.name) (\(voice.language))").tag(voice.identifier)
+                    }
+                } label: {
+                    Label("voice", systemImage: "bubble")
+                }
+                .onChange(of: speechSynthesizer.voice, speak)
+            }
+
+            HStack(spacing: 16) {
+                Text("volume: 1.00")
+                    .hidden()
+                    .overlay(alignment: .leading) {
+                        Text("rate: \(speechSynthesizer.rate, specifier: "%.2f")")
+                    }
+                Slider(value: $speechSynthesizer.rate, in: Double(AVSpeechUtteranceMinimumSpeechRate)...Double(AVSpeechUtteranceMaximumSpeechRate), onEditingChanged: handleEditingChanged)
+            }
+
+            HStack(spacing: 16) {
+                Text("volume: 1.00")
+                    .hidden()
+                    .overlay(alignment: .leading) {
+                        Text("pitch: \(speechSynthesizer.pitch, specifier: "%.2f")")
+                    }
+                Slider(value: $speechSynthesizer.pitch, in: 0.5...2.0, onEditingChanged: handleEditingChanged)
+            }
+
+            HStack(spacing: 16) {
+                Text("volume: \(speechSynthesizer.volume, specifier: "%.2f")")
+                Slider(value: $speechSynthesizer.volume, in: 0.0...1.0, onEditingChanged: handleEditingChanged)
+            }
+        }
+        .monospacedDigit()
+        .navigationTitle("speech")
+        .navigationBarTitleDisplayMode(.inline)
+    }
+
+    func handleEditingChanged(_ isEditing: Bool) {
+        guard !isEditing else { return }
+        speak()
+    }
+
+    private func speak() {
+        speechSynthesizer.speak(textToSpeak)
+    }
+}
+
+#Preview {
+    SpeechSettingsView()
+        .environmentObject(SpeechSynthesizer.shared)
+}

fullmoon/fullmoonApp.swift

@@ -11,13 +11,15 @@ import MLXLLM
 @main
 struct fullmoonApp: App {
     @StateObject var appManager = AppManager()
+    @StateObject var speechSynthesizer = SpeechSynthesizer.shared
     @State var llm = LLMEvaluator()
 
     var body: some Scene {
         WindowGroup {
             ContentView()
                 .modelContainer(for: [Thread.self, Message.self])
                 .environmentObject(appManager)
+                .environmentObject(speechSynthesizer)
                 .environment(llm)
                 .environment(DeviceStat())
         }