iOS screen share audio (#576)

ladvoc · hiroshihorie · web-flow · commit 4f2f5c28b701 · 2025-02-24T13:20:48.000+09:00
This PR adds support for capturing application audio during screen share when using a broadcast extension. Note: PR #598 should be merged first. --------- Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com>
diff --git a/.nanpa/ios-screen-share-audio.kdl b/.nanpa/ios-screen-share-audio.kdl
@@ -0,0 +1 @@
+patch type="added" "Add support for screen share audio on iOS when using a broadcast extension"
diff --git a/Docs/ios-screen-sharing.md b/Docs/ios-screen-sharing.md
@@ -7,7 +7,7 @@ LiveKit integrates with [ReplayKit](https://developer.apple.com/documentation/re
 
 ## In-app Capture
 
-By default, LiveKit uses the In-app Capture mode, which requires no additional configuration. In this mode, when screen sharing is enabled, the system prompts the user with a screen recording permission dialog. Once granted, a screen share track is published. The user only needs to grant permission once per app execution.
+By default, LiveKit uses the In-app Capture mode, which requires no additional configuration. In this mode, when screen sharing is enabled, the system prompts the user with a screen recording permission dialog. Once granted, a screen share track is published. The user only needs to grant permission once per app execution. Application audio is not supported with the In-App Capture mode.
 
 <center>
     <figure>
@@ -88,6 +88,32 @@ try await room.localParticipant.setScreenShare(enabled: true)
 
 <small>Note: When using broadcast capture, custom capture options must be set as room defaults rather than passed when enabling screen share with `set(source:enabled:captureOptions:publishOptions:)`.</small>
 
+### Application Audio
+
+When using Broadcast Capture, you can capture app audio even when the user navigates away from your app. When enabled, the captured app audio is mixed with the local participant's microphone track. To enable this feature, set the default screen share capture options when connecting to the room:
+
+
+```swift
+let roomOptions = RoomOptions(
+    defaultScreenShareCaptureOptions: ScreenShareCaptureOptions(
+        appAudio: true // enables capture of app audio
+    )
+)
+
+// Option 1: Using SwiftUI RoomScope component
+RoomScope(url: wsURL, token: token, enableMicrophone: true, roomOptions: roomOptions) {
+    // your components here
+}
+
+// Option 2: Using Room object directly
+try await room.connect(
+    url: wsURL,
+    token: token,
+    roomOptions: roomOptions
+)
+try await room.localParticipant.setMicrophone(enabled: true)
+```
+
 ### Troubleshooting
 
 While running your app in a debug session in Xcode, check the debug console for errors and use the Console app to inspect logs from the broadcast extension:
diff --git a/Sources/LiveKit/Broadcast/BroadcastScreenCapturer.swift b/Sources/LiveKit/Broadcast/BroadcastScreenCapturer.swift
@@ -29,6 +29,7 @@ internal import LiveKitWebRTC
 #endif
 
 class BroadcastScreenCapturer: BufferCapturer {
+    private let appAudio: Bool
     private var receiver: BroadcastReceiver?
 
     override func startCapture() async throws -> Bool {
@@ -52,30 +53,32 @@ class BroadcastScreenCapturer: BufferCapturer {
     }
 
     private func createReceiver() -> Bool {
-        guard receiver == nil else {
-            return false
-        }
         guard let socketPath = BroadcastBundleInfo.socketPath else {
             logger.error("Bundle settings improperly configured for screen capture")
             return false
         }
         Task { [weak self] in
+            guard let self else { return }
             do {
                 let receiver = try await BroadcastReceiver(socketPath: socketPath)
                 logger.debug("Broadcast receiver connected")
-                self?.receiver = receiver
+                self.receiver = receiver
+
+                if self.appAudio {
+                    try await receiver.enableAudio()
+                }
 
                 for try await sample in receiver.incomingSamples {
                     switch sample {
-                    case let .image(imageBuffer, rotation):
-                        self?.capture(imageBuffer, rotation: rotation)
+                    case let .image(buffer, rotation): self.capture(buffer, rotation: rotation)
+                    case let .audio(buffer): AudioManager.shared.mixer.capture(appAudio: buffer)
                     }
                 }
                 logger.debug("Broadcast receiver closed")
             } catch {
                 logger.error("Broadcast receiver error: \(error)")
             }
-            _ = try? await self?.stopCapture()
+            _ = try? await self.stopCapture()
         }
         return true
     }
@@ -88,6 +91,11 @@ class BroadcastScreenCapturer: BufferCapturer {
         receiver?.close()
         return true
     }
+
+    init(delegate: LKRTCVideoCapturerDelegate, options: ScreenShareCaptureOptions) {
+        appAudio = options.appAudio
+        super.init(delegate: delegate, options: BufferCaptureOptions(from: options))
+    }
 }
 
 public extension LocalVideoTrack {
@@ -98,7 +106,7 @@ public extension LocalVideoTrack {
                                                    reportStatistics: Bool = false) -> LocalVideoTrack
     {
         let videoSource = RTC.createVideoSource(forScreenShare: true)
-        let capturer = BroadcastScreenCapturer(delegate: videoSource, options: BufferCaptureOptions(from: options))
+        let capturer = BroadcastScreenCapturer(delegate: videoSource, options: options)
         return LocalVideoTrack(
             name: name,
             source: source,
diff --git a/Sources/LiveKit/Broadcast/IPC/BroadcastAudioCodec.swift b/Sources/LiveKit/Broadcast/IPC/BroadcastAudioCodec.swift
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#if os(iOS)
+
+import AVFoundation
+
+/// Encode and decode audio samples for transport.
+struct BroadcastAudioCodec {
+    struct Metadata: Codable {
+        let sampleCount: Int32
+        let description: AudioStreamBasicDescription
+    }
+
+    enum Error: Swift.Error {
+        case encodingFailed
+        case decodingFailed
+    }
+
+    func encode(_ audioBuffer: CMSampleBuffer) throws -> (Metadata, Data) {
+        guard let formatDescription = audioBuffer.formatDescription,
+              let basicDescription = formatDescription.audioStreamBasicDescription,
+              let blockBuffer = audioBuffer.dataBuffer
+        else {
+            throw Error.encodingFailed
+        }
+
+        var count = 0
+        var dataPointer: UnsafeMutablePointer<Int8>?
+
+        guard CMBlockBufferGetDataPointer(
+            blockBuffer,
+            atOffset: 0,
+            lengthAtOffsetOut: nil,
+            totalLengthOut: &count,
+            dataPointerOut: &dataPointer
+        ) == kCMBlockBufferNoErr, let dataPointer else {
+            throw Error.encodingFailed
+        }
+
+        let data = Data(bytes: dataPointer, count: count)
+        let metadata = Metadata(
+            sampleCount: Int32(audioBuffer.numSamples),
+            description: basicDescription
+        )
+        return (metadata, data)
+    }
+
+    func decode(_ encodedData: Data, with metadata: Metadata) throws -> AVAudioPCMBuffer {
+        guard !encodedData.isEmpty else {
+            throw Error.decodingFailed
+        }
+
+        var description = metadata.description
+        guard let format = AVAudioFormat(streamDescription: &description) else {
+            throw Error.decodingFailed
+        }
+
+        let sampleCount = AVAudioFrameCount(metadata.sampleCount)
+        guard let pcmBuffer = AVAudioPCMBuffer(
+            pcmFormat: format,
+            frameCapacity: sampleCount
+        ) else {
+            throw Error.decodingFailed
+        }
+        pcmBuffer.frameLength = sampleCount
+
+        guard format.isInterleaved else {
+            throw Error.decodingFailed
+        }
+
+        guard let mData = pcmBuffer.audioBufferList.pointee.mBuffers.mData else {
+            throw Error.decodingFailed
+        }
+        encodedData.copyBytes(
+            to: mData.assumingMemoryBound(to: UInt8.self),
+            count: encodedData.count
+        )
+        return pcmBuffer
+    }
+}
+
+extension AudioStreamBasicDescription: Codable {
+    public func encode(to encoder: any Encoder) throws {
+        var container = encoder.unkeyedContainer()
+        try container.encode(mSampleRate)
+        try container.encode(mFormatID)
+        try container.encode(mFormatFlags)
+        try container.encode(mBytesPerPacket)
+        try container.encode(mFramesPerPacket)
+        try container.encode(mBytesPerFrame)
+        try container.encode(mChannelsPerFrame)
+        try container.encode(mBitsPerChannel)
+    }
+
+    public init(from decoder: any Decoder) throws {
+        var container = try decoder.unkeyedContainer()
+        try self.init(
+            mSampleRate: container.decode(Float64.self),
+            mFormatID: container.decode(AudioFormatID.self),
+            mFormatFlags: container.decode(AudioFormatFlags.self),
+            mBytesPerPacket: container.decode(UInt32.self),
+            mFramesPerPacket: container.decode(UInt32.self),
+            mBytesPerFrame: container.decode(UInt32.self),
+            mChannelsPerFrame: container.decode(UInt32.self),
+            mBitsPerChannel: container.decode(UInt32.self),
+            mReserved: 0 // as per documentation
+        )
+    }
+}
+
+#endif
diff --git a/Sources/LiveKit/Broadcast/IPC/BroadcastIPCHeader.swift b/Sources/LiveKit/Broadcast/IPC/BroadcastIPCHeader.swift
@@ -20,6 +20,12 @@
 enum BroadcastIPCHeader: Codable {
     /// Image sample sent by uploader.
     case image(BroadcastImageCodec.Metadata, VideoRotation)
+
+    /// Audio sample sent by uploader.
+    case audio(BroadcastAudioCodec.Metadata)
+
+    /// Request sent by receiver to set audio demand.
+    case wantsAudio(Bool)
 }
 
 #endif
diff --git a/Sources/LiveKit/Broadcast/IPC/BroadcastReceiver.swift b/Sources/LiveKit/Broadcast/IPC/BroadcastReceiver.swift
@@ -16,14 +16,15 @@
 
 #if os(iOS)
 
+import AVFoundation
 import CoreImage
-import Foundation
 
 /// Receives broadcast samples from another process.
 final class BroadcastReceiver: Sendable {
     /// Sample received from the other process with associated metadata.
     enum IncomingSample {
         case image(CVImageBuffer, VideoRotation)
+        case audio(AVAudioPCMBuffer)
     }
 
     enum Error: Swift.Error {
@@ -49,18 +50,27 @@ final class BroadcastReceiver: Sendable {
 
     struct AsyncSampleSequence: AsyncSequence, AsyncIteratorProtocol {
         fileprivate let upstream: IPCChannel.AsyncMessageSequence<BroadcastIPCHeader>
+
         private let imageCodec = BroadcastImageCodec()
+        private let audioCodec = BroadcastAudioCodec()
 
         func next() async throws -> IncomingSample? {
-            guard let (header, payload) = try await upstream.next() else {
-                return nil
-            }
-            switch header {
-            case let .image(metadata, rotation):
-                guard let payload else { throw Error.missingSampleData }
-                let imageBuffer = try imageCodec.decode(payload, with: metadata)
-                return IncomingSample.image(imageBuffer, rotation)
+            while let (header, payload) = try await upstream.next(), let payload {
+                switch header {
+                case let .image(metadata, rotation):
+                    let imageBuffer = try imageCodec.decode(payload, with: metadata)
+                    return IncomingSample.image(imageBuffer, rotation)
+
+                case let .audio(metadata):
+                    let audioBuffer = try audioCodec.decode(payload, with: metadata)
+                    return IncomingSample.audio(audioBuffer)
+
+                default:
+                    logger.debug("Unhandled incoming message: \(header)")
+                    continue
+                }
             }
+            return nil
         }
 
         func makeAsyncIterator() -> Self { self }
@@ -74,6 +84,16 @@ final class BroadcastReceiver: Sendable {
     var incomingSamples: AsyncSampleSequence {
         AsyncSampleSequence(upstream: channel.incomingMessages(BroadcastIPCHeader.self))
     }
+
+    /// Tells the uploader to begin sending audio samples.
+    func enableAudio() async throws {
+        try await channel.send(header: BroadcastIPCHeader.wantsAudio(true))
+    }
+
+    /// Tells the uploader to stop sending audio samples.
+    func disableAudio() async throws {
+        try await channel.send(header: BroadcastIPCHeader.wantsAudio(false))
+    }
 }
 
 #endif
diff --git a/Sources/LiveKit/Broadcast/IPC/BroadcastUploader.swift b/Sources/LiveKit/Broadcast/IPC/BroadcastUploader.swift
@@ -22,10 +22,13 @@ import ReplayKit
 /// Uploads broadcast samples to another process.
 final class BroadcastUploader: Sendable {
     private let channel: IPCChannel
+
     private let imageCodec = BroadcastImageCodec()
+    private let audioCodec = BroadcastAudioCodec()
 
     private struct State {
         var isUploadingImage = false
+        var shouldUploadAudio = false
     }
 
     private let state = StateSync(State())
@@ -38,6 +41,7 @@ final class BroadcastUploader: Sendable {
     /// Creates an uploader with an open connection to another process.
     init(socketPath: SocketPath) async throws {
         channel = try await IPCChannel(connectingTo: socketPath)
+        Task { try await handleIncomingMessages() }
     }
 
     /// Whether or not the connection to the receiver has been closed.
@@ -76,10 +80,29 @@ final class BroadcastUploader: Sendable {
                 state.mutate { $0.isUploadingImage = false }
                 throw error
             }
+        case .audioApp:
+            guard state.shouldUploadAudio else { return }
+            let (metadata, audioData) = try audioCodec.encode(sampleBuffer)
+            Task {
+                let header = BroadcastIPCHeader.audio(metadata)
+                try await channel.send(header: header, payload: audioData)
+            }
         default:
             throw Error.unsupportedSample
         }
     }
+
+    private func handleIncomingMessages() async throws {
+        for try await (header, _) in channel.incomingMessages(BroadcastIPCHeader.self) {
+            switch header {
+            case let .wantsAudio(wantsAudio):
+                state.mutate { $0.shouldUploadAudio = wantsAudio }
+            default:
+                logger.debug("Unhandled incoming message: \(header)")
+                continue
+            }
+        }
+    }
 }
 
 private extension CMSampleBuffer {
diff --git a/Tests/LiveKitTests/Broadcast/BroadcastAudioCodecTests.swift b/Tests/LiveKitTests/Broadcast/BroadcastAudioCodecTests.swift

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+patch type="added" "Add support for screen share audio on iOS when using a broadcast extension"`