Skip to content

Commit ee1d385

Browse files
davidliudavidliu
committed
feat: native audio processing apis and various useTrackVolume hooks
1 parent e479393 commit ee1d385

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+2498
-125
lines changed

‎android/build.gradle

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ dependencies {
131131
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
132132
api 'com.github.davidliu:audioswitch:89582c47c9a04c62f90aa5e57251af4800a62c9a'
133133
api 'io.github.webrtc-sdk:android:125.6422.02'
134+
implementation "com.github.paramsen:noise:2.0.0"
134135
implementation project(':livekit_react-native-webrtc')
135-
implementation "androidx.annotation:annotation:1.4.0"
136+
implementation "androidx.annotation:annotation:1.9.1"
136137
}

‎android/src/main/java/com/livekit/reactnative/LiveKitReactNative.kt

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,54 @@
11
package com.livekit.reactnative
22

3+
import android.annotation.SuppressLint
34
import android.app.Application
45
import android.content.Context
56
import android.os.Build
67
import com.livekit.reactnative.audio.AudioType
7-
import com.livekit.reactnative.video.CustomVideoEncoderFactory
8+
import com.livekit.reactnative.audio.processing.AudioProcessingController
9+
import com.livekit.reactnative.audio.processing.AudioRecordSamplesDispatcher
10+
import com.livekit.reactnative.audio.processing.CustomAudioProcessingController
811
import com.livekit.reactnative.video.CustomVideoDecoderFactory
12+
import com.livekit.reactnative.video.CustomVideoEncoderFactory
913
import com.oney.WebRTCModule.WebRTCModuleOptions
1014
import org.webrtc.audio.JavaAudioDeviceModule
15+
import java.util.concurrent.Callable
1116

1217
object LiveKitReactNative {
1318

19+
@SuppressLint("StaticFieldLeak")
1420
private lateinit var adm: JavaAudioDeviceModule
1521

1622
val audioDeviceModule: JavaAudioDeviceModule
1723
get() {
18-
if(!::adm.isInitialized) {
24+
if (!::adm.isInitialized) {
1925
throw IllegalStateException("Audio device module is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
2026
}
21-
2227
return adm
2328
}
2429

30+
private lateinit var _audioProcessingController: AudioProcessingController
31+
32+
val audioProcessingController: AudioProcessingController
33+
get() {
34+
if (!::_audioProcessingController.isInitialized) {
35+
throw IllegalStateException("audioProcessingController is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
36+
}
37+
return _audioProcessingController
38+
}
39+
40+
41+
lateinit var _audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher
42+
43+
val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher
44+
get() {
45+
if (!::_audioRecordSamplesDispatcher.isInitialized) {
46+
throw IllegalStateException("audioRecordSamplesDispatcher is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
47+
}
48+
return _audioRecordSamplesDispatcher
49+
}
50+
51+
2552
/**
2653
* Initializes components required for LiveKit to work on Android.
2754
*
@@ -34,6 +61,8 @@ object LiveKitReactNative {
3461
context: Context,
3562
audioType: AudioType = AudioType.CommunicationAudioType()
3663
) {
64+
_audioRecordSamplesDispatcher = AudioRecordSamplesDispatcher()
65+
3766
val options = WebRTCModuleOptions.getInstance()
3867
options.videoEncoderFactory = CustomVideoEncoderFactory(null, true, true)
3968
options.videoDecoderFactory = CustomVideoDecoderFactory()
@@ -45,8 +74,17 @@ object LiveKitReactNative {
4574
.setUseHardwareAcousticEchoCanceler(useHardwareAudioProcessing)
4675
.setUseHardwareNoiseSuppressor(useHardwareAudioProcessing)
4776
.setAudioAttributes(audioType.audioAttributes)
77+
.setSamplesReadyCallback(audioRecordSamplesDispatcher)
4878
.createAudioDeviceModule()
4979

5080
options.audioDeviceModule = adm
81+
82+
// CustomAudioProcessingController can't be instantiated before WebRTC is loaded.
83+
options.audioProcessingFactoryFactory = Callable {
84+
val apc = CustomAudioProcessingController()
85+
_audioProcessingController = apc
86+
return@Callable apc.externalAudioProcessor
87+
}
88+
5189
}
5290
}

‎android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt

Lines changed: 77 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,26 @@
11
package com.livekit.reactnative
22

3-
import android.annotation.SuppressLint
4-
import android.content.Context
53
import android.media.AudioAttributes
6-
import com.facebook.react.bridge.*
4+
import android.util.Log
5+
import com.facebook.react.bridge.Arguments
6+
import com.facebook.react.bridge.Promise
7+
import com.facebook.react.bridge.ReactApplicationContext
8+
import com.facebook.react.bridge.ReactContextBaseJavaModule
9+
import com.facebook.react.bridge.ReactMethod
10+
import com.facebook.react.bridge.ReadableMap
711
import com.livekit.reactnative.audio.AudioDeviceKind
812
import com.livekit.reactnative.audio.AudioManagerUtils
913
import com.livekit.reactnative.audio.AudioSwitchManager
14+
import com.livekit.reactnative.audio.processing.AudioSinkManager
15+
import com.livekit.reactnative.audio.processing.MultibandVolumeProcessor
16+
import com.livekit.reactnative.audio.processing.VolumeProcessor
1017
import org.webrtc.audio.WebRtcAudioTrackHelper
18+
import kotlin.time.Duration.Companion.milliseconds
1119

1220

1321
class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) {
1422

23+
val audioSinkManager = AudioSinkManager(reactContext)
1524
val audioManager = AudioSwitchManager(reactContext.applicationContext)
1625
override fun getName(): String {
1726
return "LivekitReactNative"
@@ -115,9 +124,73 @@ class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactCon
115124
promise.resolve(Arguments.makeNativeArray(deviceIds))
116125
}
117126

118-
@ReactMethod
127+
@ReactMethod(isBlockingSynchronousMethod = true)
119128
fun selectAudioOutput(deviceId: String, promise: Promise) {
120129
audioManager.selectAudioOutput(AudioDeviceKind.fromTypeName(deviceId))
121130
promise.resolve(null)
122131
}
132+
133+
@ReactMethod(isBlockingSynchronousMethod = true)
134+
fun createVolumeProcessor(pcId: Int, trackId: String): String {
135+
val processor = VolumeProcessor(reactApplicationContext)
136+
val reactTag = audioSinkManager.registerSink(processor)
137+
audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
138+
processor.reactTag = reactTag
139+
140+
return reactTag
141+
}
142+
143+
@ReactMethod(isBlockingSynchronousMethod = true)
144+
fun deleteVolumeProcessor(reactTag: String, pcId: Int, trackId: String) {
145+
audioSinkManager.detachSinkFromTrack(reactTag, pcId, trackId)
146+
audioSinkManager.unregisterSink(reactTag)
147+
}
148+
149+
@ReactMethod(isBlockingSynchronousMethod = true)
150+
fun createMultibandVolumeProcessor(options: ReadableMap, pcId: Int, trackId: String): String {
151+
val bands = options.getInt("bands")
152+
val minFrequency = options.getDouble("minFrequency")
153+
val maxFrequency = options.getDouble("maxFrequency")
154+
val intervalMs = options.getDouble("updateInterval")
155+
156+
val processor = MultibandVolumeProcessor(
157+
minFrequency = minFrequency.toFloat(),
158+
maxFrequency = maxFrequency.toFloat(),
159+
barCount = bands,
160+
interval = intervalMs.milliseconds,
161+
reactContext = reactApplicationContext
162+
)
163+
val reactTag = audioSinkManager.registerSink(processor)
164+
processor.reactTag = reactTag
165+
audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
166+
167+
processor.start()
168+
169+
return reactTag
170+
}
171+
172+
@ReactMethod(isBlockingSynchronousMethod = true)
173+
fun deleteMultibandVolumeProcessor(reactTag: String, pcId: Int, trackId: String) {
174+
val volumeProcessor =
175+
audioSinkManager.getSink(reactTag) ?: throw IllegalArgumentException("Can't find volume processor for $reactTag")
176+
audioSinkManager.detachSinkFromTrack(volumeProcessor, pcId, trackId)
177+
audioSinkManager.unregisterSink(volumeProcessor)
178+
val multibandVolumeProcessor = volumeProcessor as? MultibandVolumeProcessor
179+
180+
if (multibandVolumeProcessor != null) {
181+
multibandVolumeProcessor.release()
182+
} else {
183+
Log.w(name, "deleteMultibandVolumeProcessor called, but non-MultibandVolumeProcessor found?!")
184+
}
185+
}
186+
187+
@ReactMethod
188+
fun addListener(eventName: String?) {
189+
// Keep: Required for RN built in Event Emitter Calls.
190+
}
191+
192+
@ReactMethod
193+
fun removeListeners(count: Int?) {
194+
// Keep: Required for RN built in Event Emitter Calls.
195+
}
123196
}

‎android/src/main/java/com/livekit/reactnative/audio/events/Events.kt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package com.livekit.reactnative.audio.events
2+
3+
enum class Events {
4+
LK_VOLUME_PROCESSED,
5+
LK_MULTIBAND_PROCESSED,
6+
}

‎android/src/main/java/com/livekit/reactnative/audio/processing/AudioFormat.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
package com.livekit.reactnative.audio.processing
2+
data class AudioFormat(val bitsPerSample: Int, val sampleRate: Int, val numberOfChannels: Int)

‎android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessingController.kt

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package com.livekit.reactnative.audio.processing
2+
3+
/**
4+
* Interface for controlling external audio processing.
5+
*/
6+
interface AudioProcessingController {
7+
/**
8+
* the audio processor to be used for capture post processing.
9+
*/
10+
var capturePostProcessor: AudioProcessorInterface?
11+
12+
/**
13+
* the audio processor to be used for render pre processing.
14+
*/
15+
var renderPreProcessor: AudioProcessorInterface?
16+
17+
/**
18+
* whether to bypass mode the render pre processing.
19+
*/
20+
var bypassRenderPreProcessing: Boolean
21+
22+
/**
23+
* whether to bypass the capture post processing.
24+
*/
25+
var bypassCapturePostProcessing: Boolean
26+
27+
}

‎android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessorInterface.kt

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package com.livekit.reactnative.audio.processing
2+
3+
import java.nio.ByteBuffer
4+
5+
/**
6+
* Interface for external audio processing.
7+
*/
8+
interface AudioProcessorInterface {
9+
/**
10+
* Check if the audio processing is enabled.
11+
*/
12+
fun isEnabled(): Boolean
13+
14+
/**
15+
* Get the name of the audio processing.
16+
*/
17+
fun getName(): String
18+
19+
/**
20+
* Initialize the audio processing.
21+
*
22+
* Note: audio processing methods will be called regardless of whether
23+
* [isEnabled] returns true or not.
24+
*/
25+
fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int)
26+
27+
/**
28+
* Called when the sample rate has changed.
29+
*
30+
* Note: audio processing methods will be called regardless of whether
31+
* [isEnabled] returns true or not.
32+
*/
33+
fun resetAudioProcessing(newRate: Int)
34+
35+
/**
36+
* Process the audio frame (10ms).
37+
*
38+
* Note: audio processing methods will be called regardless of whether
39+
* [isEnabled] returns true or not.
40+
*/
41+
fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer)
42+
}
43+
44+
/**
45+
* @suppress
46+
*/
47+
interface AuthedAudioProcessorInterface : AudioProcessorInterface {
48+
/**
49+
* @suppress
50+
*/
51+
fun authenticate(url: String, token: String)
52+
}

‎android/src/main/java/com/livekit/reactnative/audio/processing/AudioRecordSamplesDispatcher.kt

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Copyright 2024 LiveKit, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.livekit.reactnative.audio.processing
18+
19+
import android.media.AudioFormat
20+
import android.os.SystemClock
21+
import org.webrtc.AudioTrackSink
22+
import org.webrtc.audio.JavaAudioDeviceModule
23+
import java.nio.ByteBuffer
24+
25+
/**
26+
* Dispatches recorded audio samples from the local microphone.
27+
*/
28+
class AudioRecordSamplesDispatcher : JavaAudioDeviceModule.SamplesReadyCallback {
29+
30+
private val sinks = mutableSetOf<AudioTrackSink>()
31+
32+
@Synchronized
33+
fun registerSink(sink: AudioTrackSink) {
34+
sinks.add(sink)
35+
}
36+
37+
@Synchronized
38+
fun unregisterSink(sink: AudioTrackSink) {
39+
sinks.remove(sink)
40+
}
41+
42+
// Reference from Android code, AudioFormat.getBytesPerSample. BitPerSample / 8
43+
// Default audio data format is PCM 16 bits per sample.
44+
// Guaranteed to be supported by all devices
45+
private fun getBytesPerSample(audioFormat: Int): Int {
46+
return when (audioFormat) {
47+
AudioFormat.ENCODING_PCM_8BIT -> 1
48+
AudioFormat.ENCODING_PCM_16BIT, AudioFormat.ENCODING_IEC61937, AudioFormat.ENCODING_DEFAULT -> 2
49+
AudioFormat.ENCODING_PCM_FLOAT -> 4
50+
AudioFormat.ENCODING_INVALID -> throw IllegalArgumentException("Bad audio format $audioFormat")
51+
else -> throw IllegalArgumentException("Bad audio format $audioFormat")
52+
}
53+
}
54+
55+
@Synchronized
56+
override fun onWebRtcAudioRecordSamplesReady(samples: JavaAudioDeviceModule.AudioSamples) {
57+
val bitsPerSample = getBytesPerSample(samples.audioFormat) * 8
58+
val numFrames = samples.sampleRate / 100 // 10ms worth of samples.
59+
val timestamp = SystemClock.elapsedRealtime()
60+
for (sink in sinks) {
61+
val byteBuffer = ByteBuffer.wrap(samples.data)
62+
sink.onData(
63+
byteBuffer,
64+
bitsPerSample,
65+
samples.sampleRate,
66+
samples.channelCount,
67+
numFrames,
68+
timestamp,
69+
)
70+
}
71+
}
72+
}

0 commit comments

Comments
 (0)