diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index dc9bd64a76..03e5013dd9 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -392,6 +392,18 @@ if (is_ios || is_mac) { } } + rtc_source_set("audio_device_api_objc") { + visibility = [ "*" ] + + sources = [ "objc/components/audio/RTCAudioDevice.h" ] + + public_configs = [ ":common_config_objc" ] + + frameworks = [ "AudioToolbox.framework" ] + + deps = [ ":base_objc" ] + } + rtc_library("audio_device_objc") { visibility = [ "*" ] @@ -401,6 +413,24 @@ if (is_ios || is_mac) { ] deps = [ + ":audio_device_api_objc", + "../modules/audio_device:audio_device_api", + "../rtc_base:logging", + ] + } + + rtc_library("objc_audio_device_module") { + visibility = [ "*" ] + + sources = [ + "objc/native/api/objc_audio_device_module.h", + "objc/native/api/objc_audio_device_module.mm", + ] + + deps = [ + ":audio_device_api_objc", + ":audio_device_objc", + "../api:make_ref_counted", "../modules/audio_device:audio_device_api", "../rtc_base:logging", ] @@ -1001,6 +1031,7 @@ if (is_ios || is_mac) { public_configs = [ ":common_config_objc" ] deps = [ + ":audio_device_api_objc", ":base_native_additions_objc", ":base_objc", ":file_logger_objc", @@ -1009,6 +1040,7 @@ if (is_ios || is_mac) { ":mediasource_objc", ":native_api", ":native_video", + ":objc_audio_device_module", ":videoframebuffer_objc", ":videorendereradapter_objc", ":videosource_objc", @@ -1236,6 +1268,7 @@ if (is_ios || is_mac) { "objc/base/RTCVideoFrameBuffer.h", "objc/base/RTCVideoRenderer.h", "objc/base/RTCYUVPlanarBuffer.h", + "objc/components/audio/RTCAudioDevice.h", "objc/components/audio/RTCAudioSession.h", "objc/components/audio/RTCAudioSessionConfiguration.h", "objc/components/capturer/RTCCameraVideoCapturer.h", diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h index 88aac990f2..5575af98c9 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h @@ -31,18 +31,26 @@ NS_ASSUME_NONNULL_BEGIN (RTCVideoEncoderFactory); @protocol RTC_OBJC_TYPE (RTCSSLCertificateVerifier); +@protocol RTC_OBJC_TYPE +(RTCAudioDevice); RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCPeerConnectionFactory) : NSObject -/* Initialize object with default H264 video encoder/decoder factories */ +/* Initialize object with default H264 video encoder/decoder factories and default ADM */ - (instancetype)init; -/* Initialize object with injectable video encoder/decoder factories */ +/* Initialize object with injectable video encoder/decoder factories and default ADM */ - (instancetype) initWithEncoderFactory:(nullable id)encoderFactory decoderFactory:(nullable id)decoderFactory; +/* Initialize object with injectable video encoder/decoder factories and injectable ADM */ +- (instancetype) + initWithEncoderFactory:(nullable id)encoderFactory + decoderFactory:(nullable id)decoderFactory + audioDevice:(nullable id)audioDevice; + /** Initialize an RTCAudioSource with constraints. */ - (RTC_OBJC_TYPE(RTCAudioSource) *)audioSourceWithConstraints: (nullable RTC_OBJC_TYPE(RTCMediaConstraints) *)constraints; diff --git a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm index e2215c8b45..c4d89e911d 100644 --- a/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm +++ b/sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm @@ -41,6 +41,7 @@ #include "modules/audio_device/include/audio_device.h" #include "modules/audio_processing/include/audio_processing.h" +#include "sdk/objc/native/api/objc_audio_device_module.h" #include "sdk/objc/native/api/video_decoder_factory.h" #include "sdk/objc/native/api/video_encoder_factory.h" #include "sdk/objc/native/src/objc_video_decoder_factory.h" @@ -82,6 +83,16 @@ - (instancetype) initWithEncoderFactory:(nullable id)encoderFactory decoderFactory:(nullable id)decoderFactory { + return [self initWithEncoderFactory:encoderFactory decoderFactory:decoderFactory audioDevice:nil]; +} + +- (instancetype) + initWithEncoderFactory:(nullable id)encoderFactory + decoderFactory:(nullable id)decoderFactory + audioDevice:(nullable id)audioDevice { +#ifdef HAVE_NO_MEDIA + return [self initWithNoMedia]; +#else std::unique_ptr native_encoder_factory; std::unique_ptr native_decoder_factory; if (encoderFactory) { @@ -90,13 +101,21 @@ if (decoderFactory) { native_decoder_factory = webrtc::ObjCToNativeVideoDecoderFactory(decoderFactory); } + rtc::scoped_refptr audio_device_module; + if (audioDevice) { + audio_device_module = webrtc::CreateAudioDeviceModule(audioDevice); + } else { + audio_device_module = [self audioDeviceModule]; + } return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory() nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory() nativeVideoEncoderFactory:std::move(native_encoder_factory) nativeVideoDecoderFactory:std::move(native_decoder_factory) - audioDeviceModule:[self audioDeviceModule].get() + audioDeviceModule:audio_device_module.get() audioProcessingModule:nullptr]; +#endif } + - (instancetype)initNative { if (self = [super init]) { _networkThread = rtc::Thread::CreateWithSocketServer(); diff --git a/sdk/objc/components/audio/RTCAudioDevice.h b/sdk/objc/components/audio/RTCAudioDevice.h new file mode 100644 index 0000000000..7f4509ebb0 --- /dev/null +++ b/sdk/objc/components/audio/RTCAudioDevice.h @@ -0,0 +1,308 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +#import "RTCMacros.h" + +NS_ASSUME_NONNULL_BEGIN + +typedef OSStatus (^RTC_OBJC_TYPE(RTCAudioDeviceGetPlayoutDataBlock))( + AudioUnitRenderActionFlags *_Nonnull actionFlags, + const AudioTimeStamp *_Nonnull timestamp, + NSInteger inputBusNumber, + UInt32 frameCount, + AudioBufferList *_Nonnull outputData); + +typedef OSStatus (^RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock))( + AudioUnitRenderActionFlags *_Nonnull actionFlags, + const AudioTimeStamp *_Nonnull timestamp, + NSInteger inputBusNumber, + UInt32 frameCount, + AudioBufferList *_Nonnull inputData, + void *_Nullable renderContext); + +typedef OSStatus (^RTC_OBJC_TYPE(RTCAudioDeviceDeliverRecordedDataBlock))( + AudioUnitRenderActionFlags *_Nonnull actionFlags, + const AudioTimeStamp *_Nonnull timestamp, + NSInteger inputBusNumber, + UInt32 frameCount, + const AudioBufferList *_Nullable inputData, + void *_Nullable renderContext, + NS_NOESCAPE RTC_OBJC_TYPE(RTCAudioDeviceRenderRecordedDataBlock) _Nullable renderBlock); + +/** + * Delegate object provided by native ADM during RTCAudioDevice initialization. + * Provides blocks to poll playback audio samples from native ADM and to feed + * recorded audio samples into native ADM. + */ +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE +(RTCAudioDeviceDelegate) + /** + * Implementation of RTCAudioSource should call this block to feed recorded PCM (16-bit integer) + * into native ADM. Stereo data is expected to be interleaved starting with the left channel. + * Either `inputData` with pre-filled audio data must be provided during block + * call or `renderBlock` must be provided which must fill provided audio buffer with recorded + * samples. + * + * NOTE: Implementation of RTCAudioDevice is expected to call the block on the same thread until + * `notifyAudioInterrupted` is called. When `notifyAudioInterrupted` is called implementation + * can call the block on a different thread. + */ + @property(readonly, nonnull) + RTC_OBJC_TYPE(RTCAudioDeviceDeliverRecordedDataBlock) deliverRecordedData; + +/** + * Provides input sample rate preference as it preferred by native ADM. + */ +@property(readonly) double preferredInputSampleRate; + +/** + * Provides input IO buffer duration preference as it preferred by native ADM. + */ +@property(readonly) NSTimeInterval preferredInputIOBufferDuration; + +/** + * Provides output sample rate preference as it preferred by native ADM. + */ +@property(readonly) double preferredOutputSampleRate; + +/** + * Provides output IO buffer duration preference as it preferred by native ADM. + */ +@property(readonly) NSTimeInterval preferredOutputIOBufferDuration; + +/** + * Implementation of RTCAudioDevice should call this block to request PCM (16-bit integer) + * from native ADM to play. Stereo data is interleaved starting with the left channel. + * + * NOTE: Implementation of RTCAudioDevice is expected to invoke of this block on the + * same thread until `notifyAudioInterrupted` is called. When `notifyAudioInterrupted` is called + * implementation can call the block from a different thread. + */ +@property(readonly, nonnull) RTC_OBJC_TYPE(RTCAudioDeviceGetPlayoutDataBlock) getPlayoutData; + +/** + * Notifies native ADM that some of the audio input parameters of RTCAudioDevice like + * samle rate and/or IO buffer duration and/or IO latency had possibly changed. + * Native ADM will adjust its audio input buffer to match current parameters of audio device. + * + * NOTE: Must be called within block executed via `dispatchAsync` or `dispatchSync`. + */ +- (void)notifyAudioInputParametersChange; + +/** + * Notifies native ADM that some of the audio output parameters of RTCAudioDevice like + * samle rate and/or IO buffer duration and/or IO latency had possibly changed. + * Native ADM will adjust its audio output buffer to match current parameters of audio device. + * + * NOTE: Must be called within block executed via `dispatchAsync` or `dispatchSync`. + */ +- (void)notifyAudioOutputParametersChange; + +/** + * Notifies native ADM that audio input is interrupted and further audio playout + * and recording might happen on a different thread. + * + * NOTE: Must be called within block executed via `dispatchAsync` or `dispatchSync`. + */ +- (void)notifyAudioInputInterrupted; + +/** + * Notifies native ADM that audio output is interrupted and further audio playout + * and recording might happen on a different thread. + * + * NOTE: Must be called within block executed via `dispatchAsync` or `dispatchSync`. + */ +- (void)notifyAudioOutputInterrupted; + +/** + * Asynchronously execute block of code within the context of + * thread which owns native ADM. + * + * NOTE: Intended to be used to invoke `notifyAudioInputParametersChange`, + * `notifyAudioOutputParametersChange`, `notifyAudioInputInterrupted`, + * `notifyAudioOutputInterrupted` on native ADM thread. + * Also could be used by `RTCAudioDevice` implementation to tie + * mutations of underlying audio objects (AVAudioEngine, AudioUnit, etc) + * to the native ADM thread. Could be useful to handle events like audio route change, which + * could lead to audio parameters change. + */ +- (void)dispatchAsync:(dispatch_block_t)block; + +/** + * Synchronously execute block of code within the context of + * thread which owns native ADM. Allows reentrancy. + * + * NOTE: Intended to be used to invoke `notifyAudioInputParametersChange`, + * `notifyAudioOutputParametersChange`, `notifyAudioInputInterrupted`, + * `notifyAudioOutputInterrupted` on native ADM thread and make sure + * aforementioned is completed before `dispatchSync` returns. Could be useful + * when implementation of `RTCAudioDevice` tie mutation to underlying audio objects (AVAudioEngine, + * AudioUnit, etc) to own thread to satisfy requirement that native ADM audio parameters + * must be kept in sync with current audio parameters before audio is actually played or recorded. + */ +- (void)dispatchSync:(dispatch_block_t)block; + +@end + +/** + * Protocol to abstract platform specific ways to implement playback and recording. + * + * NOTE: All the members of protocol are called by native ADM from the same thread + * between calls to `initializeWithDelegate` and `terminate`. + * NOTE: Implementation is fully responsible for configuring application's AVAudioSession. + * An example implementation of RTCAudioDevice: https://github.com/mstyura/RTCAudioDevice + * TODO(yura.yaroshevich): Implement custom RTCAudioDevice for AppRTCMobile demo app. + */ +RTC_OBJC_EXPORT @protocol RTC_OBJC_TYPE +(RTCAudioDevice) + + /** + * Indicates current sample rate of audio recording. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate + * notifyAudioParametersChange]`. + */ + @property(readonly) double inputSampleRate; + +/** + * Indicates current size of record buffer. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) NSTimeInterval inputIOBufferDuration; + +/** + * Indicates current number of recorded audio channels. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) NSInteger inputNumberOfChannels; + +/** + * Indicates current input latency + */ +@property(readonly) NSTimeInterval inputLatency; + +/** + * Indicates current sample rate of audio playback. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) double outputSampleRate; + +/** + * Indicates current size of playback buffer. Changes to this property + * must be notified back to native ADM via `-[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) NSTimeInterval outputIOBufferDuration; + +/** + * Indicates current number of playback audio channels. Changes to this property + * must be notified back to WebRTC via `[RTCAudioDeviceDelegate notifyAudioParametersChange]`. + */ +@property(readonly) NSInteger outputNumberOfChannels; + +/** + * Indicates current output latency + */ +@property(readonly) NSTimeInterval outputLatency; + +/** + * Indicates if invocation of `initializeWithDelegate` required before usage of RTCAudioDevice. + * YES indicates that `initializeWithDelegate` was called earlier without subsequent call to + * `terminate`. NO indicates that either `initializeWithDelegate` not called or `terminate` called. + */ +@property(readonly) BOOL isInitialized; + +/** + * Initializes RTCAudioDevice with RTCAudioDeviceDelegate. + * Implementation must return YES if RTCAudioDevice initialized successfully and NO otherwise. + */ +- (BOOL)initializeWithDelegate:(id)delegate; + +/** + * De-initializes RTCAudioDevice. Implementation should forget about `delegate` provided in + * `initializeWithDelegate`. + */ +- (BOOL)terminate; + +/** + * Property to indicate if `initializePlayout` call required before invocation of `startPlayout`. + * YES indicates that `initializePlayout` was successfully invoked earlier or not necessary, + * NO indicates that `initializePlayout` invocation required. + */ +@property(readonly) BOOL isPlayoutInitialized; + +/** + * Prepares RTCAudioDevice to play audio. + * Called by native ADM before invocation of `startPlayout`. + * Implementation is expected to return YES in case of successful playout initialization and NO + * otherwise. + */ +- (BOOL)initializePlayout; + +/** + * Property to indicate if RTCAudioDevice should be playing according to + * earlier calls of `startPlayout` and `stopPlayout`. + */ +@property(readonly) BOOL isPlaying; + +/** + * Method is called when native ADM wants to play audio. + * Implementation is expected to return YES if playback start request + * successfully handled and NO otherwise. + */ +- (BOOL)startPlayout; + +/** + * Method is called when native ADM no longer needs to play audio. + * Implementation is expected to return YES if playback stop request + * successfully handled and NO otherwise. + */ +- (BOOL)stopPlayout; + +/** + * Property to indicate if `initializeRecording` call required before usage of `startRecording`. + * YES indicates that `initializeRecording` was successfully invoked earlier or not necessary, + * NO indicates that `initializeRecording` invocation required. + */ +@property(readonly) BOOL isRecordingInitialized; + +/** + * Prepares RTCAudioDevice to record audio. + * Called by native ADM before invocation of `startRecording`. + * Implementation may use this method to prepare resources required to record audio. + * Implementation is expected to return YES in case of successful record initialization and NO + * otherwise. + */ +- (BOOL)initializeRecording; + +/** + * Property to indicate if RTCAudioDevice should record audio according to + * earlier calls to `startRecording` and `stopRecording`. + */ +@property(readonly) BOOL isRecording; + +/** + * Method is called when native ADM wants to record audio. + * Implementation is expected to return YES if recording start request + * successfully handled and NO otherwise. + */ +- (BOOL)startRecording; + +/** + * Method is called when native ADM no longer needs to record audio. + * Implementation is expected to return YES if recording stop request + * successfully handled and NO otherwise. + */ +- (BOOL)stopRecording; + +@end + +NS_ASSUME_NONNULL_END diff --git a/sdk/objc/native/api/objc_audio_device_module.h b/sdk/objc/native/api/objc_audio_device_module.h new file mode 100644 index 0000000000..0fe2dda4a0 --- /dev/null +++ b/sdk/objc/native/api/objc_audio_device_module.h @@ -0,0 +1,24 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_OBJC_NATIVE_API_OBJC_AUDIO_DEVICE_MODULE_H_ +#define SDK_OBJC_NATIVE_API_OBJC_AUDIO_DEVICE_MODULE_H_ + +#import "components/audio/RTCAudioDevice.h" +#include "modules/audio_device/include/audio_device.h" + +namespace webrtc { + +rtc::scoped_refptr CreateAudioDeviceModule( + id audio_device); + +} // namespace webrtc + +#endif // SDK_OBJC_NATIVE_API_OBJC_AUDIO_DEVICE_MODULE_H_ diff --git a/sdk/objc/native/api/objc_audio_device_module.mm b/sdk/objc/native/api/objc_audio_device_module.mm new file mode 100644 index 0000000000..76edd45605 --- /dev/null +++ b/sdk/objc/native/api/objc_audio_device_module.mm @@ -0,0 +1,26 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "objc_audio_device_module.h" + +#include "api/make_ref_counted.h" +#include "rtc_base/logging.h" + +#include "sdk/objc/native/src/objc_audio_device.h" + +namespace webrtc { + +rtc::scoped_refptr CreateAudioDeviceModule( + id audio_device) { + RTC_DLOG(LS_INFO) << __FUNCTION__; + return rtc::make_ref_counted(audio_device); +} + +} // namespace webrtc diff --git a/sdk/objc/native/src/objc_audio_device.h b/sdk/objc/native/src/objc_audio_device.h index 88628894ae..7700855f74 100644 --- a/sdk/objc/native/src/objc_audio_device.h +++ b/sdk/objc/native/src/objc_audio_device.h @@ -11,6 +11,7 @@ #ifndef SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_ #define SDK_OBJC_NATIVE_SRC_OBJC_AUDIO_DEVICE_H_ +#import "components/audio/RTCAudioDevice.h" #include "modules/audio_device/include/audio_device.h" namespace webrtc { @@ -19,7 +20,8 @@ namespace objc_adm { class ObjCAudioDeviceModule : public AudioDeviceModule { public: - ObjCAudioDeviceModule(); + explicit ObjCAudioDeviceModule( + id audio_device); ~ObjCAudioDeviceModule() override; // Retrieve the currently utilized audio layer @@ -123,6 +125,9 @@ class ObjCAudioDeviceModule : public AudioDeviceModule { int GetPlayoutAudioParameters(AudioParameters* params) const override; int GetRecordAudioParameters(AudioParameters* params) const override; #endif // WEBRTC_IOS + + private: + id audio_device_; }; } // namespace objc_adm diff --git a/sdk/objc/native/src/objc_audio_device.mm b/sdk/objc/native/src/objc_audio_device.mm index b252031bbf..1f8a5f8159 100644 --- a/sdk/objc/native/src/objc_audio_device.mm +++ b/sdk/objc/native/src/objc_audio_device.mm @@ -15,8 +15,10 @@ namespace webrtc { namespace objc_adm { -ObjCAudioDeviceModule::ObjCAudioDeviceModule() { +ObjCAudioDeviceModule::ObjCAudioDeviceModule(id audio_device) + : audio_device_(audio_device) { RTC_DLOG_F(LS_VERBOSE) << ""; + RTC_DCHECK(audio_device_); } ObjCAudioDeviceModule::~ObjCAudioDeviceModule() {