From d643be9fdce42d1a46a6445eb8c0253bd6025234 Mon Sep 17 00:00:00 2001 From: Peter Hanspers Date: Thu, 30 Jan 2025 10:38:57 +0100 Subject: [PATCH] Add a render error callback from AudioDeviceIOS to AudioDeviceModuleIOS. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change expands on https://webrtc-review.googlesource.com/c/src/+/374420 to cover the error produced when copying microphone samples. Change-Id: I7aa58c9c9ac175d5f4cfdb60bbd3f16334c03c1b Bug: webrtc:390314937 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/375540 Reviewed-by: Jakob Ivarsson‎ Reviewed-by: Henrik Andreassson Commit-Queue: Peter Hanspers Cr-Commit-Position: refs/heads/main@{#43826} --- modules/audio_device/audio_device_impl.cc | 3 ++- sdk/objc/native/src/audio/audio_device_ios.h | 15 ++++++++++++++- sdk/objc/native/src/audio/audio_device_ios.mm | 12 ++++++++++-- .../native/src/audio/audio_device_module_ios.mm | 7 +++++-- sdk/objc/unittests/RTCAudioDevice_xctest.mm | 9 ++++++--- 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/modules/audio_device/audio_device_impl.cc b/modules/audio_device/audio_device_impl.cc index b084ff2d5d..45605292ff 100644 --- a/modules/audio_device/audio_device_impl.cc +++ b/modules/audio_device/audio_device_impl.cc @@ -243,7 +243,8 @@ int32_t AudioDeviceModuleImpl::CreatePlatformSpecificObjects() { if (audio_layer == kPlatformDefaultAudio) { audio_device_.reset(new ios_adm::AudioDeviceIOS( /*bypass_voice_processing=*/false, - /*muted_speech_event_handler=*/nullptr)); + /*muted_speech_event_handler=*/nullptr, + /*render_error_handler=*/nullptr)); RTC_LOG(LS_INFO) << "iPhone Audio APIs will be utilized."; } // END #if defined(WEBRTC_IOS) diff --git a/sdk/objc/native/src/audio/audio_device_ios.h b/sdk/objc/native/src/audio/audio_device_ios.h index caecf12994..bbb4025694 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.h +++ b/sdk/objc/native/src/audio/audio_device_ios.h @@ -33,6 +33,11 @@ class FineAudioBuffer; namespace ios_adm { +// A callback handler for audio device rendering errors. +// Note: Called on a realtime thread. +// Note: Only applies to input rendering errors, not output. +typedef void (^AudioDeviceIOSRenderErrorHandler)(OSStatus error); + // Implements full duplex 16-bit mono PCM audio support for iOS using a // Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit // supports audio echo cancellation. It also adds automatic gain control, @@ -52,7 +57,8 @@ class AudioDeviceIOS : public AudioDeviceGeneric, public: explicit AudioDeviceIOS( bool bypass_voice_processing, - AudioDeviceModule::MutedSpeechEventHandler muted_speech_event_handler); + AudioDeviceModule::MutedSpeechEventHandler muted_speech_event_handler, + AudioDeviceIOSRenderErrorHandler render_error_handler); ~AudioDeviceIOS() override; void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override; @@ -220,6 +226,13 @@ class AudioDeviceIOS : public AudioDeviceGeneric, // Handle a user speaking during muted event AudioDeviceModule::MutedSpeechEventHandler muted_speech_event_handler_; + // Handle microphone rendering errors. + AudioDeviceIOSRenderErrorHandler render_error_handler_; + + // Copying microphone data (rendering to a buffer) may keep failing. This + // field makes sure subsequent errors are not reported. + bool disregard_next_render_error_; + // Native I/O audio thread checker. SequenceChecker io_thread_checker_; diff --git a/sdk/objc/native/src/audio/audio_device_ios.mm b/sdk/objc/native/src/audio/audio_device_ios.mm index 49734a2ec9..dd7dcc4201 100644 --- a/sdk/objc/native/src/audio/audio_device_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_ios.mm @@ -97,9 +97,12 @@ static void LogDeviceInfo() { AudioDeviceIOS::AudioDeviceIOS( bool bypass_voice_processing, - AudioDeviceModule::MutedSpeechEventHandler muted_speech_event_handler) + AudioDeviceModule::MutedSpeechEventHandler muted_speech_event_handler, + AudioDeviceIOSRenderErrorHandler render_error_handler) : bypass_voice_processing_(bypass_voice_processing), muted_speech_event_handler_(muted_speech_event_handler), + render_error_handler_(render_error_handler), + disregard_next_render_error_(false), audio_device_buffer_(nullptr), audio_unit_(nullptr), recording_(0), @@ -431,13 +434,18 @@ OSStatus AudioDeviceIOS::OnDeliverRecordedData( // to the preallocated audio buffer list that the audio unit renders into. // We can make the audio unit provide a buffer instead in io_data, but we // currently just use our own. - // TODO(henrika): should error handling be improved? result = audio_unit_->Render( flags, time_stamp, bus_number, num_frames, &audio_buffer_list); if (result != noErr) { RTCLogError(@"Failed to render audio."); + if (render_error_handler_ && !disregard_next_render_error_) { + disregard_next_render_error_ = true; + thread_->PostTask( + SafeTask(safety_, [this, result] { render_error_handler_(result); })); + } return result; } + disregard_next_render_error_ = false; // Get a pointer to the recorded audio and send it to the WebRTC ADB. // Use the FineAudioBuffer instance to convert between native buffer size diff --git a/sdk/objc/native/src/audio/audio_device_module_ios.mm b/sdk/objc/native/src/audio/audio_device_module_ios.mm index 3a7abb479c..3b338f2399 100644 --- a/sdk/objc/native/src/audio/audio_device_module_ios.mm +++ b/sdk/objc/native/src/audio/audio_device_module_ios.mm @@ -84,10 +84,13 @@ int32_t AudioDeviceModuleIOS::Init() { RTC_DLOG(LS_INFO) << __FUNCTION__; if (initialized_) return 0; + AudioDeviceIOSRenderErrorHandler error_handler = ^(OSStatus error) { + ReportError(kRecordingDeviceFailed); + }; audio_device_buffer_.reset( new webrtc::AudioDeviceBuffer(task_queue_factory_.get())); - audio_device_.reset(new ios_adm::AudioDeviceIOS(bypass_voice_processing_, - muted_speech_event_handler_)); + audio_device_.reset(new ios_adm::AudioDeviceIOS( + bypass_voice_processing_, muted_speech_event_handler_, error_handler)); RTC_CHECK(audio_device_); this->AttachAudioBuffer(); diff --git a/sdk/objc/unittests/RTCAudioDevice_xctest.mm b/sdk/objc/unittests/RTCAudioDevice_xctest.mm index 299e5cb5d9..1d486770b7 100644 --- a/sdk/objc/unittests/RTCAudioDevice_xctest.mm +++ b/sdk/objc/unittests/RTCAudioDevice_xctest.mm @@ -48,7 +48,8 @@ _audioDeviceModule = webrtc::CreateAudioDeviceModule(); _audio_device.reset(new webrtc::ios_adm::AudioDeviceIOS( /*bypass_voice_processing=*/false, - /*muted_speech_event_handler=*/nullptr)); + /*muted_speech_event_handler=*/nullptr, + /*render_error_handler=*/nullptr)); self.audioSession = [RTC_OBJC_TYPE(RTCAudioSession) sharedInstance]; NSError *error = nil; @@ -149,7 +150,8 @@ _audio_device.reset(new webrtc::ios_adm::AudioDeviceIOS( /*bypass_voice_processing=*/false, - /*muted_speech_event_handler=*/muted_speech_event_handler)); + /*muted_speech_event_handler=*/muted_speech_event_handler, + /*render_error_handler=*/nullptr)); _audio_device->OnReceivedMutedSpeechActivity( kAUVoiceIOSpeechActivityHasStarted); @@ -168,7 +170,8 @@ _audio_device.reset(new webrtc::ios_adm::AudioDeviceIOS( /*bypass_voice_processing=*/false, - /*muted_speech_event_handler=*/muted_speech_event_handler)); + /*muted_speech_event_handler=*/muted_speech_event_handler, + /*render_error_handler=*/nullptr)); _audio_device->OnReceivedMutedSpeechActivity( kAUVoiceIOSpeechActivityHasEnded); [self waitForExpectations:@[ handlerExpectation ] timeout:10.0];