diff --git a/webrtc/modules/audio_device/android/opensles_player.cc b/webrtc/modules/audio_device/android/opensles_player.cc index 2d305f0ff7..f79f4f6ee7 100644 --- a/webrtc/modules/audio_device/android/opensles_player.cc +++ b/webrtc/modules/audio_device/android/opensles_player.cc @@ -12,6 +12,7 @@ #include +#include "webrtc/base/array_view.h" #include "webrtc/base/arraysize.h" #include "webrtc/base/checks.h" #include "webrtc/base/format_macros.h" @@ -209,9 +210,9 @@ void OpenSLESPlayer::AllocateDataBuffers() { ALOGD("native buffer size: %" PRIuS, buffer_size_in_bytes); ALOGD("native buffer size in ms: %.2f", audio_parameters_.GetBufferSizeInMilliseconds()); - fine_audio_buffer_.reset( - new FineAudioBuffer(audio_device_buffer_, buffer_size_in_bytes, - audio_parameters_.sample_rate())); + fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_, + audio_parameters_.sample_rate(), + 2 * buffer_size_in_bytes)); // Allocated memory for audio buffers. for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) { audio_buffers_[i].reset(new SLint8[buffer_size_in_bytes]); @@ -398,7 +399,8 @@ void OpenSLESPlayer::EnqueuePlayoutData(bool silence) { // Read audio data from the WebRTC source using the FineAudioBuffer object // to adjust for differences in buffer size between WebRTC (10ms) and native // OpenSL ES. - fine_audio_buffer_->GetPlayoutData(audio_ptr); + fine_audio_buffer_->GetPlayoutData(rtc::ArrayView( + audio_ptr, audio_parameters_.GetBytesPerBuffer())); } // Enqueue the decoded audio buffer for playback. SLresult err = (*simple_buffer_queue_) diff --git a/webrtc/modules/audio_device/android/opensles_recorder.cc b/webrtc/modules/audio_device/android/opensles_recorder.cc index 5178d2c149..9f5de20774 100644 --- a/webrtc/modules/audio_device/android/opensles_recorder.cc +++ b/webrtc/modules/audio_device/android/opensles_recorder.cc @@ -12,6 +12,7 @@ #include +#include "webrtc/base/array_view.h" #include "webrtc/base/arraysize.h" #include "webrtc/base/checks.h" #include "webrtc/base/format_macros.h" @@ -335,9 +336,9 @@ void OpenSLESRecorder::AllocateDataBuffers() { audio_parameters_.GetBytesPerBuffer()); ALOGD("native sample rate: %d", audio_parameters_.sample_rate()); RTC_DCHECK(audio_device_buffer_); - fine_audio_buffer_.reset(new FineAudioBuffer( - audio_device_buffer_, audio_parameters_.GetBytesPerBuffer(), - audio_parameters_.sample_rate())); + fine_audio_buffer_.reset( + new FineAudioBuffer(audio_device_buffer_, audio_parameters_.sample_rate(), + 2 * audio_parameters_.GetBytesPerBuffer())); // Allocate queue of audio buffers that stores recorded audio samples. const int data_size_bytes = audio_parameters_.GetBytesPerBuffer(); audio_buffers_.reset(new std::unique_ptr[kNumOfOpenSLESBuffers]); @@ -371,7 +372,8 @@ void OpenSLESRecorder::ReadBufferQueue() { static_cast(audio_parameters_.GetBytesPerBuffer()); const int8_t* data = static_cast(audio_buffers_[buffer_index_].get()); - fine_audio_buffer_->DeliverRecordedData(data, size_in_bytes, 25, 25); + fine_audio_buffer_->DeliverRecordedData( + rtc::ArrayView(data, size_in_bytes), 25, 25); // Enqueue the utilized audio buffer and use if for recording again. EnqueueAudioBuffer(); } diff --git a/webrtc/modules/audio_device/fine_audio_buffer.cc b/webrtc/modules/audio_device/fine_audio_buffer.cc index 83775741d8..27d1937868 100644 --- a/webrtc/modules/audio_device/fine_audio_buffer.cc +++ b/webrtc/modules/audio_device/fine_audio_buffer.cc @@ -21,14 +21,15 @@ namespace webrtc { FineAudioBuffer::FineAudioBuffer(AudioDeviceBuffer* device_buffer, - size_t desired_frame_size_bytes, - int sample_rate) + int sample_rate, + size_t capacity) : device_buffer_(device_buffer), - desired_frame_size_bytes_(desired_frame_size_bytes), sample_rate_(sample_rate), samples_per_10_ms_(static_cast(sample_rate_ * 10 / 1000)), - bytes_per_10_ms_(samples_per_10_ms_ * sizeof(int16_t)) { - LOG(INFO) << "desired_frame_size_bytes:" << desired_frame_size_bytes; + bytes_per_10_ms_(samples_per_10_ms_ * sizeof(int16_t)), + playout_buffer_(0, capacity), + record_buffer_(0, capacity) { + LOG(INFO) << "samples_per_10_ms_:" << samples_per_10_ms_; } FineAudioBuffer::~FineAudioBuffer() {} @@ -41,11 +42,11 @@ void FineAudioBuffer::ResetRecord() { record_buffer_.Clear(); } -void FineAudioBuffer::GetPlayoutData(int8_t* buffer) { - const size_t num_bytes = desired_frame_size_bytes_; +void FineAudioBuffer::GetPlayoutData(rtc::ArrayView audio_buffer) { // Ask WebRTC for new data in chunks of 10ms until we have enough to // fulfill the request. It is possible that the buffer already contains // enough samples from the last round. + const size_t num_bytes = audio_buffer.size(); while (playout_buffer_.size() < num_bytes) { // Get 10ms decoded audio from WebRTC. device_buffer_->RequestPlayoutData(samples_per_10_ms_); @@ -61,19 +62,19 @@ void FineAudioBuffer::GetPlayoutData(int8_t* buffer) { RTC_DCHECK_EQ(bytes_per_10_ms_, bytes_written); } // Provide the requested number of bytes to the consumer. - memcpy(buffer, playout_buffer_.data(), num_bytes); + memcpy(audio_buffer.data(), playout_buffer_.data(), num_bytes); // Move remaining samples to start of buffer to prepare for next round. memmove(playout_buffer_.data(), playout_buffer_.data() + num_bytes, playout_buffer_.size() - num_bytes); playout_buffer_.SetSize(playout_buffer_.size() - num_bytes); } -void FineAudioBuffer::DeliverRecordedData(const int8_t* buffer, - size_t size_in_bytes, - int playout_delay_ms, - int record_delay_ms) { +void FineAudioBuffer::DeliverRecordedData( + rtc::ArrayView audio_buffer, + int playout_delay_ms, + int record_delay_ms) { // Always append new data and grow the buffer if needed. - record_buffer_.AppendData(buffer, size_in_bytes); + record_buffer_.AppendData(audio_buffer.data(), audio_buffer.size()); // Consume samples from buffer in chunks of 10ms until there is not // enough data left. The number of remaining bytes in the cache is given by // the new size of the buffer. diff --git a/webrtc/modules/audio_device/fine_audio_buffer.h b/webrtc/modules/audio_device/fine_audio_buffer.h index 306f9d24d3..9f3bb5e2b7 100644 --- a/webrtc/modules/audio_device/fine_audio_buffer.h +++ b/webrtc/modules/audio_device/fine_audio_buffer.h @@ -13,6 +13,7 @@ #include +#include "webrtc/base/array_view.h" #include "webrtc/base/buffer.h" #include "webrtc/typedefs.h" @@ -28,40 +29,38 @@ class AudioDeviceBuffer; // in 10ms chunks when the size of the provided audio buffers differs from 10ms. // As an example: calling DeliverRecordedData() with 5ms buffers will deliver // accumulated 10ms worth of data to the ADB every second call. +// TODO(henrika): add support for stereo when mobile platforms need it. class FineAudioBuffer { public: // |device_buffer| is a buffer that provides 10ms of audio data. - // |desired_frame_size_bytes| is the number of bytes of audio data - // GetPlayoutData() should return on success. It is also the required size of - // each recorded buffer used in DeliverRecordedData() calls. // |sample_rate| is the sample rate of the audio data. This is needed because // |device_buffer| delivers 10ms of data. Given the sample rate the number - // of samples can be calculated. + // of samples can be calculated. The |capacity| ensures that the buffer size + // can be increased to at least capacity without further reallocation. FineAudioBuffer(AudioDeviceBuffer* device_buffer, - size_t desired_frame_size_bytes, - int sample_rate); + int sample_rate, + size_t capacity); ~FineAudioBuffer(); // Clears buffers and counters dealing with playour and/or recording. void ResetPlayout(); void ResetRecord(); - // |buffer| must be of equal or greater size than what is returned by - // RequiredBufferSize(). This is to avoid unnecessary memcpy. - void GetPlayoutData(int8_t* buffer); + // Copies audio samples into |audio_buffer| where number of requested + // elements is specified by |audio_buffer.size()|. The producer will always + // fill up the audio buffer and if no audio exists, the buffer will contain + // silence instead. + void GetPlayoutData(rtc::ArrayView audio_buffer); - // Consumes the audio data in |buffer| and sends it to the WebRTC layer in - // chunks of 10ms. The provided delay estimates in |playout_delay_ms| and + // Consumes the audio data in |audio_buffer| and sends it to the WebRTC layer + // in chunks of 10ms. The provided delay estimates in |playout_delay_ms| and // |record_delay_ms| are given to the AEC in the audio processing module. // They can be fixed values on most platforms and they are ignored if an // external (hardware/built-in) AEC is used. - // The size of |buffer| is given by |size_in_bytes| and must be equal to - // |desired_frame_size_bytes_|. // Example: buffer size is 5ms => call #1 stores 5ms of data, call #2 stores // 5ms of data and sends a total of 10ms to WebRTC and clears the intenal // cache. Call #3 restarts the scheme above. - void DeliverRecordedData(const int8_t* buffer, - size_t size_in_bytes, + void DeliverRecordedData(rtc::ArrayView audio_buffer, int playout_delay_ms, int record_delay_ms); @@ -73,15 +72,14 @@ class FineAudioBuffer { // class and the owner must ensure that the pointer is valid during the life- // time of this object. AudioDeviceBuffer* const device_buffer_; - // Number of bytes delivered by GetPlayoutData() call and provided to - // DeliverRecordedData(). - const size_t desired_frame_size_bytes_; // Sample rate in Hertz. const int sample_rate_; // Number of audio samples per 10ms. const size_t samples_per_10_ms_; // Number of audio bytes per 10ms. const size_t bytes_per_10_ms_; + // Storage for output samples from which a consumer can read audio buffers + // in any size using GetPlayoutData(). rtc::BufferT playout_buffer_; // Storage for input samples that are about to be delivered to the WebRTC // ADB or remains from the last successful delivery of a 10ms audio buffer. diff --git a/webrtc/modules/audio_device/fine_audio_buffer_unittest.cc b/webrtc/modules/audio_device/fine_audio_buffer_unittest.cc index 535f16816c..c7fef57097 100644 --- a/webrtc/modules/audio_device/fine_audio_buffer_unittest.cc +++ b/webrtc/modules/audio_device/fine_audio_buffer_unittest.cc @@ -13,6 +13,7 @@ #include #include +#include "webrtc/base/array_view.h" #include "webrtc/modules/audio_device/mock_audio_device_buffer.h" #include "webrtc/test/gmock.h" #include "webrtc/test/gtest.h" @@ -24,6 +25,9 @@ using ::testing::Return; namespace webrtc { +const int kSampleRate = 44100; +const int kSamplesPer10Ms = kSampleRate * 10 / 1000; + // The fake audio data is 0,1,..SCHAR_MAX-1,0,1,... This is to make it easy // to detect errors. This function verifies that the buffers contain such data. // E.g. if there are two buffers of size 3, buffer 1 would contain 0,1,2 and @@ -80,8 +84,7 @@ ACTION_P2(VerifyInputBuffer, iteration, samples_per_10_ms) { return 0; } -void RunFineBufferTest(int sample_rate, int frame_size_in_samples) { - const int kSamplesPer10Ms = sample_rate * 10 / 1000; +void RunFineBufferTest(int frame_size_in_samples) { const int kFrameSizeBytes = frame_size_in_samples * static_cast(sizeof(int16_t)); const int kNumberOfFrames = 5; @@ -114,33 +117,29 @@ void RunFineBufferTest(int sample_rate, int frame_size_in_samples) { .Times(kNumberOfUpdateBufferCalls - 1) .WillRepeatedly(Return(kSamplesPer10Ms)); - FineAudioBuffer fine_buffer(&audio_device_buffer, kFrameSizeBytes, - sample_rate); + FineAudioBuffer fine_buffer(&audio_device_buffer, kSampleRate, + kFrameSizeBytes); + std::unique_ptr out_buffer(new int8_t[kFrameSizeBytes]); + std::unique_ptr in_buffer(new int8_t[kFrameSizeBytes]); - std::unique_ptr out_buffer; - out_buffer.reset(new int8_t[kFrameSizeBytes]); - std::unique_ptr in_buffer; - in_buffer.reset(new int8_t[kFrameSizeBytes]); for (int i = 0; i < kNumberOfFrames; ++i) { - fine_buffer.GetPlayoutData(out_buffer.get()); + fine_buffer.GetPlayoutData( + rtc::ArrayView(out_buffer.get(), kFrameSizeBytes)); EXPECT_TRUE(VerifyBuffer(out_buffer.get(), i, kFrameSizeBytes)); UpdateInputBuffer(in_buffer.get(), i, kFrameSizeBytes); - fine_buffer.DeliverRecordedData(in_buffer.get(), kFrameSizeBytes, 0, 0); + fine_buffer.DeliverRecordedData( + rtc::ArrayView(in_buffer.get(), kFrameSizeBytes), 0, 0); } } TEST(FineBufferTest, BufferLessThan10ms) { - const int kSampleRate = 44100; - const int kSamplesPer10Ms = kSampleRate * 10 / 1000; const int kFrameSizeSamples = kSamplesPer10Ms - 50; - RunFineBufferTest(kSampleRate, kFrameSizeSamples); + RunFineBufferTest(kFrameSizeSamples); } TEST(FineBufferTest, GreaterThan10ms) { - const int kSampleRate = 44100; - const int kSamplesPer10Ms = kSampleRate * 10 / 1000; const int kFrameSizeSamples = kSamplesPer10Ms + 50; - RunFineBufferTest(kSampleRate, kFrameSizeSamples); + RunFineBufferTest(kFrameSizeSamples); } } // namespace webrtc diff --git a/webrtc/modules/audio_device/ios/audio_device_ios.h b/webrtc/modules/audio_device/ios/audio_device_ios.h index 5710299bdb..f323ad41aa 100644 --- a/webrtc/modules/audio_device/ios/audio_device_ios.h +++ b/webrtc/modules/audio_device/ios/audio_device_ios.h @@ -260,10 +260,6 @@ class AudioDeviceIOS : public AudioDeviceGeneric, // to WebRTC and the remaining part is stored. std::unique_ptr fine_audio_buffer_; - // Extra audio buffer to be used by the playout side for rendering audio. - // The buffer size is given by FineAudioBuffer::RequiredBufferSizeBytes(). - std::unique_ptr playout_audio_buffer_; - // Provides a mechanism for encapsulating one or more buffers of audio data. // Only used on the recording side. AudioBufferList audio_record_buffer_list_; diff --git a/webrtc/modules/audio_device/ios/audio_device_ios.mm b/webrtc/modules/audio_device/ios/audio_device_ios.mm index 3add4b9166..b75e5f7a02 100644 --- a/webrtc/modules/audio_device/ios/audio_device_ios.mm +++ b/webrtc/modules/audio_device/ios/audio_device_ios.mm @@ -15,6 +15,7 @@ #include +#include "webrtc/base/array_view.h" #include "webrtc/base/atomicops.h" #include "webrtc/base/bind.h" #include "webrtc/base/checks.h" @@ -69,6 +70,11 @@ enum AudioDeviceMessageType : uint32_t { using ios::CheckAndLogError; #if !defined(NDEBUG) +// Returns true when the code runs on a device simulator. +static bool DeviceIsSimulator() { + return ios::GetDeviceName() == "x86_64"; +} + // Helper method that logs essential device information strings. static void LogDeviceInfo() { LOG(LS_INFO) << "LogDeviceInfo"; @@ -86,6 +92,10 @@ static void LogDeviceInfo() { && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_9_0 LOG(LS_INFO) << " low power mode: " << ios::GetLowPowerModeEnabled(); #endif +#if TARGET_IPHONE_SIMULATOR + LOG(LS_INFO) << " TARGET_IPHONE_SIMULATOR is defined"; +#endif + LOG(LS_INFO) << " DeviceIsSimulator: " << DeviceIsSimulator(); } } #endif // !defined(NDEBUG) @@ -395,7 +405,7 @@ OSStatus AudioDeviceIOS::OnDeliverRecordedData( RTC_CHECK_EQ(size_in_bytes / VoiceProcessingAudioUnit::kBytesPerSample, num_frames); int8_t* data = static_cast(audio_buffer->mData); - fine_audio_buffer_->DeliverRecordedData(data, size_in_bytes, + fine_audio_buffer_->DeliverRecordedData(rtc::ArrayView(data, size_in_bytes), kFixedPlayoutDelayEstimate, kFixedRecordDelayEstimate); return noErr; @@ -423,26 +433,11 @@ OSStatus AudioDeviceIOS::OnGetPlayoutData(AudioUnitRenderActionFlags* flags, memset(destination, 0, size_in_bytes); return noErr; } - // Produce silence and log a warning message for the case when Core Audio is - // asking for an invalid number of audio frames. I don't expect this to happen - // but it is done as a safety measure to avoid bad audio if such as case would - // ever be triggered e.g. in combination with BT devices. - const size_t frames_per_buffer = playout_parameters_.frames_per_buffer(); - if (num_frames != frames_per_buffer) { - RTCLogWarning(@"Expected %u frames but got %u", - static_cast(frames_per_buffer), - static_cast(num_frames)); - *flags |= kAudioUnitRenderAction_OutputIsSilence; - memset(destination, 0, size_in_bytes); - return noErr; - } // Read decoded 16-bit PCM samples from WebRTC (using a size that matches - // the native I/O audio unit) to a preallocated intermediate buffer and - // copy the result to the audio buffer in the |io_data| destination. - int8_t* source = playout_audio_buffer_.get(); - fine_audio_buffer_->GetPlayoutData(source); - memcpy(destination, source, size_in_bytes); + // the native I/O audio unit) and copy the result to the audio buffer in the + // |io_data| destination. + fine_audio_buffer_->GetPlayoutData(rtc::ArrayView(destination, size_in_bytes)); return noErr; } @@ -632,13 +627,12 @@ void AudioDeviceIOS::SetupAudioBuffersForActiveAudioSession() { // Create a modified audio buffer class which allows us to ask for, // or deliver, any number of samples (and not only multiple of 10ms) to match - // the native audio unit buffer size. + // the native audio unit buffer size. Use a reasonable capacity to avoid + // reallocations while audio is played to reduce risk of glitches. RTC_DCHECK(audio_device_buffer_); - const size_t buffer_size_in_bytes = playout_parameters_.GetBytesPerBuffer(); + const size_t capacity_in_bytes = 2 * playout_parameters_.GetBytesPerBuffer(); fine_audio_buffer_.reset(new FineAudioBuffer( - audio_device_buffer_, buffer_size_in_bytes, - playout_parameters_.sample_rate())); - playout_audio_buffer_.reset(new SInt8[buffer_size_in_bytes]); + audio_device_buffer_, playout_parameters_.sample_rate(), capacity_in_bytes)); // Allocate AudioBuffers to be used as storage for the received audio. // The AudioBufferList structure works as a placeholder for the