admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor the AudioDevice for iOS and improve the performance and stability

Browse Source 
This CL contains major modifications of the audio output parts for WebRTC on iOS:
- general code cleanup
- improves thread handling (added thread checks, remove critical section, atomic ops etc.)
- reduces loopback latency of iPhone 6 from ~90ms to ~60ms ;-)
- improves selection of audio parameters on iOS
- reduces complexity by removing complex and redundant delay estimates
- now instead uses fixed delay estimates if for some reason the SW EAC must be used
- adds AudioFineBuffer to compensate for differences in native output buffer size and
  the 10ms size used by WebRTC. Same class as is used today on Android and we have unit tests for
  this class (the old code was buggy and we have several issue reports of crashes related to it)

Similar improvements will be done for the recording sid as well in a separate CL.
I will also add support for 48kHz in an upcoming CL since that will improve Opus performance.

BUG=webrtc:4796,webrtc:4817,webrtc:4954, webrtc:4212
TEST=AppRTC demo and iOS modules_unittests using --gtest_filter=AudioDevice*
R=pbos@webrtc.org, tkchin@webrtc.org

Review URL: https://codereview.webrtc.org/1254883002 .

Cr-Commit-Position: refs/heads/master@{#9875}
This commit is contained in:
henrika 2015-09-07 16:09:50 +02:00
parent 05cfcd3469
commit 86d907cffd
16 changed files with 944 additions and 1025 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
talk/media/webrtc/webrtcvoiceengine.cc
View File

@ -645,6 +645,7 @@ bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) {
// On iOS, VPIO provides built-in EC and AGC.
options.echo_cancellation.Set(false);
options.auto_gain_control.Set(false);
LOG(LS_INFO) << "Always disable AEC and AGC on iOS. Use built-in instead.";
#elif defined(ANDROID)
ec_mode = webrtc::kEcAecm;
#endif
@ -702,8 +703,8 @@ bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) {
LOG_RTCERR2(SetEcStatus, echo_cancellation, ec_mode);
return false;
} else {
LOG(LS_VERBOSE) << "Echo control set to " << echo_cancellation
<< " with mode " << ec_mode;
LOG(LS_INFO) << "Echo control set to " << echo_cancellation
<< " with mode " << ec_mode;
}
#if !defined(ANDROID)
// TODO(ajm): Remove the error return on Android from webrtc.
@ -726,8 +727,8 @@ bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) {
LOG_RTCERR2(SetAgcStatus, auto_gain_control, agc_mode);
return false;
} else {
LOG(LS_VERBOSE) << "Auto gain set to " << auto_gain_control
<< " with mode " << agc_mode;
LOG(LS_INFO) << "Auto gain set to " << auto_gain_control << " with mode "
<< agc_mode;
}
}

4
webrtc/modules/audio_device/BUILD.gn
View File

@ -27,6 +27,8 @@ source_set("audio_device") {
"dummy/audio_device_dummy.h",
"dummy/file_audio_device.cc",
"dummy/file_audio_device.h",
"fine_audio_buffer.cc",
"fine_audio_buffer.h",
"include/audio_device.h",
"include/audio_device_defines.h",
]
@ -57,8 +59,6 @@ source_set("audio_device") {
"android/audio_record_jni.h",
"android/audio_track_jni.cc",
"android/audio_track_jni.h",
"android/fine_audio_buffer.cc",
"android/fine_audio_buffer.h",
"android/opensles_common.cc",
"android/opensles_common.h",
"android/opensles_player.cc",

89
webrtc/modules/audio_device/android/fine_audio_buffer.cc
View File

@ -1,89 +0,0 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/audio_device/android/fine_audio_buffer.h"
#include <memory.h>
#include <stdio.h>
#include <algorithm>
#include "webrtc/base/checks.h"
#include "webrtc/modules/audio_device/audio_device_buffer.h"
namespace webrtc {
FineAudioBuffer::FineAudioBuffer(AudioDeviceBuffer* device_buffer,
size_t desired_frame_size_bytes,
int sample_rate)
: device_buffer_(device_buffer),
desired_frame_size_bytes_(desired_frame_size_bytes),
sample_rate_(sample_rate),
samples_per_10_ms_(static_cast<size_t>(sample_rate_ * 10 / 1000)),
bytes_per_10_ms_(samples_per_10_ms_ * sizeof(int16_t)),
cached_buffer_start_(0),
cached_bytes_(0) {
cache_buffer_.reset(new int8_t[bytes_per_10_ms_]);
}
FineAudioBuffer::~FineAudioBuffer() {
}
size_t FineAudioBuffer::RequiredBufferSizeBytes() {
// It is possible that we store the desired frame size - 1 samples. Since new
// audio frames are pulled in chunks of 10ms we will need a buffer that can
// hold desired_frame_size - 1 + 10ms of data. We omit the - 1.
return desired_frame_size_bytes_ + bytes_per_10_ms_;
}
void FineAudioBuffer::GetBufferData(int8_t* buffer) {
if (desired_frame_size_bytes_ <= cached_bytes_) {
memcpy(buffer, &cache_buffer_.get()[cached_buffer_start_],
desired_frame_size_bytes_);
cached_buffer_start_ += desired_frame_size_bytes_;
cached_bytes_ -= desired_frame_size_bytes_;
CHECK_LT(cached_buffer_start_ + cached_bytes_, bytes_per_10_ms_);
return;
}
memcpy(buffer, &cache_buffer_.get()[cached_buffer_start_], cached_bytes_);
// Push another n*10ms of audio to |buffer|. n > 1 if
// |desired_frame_size_bytes_| is greater than 10ms of audio. Note that we
// write the audio after the cached bytes copied earlier.
int8_t* unwritten_buffer = &buffer[cached_bytes_];
int bytes_left = static_cast<int>(desired_frame_size_bytes_ - cached_bytes_);
// Ceiling of integer division: 1 + ((x - 1) / y)
size_t number_of_requests = 1 + (bytes_left - 1) / (bytes_per_10_ms_);
for (size_t i = 0; i < number_of_requests; ++i) {
device_buffer_->RequestPlayoutData(samples_per_10_ms_);
int num_out = device_buffer_->GetPlayoutData(unwritten_buffer);
if (static_cast<size_t>(num_out) != samples_per_10_ms_) {
CHECK_EQ(num_out, 0);
cached_bytes_ = 0;
return;
}
unwritten_buffer += bytes_per_10_ms_;
CHECK_GE(bytes_left, 0);
bytes_left -= bytes_per_10_ms_;
}
CHECK_LE(bytes_left, 0);
// Put the samples that were written to |buffer| but are not used in the
// cache.
size_t cache_location = desired_frame_size_bytes_;
int8_t* cache_ptr = &buffer[cache_location];
cached_bytes_ = number_of_requests * bytes_per_10_ms_ -
(desired_frame_size_bytes_ - cached_bytes_);
// If cached_bytes_ is larger than the cache buffer, uninitialized memory
// will be read.
CHECK_LE(cached_bytes_, bytes_per_10_ms_);
CHECK_EQ(static_cast<size_t>(-bytes_left), cached_bytes_);
cached_buffer_start_ = 0;
memcpy(cache_buffer_.get(), cache_ptr, cached_bytes_);
}
} // namespace webrtc

69
webrtc/modules/audio_device/android/fine_audio_buffer.h
View File

@ -1,69 +0,0 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_FINE_AUDIO_BUFFER_H_
#define WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_FINE_AUDIO_BUFFER_H_
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/typedefs.h"
namespace webrtc {
class AudioDeviceBuffer;
// FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data
// corresponding to 10ms of data. It then allows for this data to be pulled in
// a finer or coarser granularity. I.e. interacting with this class instead of
// directly with the AudioDeviceBuffer one can ask for any number of audio data
// samples.
class FineAudioBuffer {
public:
// |device_buffer| is a buffer that provides 10ms of audio data.
// |desired_frame_size_bytes| is the number of bytes of audio data
// (not samples) |GetBufferData| should return on success.
// |sample_rate| is the sample rate of the audio data. This is needed because
// |device_buffer| delivers 10ms of data. Given the sample rate the number
// of samples can be calculated.
FineAudioBuffer(AudioDeviceBuffer* device_buffer,
size_t desired_frame_size_bytes,
int sample_rate);
~FineAudioBuffer();
// Returns the required size of |buffer| when calling GetBufferData. If the
// buffer is smaller memory trampling will happen.
// |desired_frame_size_bytes| and |samples_rate| are as described in the
// constructor.
size_t RequiredBufferSizeBytes();
// |buffer| must be of equal or greater size than what is returned by
// RequiredBufferSize. This is to avoid unnecessary memcpy.
void GetBufferData(int8_t* buffer);
private:
// Device buffer that provides 10ms chunks of data.
AudioDeviceBuffer* device_buffer_;
// Number of bytes delivered per GetBufferData
size_t desired_frame_size_bytes_;
int sample_rate_;
size_t samples_per_10_ms_;
// Convenience parameter to avoid converting from samples
size_t bytes_per_10_ms_;
// Storage for samples that are not yet asked for.
rtc::scoped_ptr<int8_t[]> cache_buffer_;
// Location of first unread sample.
size_t cached_buffer_start_;
// Number of bytes stored in cache.
size_t cached_bytes_;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_AUDIO_DEVICE_ANDROID_FINE_AUDIO_BUFFER_H_

7
webrtc/modules/audio_device/android/opensles_player.cc
View File

@ -16,7 +16,7 @@
#include "webrtc/base/checks.h"
#include "webrtc/base/format_macros.h"
#include "webrtc/modules/audio_device/android/audio_manager.h"
#include "webrtc/modules/audio_device/android/fine_audio_buffer.h"
#include "webrtc/modules/audio_device/fine_audio_buffer.h"
#define TAG "OpenSLESPlayer"
#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
@ -242,7 +242,8 @@ void OpenSLESPlayer::AllocateDataBuffers() {
audio_parameters_.sample_rate()));
// Each buffer must be of this size to avoid unnecessary memcpy while caching
// data between successive callbacks.
const size_t required_buffer_size = fine_buffer_->RequiredBufferSizeBytes();
const size_t required_buffer_size =
fine_buffer_->RequiredPlayoutBufferSizeBytes();
ALOGD("required buffer size: %" PRIuS, required_buffer_size);
for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
audio_buffers_[i].reset(new SLint8[required_buffer_size]);
@ -420,7 +421,7 @@ void OpenSLESPlayer::EnqueuePlayoutData() {
// to adjust for differences in buffer size between WebRTC (10ms) and native
// OpenSL ES.
SLint8* audio_ptr = audio_buffers_[buffer_index_].get();
fine_buffer_->GetBufferData(audio_ptr);
fine_buffer_->GetPlayoutData(audio_ptr);
// Enqueue the decoded audio buffer for playback.
SLresult err =
(*simple_buffer_queue_)

4
webrtc/modules/audio_device/audio_device.gypi
View File

@ -43,6 +43,8 @@
'dummy/audio_device_dummy.h',
'dummy/file_audio_device.cc',
'dummy/file_audio_device.h',
'fine_audio_buffer.cc',
'fine_audio_buffer.h',
],
'conditions': [
['OS=="linux"', {
@ -93,8 +95,6 @@
'android/audio_track_jni.h',
'android/build_info.cc',
'android/build_info.h',
'android/fine_audio_buffer.cc',
'android/fine_audio_buffer.h',
'android/opensles_common.cc',
'android/opensles_common.h',
'android/opensles_player.cc',

150
webrtc/modules/audio_device/fine_audio_buffer.cc Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/audio_device/fine_audio_buffer.h"
#include <memory.h>
#include <stdio.h>
#include <algorithm>
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/modules/audio_device/audio_device_buffer.h"
namespace webrtc {
FineAudioBuffer::FineAudioBuffer(AudioDeviceBuffer* device_buffer,
size_t desired_frame_size_bytes,
int sample_rate)
: device_buffer_(device_buffer),
desired_frame_size_bytes_(desired_frame_size_bytes),
sample_rate_(sample_rate),
samples_per_10_ms_(static_cast<size_t>(sample_rate_ * 10 / 1000)),
bytes_per_10_ms_(samples_per_10_ms_ * sizeof(int16_t)),
playout_cached_buffer_start_(0),
playout_cached_bytes_(0),
// Allocate extra space on the recording side to reduce the number of
// memmove() calls.
required_record_buffer_size_bytes_(
5 * (desired_frame_size_bytes + bytes_per_10_ms_)),
record_cached_bytes_(0),
record_read_pos_(0),
record_write_pos_(0) {
playout_cache_buffer_.reset(new int8_t[bytes_per_10_ms_]);
record_cache_buffer_.reset(new int8_t[required_record_buffer_size_bytes_]);
memset(record_cache_buffer_.get(), 0, required_record_buffer_size_bytes_);
}
FineAudioBuffer::~FineAudioBuffer() {}
size_t FineAudioBuffer::RequiredPlayoutBufferSizeBytes() {
// It is possible that we store the desired frame size - 1 samples. Since new
// audio frames are pulled in chunks of 10ms we will need a buffer that can
// hold desired_frame_size - 1 + 10ms of data. We omit the - 1.
return desired_frame_size_bytes_ + bytes_per_10_ms_;
}
void FineAudioBuffer::ResetPlayout() {
playout_cached_buffer_start_ = 0;
playout_cached_bytes_ = 0;
memset(playout_cache_buffer_.get(), 0, bytes_per_10_ms_);
}
void FineAudioBuffer::ResetRecord() {
record_cached_bytes_ = 0;
record_read_pos_ = 0;
record_write_pos_ = 0;
memset(record_cache_buffer_.get(), 0, required_record_buffer_size_bytes_);
}
void FineAudioBuffer::GetPlayoutData(int8_t* buffer) {
if (desired_frame_size_bytes_ <= playout_cached_bytes_) {
memcpy(buffer, &playout_cache_buffer_.get()[playout_cached_buffer_start_],
desired_frame_size_bytes_);
playout_cached_buffer_start_ += desired_frame_size_bytes_;
playout_cached_bytes_ -= desired_frame_size_bytes_;
CHECK_LT(playout_cached_buffer_start_ + playout_cached_bytes_,
bytes_per_10_ms_);
return;
}
memcpy(buffer, &playout_cache_buffer_.get()[playout_cached_buffer_start_],
playout_cached_bytes_);
// Push another n*10ms of audio to |buffer|. n > 1 if
// |desired_frame_size_bytes_| is greater than 10ms of audio. Note that we
// write the audio after the cached bytes copied earlier.
int8_t* unwritten_buffer = &buffer[playout_cached_bytes_];
int bytes_left =
static_cast<int>(desired_frame_size_bytes_ - playout_cached_bytes_);
// Ceiling of integer division: 1 + ((x - 1) / y)
size_t number_of_requests = 1 + (bytes_left - 1) / (bytes_per_10_ms_);
for (size_t i = 0; i < number_of_requests; ++i) {
device_buffer_->RequestPlayoutData(samples_per_10_ms_);
int num_out = device_buffer_->GetPlayoutData(unwritten_buffer);
if (static_cast<size_t>(num_out) != samples_per_10_ms_) {
CHECK_EQ(num_out, 0);
playout_cached_bytes_ = 0;
return;
}
unwritten_buffer += bytes_per_10_ms_;
CHECK_GE(bytes_left, 0);
bytes_left -= static_cast<int>(bytes_per_10_ms_);
}
CHECK_LE(bytes_left, 0);
// Put the samples that were written to |buffer| but are not used in the
// cache.
size_t cache_location = desired_frame_size_bytes_;
int8_t* cache_ptr = &buffer[cache_location];
playout_cached_bytes_ = number_of_requests * bytes_per_10_ms_ -
(desired_frame_size_bytes_ - playout_cached_bytes_);
// If playout_cached_bytes_ is larger than the cache buffer, uninitialized
// memory will be read.
CHECK_LE(playout_cached_bytes_, bytes_per_10_ms_);
CHECK_EQ(static_cast<size_t>(-bytes_left), playout_cached_bytes_);
playout_cached_buffer_start_ = 0;
memcpy(playout_cache_buffer_.get(), cache_ptr, playout_cached_bytes_);
}
void FineAudioBuffer::DeliverRecordedData(const int8_t* buffer,
size_t size_in_bytes,
int playout_delay_ms,
int record_delay_ms) {
CHECK_EQ(size_in_bytes, desired_frame_size_bytes_);
// Check if the temporary buffer can store the incoming buffer. If not,
// move the remaining (old) bytes to the beginning of the temporary buffer
// and start adding new samples after the old samples.
if (record_write_pos_ + size_in_bytes > required_record_buffer_size_bytes_) {
if (record_cached_bytes_ > 0) {
memmove(record_cache_buffer_.get(),
record_cache_buffer_.get() + record_read_pos_,
record_cached_bytes_);
}
record_write_pos_ = record_cached_bytes_;
record_read_pos_ = 0;
}
// Add recorded samples to a temporary buffer.
memcpy(record_cache_buffer_.get() + record_write_pos_, buffer, size_in_bytes);
record_write_pos_ += size_in_bytes;
record_cached_bytes_ += size_in_bytes;
// Consume samples in temporary buffer in chunks of 10ms until there is not
// enough data left. The number of remaining bytes in the cache is given by
// |record_cached_bytes_| after this while loop is done.
while (record_cached_bytes_ >= bytes_per_10_ms_) {
device_buffer_->SetRecordedBuffer(
record_cache_buffer_.get() + record_read_pos_, samples_per_10_ms_);
device_buffer_->SetVQEData(playout_delay_ms, record_delay_ms, 0);
device_buffer_->DeliverRecordedData();
// Read next chunk of 10ms data.
record_read_pos_ += bytes_per_10_ms_;
// Reduce number of cached bytes with the consumed amount.
record_cached_bytes_ -= bytes_per_10_ms_;
}
}
} // namespace webrtc

107
webrtc/modules/audio_device/fine_audio_buffer.h Normal file
View File

@ -0,0 +1,107 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_AUDIO_DEVICE_FINE_AUDIO_BUFFER_H_
#define WEBRTC_MODULES_AUDIO_DEVICE_FINE_AUDIO_BUFFER_H_
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/typedefs.h"
namespace webrtc {
class AudioDeviceBuffer;
// FineAudioBuffer takes an AudioDeviceBuffer (ADB) which deals with audio data
// corresponding to 10ms of data. It then allows for this data to be pulled in
// a finer or coarser granularity. I.e. interacting with this class instead of
// directly with the AudioDeviceBuffer one can ask for any number of audio data
// samples. This class also ensures that audio data can be delivered to the ADB
// in 10ms chunks when the size of the provided audio buffers differs from 10ms.
// As an example: calling DeliverRecordedData() with 5ms buffers will deliver
// accumulated 10ms worth of data to the ADB every second call.
class FineAudioBuffer {
public:
// |device_buffer| is a buffer that provides 10ms of audio data.
// |desired_frame_size_bytes| is the number of bytes of audio data
// GetPlayoutData() should return on success. It is also the required size of
// each recorded buffer used in DeliverRecordedData() calls.
// |sample_rate| is the sample rate of the audio data. This is needed because
// |device_buffer| delivers 10ms of data. Given the sample rate the number
// of samples can be calculated.
FineAudioBuffer(AudioDeviceBuffer* device_buffer,
size_t desired_frame_size_bytes,
int sample_rate);
~FineAudioBuffer();
// Returns the required size of |buffer| when calling GetPlayoutData(). If
// the buffer is smaller memory trampling will happen.
size_t RequiredPlayoutBufferSizeBytes();
// Clears buffers and counters dealing with playour and/or recording.
void ResetPlayout();
void ResetRecord();
// |buffer| must be of equal or greater size than what is returned by
// RequiredBufferSize(). This is to avoid unnecessary memcpy.
void GetPlayoutData(int8_t* buffer);
// Consumes the audio data in |buffer| and sends it to the WebRTC layer in
// chunks of 10ms. The provided delay estimates in |playout_delay_ms| and
// |record_delay_ms| are given to the AEC in the audio processing module.
// They can be fixed values on most platforms and they are ignored if an
// external (hardware/built-in) AEC is used.
// The size of |buffer| is given by |size_in_bytes| and must be equal to
// |desired_frame_size_bytes_|. A CHECK will be hit if this is not the case.
// Example: buffer size is 5ms => call #1 stores 5ms of data, call #2 stores
// 5ms of data and sends a total of 10ms to WebRTC and clears the intenal
// cache. Call #3 restarts the scheme above.
void DeliverRecordedData(const int8_t* buffer,
size_t size_in_bytes,
int playout_delay_ms,
int record_delay_ms);
private:
// Device buffer that works with 10ms chunks of data both for playout and
// for recording. I.e., the WebRTC side will always be asked for audio to be
// played out in 10ms chunks and recorded audio will be sent to WebRTC in
// 10ms chunks as well. This pointer is owned by the constructor of this
// class and the owner must ensure that the pointer is valid during the life-
// time of this object.
AudioDeviceBuffer* const device_buffer_;
// Number of bytes delivered by GetPlayoutData() call and provided to
// DeliverRecordedData().
const size_t desired_frame_size_bytes_;
// Sample rate in Hertz.
const int sample_rate_;
// Number of audio samples per 10ms.
const size_t samples_per_10_ms_;
// Number of audio bytes per 10ms.
const size_t bytes_per_10_ms_;
// Storage for output samples that are not yet asked for.
rtc::scoped_ptr<int8_t[]> playout_cache_buffer_;
// Location of first unread output sample.
size_t playout_cached_buffer_start_;
// Number of bytes stored in output (contain samples to be played out) cache.
size_t playout_cached_bytes_;
// Storage for input samples that are about to be delivered to the WebRTC
// ADB or remains from the last successful delivery of a 10ms audio buffer.
rtc::scoped_ptr<int8_t[]> record_cache_buffer_;
// Required (max) size in bytes of the |record_cache_buffer_|.
const size_t required_record_buffer_size_bytes_;
// Number of bytes in input (contains recorded samples) cache.
size_t record_cached_bytes_;
// Read and write pointers used in the buffering scheme on the recording side.
size_t record_read_pos_;
size_t record_write_pos_;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_AUDIO_DEVICE_FINE_AUDIO_BUFFER_H_

61
webrtc/modules/audio_device/android/fine_audio_buffer_unittest.cc → webrtc/modules/audio_device/fine_audio_buffer_unittest.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/audio_device/android/fine_audio_buffer.h"
#include "webrtc/modules/audio_device/fine_audio_buffer.h"
#include <limits.h>
#include <memory>
@ -19,6 +19,7 @@
#include "webrtc/modules/audio_device/mock_audio_device_buffer.h"
using ::testing::_;
using ::testing::AtLeast;
using ::testing::InSequence;
using ::testing::Return;
@ -40,10 +41,10 @@ bool VerifyBuffer(const int8_t* buffer, int buffer_number, int size) {
return true;
}
// This function replaces GetPlayoutData when it's called (which is done
// implicitly when calling GetBufferData). It writes the sequence
// 0,1,..SCHAR_MAX-1,0,1,... to the buffer. Note that this is likely a buffer of
// different size than the one VerifyBuffer verifies.
// This function replaces the real AudioDeviceBuffer::GetPlayoutData when it's
// called (which is done implicitly when calling GetBufferData). It writes the
// sequence 0,1,..SCHAR_MAX-1,0,1,... to the buffer. Note that this is likely a
// buffer of different size than the one VerifyBuffer verifies.
// |iteration| is the number of calls made to UpdateBuffer prior to this call.
// |samples_per_10_ms| is the number of samples that should be written to the
// buffer (|arg0|).
@ -57,10 +58,33 @@ ACTION_P2(UpdateBuffer, iteration, samples_per_10_ms) {
return samples_per_10_ms;
}
// Writes a periodic ramp pattern to the supplied |buffer|. See UpdateBuffer()
// for details.
void UpdateInputBuffer(int8_t* buffer, int iteration, int size) {
int start_value = (iteration * size) % SCHAR_MAX;
for (int i = 0; i < size; ++i) {
buffer[i] = (i + start_value) % SCHAR_MAX;
}
}
// Action macro which verifies that the recorded 10ms chunk of audio data
// (in |arg0|) contains the correct reference values even if they have been
// supplied using a buffer size that is smaller or larger than 10ms.
// See VerifyBuffer() for details.
ACTION_P2(VerifyInputBuffer, iteration, samples_per_10_ms) {
const int8_t* buffer = static_cast<const int8_t*>(arg0);
int bytes_per_10_ms = samples_per_10_ms * static_cast<int>(sizeof(int16_t));
int start_value = (iteration * bytes_per_10_ms) % SCHAR_MAX;
for (int i = 0; i < bytes_per_10_ms; ++i) {
EXPECT_EQ(buffer[i], (i + start_value) % SCHAR_MAX);
}
return 0;
}
void RunFineBufferTest(int sample_rate, int frame_size_in_samples) {
const int kSamplesPer10Ms = sample_rate * 10 / 1000;
const int kFrameSizeBytes = frame_size_in_samples *
static_cast<int>(sizeof(int16_t));
const int kFrameSizeBytes =
frame_size_in_samples * static_cast<int>(sizeof(int16_t));
const int kNumberOfFrames = 5;
// Ceiling of integer division: 1 + ((x - 1) / y)
const int kNumberOfUpdateBufferCalls =
@ -77,15 +101,32 @@ void RunFineBufferTest(int sample_rate, int frame_size_in_samples) {
.RetiresOnSaturation();
}
}
{
InSequence s;
for (int j = 0; j < kNumberOfUpdateBufferCalls - 1; ++j) {
EXPECT_CALL(audio_device_buffer, SetRecordedBuffer(_, kSamplesPer10Ms))
.WillOnce(VerifyInputBuffer(j, kSamplesPer10Ms))
.RetiresOnSaturation();
}
}
EXPECT_CALL(audio_device_buffer, SetVQEData(_, _, _))
.Times(kNumberOfUpdateBufferCalls - 1);
EXPECT_CALL(audio_device_buffer, DeliverRecordedData())
.Times(kNumberOfUpdateBufferCalls - 1)
.WillRepeatedly(Return(kSamplesPer10Ms));
FineAudioBuffer fine_buffer(&audio_device_buffer, kFrameSizeBytes,
sample_rate);
rtc::scoped_ptr<int8_t[]> out_buffer;
out_buffer.reset(
new int8_t[fine_buffer.RequiredBufferSizeBytes()]);
out_buffer.reset(new int8_t[fine_buffer.RequiredPlayoutBufferSizeBytes()]);
rtc::scoped_ptr<int8_t[]> in_buffer;
in_buffer.reset(new int8_t[kFrameSizeBytes]);
for (int i = 0; i < kNumberOfFrames; ++i) {
fine_buffer.GetBufferData(out_buffer.get());
fine_buffer.GetPlayoutData(out_buffer.get());
EXPECT_TRUE(VerifyBuffer(out_buffer.get(), i, kFrameSizeBytes));
UpdateInputBuffer(in_buffer.get(), i, kFrameSizeBytes);
fine_buffer.DeliverRecordedData(in_buffer.get(), kFrameSizeBytes, 0, 0);
}
}

35
webrtc/modules/audio_device/include/audio_device_defines.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_DEFINES_H
#define WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_DEFINES_H
#ifndef WEBRTC_MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DEFINES_H_
#define WEBRTC_MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DEFINES_H_
#include <stddef.h>
@ -161,24 +161,41 @@ class AudioParameters {
frames_per_10ms_buffer_ = static_cast<size_t>(sample_rate / 100);
}
size_t bits_per_sample() const { return kBitsPerSample; }
void reset(int sample_rate, int channels, double ms_per_buffer) {
reset(sample_rate, channels,
static_cast<size_t>(sample_rate * ms_per_buffer + 0.5));
}
void reset(int sample_rate, int channels) {
reset(sample_rate, channels, static_cast<size_t>(0));
}
int sample_rate() const { return sample_rate_; }
int channels() const { return channels_; }
size_t frames_per_buffer() const { return frames_per_buffer_; }
size_t frames_per_10ms_buffer() const { return frames_per_10ms_buffer_; }
bool is_valid() const {
return ((sample_rate_ > 0) && (channels_ > 0) && (frames_per_buffer_ > 0));
}
size_t GetBytesPerFrame() const { return channels_ * kBitsPerSample / 8; }
size_t GetBytesPerBuffer() const {
return frames_per_buffer_ * GetBytesPerFrame();
}
// The WebRTC audio device buffer (ADB) only requires that the sample rate
// and number of channels are configured. Hence, to be "valid", only these
// two attributes must be set.
bool is_valid() const { return ((sample_rate_ > 0) && (channels_ > 0)); }
// Most platforms also require that a native buffer size is defined.
// An audio parameter instance is considered to be "complete" if it is both
// "valid" (can be used by the ADB) and also has a native frame size.
bool is_complete() const { return (is_valid() && (frames_per_buffer_ > 0)); }
size_t GetBytesPer10msBuffer() const {
return frames_per_10ms_buffer_ * GetBytesPerFrame();
}
float GetBufferSizeInMilliseconds() const {
double GetBufferSizeInMilliseconds() const {
if (sample_rate_ == 0)
return 0.0f;
return frames_per_buffer_ / (sample_rate_ / 1000.0f);
return 0.0;
return frames_per_buffer_ / (sample_rate_ / 1000.0);
}
double GetBufferSizeInSeconds() const {
if (sample_rate_ == 0)
return 0.0;
return static_cast<double>(frames_per_buffer_) / (sample_rate_);
}
private:
@ -190,4 +207,4 @@ class AudioParameters {
} // namespace webrtc
#endif // WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_DEFINES_H
#endif // WEBRTC_MODULES_AUDIO_DEVICE_INCLUDE_AUDIO_DEVICE_DEFINES_H_

220
webrtc/modules/audio_device/ios/audio_device_ios.h
View File

@ -8,26 +8,32 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_IOS_H
#define WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_IOS_H
#ifndef WEBRTC_MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_
#define WEBRTC_MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_
#include <AudioUnit/AudioUnit.h>
#include "webrtc/base/scoped_ptr.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/modules/audio_device/audio_device_generic.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/thread_wrapper.h"
namespace webrtc {
const uint32_t N_REC_SAMPLES_PER_SEC = 44100;
const uint32_t N_PLAY_SAMPLES_PER_SEC = 44100;
const uint32_t ENGINE_REC_BUF_SIZE_IN_SAMPLES = (N_REC_SAMPLES_PER_SEC / 100);
const uint32_t ENGINE_PLAY_BUF_SIZE_IN_SAMPLES = (N_PLAY_SAMPLES_PER_SEC / 100);
// Number of 10 ms recording blocks in recording buffer
const uint16_t N_REC_BUFFERS = 20;
class FineAudioBuffer;
// Implements full duplex 16-bit mono PCM audio support for iOS using a
// Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit
// supports audio echo cancellation. It also adds automatic gain control,
// adjustment of voice-processing quality and muting.
//
// An instance must be created and destroyed on one and the same thread.
// All supported public methods must also be called on the same thread.
// A thread checker will DCHECK if any supported method is called on an invalid
// thread.
//
// Recorded audio will be delivered on a real-time internal I/O thread in the
// audio unit. The audio unit will also ask for audio data to play out on this
// same thread.
class AudioDeviceIOS : public AudioDeviceGeneric {
public:
AudioDeviceIOS();
@ -56,23 +62,28 @@ class AudioDeviceIOS : public AudioDeviceGeneric {
int32_t SetLoudspeakerStatus(bool enable) override;
int32_t GetLoudspeakerStatus(bool& enabled) const override;
// TODO(henrika): investigate if we can reduce the complexity here.
// Do we even need delay estimates?
// These methods returns hard-coded delay values and not dynamic delay
// estimates. The reason is that iOS supports a built-in AEC and the WebRTC
// AEC will always be disabled in the Libjingle layer to avoid running two
// AEC implementations at the same time. And, it saves resources to avoid
// updating these delay values continuously.
// TODO(henrika): it would be possible to mark these two methods as not
// implemented since they are only called for A/V-sync purposes today and
// A/V-sync is not supported on iOS. However, we avoid adding error messages
// the log by using these dummy implementations instead.
int32_t PlayoutDelay(uint16_t& delayMS) const override;
int32_t RecordingDelay(uint16_t& delayMS) const override;
int32_t PlayoutBuffer(AudioDeviceModule::BufferType& type,
uint16_t& sizeMS) const override;
// These methods are unique for the iOS implementation.
// Native audio parameters stored during construction.
// These methods are unique for the iOS implementation.
int GetPlayoutAudioParameters(AudioParameters* params) const override;
int GetRecordAudioParameters(AudioParameters* params) const override;
// These methods are currently not implemented on iOS.
// See audio_device_not_implemented_ios.mm for dummy implementations.
// These methods are currently not fully implemented on iOS:
// See audio_device_not_implemented.cc for trivial implementations.
int32_t PlayoutBuffer(AudioDeviceModule::BufferType& type,
uint16_t& sizeMS) const override;
int32_t ActiveAudioLayer(AudioDeviceModule::AudioLayer& audioLayer) const;
int32_t ResetAudioDevice() override;
int32_t PlayoutIsAvailable(bool& available) override;
@ -140,97 +151,132 @@ class AudioDeviceIOS : public AudioDeviceGeneric {
void ClearRecordingError() override{};
private:
// TODO(henrika): try to remove these.
void Lock() {
_critSect.Enter();
}
// Uses current |_playoutParameters| and |_recordParameters| to inform the
// audio device buffer (ADB) about our internal audio parameters.
void UpdateAudioDeviceBuffer();
void UnLock() {
_critSect.Leave();
}
// Since the preferred audio parameters are only hints to the OS, the actual
// values may be different once the AVAudioSession has been activated.
// This method asks for the current hardware parameters and takes actions
// if they should differ from what we have asked for initially. It also
// defines |_playoutParameters| and |_recordParameters|.
void SetupAudioBuffersForActiveAudioSession();
// Init and shutdown
int32_t InitPlayOrRecord();
int32_t ShutdownPlayOrRecord();
// Creates a Voice-Processing I/O unit and configures it for full-duplex
// audio. The selected stream format is selected to avoid internal resampling
// and to match the 10ms callback rate for WebRTC as well as possible.
// This method also initializes the created audio unit.
bool SetupAndInitializeVoiceProcessingAudioUnit();
void UpdateRecordingDelay();
void UpdatePlayoutDelay();
// Activates our audio session, creates and initilizes the voice-processing
// audio unit and verifies that we got the preferred native audio parameters.
bool InitPlayOrRecord();
static OSStatus RecordProcess(void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *timeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData);
// Closes and deletes the voice-processing I/O unit.
bool ShutdownPlayOrRecord();
static OSStatus PlayoutProcess(void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *timeStamp,
// Callback function called on a real-time priority I/O thread from the audio
// unit. This method is used to signal that recorded audio is available.
static OSStatus RecordedDataIsAvailable(
void* inRefCon,
AudioUnitRenderActionFlags* ioActionFlags,
const AudioTimeStamp* timeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList* ioData);
OSStatus OnRecordedDataIsAvailable(AudioUnitRenderActionFlags* ioActionFlags,
const AudioTimeStamp* timeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames);
// Callback function called on a real-time priority I/O thread from the audio
// unit. This method is used to provide audio samples to the audio unit.
static OSStatus GetPlayoutData(void* inRefCon,
AudioUnitRenderActionFlags* ioActionFlags,
const AudioTimeStamp* timeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData);
OSStatus RecordProcessImpl(AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *timeStamp,
uint32_t inBusNumber,
uint32_t inNumberFrames);
OSStatus PlayoutProcessImpl(uint32_t inNumberFrames,
AudioBufferList *ioData);
static bool RunCapture(void* ptrThis);
bool CaptureWorkerThread();
AudioBufferList* ioData);
OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* ioActionFlags,
UInt32 inNumberFrames,
AudioBufferList* ioData);
private:
rtc::ThreadChecker thread_checker_;
// Ensures that methods are called from the same thread as this object is
// created on.
rtc::ThreadChecker _threadChecker;
// Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the
// AudioDeviceModuleImpl class and called by AudioDeviceModuleImpl::Create().
// The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance
// and therefore outlives this object.
AudioDeviceBuffer* audio_device_buffer_;
AudioDeviceBuffer* _audioDeviceBuffer;
CriticalSectionWrapper& _critSect;
// Contains audio parameters (sample rate, #channels, buffer size etc.) for
// the playout and recording sides. These structure is set in two steps:
// first, native sample rate and #channels are defined in Init(). Next, the
// audio session is activated and we verify that the preferred parameters
// were granted by the OS. At this stage it is also possible to add a third
// component to the parameters; the native I/O buffer duration.
// A CHECK will be hit if we for some reason fail to open an audio session
// using the specified parameters.
AudioParameters _playoutParameters;
AudioParameters _recordParameters;
AudioParameters playout_parameters_;
AudioParameters record_parameters_;
// The Voice-Processing I/O unit has the same characteristics as the
// Remote I/O unit (supports full duplex low-latency audio input and output)
// and adds AEC for for two-way duplex communication. It also adds AGC,
// adjustment of voice-processing quality, and muting. Hence, ideal for
// VoIP applications.
AudioUnit _vpioUnit;
rtc::scoped_ptr<ThreadWrapper> _captureWorkerThread;
// FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data
// in chunks of 10ms. It then allows for this data to be pulled in
// a finer or coarser granularity. I.e. interacting with this class instead
// of directly with the AudioDeviceBuffer one can ask for any number of
// audio data samples. Is also supports a similar scheme for the recording
// side.
// Example: native buffer size can be 128 audio frames at 16kHz sample rate.
// WebRTC will provide 480 audio frames per 10ms but iOS asks for 128
// in each callback (one every 8ms). This class can then ask for 128 and the
// FineAudioBuffer will ask WebRTC for new data only when needed and also
// cache non-utilized audio between callbacks. On the recording side, iOS
// can provide audio data frames of size 128 and these are accumulated until
// enough data to supply one 10ms call exists. This 10ms chunk is then sent
// to WebRTC and the remaining part is stored.
rtc::scoped_ptr<FineAudioBuffer> _fineAudioBuffer;
AudioUnit _auVoiceProcessing;
void* _audioInterruptionObserver;
// Extra audio buffer to be used by the playout side for rendering audio.
// The buffer size is given by FineAudioBuffer::RequiredBufferSizeBytes().
rtc::scoped_ptr<SInt8[]> _playoutAudioBuffer;
// Provides a mechanism for encapsulating one or more buffers of audio data.
// Only used on the recording side.
AudioBufferList _audioRecordBufferList;
// Temporary storage for recorded data. AudioUnitRender() renders into this
// array as soon as a frame of the desired buffer size has been recorded.
rtc::scoped_ptr<SInt8[]> _recordAudioBuffer;
// Set to 1 when recording is active and 0 otherwise.
volatile int _recording;
// Set to 1 when playout is active and 0 otherwise.
volatile int _playing;
// Set to true after successful call to Init(), false otherwise.
bool _initialized;
bool _isShutDown;
bool _recording;
bool _playing;
// Set to true after successful call to InitRecording(), false otherwise.
bool _recIsInitialized;
// Set to true after successful call to InitPlayout(), false otherwise.
bool _playIsInitialized;
// The sampling rate to use with Audio Device Buffer
int _adbSampFreq;
// Delay calculation
uint32_t _recordingDelay;
uint32_t _playoutDelay;
uint32_t _playoutDelayMeasurementCounter;
uint32_t _recordingDelayHWAndOS;
uint32_t _recordingDelayMeasurementCounter;
// Playout buffer, needed for 44.0 / 44.1 kHz mismatch
int16_t _playoutBuffer[ENGINE_PLAY_BUF_SIZE_IN_SAMPLES];
uint32_t _playoutBufferUsed; // How much is filled
// Recording buffers
int16_t _recordingBuffer[N_REC_BUFFERS][ENGINE_REC_BUF_SIZE_IN_SAMPLES];
uint32_t _recordingLength[N_REC_BUFFERS];
uint32_t _recordingSeqNumber[N_REC_BUFFERS];
uint32_t _recordingCurrentSeq;
// Current total size all data in buffers, used for delay estimate
uint32_t _recordingBufferTotalSize;
// Audio interruption observer instance.
void* _audioInterruptionObserver;
};
} // namespace webrtc
#endif // WEBRTC_AUDIO_DEVICE_AUDIO_DEVICE_IOS_H
#endif // WEBRTC_MODULES_AUDIO_DEVICE_IOS_AUDIO_DEVICE_IOS_H_

1193
webrtc/modules/audio_device/ios/audio_device_ios.mm
View File

File diff suppressed because it is too large Load Diff

6
webrtc/modules/audio_device/ios/audio_device_not_implemented_ios.mm
View File

@ -15,6 +15,12 @@
namespace webrtc {
int32_t AudioDeviceIOS::PlayoutBuffer(AudioDeviceModule::BufferType& type,
uint16_t& sizeMS) const {
RTC_NOTREACHED() << "Not implemented";
return -1;
}
int32_t AudioDeviceIOS::ActiveAudioLayer(
AudioDeviceModule::AudioLayer& audioLayer) const {
audioLayer = AudioDeviceModule::kPlatformDefaultAudio;

7
webrtc/modules/audio_device/ios/audio_device_unittest_ios.cc
View File

@ -507,7 +507,6 @@ class AudioDeviceTest : public ::testing::Test {
rtc::LogMessage::LogToDebug(old_sev_);
}
// TODO(henrika): don't use hardcoded values below.
int playout_sample_rate() const { return playout_parameters_.sample_rate(); }
int record_sample_rate() const { return record_parameters_.sample_rate(); }
int playout_channels() const { return playout_parameters_.channels(); }
@ -519,11 +518,6 @@ class AudioDeviceTest : public ::testing::Test {
return record_parameters_.frames_per_10ms_buffer();
}
int total_delay_ms() const {
// TODO(henrika): improve this part.
return 100;
}
rtc::scoped_refptr<AudioDeviceModule> audio_device() const {
return audio_device_;
}
@ -609,7 +603,6 @@ TEST_F(AudioDeviceTest, ConstructDestruct) {
TEST_F(AudioDeviceTest, InitTerminate) {
// Initialization is part of the test fixture.
EXPECT_TRUE(audio_device()->Initialized());
// webrtc::SleepMs(5 * 1000);
EXPECT_EQ(0, audio_device()->Terminate());
EXPECT_FALSE(audio_device()->Initialized());
}

6
webrtc/modules/audio_device/mock_audio_device_buffer.h
View File

@ -20,9 +20,13 @@ class MockAudioDeviceBuffer : public AudioDeviceBuffer {
public:
MockAudioDeviceBuffer() {}
virtual ~MockAudioDeviceBuffer() {}
MOCK_METHOD1(RequestPlayoutData, int32_t(size_t nSamples));
MOCK_METHOD1(GetPlayoutData, int32_t(void* audioBuffer));
MOCK_METHOD2(SetRecordedBuffer,
int32_t(const void* audioBuffer, size_t nSamples));
MOCK_METHOD3(SetVQEData,
void(int playDelayMS, int recDelayMS, int clockDrift));
MOCK_METHOD0(DeliverRecordedData, int32_t());
};
} // namespace webrtc

2
webrtc/modules/modules.gyp
View File

@ -160,6 +160,7 @@
'audio_coding/neteq/tools/input_audio_file_unittest.cc',
'audio_coding/neteq/tools/packet_unittest.cc',
'audio_conference_mixer/test/audio_conference_mixer_unittest.cc',
'audio_device/fine_audio_buffer_unittest.cc',
'audio_processing/aec/echo_cancellation_unittest.cc',
'audio_processing/aec/system_delay_unittest.cc',
# TODO(ajm): Fix to match new interface.
@ -356,7 +357,6 @@
'audio_device/android/audio_manager_unittest.cc',
'audio_device/android/ensure_initialized.cc',
'audio_device/android/ensure_initialized.h',
'audio_device/android/fine_audio_buffer_unittest.cc',
],
}],
['OS=="ios"', {