From e71b24e42163afb48f24832ed079a92ee83f01b8 Mon Sep 17 00:00:00 2001 From: henrika Date: Thu, 12 Nov 2015 01:48:32 -0800 Subject: [PATCH] OpenSL ES stability improvements. This CL does two things: 1) Improves stability in the existing OpenSL ES implementation for devices that supports OpenSL ES. The cost is a slight increase in latency since the focus here has been on avoiding audio glitches. 2) Adds a new Java API to exclude usage of OpenSL ES to enable comparisons between OpenSL ES and Java based audio backends. BUG=b/22452539 Review URL: https://codereview.webrtc.org/1440623002 Cr-Commit-Position: refs/heads/master@{#10618} --- .../voiceengine/WebRtcAudioManager.java | 18 +++++++++++-- .../audio_device/android/opensles_player.cc | 25 +++++++++++++++++-- .../audio_device/android/opensles_player.h | 5 +++- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java b/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java index 10fe8ca33f..7359486a3f 100644 --- a/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java +++ b/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java @@ -33,11 +33,24 @@ import java.lang.Math; // recommended to always use AudioManager.MODE_IN_COMMUNICATION. // This class also adds support for output volume control of the // STREAM_VOICE_CALL-type stream. -class WebRtcAudioManager { +public class WebRtcAudioManager { private static final boolean DEBUG = false; private static final String TAG = "WebRtcAudioManager"; + private static boolean blacklistDeviceForOpenSLESUsage = false; + private static boolean blacklistDeviceForOpenSLESUsageIsOverridden = false; + + // Call this method to override the deault list of blacklisted devices + // specified in WebRtcAudioUtils.BLACKLISTED_OPEN_SL_ES_MODELS. + // Allows an app to take control over which devices to exlude from using + // the OpenSL ES audio output path + public static synchronized void setBlacklistDeviceForOpenSLESUsage( + boolean enable) { + blacklistDeviceForOpenSLESUsageIsOverridden = true; + blacklistDeviceForOpenSLESUsage = enable; + } + // Default audio data format is PCM 16 bit per sample. // Guaranteed to be supported by all devices. private static final int BITS_PER_SAMPLE = 16; @@ -110,7 +123,8 @@ class WebRtcAudioManager { } private boolean isDeviceBlacklistedForOpenSLESUsage() { - boolean blacklisted = + boolean blacklisted = blacklistDeviceForOpenSLESUsageIsOverridden ? + blacklistDeviceForOpenSLESUsage : WebRtcAudioUtils.deviceIsBlacklistedForOpenSLESUsage(); if (blacklisted) { Logging.e(TAG, Build.MODEL + " is blacklisted for OpenSL ES usage!"); diff --git a/webrtc/modules/audio_device/android/opensles_player.cc b/webrtc/modules/audio_device/android/opensles_player.cc index b9ccfd594d..d1edef227e 100644 --- a/webrtc/modules/audio_device/android/opensles_player.cc +++ b/webrtc/modules/audio_device/android/opensles_player.cc @@ -15,6 +15,7 @@ #include "webrtc/base/arraysize.h" #include "webrtc/base/checks.h" #include "webrtc/base/format_macros.h" +#include "webrtc/base/timeutils.h" #include "webrtc/modules/audio_device/android/audio_manager.h" #include "webrtc/modules/audio_device/fine_audio_buffer.h" @@ -46,7 +47,8 @@ OpenSLESPlayer::OpenSLESPlayer(AudioManager* audio_manager) engine_(nullptr), player_(nullptr), simple_buffer_queue_(nullptr), - volume_(nullptr) { + volume_(nullptr), + last_play_time_(0) { ALOGD("ctor%s", GetThreadInfo().c_str()); // Use native audio output parameters provided by the audio manager and // define the PCM format structure. @@ -95,6 +97,7 @@ int OpenSLESPlayer::InitPlayout() { CreateMix(); initialized_ = true; buffer_index_ = 0; + last_play_time_ = rtc::Time(); return 0; } @@ -233,7 +236,16 @@ void OpenSLESPlayer::AllocateDataBuffers() { RTC_DCHECK(thread_checker_.CalledOnValidThread()); RTC_DCHECK(!simple_buffer_queue_); RTC_CHECK(audio_device_buffer_); - bytes_per_buffer_ = audio_parameters_.GetBytesPerBuffer(); + // Don't use the lowest possible size as native buffer size. Instead, + // use 10ms to better match the frame size that WebRTC uses. It will result + // in a reduced risk for audio glitches and also in a more "clean" sequence + // of callbacks from the OpenSL ES thread in to WebRTC when asking for audio + // to render. + ALOGD("lowest possible buffer size: %" PRIuS, + audio_parameters_.GetBytesPerBuffer()); + bytes_per_buffer_ = audio_parameters_.GetBytesPerFrame() * + audio_parameters_.frames_per_10ms_buffer(); + RTC_DCHECK_GT(bytes_per_buffer_, audio_parameters_.GetBytesPerBuffer()); ALOGD("native buffer size: %" PRIuS, bytes_per_buffer_); // Create a modified audio buffer class which allows us to ask for any number // of samples (and not only multiple of 10ms) to match the native OpenSL ES @@ -418,6 +430,15 @@ void OpenSLESPlayer::FillBufferQueue() { } void OpenSLESPlayer::EnqueuePlayoutData() { + // Check delta time between two successive callbacks and provide a warning + // if it becomes very large. + // TODO(henrika): using 100ms as upper limit but this value is rather random. + const uint32_t current_time = rtc::Time(); + const uint32_t diff = current_time - last_play_time_; + if (diff > 100) { + ALOGW("Bad OpenSL ES playout timing, dT=%u [ms]", diff); + } + last_play_time_ = current_time; // Read audio data from the WebRTC source using the FineAudioBuffer object // to adjust for differences in buffer size between WebRTC (10ms) and native // OpenSL ES. diff --git a/webrtc/modules/audio_device/android/opensles_player.h b/webrtc/modules/audio_device/android/opensles_player.h index 4c4d72434d..c9aa08693f 100644 --- a/webrtc/modules/audio_device/android/opensles_player.h +++ b/webrtc/modules/audio_device/android/opensles_player.h @@ -52,7 +52,7 @@ class OpenSLESPlayer { // buffer count of 2 or more, and a buffer size and sample rate that are // compatible with the device's native output configuration provided via the // audio manager at construction. - static const int kNumOfOpenSLESBuffers = 2; + static const int kNumOfOpenSLESBuffers = 4; // There is no need for this class to use JNI. static int32_t SetAndroidAudioDeviceObjects(void* javaVM, void* context) { @@ -195,6 +195,9 @@ class OpenSLESPlayer { // This interface exposes controls for manipulating the object’s audio volume // properties. This interface is supported on the Audio Player object. SLVolumeItf volume_; + + // Last time the OpenSL ES layer asked for audio data to play out. + uint32_t last_play_time_; }; } // namespace webrtc