From 521f7a8db7dc4125e2eda72801188a1be2dce436 Mon Sep 17 00:00:00 2001 From: henrika Date: Tue, 31 May 2016 07:03:17 -0700 Subject: [PATCH] Moves ownership of OpenSL engine object to Android audio manager with the goal of adding support for OpenSL ES based audio capture. BUG=webrtc:5925 Review-Url: https://codereview.webrtc.org/2019223004 Cr-Commit-Position: refs/heads/master@{#12975} --- .../audio_device/android/audio_manager.cc | 41 +++++++++++- .../audio_device/android/audio_manager.h | 23 +++++++ .../android/audio_manager_unittest.cc | 51 +++++++++++++++ .../audio_device/android/opensles_common.cc | 31 ++++++++++ .../audio_device/android/opensles_common.h | 4 ++ .../audio_device/android/opensles_player.cc | 62 ++++++++----------- .../audio_device/android/opensles_player.h | 23 +++---- .../audio_device/include/audio_device.h | 3 +- 8 files changed, 187 insertions(+), 51 deletions(-) diff --git a/webrtc/modules/audio_device/android/audio_manager.cc b/webrtc/modules/audio_device/android/audio_manager.cc index d7108dca48..9b1ee0a67b 100644 --- a/webrtc/modules/audio_device/android/audio_manager.cc +++ b/webrtc/modules/audio_device/android/audio_manager.cc @@ -101,7 +101,7 @@ void AudioManager::SetActiveAudioLayer( ALOGD("SetActiveAudioLayer(%d)%s", audio_layer, GetThreadInfo().c_str()); RTC_DCHECK(thread_checker_.CalledOnValidThread()); RTC_DCHECK(!initialized_); - // Store the currenttly utilized audio layer. + // Store the currently utilized audio layer. audio_layer_ = audio_layer; // The delay estimate can take one of two fixed values depending on if the // device supports low-latency output or not. However, it is also possible @@ -114,6 +114,45 @@ void AudioManager::SetActiveAudioLayer( ALOGD("delay_estimate_in_milliseconds: %d", delay_estimate_in_milliseconds_); } +SLObjectItf AudioManager::GetOpenSLEngine() { + ALOGD("GetOpenSLEngine%s", GetThreadInfo().c_str()); + RTC_DCHECK(thread_checker_.CalledOnValidThread()); + // Only allow usage of OpenSL ES if such an audio layer has been specified. + if (audio_layer_ != AudioDeviceModule::kAndroidOpenSLESAudio && + audio_layer_ != + AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio) { + ALOGI("Unable to create OpenSL engine for the current audio layer: %d", + audio_layer_); + return nullptr; + } + // OpenSL ES for Android only supports a single engine per application. + // If one already has been created, return existing object instead of + // creating a new. + if (engine_object_.Get() != nullptr) { + ALOGI("The OpenSL ES engine object has already been created"); + return engine_object_.Get(); + } + // Create the engine object in thread safe mode. + const SLEngineOption option[] = { + {SL_ENGINEOPTION_THREADSAFE, static_cast(SL_BOOLEAN_TRUE)}}; + SLresult result = + slCreateEngine(engine_object_.Receive(), 1, option, 0, NULL, NULL); + if (result != SL_RESULT_SUCCESS) { + ALOGE("slCreateEngine() failed: %s", GetSLErrorString(result)); + engine_object_.Reset(); + return nullptr; + } + // Realize the SL Engine in synchronous mode. + result = engine_object_->Realize(engine_object_.Get(), SL_BOOLEAN_FALSE); + if (result != SL_RESULT_SUCCESS) { + ALOGE("Realize() failed: %s", GetSLErrorString(result)); + engine_object_.Reset(); + return nullptr; + } + // Finally return the SLObjectItf interface of the engine object. + return engine_object_.Get(); +} + bool AudioManager::Init() { ALOGD("Init%s", GetThreadInfo().c_str()); RTC_DCHECK(thread_checker_.CalledOnValidThread()); diff --git a/webrtc/modules/audio_device/android/audio_manager.h b/webrtc/modules/audio_device/android/audio_manager.h index aa51d092d0..808417ca9d 100644 --- a/webrtc/modules/audio_device/android/audio_manager.h +++ b/webrtc/modules/audio_device/android/audio_manager.h @@ -14,11 +14,13 @@ #include #include +#include #include "webrtc/base/thread_checker.h" #include "webrtc/modules/audio_device/android/audio_common.h" #include "webrtc/modules/audio_device/audio_device_config.h" #include "webrtc/modules/audio_device/include/audio_device_defines.h" +#include "webrtc/modules/audio_device/android/opensles_common.h" #include "webrtc/modules/audio_device/audio_device_generic.h" #include "webrtc/modules/utility/include/helpers_android.h" #include "webrtc/modules/utility/include/jvm_android.h" @@ -63,6 +65,18 @@ class AudioManager { // Init(). void SetActiveAudioLayer(AudioDeviceModule::AudioLayer audio_layer); + // Creates and realizes the main (global) Open SL engine object and returns + // a reference to it. The engine object is only created at the first call + // since OpenSL ES for Android only supports a single engine per application. + // Subsequent calls returns the already created engine. The SL engine object + // is destroyed when the AudioManager object is deleted. It means that the + // engine object will be the first OpenSL ES object to be created and last + // object to be destroyed. + // Note that NULL will be returned unless the audio layer is specified as + // AudioDeviceModule::kAndroidOpenSLESAudio or + // AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio. + SLObjectItf GetOpenSLEngine(); + // Initializes the audio manager and stores the current audio mode. bool Init(); // Revert any setting done by Init(). @@ -143,8 +157,17 @@ class AudioManager { // Wraps the Java specific parts of the AudioManager. std::unique_ptr j_audio_manager_; + // Contains the selected audio layer specified by the AudioLayer enumerator + // in the AudioDeviceModule class. AudioDeviceModule::AudioLayer audio_layer_; + // This object is the global entry point of the OpenSL ES API. + // After creating the engine object, the application can obtain this object‘s + // SLEngineItf interface. This interface contains creation methods for all + // the other object types in the API. None of these interface are realized + // by this class. It only provides access to the global engine object. + webrtc::ScopedSLObjectItf engine_object_; + // Set to true by Init() and false by Close(). bool initialized_; diff --git a/webrtc/modules/audio_device/android/audio_manager_unittest.cc b/webrtc/modules/audio_device/android/audio_manager_unittest.cc index 3abfc5a8ce..0249ab9473 100644 --- a/webrtc/modules/audio_device/android/audio_manager_unittest.cc +++ b/webrtc/modules/audio_device/android/audio_manager_unittest.cc @@ -9,8 +9,10 @@ */ #include +#include #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/arraysize.h" #include "webrtc/base/format_macros.h" #include "webrtc/modules/audio_device/android/build_info.h" #include "webrtc/modules/audio_device/android/audio_manager.h" @@ -44,6 +46,29 @@ class AudioManagerTest : public ::testing::Test { EXPECT_NE(0, audio_manager()->GetDelayEstimateInMilliseconds()); } + // One way to ensure that the engine object is valid is to create an + // SL Engine interface since it exposes creation methods of all the OpenSL ES + // object types and it is only supported on the engine object. This method + // also verifies that the engine interface supports at least one interface. + // Note that, the test below is not a full test of the SLEngineItf object + // but only a simple sanity test to check that the global engine object is OK. + void ValidateSLEngine(SLObjectItf engine_object) { + EXPECT_NE(nullptr, engine_object); + // Get the SL Engine interface which is exposed by the engine object. + SLEngineItf engine; + SLresult result = + (*engine_object)->GetInterface(engine_object, SL_IID_ENGINE, &engine); + EXPECT_EQ(result, SL_RESULT_SUCCESS) << "GetInterface() on engine failed"; + // Ensure that the SL Engine interface exposes at least one interface. + SLuint32 object_id = SL_OBJECTID_ENGINE; + SLuint32 num_supported_interfaces = 0; + result = (*engine)->QueryNumSupportedInterfaces(engine, object_id, + &num_supported_interfaces); + EXPECT_EQ(result, SL_RESULT_SUCCESS) + << "QueryNumSupportedInterfaces() failed"; + EXPECT_GE(num_supported_interfaces, 1u); + } + std::unique_ptr audio_manager_; AudioParameters playout_parameters_; AudioParameters record_parameters_; @@ -52,6 +77,32 @@ class AudioManagerTest : public ::testing::Test { TEST_F(AudioManagerTest, ConstructDestruct) { } +// It should not be possible to create an OpenSL engine object if Java based +// audio is requested in both directions. +TEST_F(AudioManagerTest, GetOpenSLEngineShouldFailForJavaAudioLayer) { + audio_manager()->SetActiveAudioLayer(AudioDeviceModule::kAndroidJavaAudio); + SLObjectItf engine_object = audio_manager()->GetOpenSLEngine(); + EXPECT_EQ(nullptr, engine_object); +} + +// It should be possible to create an OpenSL engine object if OpenSL ES based +// audio is requested in any direction. +TEST_F(AudioManagerTest, GetOpenSLEngineShouldSucceedForOpenSLESAudioLayer) { + // List of supported audio layers that uses OpenSL ES audio. + const AudioDeviceModule::AudioLayer opensles_audio[] = { + AudioDeviceModule::kAndroidOpenSLESAudio, + AudioDeviceModule::kAndroidJavaInputAndOpenSLESOutputAudio}; + // Verify that the global (singleton) OpenSL Engine can be acquired for all + // audio layes that uses OpenSL ES. Note that the engine is only created once. + for (const AudioDeviceModule::AudioLayer audio_layer : opensles_audio) { + audio_manager()->SetActiveAudioLayer(audio_layer); + SLObjectItf engine_object = audio_manager()->GetOpenSLEngine(); + EXPECT_NE(nullptr, engine_object); + // Perform a simple sanity check of the created engine object. + ValidateSLEngine(engine_object); + } +} + TEST_F(AudioManagerTest, InitClose) { EXPECT_TRUE(audio_manager()->Init()); EXPECT_TRUE(audio_manager()->Close()); diff --git a/webrtc/modules/audio_device/android/opensles_common.cc b/webrtc/modules/audio_device/android/opensles_common.cc index da1e25483c..9e3cbf7b9a 100644 --- a/webrtc/modules/audio_device/android/opensles_common.cc +++ b/webrtc/modules/audio_device/android/opensles_common.cc @@ -11,13 +11,44 @@ #include "webrtc/modules/audio_device/android/opensles_common.h" #include +#include +#include "webrtc/base/arraysize.h" #include "webrtc/modules/audio_device/android/audio_common.h" using webrtc::kNumChannels; namespace webrtc { +// Returns a string representation given an integer SL_RESULT_XXX code. +// The mapping can be found in . +const char* GetSLErrorString(size_t code) { + static const char* sl_error_strings[] = { + "SL_RESULT_SUCCESS", // 0 + "SL_RESULT_PRECONDITIONS_VIOLATED", // 1 + "SL_RESULT_PARAMETER_INVALID", // 2 + "SL_RESULT_MEMORY_FAILURE", // 3 + "SL_RESULT_RESOURCE_ERROR", // 4 + "SL_RESULT_RESOURCE_LOST", // 5 + "SL_RESULT_IO_ERROR", // 6 + "SL_RESULT_BUFFER_INSUFFICIENT", // 7 + "SL_RESULT_CONTENT_CORRUPTED", // 8 + "SL_RESULT_CONTENT_UNSUPPORTED", // 9 + "SL_RESULT_CONTENT_NOT_FOUND", // 10 + "SL_RESULT_PERMISSION_DENIED", // 11 + "SL_RESULT_FEATURE_UNSUPPORTED", // 12 + "SL_RESULT_INTERNAL_ERROR", // 13 + "SL_RESULT_UNKNOWN_ERROR", // 14 + "SL_RESULT_OPERATION_ABORTED", // 15 + "SL_RESULT_CONTROL_LOST", // 16 + }; + + if (code >= arraysize(sl_error_strings)) { + return "SL_RESULT_UNKNOWN_ERROR"; + } + return sl_error_strings[code]; +} + SLDataFormat_PCM CreatePcmConfiguration(int sample_rate) { SLDataFormat_PCM configuration; configuration.formatType = SL_DATAFORMAT_PCM; diff --git a/webrtc/modules/audio_device/android/opensles_common.h b/webrtc/modules/audio_device/android/opensles_common.h index a4487b095c..5ff295b69d 100644 --- a/webrtc/modules/audio_device/android/opensles_common.h +++ b/webrtc/modules/audio_device/android/opensles_common.h @@ -17,6 +17,10 @@ namespace webrtc { +// Returns a string representation given an integer SL_RESULT_XXX code. +// The mapping can be found in . +const char* GetSLErrorString(size_t code); + SLDataFormat_PCM CreatePcmConfiguration(int sample_rate); // Helper class for using SLObjectItf interfaces. diff --git a/webrtc/modules/audio_device/android/opensles_player.cc b/webrtc/modules/audio_device/android/opensles_player.cc index d2bff4905e..b3ad64e424 100644 --- a/webrtc/modules/audio_device/android/opensles_player.cc +++ b/webrtc/modules/audio_device/android/opensles_player.cc @@ -16,6 +16,7 @@ #include "webrtc/base/checks.h" #include "webrtc/base/format_macros.h" #include "webrtc/base/timeutils.h" +#include "webrtc/modules/audio_device/android/audio_common.h" #include "webrtc/modules/audio_device/android/audio_manager.h" #include "webrtc/modules/audio_device/fine_audio_buffer.h" @@ -26,20 +27,21 @@ #define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) #define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) -#define RETURN_ON_ERROR(op, ...) \ - do { \ - SLresult err = (op); \ - if (err != SL_RESULT_SUCCESS) { \ - ALOGE("%s failed: %d", #op, err); \ - return __VA_ARGS__; \ - } \ +#define RETURN_ON_ERROR(op, ...) \ + do { \ + SLresult err = (op); \ + if (err != SL_RESULT_SUCCESS) { \ + ALOGE("%s failed: %s", #op, GetSLErrorString(err)); \ + return __VA_ARGS__; \ + } \ } while (0) namespace webrtc { OpenSLESPlayer::OpenSLESPlayer(AudioManager* audio_manager) - : audio_parameters_(audio_manager->GetPlayoutAudioParameters()), - audio_device_buffer_(NULL), + : audio_manager_(audio_manager), + audio_parameters_(audio_manager->GetPlayoutAudioParameters()), + audio_device_buffer_(nullptr), initialized_(false), playing_(false), bytes_per_buffer_(0), @@ -66,8 +68,7 @@ OpenSLESPlayer::~OpenSLESPlayer() { Terminate(); DestroyAudioPlayer(); DestroyMix(); - DestroyEngine(); - RTC_DCHECK(!engine_object_.Get()); + engine_ = nullptr; RTC_DCHECK(!engine_); RTC_DCHECK(!output_mix_.Get()); RTC_DCHECK(!player_); @@ -93,7 +94,7 @@ int OpenSLESPlayer::InitPlayout() { RTC_DCHECK(thread_checker_.CalledOnValidThread()); RTC_DCHECK(!initialized_); RTC_DCHECK(!playing_); - CreateEngine(); + ObtainEngineInterface(); CreateMix(); initialized_ = true; buffer_index_ = 0; @@ -263,34 +264,24 @@ void OpenSLESPlayer::AllocateDataBuffers() { } } -bool OpenSLESPlayer::CreateEngine() { - ALOGD("CreateEngine"); +bool OpenSLESPlayer::ObtainEngineInterface() { + ALOGD("ObtainEngineInterface"); RTC_DCHECK(thread_checker_.CalledOnValidThread()); - if (engine_object_.Get()) - return true; RTC_DCHECK(!engine_); - const SLEngineOption option[] = { - {SL_ENGINEOPTION_THREADSAFE, static_cast(SL_BOOLEAN_TRUE)}}; + // Get access to (or create if not already existing) the global OpenSL Engine + // object. + SLObjectItf engine_object = audio_manager_->GetOpenSLEngine(); + if (engine_object == nullptr) { + ALOGE("Failed to access the global OpenSL engine"); + return false; + } + // Get the SL Engine Interface which is implicit. RETURN_ON_ERROR( - slCreateEngine(engine_object_.Receive(), 1, option, 0, NULL, NULL), + (*engine_object)->GetInterface(engine_object, SL_IID_ENGINE, &engine_), false); - RETURN_ON_ERROR( - engine_object_->Realize(engine_object_.Get(), SL_BOOLEAN_FALSE), false); - RETURN_ON_ERROR(engine_object_->GetInterface(engine_object_.Get(), - SL_IID_ENGINE, &engine_), - false); return true; } -void OpenSLESPlayer::DestroyEngine() { - ALOGD("DestroyEngine"); - RTC_DCHECK(thread_checker_.CalledOnValidThread()); - if (!engine_object_.Get()) - return; - engine_ = nullptr; - engine_object_.Reset(); -} - bool OpenSLESPlayer::CreateMix() { ALOGD("CreateMix"); RTC_DCHECK(thread_checker_.CalledOnValidThread()); @@ -300,7 +291,7 @@ bool OpenSLESPlayer::CreateMix() { // Create the ouput mix on the engine object. No interfaces will be used. RETURN_ON_ERROR((*engine_)->CreateOutputMix(engine_, output_mix_.Receive(), 0, - NULL, NULL), + nullptr, nullptr), false); RETURN_ON_ERROR(output_mix_->Realize(output_mix_.Get(), SL_BOOLEAN_FALSE), false); @@ -318,7 +309,6 @@ void OpenSLESPlayer::DestroyMix() { bool OpenSLESPlayer::CreateAudioPlayer() { ALOGD("CreateAudioPlayer"); RTC_DCHECK(thread_checker_.CalledOnValidThread()); - RTC_DCHECK(engine_object_.Get()); RTC_DCHECK(output_mix_.Get()); if (player_object_.Get()) return true; @@ -335,7 +325,7 @@ bool OpenSLESPlayer::CreateAudioPlayer() { // sink: OutputMix-based data is sink. SLDataLocator_OutputMix locator_output_mix = {SL_DATALOCATOR_OUTPUTMIX, output_mix_.Get()}; - SLDataSink audio_sink = {&locator_output_mix, NULL}; + SLDataSink audio_sink = {&locator_output_mix, nullptr}; // Define interfaces that we indend to use and realize. const SLInterfaceID interface_ids[] = { diff --git a/webrtc/modules/audio_device/android/opensles_player.h b/webrtc/modules/audio_device/android/opensles_player.h index 4058ff9994..e3978a1990 100644 --- a/webrtc/modules/audio_device/android/opensles_player.h +++ b/webrtc/modules/audio_device/android/opensles_player.h @@ -55,12 +55,6 @@ class OpenSLESPlayer { // audio manager at construction. static const int kNumOfOpenSLESBuffers = 4; - // There is no need for this class to use JNI. - static int32_t SetAndroidAudioDeviceObjects(void* javaVM, void* context) { - return 0; - } - static void ClearAndroidAudioDeviceObjects() {} - explicit OpenSLESPlayer(AudioManager* audio_manager); ~OpenSLESPlayer(); @@ -103,9 +97,10 @@ class OpenSLESPlayer { // via the SLAndroidSimpleBufferQueueItf interface. void AllocateDataBuffers(); - // Creates/destroys the main engine object and the SLEngineItf interface. - bool CreateEngine(); - void DestroyEngine(); + // Obtaines the SL Engine Interface from the existing global Engine object. + // The interface exposes creation methods of all the OpenSL ES object types. + // This method defines the |engine_| member variable. + bool ObtainEngineInterface(); // Creates/destroys the output mix object. bool CreateMix(); @@ -127,6 +122,12 @@ class OpenSLESPlayer { // Detached during construction of this object. rtc::ThreadChecker thread_checker_opensles_; + // Raw pointer to the audio manager injected at construction. Used to cache + // audio parameters and to access the global SL engine object needed by the + // ObtainEngineInterface() method. The audio manager outlives any instance of + // this class. + AudioManager* audio_manager_; + // Contains audio parameters provided to this class at construction by the // AudioManager. const AudioParameters audio_parameters_; @@ -169,10 +170,6 @@ class OpenSLESPlayer { // Example (kNumOfOpenSLESBuffers = 2): counts 0, 1, 0, 1, ... int buffer_index_; - // The engine object which provides the SLEngineItf interface. - // Created by the global Open SL ES constructor slCreateEngine(). - webrtc::ScopedSLObjectItf engine_object_; - // This interface exposes creation methods for all the OpenSL ES object types. // It is the OpenSL ES API entry point. SLEngineItf engine_; diff --git a/webrtc/modules/audio_device/include/audio_device.h b/webrtc/modules/audio_device/include/audio_device.h index 8457a6b771..b8b48c15e5 100644 --- a/webrtc/modules/audio_device/include/audio_device.h +++ b/webrtc/modules/audio_device/include/audio_device.h @@ -31,7 +31,8 @@ class AudioDeviceModule : public RefCountedModule { kLinuxAlsaAudio = 3, kLinuxPulseAudio = 4, kAndroidJavaAudio = 5, - kAndroidJavaInputAndOpenSLESOutputAudio = 6, + kAndroidOpenSLESAudio = 6, + kAndroidJavaInputAndOpenSLESOutputAudio = 7, kDummyAudio = 8 };