From 779017d989b01b53954636a7237e81dd249e5b84 Mon Sep 17 00:00:00 2001 From: henrika Date: Wed, 16 Nov 2016 06:30:46 -0800 Subject: [PATCH] Adds stereo support for Java-based input and output audio on Android BUG=webrtc:6718 Review-Url: https://codereview.webrtc.org/2499613002 Cr-Commit-Position: refs/heads/master@{#15104} --- .../audio_device/android/audio_common.h | 4 -- .../audio_device/android/audio_manager.cc | 22 +++--- .../audio_device/android/audio_manager.h | 6 +- .../audio_device/android/audio_record_jni.cc | 3 +- .../audio_device/android/audio_track_jni.cc | 8 ++- .../voiceengine/WebRtcAudioManager.java | 67 ++++++++++++------- .../webrtc/voiceengine/WebRtcAudioRecord.java | 15 +++-- .../webrtc/voiceengine/WebRtcAudioTrack.java | 16 +++-- 8 files changed, 84 insertions(+), 57 deletions(-) diff --git a/webrtc/modules/audio_device/android/audio_common.h b/webrtc/modules/audio_device/android/audio_common.h index 4eecae4b70..53618656a8 100644 --- a/webrtc/modules/audio_device/android/audio_common.h +++ b/webrtc/modules/audio_device/android/audio_common.h @@ -14,10 +14,6 @@ namespace webrtc { const int kDefaultSampleRate = 44100; -const int kNumChannels = 1; -// Number of bytes per audio frame. -// Example: 16-bit PCM in mono => 1*(16/8)=2 [bytes/frame] -const size_t kBytesPerFrame = kNumChannels * (16 / 8); // Delay estimates for the two different supported modes. These values are based // on real-time round-trip delay estimates on a large set of devices and they // are lower bounds since the filter length is 128 ms, so the AEC works for diff --git a/webrtc/modules/audio_device/android/audio_manager.cc b/webrtc/modules/audio_device/android/audio_manager.cc index d8b7640c9b..6925c3d614 100644 --- a/webrtc/modules/audio_device/android/audio_manager.cc +++ b/webrtc/modules/audio_device/android/audio_manager.cc @@ -78,7 +78,7 @@ AudioManager::AudioManager() ALOGD("ctor%s", GetThreadInfo().c_str()); RTC_CHECK(j_environment_); JNINativeMethod native_methods[] = { - {"nativeCacheAudioParameters", "(IIZZZZZZIIJ)V", + {"nativeCacheAudioParameters", "(IIIZZZZZZIIJ)V", reinterpret_cast(&webrtc::AudioManager::CacheAudioParameters)}}; j_native_registration_ = j_environment_->RegisterNatives( "org/webrtc/voiceengine/WebRtcAudioManager", native_methods, @@ -228,7 +228,8 @@ int AudioManager::GetDelayEstimateInMilliseconds() const { void JNICALL AudioManager::CacheAudioParameters(JNIEnv* env, jobject obj, jint sample_rate, - jint channels, + jint output_channels, + jint input_channels, jboolean hardware_aec, jboolean hardware_agc, jboolean hardware_ns, @@ -241,14 +242,15 @@ void JNICALL AudioManager::CacheAudioParameters(JNIEnv* env, webrtc::AudioManager* this_object = reinterpret_cast(native_audio_manager); this_object->OnCacheAudioParameters( - env, sample_rate, channels, hardware_aec, hardware_agc, hardware_ns, - low_latency_output, low_latency_input, pro_audio, output_buffer_size, - input_buffer_size); + env, sample_rate, output_channels, input_channels, hardware_aec, + hardware_agc, hardware_ns, low_latency_output, low_latency_input, + pro_audio, output_buffer_size, input_buffer_size); } void AudioManager::OnCacheAudioParameters(JNIEnv* env, jint sample_rate, - jint channels, + jint output_channels, + jint input_channels, jboolean hardware_aec, jboolean hardware_agc, jboolean hardware_ns, @@ -265,7 +267,8 @@ void AudioManager::OnCacheAudioParameters(JNIEnv* env, ALOGD("low_latency_input: %d", low_latency_input); ALOGD("pro_audio: %d", pro_audio); ALOGD("sample_rate: %d", sample_rate); - ALOGD("channels: %d", channels); + ALOGD("output_channels: %d", output_channels); + ALOGD("input_channels: %d", input_channels); ALOGD("output_buffer_size: %d", output_buffer_size); ALOGD("input_buffer_size: %d", input_buffer_size); RTC_DCHECK(thread_checker_.CalledOnValidThread()); @@ -275,10 +278,9 @@ void AudioManager::OnCacheAudioParameters(JNIEnv* env, low_latency_playout_ = low_latency_output; low_latency_record_ = low_latency_input; pro_audio_ = pro_audio; - // TODO(henrika): add support for stereo output. - playout_parameters_.reset(sample_rate, static_cast(channels), + playout_parameters_.reset(sample_rate, static_cast(output_channels), static_cast(output_buffer_size)); - record_parameters_.reset(sample_rate, static_cast(channels), + record_parameters_.reset(sample_rate, static_cast(input_channels), static_cast(input_buffer_size)); } diff --git a/webrtc/modules/audio_device/android/audio_manager.h b/webrtc/modules/audio_device/android/audio_manager.h index 341d426e41..6540cc505d 100644 --- a/webrtc/modules/audio_device/android/audio_manager.h +++ b/webrtc/modules/audio_device/android/audio_manager.h @@ -120,7 +120,8 @@ class AudioManager { static void JNICALL CacheAudioParameters(JNIEnv* env, jobject obj, jint sample_rate, - jint channels, + jint output_channels, + jint input_channels, jboolean hardware_aec, jboolean hardware_agc, jboolean hardware_ns, @@ -132,7 +133,8 @@ class AudioManager { jlong native_audio_manager); void OnCacheAudioParameters(JNIEnv* env, jint sample_rate, - jint channels, + jint output_channels, + jint input_channels, jboolean hardware_aec, jboolean hardware_agc, jboolean hardware_ns, diff --git a/webrtc/modules/audio_device/android/audio_record_jni.cc b/webrtc/modules/audio_device/android/audio_record_jni.cc index a02bbd5f0c..b826ef6883 100644 --- a/webrtc/modules/audio_device/android/audio_record_jni.cc +++ b/webrtc/modules/audio_device/android/audio_record_jni.cc @@ -132,8 +132,9 @@ int32_t AudioRecordJni::InitRecording() { } frames_per_buffer_ = static_cast(frames_per_buffer); ALOGD("frames_per_buffer: %" PRIuS, frames_per_buffer_); + const size_t bytes_per_frame = audio_parameters_.channels() * sizeof(int16_t); RTC_CHECK_EQ(direct_buffer_capacity_in_bytes_, - frames_per_buffer_ * kBytesPerFrame); + frames_per_buffer_ * bytes_per_frame); RTC_CHECK_EQ(frames_per_buffer_, audio_parameters_.frames_per_10ms_buffer()); initialized_ = true; return 0; diff --git a/webrtc/modules/audio_device/android/audio_track_jni.cc b/webrtc/modules/audio_device/android/audio_track_jni.cc index fc77e32e23..7072c25aa8 100644 --- a/webrtc/modules/audio_device/android/audio_track_jni.cc +++ b/webrtc/modules/audio_device/android/audio_track_jni.cc @@ -227,7 +227,8 @@ void AudioTrackJni::OnCacheDirectBufferAddress( jlong capacity = env->GetDirectBufferCapacity(byte_buffer); ALOGD("direct buffer capacity: %lld", capacity); direct_buffer_capacity_in_bytes_ = static_cast(capacity); - frames_per_buffer_ = direct_buffer_capacity_in_bytes_ / kBytesPerFrame; + const size_t bytes_per_frame = audio_parameters_.channels() * sizeof(int16_t); + frames_per_buffer_ = direct_buffer_capacity_in_bytes_ / bytes_per_frame; ALOGD("frames_per_buffer: %" PRIuS, frames_per_buffer_); } @@ -242,7 +243,8 @@ void JNICALL AudioTrackJni::GetPlayoutData( // the thread is 'AudioRecordTrack'. void AudioTrackJni::OnGetPlayoutData(size_t length) { RTC_DCHECK(thread_checker_java_.CalledOnValidThread()); - RTC_DCHECK_EQ(frames_per_buffer_, length / kBytesPerFrame); + const size_t bytes_per_frame = audio_parameters_.channels() * sizeof(int16_t); + RTC_DCHECK_EQ(frames_per_buffer_, length / bytes_per_frame); if (!audio_device_buffer_) { ALOGE("AttachAudioBuffer has not been called!"); return; @@ -257,7 +259,7 @@ void AudioTrackJni::OnGetPlayoutData(size_t length) { // Copy decoded data into common byte buffer to ensure that it can be // written to the Java based audio track. samples = audio_device_buffer_->GetPlayoutData(direct_buffer_address_); - RTC_DCHECK_EQ(length, kBytesPerFrame * samples); + RTC_DCHECK_EQ(length, bytes_per_frame * samples); } } // namespace webrtc diff --git a/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java b/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java index fb0516fa06..eb4702b444 100644 --- a/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java +++ b/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioManager.java @@ -38,27 +38,46 @@ public class WebRtcAudioManager { private static final String TAG = "WebRtcAudioManager"; + // Use mono as default for both audio directions. + private static boolean useStereoOutput = false; + private static boolean useStereoInput = false; + private static boolean blacklistDeviceForOpenSLESUsage = false; private static boolean blacklistDeviceForOpenSLESUsageIsOverridden = false; - // Call this method to override the deault list of blacklisted devices + // Call this method to override the default list of blacklisted devices // specified in WebRtcAudioUtils.BLACKLISTED_OPEN_SL_ES_MODELS. - // Allows an app to take control over which devices to exlude from using + // Allows an app to take control over which devices to exclude from using // the OpenSL ES audio output path public static synchronized void setBlacklistDeviceForOpenSLESUsage(boolean enable) { blacklistDeviceForOpenSLESUsageIsOverridden = true; blacklistDeviceForOpenSLESUsage = enable; } + // Call these methods to override the default mono audio modes for the specified direction(s) + // (input and/or output). + public static synchronized void setStereoOutput(boolean enable) { + Logging.w(TAG, "Overriding default output behavior: setStereoOutput(" + enable + ')'); + useStereoOutput = enable; + } + public static synchronized void setStereoInput(boolean enable) { + Logging.w(TAG, "Overriding default input behavior: setStereoInput(" + enable + ')'); + useStereoInput = enable; + } + + public static synchronized boolean getStereoOutput() { + return useStereoOutput; + } + public static synchronized boolean getStereoInput() { + return useStereoInput; + } + // Default audio data format is PCM 16 bit per sample. // Guaranteed to be supported by all devices. private static final int BITS_PER_SAMPLE = 16; private static final int DEFAULT_FRAME_PER_BUFFER = 256; - // TODO(henrika): add stereo support for playout. - private static final int CHANNELS = 1; - // List of possible audio modes. private static final String[] AUDIO_MODES = new String[] { "MODE_NORMAL", "MODE_RINGTONE", "MODE_IN_CALL", "MODE_IN_COMMUNICATION", @@ -132,7 +151,8 @@ public class WebRtcAudioManager { private boolean lowLatencyInput; private boolean proAudio; private int sampleRate; - private int channels; + private int outputChannels; + private int inputChannels; private int outputBufferSize; private int inputBufferSize; @@ -148,8 +168,8 @@ public class WebRtcAudioManager { } volumeLogger = new VolumeLogger(audioManager); storeAudioParameters(); - nativeCacheAudioParameters(sampleRate, channels, hardwareAEC, hardwareAGC, hardwareNS, - lowLatencyOutput, lowLatencyInput, proAudio, outputBufferSize, inputBufferSize, + nativeCacheAudioParameters(sampleRate, outputChannels, inputChannels, hardwareAEC, hardwareAGC, + hardwareNS, lowLatencyOutput, lowLatencyInput, proAudio, outputBufferSize, inputBufferSize, nativeAudioManager); } @@ -187,9 +207,8 @@ public class WebRtcAudioManager { } private void storeAudioParameters() { - // Only mono is supported currently (in both directions). - // TODO(henrika): add support for stereo playout. - channels = CHANNELS; + outputChannels = getStereoOutput() ? 2 : 1; + inputChannels = getStereoInput() ? 2 : 1; sampleRate = getNativeOutputSampleRate(); hardwareAEC = isAcousticEchoCancelerSupported(); // TODO(henrika): use of hardware AGC is no longer supported. Currently @@ -200,9 +219,9 @@ public class WebRtcAudioManager { lowLatencyInput = isLowLatencyInputSupported(); proAudio = isProAudioSupported(); outputBufferSize = lowLatencyOutput ? getLowLatencyOutputFramesPerBuffer() - : getMinOutputFrameSize(sampleRate, channels); + : getMinOutputFrameSize(sampleRate, outputChannels); inputBufferSize = lowLatencyInput ? getLowLatencyInputFramesPerBuffer() - : getMinInputFrameSize(sampleRate, channels); + : getMinInputFrameSize(sampleRate, inputChannels); } // Gets the current earpiece state. @@ -298,14 +317,8 @@ public class WebRtcAudioManager { // lacks support of low-latency output. private static int getMinOutputFrameSize(int sampleRateInHz, int numChannels) { final int bytesPerFrame = numChannels * (BITS_PER_SAMPLE / 8); - final int channelConfig; - if (numChannels == 1) { - channelConfig = AudioFormat.CHANNEL_OUT_MONO; - } else if (numChannels == 2) { - channelConfig = AudioFormat.CHANNEL_OUT_STEREO; - } else { - return -1; - } + final int channelConfig = + (numChannels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO); return AudioTrack.getMinBufferSize( sampleRateInHz, channelConfig, AudioFormat.ENCODING_PCM_16BIT) / bytesPerFrame; @@ -322,9 +335,10 @@ public class WebRtcAudioManager { // lacks support of low-latency input. private static int getMinInputFrameSize(int sampleRateInHz, int numChannels) { final int bytesPerFrame = numChannels * (BITS_PER_SAMPLE / 8); - assertTrue(numChannels == CHANNELS); + final int channelConfig = + (numChannels == 1 ? AudioFormat.CHANNEL_IN_MONO : AudioFormat.CHANNEL_IN_STEREO); return AudioRecord.getMinBufferSize( - sampleRateInHz, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) + sampleRateInHz, channelConfig, AudioFormat.ENCODING_PCM_16BIT) / bytesPerFrame; } @@ -341,7 +355,8 @@ public class WebRtcAudioManager { } } - private native void nativeCacheAudioParameters(int sampleRate, int channels, boolean hardwareAEC, - boolean hardwareAGC, boolean hardwareNS, boolean lowLatencyOutput, boolean lowLatencyInput, - boolean proAudio, int outputBufferSize, int inputBufferSize, long nativeAudioManager); + private native void nativeCacheAudioParameters(int sampleRate, int outputChannels, + int inputChannels, boolean hardwareAEC, boolean hardwareAGC, boolean hardwareNS, + boolean lowLatencyOutput, boolean lowLatencyInput, boolean proAudio, int outputBufferSize, + int inputBufferSize, long nativeAudioManager); } diff --git a/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java b/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java index 3bda070eac..c3665442b0 100644 --- a/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java +++ b/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioRecord.java @@ -170,8 +170,9 @@ public class WebRtcAudioRecord { // Get the minimum buffer size required for the successful creation of // an AudioRecord object, in byte units. // Note that this size doesn't guarantee a smooth recording under load. - int minBufferSize = AudioRecord.getMinBufferSize( - sampleRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT); + final int channelConfig = channelCountToConfiguration(channels); + int minBufferSize = + AudioRecord.getMinBufferSize(sampleRate, channelConfig, AudioFormat.ENCODING_PCM_16BIT); if (minBufferSize == AudioRecord.ERROR || minBufferSize == AudioRecord.ERROR_BAD_VALUE) { Logging.e(TAG, "AudioRecord.getMinBufferSize failed: " + minBufferSize); return -1; @@ -184,8 +185,8 @@ public class WebRtcAudioRecord { int bufferSizeInBytes = Math.max(BUFFER_SIZE_FACTOR * minBufferSize, byteBuffer.capacity()); Logging.d(TAG, "bufferSizeInBytes: " + bufferSizeInBytes); try { - audioRecord = new AudioRecord(AudioSource.VOICE_COMMUNICATION, sampleRate, - AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSizeInBytes); + audioRecord = new AudioRecord(AudioSource.VOICE_COMMUNICATION, sampleRate, channelConfig, + AudioFormat.ENCODING_PCM_16BIT, bufferSizeInBytes); } catch (IllegalArgumentException e) { Logging.e(TAG, e.getMessage()); return -1; @@ -246,7 +247,7 @@ public class WebRtcAudioRecord { // created instance uses the parameters that we asked for. private boolean areParametersValid(int sampleRate, int channels) { return (audioRecord.getAudioFormat() == AudioFormat.ENCODING_PCM_16BIT - && audioRecord.getChannelConfiguration() == AudioFormat.CHANNEL_IN_MONO + && audioRecord.getChannelConfiguration() == channelCountToConfiguration(channels) && audioRecord.getAudioSource() == AudioSource.VOICE_COMMUNICATION && audioRecord.getSampleRate() == sampleRate && audioRecord.getChannelCount() == channels); } @@ -273,6 +274,10 @@ public class WebRtcAudioRecord { } } + private int channelCountToConfiguration(int channels) { + return (channels == 1 ? AudioFormat.CHANNEL_IN_MONO : AudioFormat.CHANNEL_IN_STEREO); + } + private native void nativeCacheDirectBufferAddress(ByteBuffer byteBuffer, long nativeAudioRecord); private native void nativeDataIsRecorded(int bytes, long nativeAudioRecord); diff --git a/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java b/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java index 7af1af04c2..79961b95fc 100644 --- a/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java +++ b/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java @@ -174,8 +174,9 @@ public class WebRtcAudioTrack { // AudioTrack object to be created in the MODE_STREAM mode. // Note that this size doesn't guarantee a smooth playback under load. // TODO(henrika): should we extend the buffer size to avoid glitches? - final int minBufferSizeInBytes = AudioTrack.getMinBufferSize( - sampleRate, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT); + final int channelConfig = channelCountToConfiguration(channels); + final int minBufferSizeInBytes = + AudioTrack.getMinBufferSize(sampleRate, channelConfig, AudioFormat.ENCODING_PCM_16BIT); Logging.d(TAG, "AudioTrack.getMinBufferSize: " + minBufferSizeInBytes); // For the streaming mode, data must be written to the audio sink in // chunks of size (given by byteBuffer.capacity()) less than or equal @@ -197,9 +198,8 @@ public class WebRtcAudioTrack { // Create an AudioTrack object and initialize its associated audio buffer. // The size of this buffer determines how long an AudioTrack can play // before running out of data. - audioTrack = - new AudioTrack(AudioManager.STREAM_VOICE_CALL, sampleRate, AudioFormat.CHANNEL_OUT_MONO, - AudioFormat.ENCODING_PCM_16BIT, minBufferSizeInBytes, AudioTrack.MODE_STREAM); + audioTrack = new AudioTrack(AudioManager.STREAM_VOICE_CALL, sampleRate, channelConfig, + AudioFormat.ENCODING_PCM_16BIT, minBufferSizeInBytes, AudioTrack.MODE_STREAM); } catch (IllegalArgumentException e) { Logging.d(TAG, e.getMessage()); return false; @@ -285,7 +285,7 @@ public class WebRtcAudioTrack { private boolean areParametersValid(int sampleRate, int channels) { final int streamType = audioTrack.getStreamType(); return (audioTrack.getAudioFormat() == AudioFormat.ENCODING_PCM_16BIT - && audioTrack.getChannelConfiguration() == AudioFormat.CHANNEL_OUT_MONO + && audioTrack.getChannelConfiguration() == channelCountToConfiguration(channels) && streamType == AudioManager.STREAM_VOICE_CALL && audioTrack.getSampleRate() == sampleRate && sampleRate == audioTrack.getNativeOutputSampleRate(streamType) && audioTrack.getChannelCount() == channels); @@ -332,6 +332,10 @@ public class WebRtcAudioTrack { } } + private int channelCountToConfiguration(int channels) { + return (channels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO); + } + private native void nativeCacheDirectBufferAddress(ByteBuffer byteBuffer, long nativeAudioRecord); private native void nativeGetPlayoutData(int bytes, long nativeAudioRecord);