From 616df1e95c5ac70d288d83e694eefc8f78da6245 Mon Sep 17 00:00:00 2001 From: aleloi Date: Wed, 24 Aug 2016 01:17:12 -0700 Subject: [PATCH] Added a level indicator to new mixer. Added a level indicator to the new mixer. The level indicator is webrtc::voe::AudioLevel. It computes the current audio level, which is used all the way up to peerconnection. This is part of the project to rewrite the old conference mixer and output mixer. NOTRY=True Review-Url: https://codereview.webrtc.org/2230823004 Cr-Commit-Position: refs/heads/master@{#13878} --- webrtc/modules/audio_mixer/BUILD.gn | 1 + webrtc/modules/audio_mixer/audio_mixer.gypi | 1 + .../audio_mixer/new_audio_conference_mixer.h | 7 ++ .../new_audio_conference_mixer_impl.cc | 22 ++++- .../new_audio_conference_mixer_impl.h | 9 +++ .../audio_mixer/test/audio_mixer_unittest.cc | 80 +++++++++++++++++-- webrtc/voice_engine/level_indicator.cc | 1 + 7 files changed, 110 insertions(+), 11 deletions(-) diff --git a/webrtc/modules/audio_mixer/BUILD.gn b/webrtc/modules/audio_mixer/BUILD.gn index afd29c4f82..b6737cb220 100644 --- a/webrtc/modules/audio_mixer/BUILD.gn +++ b/webrtc/modules/audio_mixer/BUILD.gn @@ -77,5 +77,6 @@ source_set("audio_conference_mixer") { "../../modules/audio_processing", "../../modules/utility", "../../system_wrappers", + "../../voice_engine:level_indicator", ] } diff --git a/webrtc/modules/audio_mixer/audio_mixer.gypi b/webrtc/modules/audio_mixer/audio_mixer.gypi index 5b2ebe639b..38d5b917c3 100644 --- a/webrtc/modules/audio_mixer/audio_mixer.gypi +++ b/webrtc/modules/audio_mixer/audio_mixer.gypi @@ -16,6 +16,7 @@ 'webrtc_utility', '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers', '<(webrtc_root)/base/base.gyp:rtc_base_approved', + '<(webrtc_root)/voice_engine/voice_engine.gyp:level_indicator', ], 'sources': [ 'audio_frame_manipulator.cc', diff --git a/webrtc/modules/audio_mixer/new_audio_conference_mixer.h b/webrtc/modules/audio_mixer/new_audio_conference_mixer.h index 561cf88604..52d79ee75b 100644 --- a/webrtc/modules/audio_mixer/new_audio_conference_mixer.h +++ b/webrtc/modules/audio_mixer/new_audio_conference_mixer.h @@ -59,6 +59,13 @@ class NewAudioConferenceMixer { virtual bool AnonymousMixabilityStatus( const MixerAudioSource& audio_source) const = 0; + // Output level functions for VoEVolumeControl. Return value + // between 0 and 9 is returned by voe::AudioLevel. + virtual int GetOutputAudioLevel() = 0; + + // Return value between 0 and 0x7fff is returned by voe::AudioLevel. + virtual int GetOutputAudioLevelFullRange() = 0; + protected: NewAudioConferenceMixer() {} }; diff --git a/webrtc/modules/audio_mixer/new_audio_conference_mixer_impl.cc b/webrtc/modules/audio_mixer/new_audio_conference_mixer_impl.cc index 48380fc550..72f8e15184 100644 --- a/webrtc/modules/audio_mixer/new_audio_conference_mixer_impl.cc +++ b/webrtc/modules/audio_mixer/new_audio_conference_mixer_impl.cc @@ -73,7 +73,6 @@ void RemixFrame(AudioFrame* frame, size_t number_of_channels) { // |mixed_frame| always has at least as many channels as |frame|. Supports // stereo at most. // -// TODO(andrew): consider not modifying |frame| here. void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame, bool use_limiter) { RTC_DCHECK_GE(mixed_frame->num_channels_, frame->num_channels_); if (use_limiter) { @@ -237,8 +236,7 @@ void NewAudioConferenceMixerImpl::Mix(int sample_rate, time_stamp_ += static_cast(sample_size_); - use_limiter_ = num_mixed_audio_sources_ > 1 && - output_frequency_ <= AudioProcessing::kMaxNativeSampleRateHz; + use_limiter_ = num_mixed_audio_sources_ > 1; // We only use the limiter if it supports the output sample rate and // we're actually mixing multiple streams. @@ -257,6 +255,10 @@ void NewAudioConferenceMixerImpl::Mix(int sample_rate, LimitMixedAudio(audio_frame_for_mixing); } } + + // Pass the final result to the level indicator. + audio_level_.ComputeLevel(*audio_frame_for_mixing); + return; } @@ -591,4 +593,18 @@ bool NewAudioConferenceMixerImpl::LimitMixedAudio( } return true; } + +int NewAudioConferenceMixerImpl::GetOutputAudioLevel() { + const int level = audio_level_.Level(); + WEBRTC_TRACE(kTraceStateInfo, kTraceAudioMixerServer, id_, + "GetAudioOutputLevel() => level=%d", level); + return level; +} + +int NewAudioConferenceMixerImpl::GetOutputAudioLevelFullRange() { + const int level = audio_level_.LevelFullRange(); + WEBRTC_TRACE(kTraceStateInfo, kTraceAudioMixerServer, id_, + "GetAudioOutputLevelFullRange() => level=%d", level); + return level; +} } // namespace webrtc diff --git a/webrtc/modules/audio_mixer/new_audio_conference_mixer_impl.h b/webrtc/modules/audio_mixer/new_audio_conference_mixer_impl.h index 3933350fed..4356ff7ea1 100644 --- a/webrtc/modules/audio_mixer/new_audio_conference_mixer_impl.h +++ b/webrtc/modules/audio_mixer/new_audio_conference_mixer_impl.h @@ -20,6 +20,7 @@ #include "webrtc/engine_configurations.h" #include "webrtc/modules/audio_mixer/new_audio_conference_mixer.h" #include "webrtc/modules/include/module_common_types.h" +#include "webrtc/voice_engine/level_indicator.h" namespace webrtc { class AudioProcessing; @@ -125,6 +126,11 @@ class NewAudioConferenceMixerImpl : public NewAudioConferenceMixer { bool LimitMixedAudio(AudioFrame* mixedAudio) const; + // Output level functions for VoEVolumeControl. + int GetOutputAudioLevel() override; + + int GetOutputAudioLevelFullRange() override; + std::unique_ptr crit_; std::unique_ptr cb_crit_; @@ -152,6 +158,9 @@ class NewAudioConferenceMixerImpl : public NewAudioConferenceMixer { // Used for inhibiting saturation in mixing. std::unique_ptr limiter_; + + // Measures audio level for the combined signal. + voe::AudioLevel audio_level_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_mixer/test/audio_mixer_unittest.cc b/webrtc/modules/audio_mixer/test/audio_mixer_unittest.cc index 38f2b2ebff..0359b8051a 100644 --- a/webrtc/modules/audio_mixer/test/audio_mixer_unittest.cc +++ b/webrtc/modules/audio_mixer/test/audio_mixer_unittest.cc @@ -77,13 +77,14 @@ class MockMixerAudioSource : public MixerAudioSource { } private: - AudioFrame fake_frame_; + AudioFrame fake_frame_, output_frame_; AudioFrameInfo fake_audio_frame_info_; AudioFrameWithMuted FakeAudioFrameWithMuted(const int32_t id, int sample_rate_hz) { + output_frame_.CopyFrom(fake_frame_); return { - fake_frame(), // audio_frame_pointer - fake_info(), // audio_frame_info + &output_frame_, // audio_frame_pointer + fake_info(), // audio_frame_info }; } }; @@ -274,10 +275,10 @@ TEST(AudioMixer, AnonymousAndNamed) { } TEST(AudioMixer, LargestEnergyVadActiveMixed) { - const int kId = 1; - const int kAudioSources = + constexpr int kId = 1; + constexpr int kAudioSources = NewAudioConferenceMixer::kMaximumAmountOfMixedAudioSources + 3; - const int kSampleRateHz = 32000; + constexpr int kSampleRateHz = 32000; std::unique_ptr mixer( NewAudioConferenceMixer::Create(kId)); @@ -328,7 +329,7 @@ TEST(AudioMixer, LargestEnergyVadActiveMixed) { } TEST(AudioMixer, ParticipantSampleRate) { - const int kId = 1; + constexpr int kId = 1; std::unique_ptr mixer( NewAudioConferenceMixer::Create(kId)); AudioFrame frame_for_mixing; @@ -350,7 +351,7 @@ TEST(AudioMixer, ParticipantSampleRate) { } TEST(AudioMixer, ParticipantNumberOfChannels) { - const int kId = 1; + constexpr int kId = 1; std::unique_ptr mixer( NewAudioConferenceMixer::Create(kId)); AudioFrame frame_for_mixing; @@ -370,6 +371,69 @@ TEST(AudioMixer, ParticipantNumberOfChannels) { } } +// Test that the volume is reported as zero when the mixer input +// comprises only zero values. +TEST(AudioMixer, LevelIsZeroWhenMixingZeroes) { + constexpr int kId = 1; + constexpr int kSampleRateHz = 8000; + std::unique_ptr mixer( + NewAudioConferenceMixer::Create(kId)); + AudioFrame frame_for_mixing; + + MockMixerAudioSource participant; + participant.fake_frame()->sample_rate_hz_ = kSampleRateHz; + participant.fake_frame()->num_channels_ = 1; + + // Frame duration 10ms. + participant.fake_frame()->samples_per_channel_ = kSampleRateHz / 100; + + EXPECT_EQ(0, mixer->SetMixabilityStatus(&participant, true)); + for (size_t i = 0; i < 11; i++) { + EXPECT_CALL(participant, GetAudioFrameWithMuted(_, kSampleRateHz)) + .Times(Exactly(1)); + mixer->Mix(8000, 1, &frame_for_mixing); + } + + EXPECT_EQ(0, mixer->GetOutputAudioLevel()); + EXPECT_EQ(0, mixer->GetOutputAudioLevelFullRange()); +} + +// Test that the reported volume is maximal when the mixer +// input comprises frames with maximal values. +TEST(AudioMixer, LevelIsMaximalWhenMixingMaximalValues) { + constexpr int kId = 1; + constexpr int kSampleRateHz = 8000; + std::unique_ptr mixer( + NewAudioConferenceMixer::Create(kId)); + AudioFrame frame_for_mixing; + + MockMixerAudioSource participant; + participant.fake_frame()->sample_rate_hz_ = kSampleRateHz; + participant.fake_frame()->num_channels_ = 1; + + // Frame duration 10ms. + participant.fake_frame()->samples_per_channel_ = kSampleRateHz / 100; + + // Fill participant frame data with maximal sound. + std::fill(participant.fake_frame()->data_, + participant.fake_frame()->data_ + kSampleRateHz / 100, + std::numeric_limits::max()); + + EXPECT_EQ(0, mixer->SetMixabilityStatus(&participant, true)); + for (size_t i = 0; i < 11; i++) { + EXPECT_CALL(participant, GetAudioFrameWithMuted(_, kSampleRateHz)) + .Times(Exactly(1)); + mixer->Mix(8000, 1, &frame_for_mixing); + } + + // 9 is the highest possible audio level + EXPECT_EQ(9, mixer->GetOutputAudioLevel()); + + // 0x7fff = 32767 is the highest full range audio level. + EXPECT_EQ(std::numeric_limits::max(), + mixer->GetOutputAudioLevelFullRange()); +} + TEST_F(BothMixersTest, CompareInitialFrameAudio) { EXPECT_CALL(participant_, GetAudioFrameWithMuted(_, _)).Times(Exactly(1)); diff --git a/webrtc/voice_engine/level_indicator.cc b/webrtc/voice_engine/level_indicator.cc index f44ea8e3f0..92fc8efe4c 100644 --- a/webrtc/voice_engine/level_indicator.cc +++ b/webrtc/voice_engine/level_indicator.cc @@ -28,6 +28,7 @@ AudioLevel::AudioLevel() : _count(0), _currentLevel(0), _currentLevelFullRange(0) { + WebRtcSpl_Init(); } AudioLevel::~AudioLevel() {