diff --git a/modules/audio_mixer/audio_mixer_impl.cc b/modules/audio_mixer/audio_mixer_impl.cc index a1247d7338..faa2b1e1ee 100644 --- a/modules/audio_mixer/audio_mixer_impl.cc +++ b/modules/audio_mixer/audio_mixer_impl.cc @@ -27,11 +27,8 @@ namespace webrtc { struct AudioMixerImpl::SourceStatus { - SourceStatus(Source* audio_source, bool is_mixed, float gain) - : audio_source(audio_source), is_mixed(is_mixed), gain(gain) {} + explicit SourceStatus(Source* audio_source) : audio_source(audio_source) {} Source* audio_source = nullptr; - bool is_mixed = false; - float gain = 0.0f; // A frame that will be passed to audio_source->GetAudioFrameWithInfo. AudioFrame audio_frame; @@ -39,74 +36,6 @@ struct AudioMixerImpl::SourceStatus { namespace { -class SourceFrame { - public: - // Default constructor required by call to `vector::resize()` below. - SourceFrame() = default; - - SourceFrame(AudioMixerImpl::SourceStatus* source_status, - AudioFrame* audio_frame, - bool muted) - : SourceFrame(source_status, - audio_frame, - muted, - muted ? 0u : AudioMixerCalculateEnergy(*audio_frame)) {} - - SourceFrame(AudioMixerImpl::SourceStatus* source_status, - AudioFrame* audio_frame, - bool muted, - uint32_t energy) - : source_status_(source_status), - audio_frame_(audio_frame), - muted_(muted), - energy_(energy) { - RTC_DCHECK(source_status); - RTC_DCHECK(audio_frame_); - } - - AudioMixerImpl::SourceStatus* source_status() { return source_status_; } - const AudioFrame* audio_frame() const { return audio_frame_; } - AudioFrame* mutable_audio_frame() { return audio_frame_; } - bool muted() const { return muted_; } - uint32_t energy() const { return energy_; } - - private: - // The below values are never changed directly, hence only accessors are - // offered. The values can change though via implicit assignment when sorting - // vectors. Pointer values will be nullptr when default constructed as a - // result of calling `vector::resize()`. - AudioMixerImpl::SourceStatus* source_status_ = nullptr; - AudioFrame* audio_frame_ = nullptr; - bool muted_ = true; - uint32_t energy_ = 0u; -}; - -// ShouldMixBefore(a, b) is used to select mixer sources. -// Returns true if `a` is preferred over `b` as a source to be mixed. -bool ShouldMixBefore(const SourceFrame& a, const SourceFrame& b) { - if (a.muted() != b.muted()) { - return b.muted(); - } - - const auto a_activity = a.audio_frame()->vad_activity_; - const auto b_activity = b.audio_frame()->vad_activity_; - - if (a_activity != b_activity) { - return a_activity == AudioFrame::kVadActive; - } - - return a.energy() > b.energy(); -} - -void RampAndUpdateGain(rtc::ArrayView mixed_sources_and_frames) { - for (auto& source_frame : mixed_sources_and_frames) { - float target_gain = source_frame.source_status()->is_mixed ? 1.0f : 0.0f; - Ramp(source_frame.source_status()->gain, target_gain, - source_frame.mutable_audio_frame()); - source_frame.source_status()->gain = target_gain; - } -} - std::vector>::const_iterator FindSourceInList( AudioMixerImpl::Source const* audio_source, @@ -123,46 +52,34 @@ FindSourceInList( struct AudioMixerImpl::HelperContainers { void resize(size_t size) { audio_to_mix.resize(size); - audio_source_mixing_data_list.resize(size); - ramp_list.resize(size); preferred_rates.resize(size); } std::vector audio_to_mix; - std::vector audio_source_mixing_data_list; - std::vector ramp_list; std::vector preferred_rates; }; AudioMixerImpl::AudioMixerImpl( std::unique_ptr output_rate_calculator, - bool use_limiter, - int max_sources_to_mix) - : max_sources_to_mix_(max_sources_to_mix), - output_rate_calculator_(std::move(output_rate_calculator)), + bool use_limiter) + : output_rate_calculator_(std::move(output_rate_calculator)), audio_source_list_(), helper_containers_(std::make_unique()), - frame_combiner_(use_limiter) { - RTC_CHECK_GE(max_sources_to_mix, 1) << "At least one source must be mixed"; - audio_source_list_.reserve(max_sources_to_mix); - helper_containers_->resize(max_sources_to_mix); -} + frame_combiner_(use_limiter) {} AudioMixerImpl::~AudioMixerImpl() {} -rtc::scoped_refptr AudioMixerImpl::Create( - int max_sources_to_mix) { +rtc::scoped_refptr AudioMixerImpl::Create() { return Create(std::unique_ptr( new DefaultOutputRateCalculator()), - /*use_limiter=*/true, max_sources_to_mix); + /*use_limiter=*/true); } rtc::scoped_refptr AudioMixerImpl::Create( std::unique_ptr output_rate_calculator, - bool use_limiter, - int max_sources_to_mix) { + bool use_limiter) { return rtc::make_ref_counted( - std::move(output_rate_calculator), use_limiter, max_sources_to_mix); + std::move(output_rate_calculator), use_limiter); } void AudioMixerImpl::Mix(size_t number_of_channels, @@ -194,7 +111,7 @@ bool AudioMixerImpl::AddSource(Source* audio_source) { RTC_DCHECK(FindSourceInList(audio_source, &audio_source_list_) == audio_source_list_.end()) << "Source already added to mixer"; - audio_source_list_.emplace_back(new SourceStatus(audio_source, false, 0)); + audio_source_list_.emplace_back(new SourceStatus(audio_source)); helper_containers_->resize(audio_source_list_.size()); UpdateSourceCountStats(); return true; @@ -210,72 +127,27 @@ void AudioMixerImpl::RemoveSource(Source* audio_source) { rtc::ArrayView AudioMixerImpl::GetAudioFromSources( int output_frequency) { - // Get audio from the audio sources and put it in the SourceFrame vector. - int audio_source_mixing_data_count = 0; + int audio_to_mix_count = 0; for (auto& source_and_status : audio_source_list_) { const auto audio_frame_info = source_and_status->audio_source->GetAudioFrameWithInfo( output_frequency, &source_and_status->audio_frame); - - if (audio_frame_info == Source::AudioFrameInfo::kError) { - RTC_LOG_F(LS_WARNING) << "failed to GetAudioFrameWithInfo() from source"; - continue; + switch (audio_frame_info) { + case Source::AudioFrameInfo::kError: + RTC_LOG_F(LS_WARNING) + << "failed to GetAudioFrameWithInfo() from source"; + break; + case Source::AudioFrameInfo::kMuted: + break; + case Source::AudioFrameInfo::kNormal: + helper_containers_->audio_to_mix[audio_to_mix_count++] = + &source_and_status->audio_frame; } - helper_containers_ - ->audio_source_mixing_data_list[audio_source_mixing_data_count++] = - SourceFrame(source_and_status.get(), &source_and_status->audio_frame, - audio_frame_info == Source::AudioFrameInfo::kMuted); } - rtc::ArrayView audio_source_mixing_data_view( - helper_containers_->audio_source_mixing_data_list.data(), - audio_source_mixing_data_count); - - // Sort frames by sorting function. - std::sort(audio_source_mixing_data_view.begin(), - audio_source_mixing_data_view.end(), ShouldMixBefore); - - int max_audio_frame_counter = max_sources_to_mix_; - int ramp_list_length = 0; - int audio_to_mix_count = 0; - // Go through list in order and put unmuted frames in result list. - for (auto& p : audio_source_mixing_data_view) { - // Filter muted. - if (p.muted()) { - p.source_status()->is_mixed = false; - continue; - } - - // Add frame to result vector for mixing. - bool is_mixed = false; - if (max_audio_frame_counter > 0) { - --max_audio_frame_counter; - helper_containers_->audio_to_mix[audio_to_mix_count++] = - p.mutable_audio_frame(); - helper_containers_->ramp_list[ramp_list_length++] = - SourceFrame(p.source_status(), p.mutable_audio_frame(), false, -1); - is_mixed = true; - } - p.source_status()->is_mixed = is_mixed; - } - RampAndUpdateGain(rtc::ArrayView( - helper_containers_->ramp_list.data(), ramp_list_length)); return rtc::ArrayView( helper_containers_->audio_to_mix.data(), audio_to_mix_count); } -bool AudioMixerImpl::GetAudioSourceMixabilityStatusForTest( - AudioMixerImpl::Source* audio_source) const { - MutexLock lock(&mutex_); - - const auto iter = FindSourceInList(audio_source, &audio_source_list_); - if (iter != audio_source_list_.end()) { - return (*iter)->is_mixed; - } - - RTC_LOG(LS_ERROR) << "Audio source unknown"; - return false; -} - void AudioMixerImpl::UpdateSourceCountStats() { size_t current_source_count = audio_source_list_.size(); // Log to the histogram whenever the maximum number of sources increases. diff --git a/modules/audio_mixer/audio_mixer_impl.h b/modules/audio_mixer/audio_mixer_impl.h index e1040defd7..8319487018 100644 --- a/modules/audio_mixer/audio_mixer_impl.h +++ b/modules/audio_mixer/audio_mixer_impl.h @@ -35,15 +35,11 @@ class AudioMixerImpl : public AudioMixer { // AudioProcessing only accepts 10 ms frames. static const int kFrameDurationInMs = 10; - static const int kDefaultNumberOfMixedAudioSources = 3; - - static rtc::scoped_refptr Create( - int max_sources_to_mix = kDefaultNumberOfMixedAudioSources); + static rtc::scoped_refptr Create(); static rtc::scoped_refptr Create( std::unique_ptr output_rate_calculator, - bool use_limiter, - int max_sources_to_mix = kDefaultNumberOfMixedAudioSources); + bool use_limiter); ~AudioMixerImpl() override; @@ -58,24 +54,16 @@ class AudioMixerImpl : public AudioMixer { AudioFrame* audio_frame_for_mixing) override RTC_LOCKS_EXCLUDED(mutex_); - // Returns true if the source was mixed last round. Returns - // false and logs an error if the source was never added to the - // mixer. - bool GetAudioSourceMixabilityStatusForTest(Source* audio_source) const; - protected: AudioMixerImpl(std::unique_ptr output_rate_calculator, - bool use_limiter, - int max_sources_to_mix); + bool use_limiter); private: struct HelperContainers; void UpdateSourceCountStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - // Compute what audio sources to mix from audio_source_list_. Ramp - // in and out. Update mixed status. Mixes up to - // kMaximumAmountOfMixedAudioSources audio sources. + // Fetches audio frames to mix from sources. rtc::ArrayView GetAudioFromSources(int output_frequency) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); @@ -84,8 +72,6 @@ class AudioMixerImpl : public AudioMixer { // checks that mixing is done sequentially. mutable Mutex mutex_; - const int max_sources_to_mix_; - std::unique_ptr output_rate_calculator_; // List of all audio sources. diff --git a/modules/audio_mixer/audio_mixer_impl_unittest.cc b/modules/audio_mixer/audio_mixer_impl_unittest.cc index 8022332b27..641c966570 100644 --- a/modules/audio_mixer/audio_mixer_impl_unittest.cc +++ b/modules/audio_mixer/audio_mixer_impl_unittest.cc @@ -129,39 +129,6 @@ class CustomRateCalculator : public OutputRateCalculator { const int rate_; }; -// Creates participants from `frames` and `frame_info` and adds them -// to the mixer. Compares mixed status with `expected_status` -void MixAndCompare( - const std::vector& frames, - const std::vector& frame_info, - const std::vector& expected_status) { - const size_t num_audio_sources = frames.size(); - RTC_DCHECK(frames.size() == frame_info.size()); - RTC_DCHECK(frame_info.size() == expected_status.size()); - - const auto mixer = AudioMixerImpl::Create(); - std::vector participants(num_audio_sources); - - for (size_t i = 0; i < num_audio_sources; ++i) { - participants[i].fake_frame()->CopyFrom(frames[i]); - participants[i].set_fake_info(frame_info[i]); - } - - for (size_t i = 0; i < num_audio_sources; ++i) { - EXPECT_TRUE(mixer->AddSource(&participants[i])); - EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) - .Times(Exactly(1)); - } - - mixer->Mix(1, &frame_for_mixing); - - for (size_t i = 0; i < num_audio_sources; ++i) { - EXPECT_EQ(expected_status[i], - mixer->GetAudioSourceMixabilityStatusForTest(&participants[i])) - << "Mixed status of AudioSource #" << i << " wrong."; - } -} - void MixMonoAtGivenNativeRate(int native_sample_rate, AudioFrame* mix_frame, rtc::scoped_refptr mixer, @@ -174,49 +141,6 @@ void MixMonoAtGivenNativeRate(int native_sample_rate, mixer->Mix(1, mix_frame); } -TEST(AudioMixer, LargestEnergyVadActiveMixed) { - constexpr int kAudioSources = - AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 3; - - const auto mixer = AudioMixerImpl::Create(); - - MockMixerAudioSource participants[kAudioSources]; - - for (int i = 0; i < kAudioSources; ++i) { - ResetFrame(participants[i].fake_frame()); - - // We set the 80-th sample value since the first 80 samples may be - // modified by a ramped-in window. - participants[i].fake_frame()->mutable_data()[80] = i; - - EXPECT_TRUE(mixer->AddSource(&participants[i])); - EXPECT_CALL(participants[i], GetAudioFrameWithInfo(_, _)).Times(Exactly(1)); - } - - // Last participant gives audio frame with passive VAD, although it has the - // largest energy. - participants[kAudioSources - 1].fake_frame()->vad_activity_ = - AudioFrame::kVadPassive; - - AudioFrame audio_frame; - mixer->Mix(1, // number of channels - &audio_frame); - - for (int i = 0; i < kAudioSources; ++i) { - bool is_mixed = - mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]); - if (i == kAudioSources - 1 || - i < kAudioSources - 1 - - AudioMixerImpl::kDefaultNumberOfMixedAudioSources) { - EXPECT_FALSE(is_mixed) - << "Mixing status of AudioSource #" << i << " wrong."; - } else { - EXPECT_TRUE(is_mixed) - << "Mixing status of AudioSource #" << i << " wrong."; - } - } -} - TEST(AudioMixer, UpdatesSourceCountHistogram) { constexpr int kAudioSourcesGroup1 = 5; constexpr int kAudioSourcesGroup2 = 3; @@ -369,59 +293,6 @@ TEST(AudioMixer, ParticipantNumberOfChannels) { } } -// Maximal amount of participants are mixed one iteration, then -// another participant with higher energy is added. -TEST(AudioMixer, RampedOutSourcesShouldNotBeMarkedMixed) { - constexpr int kAudioSources = - AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; - - const auto mixer = AudioMixerImpl::Create(); - MockMixerAudioSource participants[kAudioSources]; - - for (int i = 0; i < kAudioSources; ++i) { - ResetFrame(participants[i].fake_frame()); - // Set the participant audio energy to increase with the index - // `i`. - participants[i].fake_frame()->mutable_data()[0] = 100 * i; - } - - // Add all participants but the loudest for mixing. - for (int i = 0; i < kAudioSources - 1; ++i) { - EXPECT_TRUE(mixer->AddSource(&participants[i])); - EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) - .Times(Exactly(1)); - } - - // First mixer iteration - mixer->Mix(1, &frame_for_mixing); - - // All participants but the loudest should have been mixed. - for (int i = 0; i < kAudioSources - 1; ++i) { - EXPECT_TRUE(mixer->GetAudioSourceMixabilityStatusForTest(&participants[i])) - << "Mixed status of AudioSource #" << i << " wrong."; - } - - // Add new participant with higher energy. - EXPECT_TRUE(mixer->AddSource(&participants[kAudioSources - 1])); - for (int i = 0; i < kAudioSources; ++i) { - EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) - .Times(Exactly(1)); - } - - mixer->Mix(1, &frame_for_mixing); - - // The most quiet participant should not have been mixed. - EXPECT_FALSE(mixer->GetAudioSourceMixabilityStatusForTest(&participants[0])) - << "Mixed status of AudioSource #0 wrong."; - - // The loudest participants should have been mixed. - for (int i = 1; i < kAudioSources; ++i) { - EXPECT_EQ(true, - mixer->GetAudioSourceMixabilityStatusForTest(&participants[i])) - << "Mixed status of AudioSource #" << i << " wrong."; - } -} - // This test checks that the initialization and participant addition // can be done on a different thread. TEST(AudioMixer, ConstructFromOtherThread) { @@ -446,127 +317,6 @@ TEST(AudioMixer, ConstructFromOtherThread) { mixer->Mix(1, &frame_for_mixing); } -TEST(AudioMixer, MutedShouldMixAfterUnmuted) { - constexpr int kAudioSources = - AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; - - std::vector frames(kAudioSources); - for (auto& frame : frames) { - ResetFrame(&frame); - } - - std::vector frame_info( - kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); - frame_info[0] = AudioMixer::Source::AudioFrameInfo::kMuted; - std::vector expected_status(kAudioSources, true); - expected_status[0] = false; - - MixAndCompare(frames, frame_info, expected_status); -} - -TEST(AudioMixer, PassiveShouldMixAfterNormal) { - constexpr int kAudioSources = - AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; - - std::vector frames(kAudioSources); - for (auto& frame : frames) { - ResetFrame(&frame); - } - - std::vector frame_info( - kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); - frames[0].vad_activity_ = AudioFrame::kVadPassive; - std::vector expected_status(kAudioSources, true); - expected_status[0] = false; - - MixAndCompare(frames, frame_info, expected_status); -} - -TEST(AudioMixer, ActiveShouldMixBeforeLoud) { - constexpr int kAudioSources = - AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; - - std::vector frames(kAudioSources); - for (auto& frame : frames) { - ResetFrame(&frame); - } - - std::vector frame_info( - kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); - frames[0].vad_activity_ = AudioFrame::kVadPassive; - int16_t* frame_data = frames[0].mutable_data(); - std::fill(frame_data, frame_data + kDefaultSampleRateHz / 100, - std::numeric_limits::max()); - std::vector expected_status(kAudioSources, true); - expected_status[0] = false; - - MixAndCompare(frames, frame_info, expected_status); -} - -TEST(AudioMixer, ShouldMixUpToSpecifiedNumberOfSourcesToMix) { - constexpr int kAudioSources = 5; - constexpr int kSourcesToMix = 2; - - std::vector frames(kAudioSources); - for (auto& frame : frames) { - ResetFrame(&frame); - } - - std::vector frame_info( - kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); - // Set up to kSourceToMix sources with kVadActive so that they're mixed. - const std::vector kVadActivities = { - AudioFrame::kVadUnknown, AudioFrame::kVadPassive, AudioFrame::kVadPassive, - AudioFrame::kVadActive, AudioFrame::kVadActive}; - // Populate VAD and frame for all sources. - for (int i = 0; i < kAudioSources; i++) { - frames[i].vad_activity_ = kVadActivities[i]; - } - - std::vector participants(kAudioSources); - for (int i = 0; i < kAudioSources; ++i) { - participants[i].fake_frame()->CopyFrom(frames[i]); - participants[i].set_fake_info(frame_info[i]); - } - - const auto mixer = AudioMixerImpl::Create(kSourcesToMix); - for (int i = 0; i < kAudioSources; ++i) { - EXPECT_TRUE(mixer->AddSource(&participants[i])); - EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _)) - .Times(Exactly(1)); - } - - mixer->Mix(1, &frame_for_mixing); - - std::vector expected_status = {false, false, false, true, true}; - for (int i = 0; i < kAudioSources; ++i) { - EXPECT_EQ(expected_status[i], - mixer->GetAudioSourceMixabilityStatusForTest(&participants[i])) - << "Wrong mix status for source #" << i << " is wrong"; - } -} - -TEST(AudioMixer, UnmutedShouldMixBeforeLoud) { - constexpr int kAudioSources = - AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1; - - std::vector frames(kAudioSources); - for (auto& frame : frames) { - ResetFrame(&frame); - } - - std::vector frame_info( - kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal); - frame_info[0] = AudioMixer::Source::AudioFrameInfo::kMuted; - int16_t* frame_data = frames[0].mutable_data(); - std::fill(frame_data, frame_data + kDefaultSampleRateHz / 100, - std::numeric_limits::max()); - std::vector expected_status(kAudioSources, true); - expected_status[0] = false; - - MixAndCompare(frames, frame_info, expected_status); -} - TEST(AudioMixer, MixingRateShouldBeDecidedByRateCalculator) { constexpr int kOutputRate = 22000; const auto mixer = @@ -728,55 +478,6 @@ TEST(AudioMixer, ShouldIncludeRtpPacketInfoFromAllMixedSources) { EXPECT_THAT(frame_for_mixing.packet_infos_, UnorderedElementsAre(p0, p1, p2)); } -TEST(AudioMixer, MixerShouldIncludeRtpPacketInfoFromMixedSourcesOnly) { - const uint32_t kSsrc0 = 10; - const uint32_t kSsrc1 = 11; - const uint32_t kSsrc2 = 21; - const uint32_t kCsrc0 = 30; - const uint32_t kCsrc1 = 31; - const uint32_t kCsrc2 = 32; - const uint32_t kCsrc3 = 33; - const int kAudioLevel0 = 10; - const absl::optional kAudioLevelMissing = absl::nullopt; - const uint32_t kRtpTimestamp0 = 300; - const uint32_t kRtpTimestamp1 = 400; - const Timestamp kReceiveTime0 = Timestamp::Millis(10); - const Timestamp kReceiveTime1 = Timestamp::Millis(20); - - RtpPacketInfo p0(kSsrc0, {kCsrc0, kCsrc1}, kRtpTimestamp0, kReceiveTime0); - p0.set_audio_level(kAudioLevel0); - RtpPacketInfo p1(kSsrc1, {kCsrc2}, kRtpTimestamp1, kReceiveTime1); - p1.set_audio_level(kAudioLevelMissing); - RtpPacketInfo p2(kSsrc2, {kCsrc3}, kRtpTimestamp1, kReceiveTime1); - p2.set_audio_level(kAudioLevelMissing); - - const auto mixer = AudioMixerImpl::Create(/*max_sources_to_mix=*/2); - - MockMixerAudioSource source1; - source1.set_packet_infos(RtpPacketInfos({p0})); - mixer->AddSource(&source1); - ResetFrame(source1.fake_frame()); - mixer->Mix(1, &frame_for_mixing); - - MockMixerAudioSource source2; - source2.set_packet_infos(RtpPacketInfos({p1})); - ResetFrame(source2.fake_frame()); - mixer->AddSource(&source2); - - // The mixer prioritizes kVadActive over kVadPassive. - // We limit the number of sources to mix to 2 and set the third source's VAD - // activity to kVadPassive so that it will not be added to the mix. - MockMixerAudioSource source3; - source3.set_packet_infos(RtpPacketInfos({p2})); - ResetFrame(source3.fake_frame()); - source3.fake_frame()->vad_activity_ = AudioFrame::kVadPassive; - mixer->AddSource(&source3); - - mixer->Mix(/*number_of_channels=*/1, &frame_for_mixing); - - EXPECT_THAT(frame_for_mixing.packet_infos_, UnorderedElementsAre(p0, p1)); -} - class HighOutputRateCalculator : public OutputRateCalculator { public: static const int kDefaultFrequency = 76000;