diff --git a/modules/audio_mixer/audio_mixer_impl.cc b/modules/audio_mixer/audio_mixer_impl.cc index 6552953dec..04a8bcf723 100644 --- a/modules/audio_mixer/audio_mixer_impl.cc +++ b/modules/audio_mixer/audio_mixer_impl.cc @@ -24,9 +24,23 @@ #include "rtc_base/ref_counted_object.h" namespace webrtc { + +struct AudioMixerImpl::SourceStatus { + SourceStatus(Source* audio_source, bool is_mixed, float gain) + : audio_source(audio_source), is_mixed(is_mixed), gain(gain) {} + Source* audio_source = nullptr; + bool is_mixed = false; + float gain = 0.0f; + + // A frame that will be passed to audio_source->GetAudioFrameWithInfo. + AudioFrame audio_frame; +}; + namespace { struct SourceFrame { + SourceFrame() = default; + SourceFrame(AudioMixerImpl::SourceStatus* source_status, AudioFrame* audio_frame, bool muted) @@ -57,6 +71,7 @@ struct SourceFrame { }; // ShouldMixBefore(a, b) is used to select mixer sources. +// Returns true if `a` is preferred over `b` as a source to be mixed. bool ShouldMixBefore(const SourceFrame& a, const SourceFrame& b) { if (a.muted != b.muted) { return b.muted; @@ -73,7 +88,7 @@ bool ShouldMixBefore(const SourceFrame& a, const SourceFrame& b) { } void RampAndUpdateGain( - const std::vector& mixed_sources_and_frames) { + rtc::ArrayView mixed_sources_and_frames) { for (const auto& source_frame : mixed_sources_and_frames) { float target_gain = source_frame.source_status->is_mixed ? 1.0f : 0.0f; Ramp(source_frame.source_status->gain, target_gain, @@ -82,9 +97,11 @@ void RampAndUpdateGain( } } -AudioMixerImpl::SourceStatusList::const_iterator FindSourceInList( +std::vector>::const_iterator +FindSourceInList( AudioMixerImpl::Source const* audio_source, - AudioMixerImpl::SourceStatusList const* audio_source_list) { + std::vector> const* + audio_source_list) { return std::find_if( audio_source_list->begin(), audio_source_list->end(), [audio_source](const std::unique_ptr& p) { @@ -93,14 +110,31 @@ AudioMixerImpl::SourceStatusList::const_iterator FindSourceInList( } } // namespace +struct AudioMixerImpl::HelperContainers { + void resize(size_t size) { + audio_to_mix.resize(size); + audio_source_mixing_data_list.resize(size); + ramp_list.resize(size); + preferred_rates.resize(size); + } + + std::vector audio_to_mix; + std::vector audio_source_mixing_data_list; + std::vector ramp_list; + std::vector preferred_rates; +}; + AudioMixerImpl::AudioMixerImpl( std::unique_ptr output_rate_calculator, bool use_limiter) : output_rate_calculator_(std::move(output_rate_calculator)), - output_frequency_(0), - sample_size_(0), audio_source_list_(), - frame_combiner_(use_limiter) {} + helper_containers_(std::make_unique()), + frame_combiner_(use_limiter) { + const int kTypicalMaxNumberOfMixedStreams = 3; + audio_source_list_.reserve(kTypicalMaxNumberOfMixedStreams); + helper_containers_->resize(kTypicalMaxNumberOfMixedStreams); +} AudioMixerImpl::~AudioMixerImpl() {} @@ -121,40 +155,23 @@ rtc::scoped_refptr AudioMixerImpl::Create( void AudioMixerImpl::Mix(size_t number_of_channels, AudioFrame* audio_frame_for_mixing) { RTC_DCHECK(number_of_channels >= 1); - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); - - CalculateOutputFrequency(); - - { - MutexLock lock(&mutex_); - const size_t number_of_streams = audio_source_list_.size(); - frame_combiner_.Combine(GetAudioFromSources(), number_of_channels, - OutputFrequency(), number_of_streams, - audio_frame_for_mixing); - } - - return; -} - -void AudioMixerImpl::CalculateOutputFrequency() { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); MutexLock lock(&mutex_); - std::vector preferred_rates; + size_t number_of_streams = audio_source_list_.size(); + std::transform(audio_source_list_.begin(), audio_source_list_.end(), - std::back_inserter(preferred_rates), + helper_containers_->preferred_rates.begin(), [&](std::unique_ptr& a) { return a->audio_source->PreferredSampleRate(); }); - output_frequency_ = - output_rate_calculator_->CalculateOutputRate(preferred_rates); - sample_size_ = (output_frequency_ * kFrameDurationInMs) / 1000; -} + int output_frequency = output_rate_calculator_->CalculateOutputRateFromRange( + rtc::ArrayView(helper_containers_->preferred_rates.data(), + number_of_streams)); -int AudioMixerImpl::OutputFrequency() const { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); - return output_frequency_; + frame_combiner_.Combine(GetAudioFromSources(output_frequency), + number_of_channels, output_frequency, + number_of_streams, audio_frame_for_mixing); } bool AudioMixerImpl::AddSource(Source* audio_source) { @@ -164,6 +181,7 @@ bool AudioMixerImpl::AddSource(Source* audio_source) { audio_source_list_.end()) << "Source already added to mixer"; audio_source_list_.emplace_back(new SourceStatus(audio_source, false, 0)); + helper_containers_->resize(audio_source_list_.size()); return true; } @@ -175,35 +193,37 @@ void AudioMixerImpl::RemoveSource(Source* audio_source) { audio_source_list_.erase(iter); } -AudioFrameList AudioMixerImpl::GetAudioFromSources() { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); - AudioFrameList result; - std::vector audio_source_mixing_data_list; - std::vector ramp_list; - +rtc::ArrayView AudioMixerImpl::GetAudioFromSources( + int output_frequency) { // Get audio from the audio sources and put it in the SourceFrame vector. + int audio_source_mixing_data_count = 0; for (auto& source_and_status : audio_source_list_) { const auto audio_frame_info = source_and_status->audio_source->GetAudioFrameWithInfo( - OutputFrequency(), &source_and_status->audio_frame); + output_frequency, &source_and_status->audio_frame); if (audio_frame_info == Source::AudioFrameInfo::kError) { RTC_LOG_F(LS_WARNING) << "failed to GetAudioFrameWithInfo() from source"; continue; } - audio_source_mixing_data_list.emplace_back( - source_and_status.get(), &source_and_status->audio_frame, - audio_frame_info == Source::AudioFrameInfo::kMuted); + helper_containers_ + ->audio_source_mixing_data_list[audio_source_mixing_data_count++] = + SourceFrame(source_and_status.get(), &source_and_status->audio_frame, + audio_frame_info == Source::AudioFrameInfo::kMuted); } + rtc::ArrayView audio_source_mixing_data_view( + helper_containers_->audio_source_mixing_data_list.data(), + audio_source_mixing_data_count); // Sort frames by sorting function. - std::sort(audio_source_mixing_data_list.begin(), - audio_source_mixing_data_list.end(), ShouldMixBefore); + std::sort(audio_source_mixing_data_view.begin(), + audio_source_mixing_data_view.end(), ShouldMixBefore); int max_audio_frame_counter = kMaximumAmountOfMixedAudioSources; - + int ramp_list_lengh = 0; + int audio_to_mix_count = 0; // Go through list in order and put unmuted frames in result list. - for (const auto& p : audio_source_mixing_data_list) { + for (const auto& p : audio_source_mixing_data_view) { // Filter muted. if (p.muted) { p.source_status->is_mixed = false; @@ -214,19 +234,21 @@ AudioFrameList AudioMixerImpl::GetAudioFromSources() { bool is_mixed = false; if (max_audio_frame_counter > 0) { --max_audio_frame_counter; - result.push_back(p.audio_frame); - ramp_list.emplace_back(p.source_status, p.audio_frame, false, -1); + helper_containers_->audio_to_mix[audio_to_mix_count++] = p.audio_frame; + helper_containers_->ramp_list[ramp_list_lengh++] = + SourceFrame(p.source_status, p.audio_frame, false, -1); is_mixed = true; } p.source_status->is_mixed = is_mixed; } - RampAndUpdateGain(ramp_list); - return result; + RampAndUpdateGain(rtc::ArrayView( + helper_containers_->ramp_list.data(), ramp_list_lengh)); + return rtc::ArrayView( + helper_containers_->audio_to_mix.data(), audio_to_mix_count); } bool AudioMixerImpl::GetAudioSourceMixabilityStatusForTest( AudioMixerImpl::Source* audio_source) const { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); MutexLock lock(&mutex_); const auto iter = FindSourceInList(audio_source, &audio_source_list_); diff --git a/modules/audio_mixer/audio_mixer_impl.h b/modules/audio_mixer/audio_mixer_impl.h index 57b1f5e4a0..0a13082725 100644 --- a/modules/audio_mixer/audio_mixer_impl.h +++ b/modules/audio_mixer/audio_mixer_impl.h @@ -16,6 +16,7 @@ #include #include +#include "api/array_view.h" #include "api/audio/audio_frame.h" #include "api/audio/audio_mixer.h" #include "api/scoped_refptr.h" @@ -28,22 +29,9 @@ namespace webrtc { -typedef std::vector AudioFrameList; - class AudioMixerImpl : public AudioMixer { public: - struct SourceStatus { - SourceStatus(Source* audio_source, bool is_mixed, float gain) - : audio_source(audio_source), is_mixed(is_mixed), gain(gain) {} - Source* audio_source = nullptr; - bool is_mixed = false; - float gain = 0.0f; - - // A frame that will be passed to audio_source->GetAudioFrameWithInfo. - AudioFrame audio_frame; - }; - - using SourceStatusList = std::vector>; + struct SourceStatus; // AudioProcessing only accepts 10 ms frames. static const int kFrameDurationInMs = 10; @@ -75,32 +63,29 @@ class AudioMixerImpl : public AudioMixer { bool use_limiter); private: - // Set mixing frequency through OutputFrequencyCalculator. - void CalculateOutputFrequency(); - // Get mixing frequency. - int OutputFrequency() const; + struct HelperContainers; // Compute what audio sources to mix from audio_source_list_. Ramp // in and out. Update mixed status. Mixes up to // kMaximumAmountOfMixedAudioSources audio sources. - AudioFrameList GetAudioFromSources() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + rtc::ArrayView GetAudioFromSources(int output_frequency) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // The critical section lock guards audio source insertion and // removal, which can be done from any thread. The race checker // checks that mixing is done sequentially. mutable Mutex mutex_; - rtc::RaceChecker race_checker_; std::unique_ptr output_rate_calculator_; - // The current sample frequency and sample size when mixing. - int output_frequency_ RTC_GUARDED_BY(race_checker_); - size_t sample_size_ RTC_GUARDED_BY(race_checker_); - // List of all audio sources. Note all lists are disjunct - SourceStatusList audio_source_list_ RTC_GUARDED_BY(mutex_); // May be mixed. + // List of all audio sources. + std::vector> audio_source_list_ + RTC_GUARDED_BY(mutex_); + const std::unique_ptr helper_containers_ + RTC_GUARDED_BY(mutex_); // Component that handles actual adding of audio frames. - FrameCombiner frame_combiner_ RTC_GUARDED_BY(race_checker_); + FrameCombiner frame_combiner_; RTC_DISALLOW_COPY_AND_ASSIGN(AudioMixerImpl); }; diff --git a/modules/audio_mixer/audio_mixer_impl_unittest.cc b/modules/audio_mixer/audio_mixer_impl_unittest.cc index 383771ce60..18a4384b66 100644 --- a/modules/audio_mixer/audio_mixer_impl_unittest.cc +++ b/modules/audio_mixer/audio_mixer_impl_unittest.cc @@ -105,7 +105,8 @@ class MockMixerAudioSource : public ::testing::NiceMock { class CustomRateCalculator : public OutputRateCalculator { public: explicit CustomRateCalculator(int rate) : rate_(rate) {} - int CalculateOutputRate(const std::vector& preferred_rates) override { + int CalculateOutputRateFromRange( + rtc::ArrayView preferred_rates) override { return rate_; } @@ -598,8 +599,8 @@ TEST(AudioMixer, MultipleChannelsManyParticipants) { class HighOutputRateCalculator : public OutputRateCalculator { public: static const int kDefaultFrequency = 76000; - int CalculateOutputRate( - const std::vector& preferred_sample_rates) override { + int CalculateOutputRateFromRange( + rtc::ArrayView preferred_sample_rates) override { return kDefaultFrequency; } ~HighOutputRateCalculator() override {} diff --git a/modules/audio_mixer/default_output_rate_calculator.cc b/modules/audio_mixer/default_output_rate_calculator.cc index 57d88b6384..5f24b653a3 100644 --- a/modules/audio_mixer/default_output_rate_calculator.cc +++ b/modules/audio_mixer/default_output_rate_calculator.cc @@ -18,14 +18,14 @@ namespace webrtc { -int DefaultOutputRateCalculator::CalculateOutputRate( - const std::vector& preferred_sample_rates) { +int DefaultOutputRateCalculator::CalculateOutputRateFromRange( + rtc::ArrayView preferred_sample_rates) { if (preferred_sample_rates.empty()) { return DefaultOutputRateCalculator::kDefaultFrequency; } using NativeRate = AudioProcessing::NativeRate; const int maximal_frequency = *std::max_element( - preferred_sample_rates.begin(), preferred_sample_rates.end()); + preferred_sample_rates.cbegin(), preferred_sample_rates.cend()); RTC_DCHECK_LE(NativeRate::kSampleRate8kHz, maximal_frequency); RTC_DCHECK_GE(NativeRate::kSampleRate48kHz, maximal_frequency); diff --git a/modules/audio_mixer/default_output_rate_calculator.h b/modules/audio_mixer/default_output_rate_calculator.h index 2d34f3489c..a7aaf681a9 100644 --- a/modules/audio_mixer/default_output_rate_calculator.h +++ b/modules/audio_mixer/default_output_rate_calculator.h @@ -13,6 +13,7 @@ #include +#include "api/array_view.h" #include "modules/audio_mixer/output_rate_calculator.h" namespace webrtc { @@ -25,8 +26,8 @@ class DefaultOutputRateCalculator : public OutputRateCalculator { // sample rates. A native rate is one in // AudioProcessing::NativeRate. If |preferred_sample_rates| is // empty, returns |kDefaultFrequency|. - int CalculateOutputRate( - const std::vector& preferred_sample_rates) override; + int CalculateOutputRateFromRange( + rtc::ArrayView preferred_sample_rates) override; ~DefaultOutputRateCalculator() override {} }; diff --git a/modules/audio_mixer/frame_combiner.cc b/modules/audio_mixer/frame_combiner.cc index f7c6a0c32e..e184506b4c 100644 --- a/modules/audio_mixer/frame_combiner.cc +++ b/modules/audio_mixer/frame_combiner.cc @@ -35,7 +35,7 @@ using MixingBuffer = std::array, FrameCombiner::kMaximumNumberOfChannels>; -void SetAudioFrameFields(const std::vector& mix_list, +void SetAudioFrameFields(rtc::ArrayView mix_list, size_t number_of_channels, int sample_rate, size_t number_of_streams, @@ -61,7 +61,7 @@ void SetAudioFrameFields(const std::vector& mix_list, } } -void MixFewFramesWithNoLimiter(const std::vector& mix_list, +void MixFewFramesWithNoLimiter(rtc::ArrayView mix_list, AudioFrame* audio_frame_for_mixing) { if (mix_list.empty()) { audio_frame_for_mixing->Mute(); @@ -74,7 +74,7 @@ void MixFewFramesWithNoLimiter(const std::vector& mix_list, audio_frame_for_mixing->mutable_data()); } -void MixToFloatFrame(const std::vector& mix_list, +void MixToFloatFrame(rtc::ArrayView mix_list, size_t samples_per_channel, size_t number_of_channels, MixingBuffer* mixing_buffer) { @@ -140,7 +140,7 @@ FrameCombiner::FrameCombiner(bool use_limiter) FrameCombiner::~FrameCombiner() = default; -void FrameCombiner::Combine(const std::vector& mix_list, +void FrameCombiner::Combine(rtc::ArrayView mix_list, size_t number_of_channels, int sample_rate, size_t number_of_streams, @@ -195,9 +195,10 @@ void FrameCombiner::Combine(const std::vector& mix_list, InterleaveToAudioFrame(mixing_buffer_view, audio_frame_for_mixing); } -void FrameCombiner::LogMixingStats(const std::vector& mix_list, - int sample_rate, - size_t number_of_streams) const { +void FrameCombiner::LogMixingStats( + rtc::ArrayView mix_list, + int sample_rate, + size_t number_of_streams) const { // Log every second. uma_logging_counter_++; if (uma_logging_counter_ > 1000 / AudioMixerImpl::kFrameDurationInMs) { diff --git a/modules/audio_mixer/frame_combiner.h b/modules/audio_mixer/frame_combiner.h index d989d02c36..9ddf81e41e 100644 --- a/modules/audio_mixer/frame_combiner.h +++ b/modules/audio_mixer/frame_combiner.h @@ -14,6 +14,7 @@ #include #include +#include "api/array_view.h" #include "api/audio/audio_frame.h" #include "modules/audio_processing/agc2/limiter.h" @@ -32,7 +33,7 @@ class FrameCombiner { // because 'mix_list' can be empty. The parameter // 'number_of_streams' is used for determining whether to pass the // data through a limiter. - void Combine(const std::vector& mix_list, + void Combine(rtc::ArrayView mix_list, size_t number_of_channels, int sample_rate, size_t number_of_streams, @@ -46,7 +47,7 @@ class FrameCombiner { kMaximumNumberOfChannels>; private: - void LogMixingStats(const std::vector& mix_list, + void LogMixingStats(rtc::ArrayView mix_list, int sample_rate, size_t number_of_streams) const; diff --git a/modules/audio_mixer/output_rate_calculator.h b/modules/audio_mixer/output_rate_calculator.h index cb3ca96b2d..d32fb292c5 100644 --- a/modules/audio_mixer/output_rate_calculator.h +++ b/modules/audio_mixer/output_rate_calculator.h @@ -13,14 +13,29 @@ #include +#include "api/array_view.h" + namespace webrtc { // Decides the sample rate of a mixing iteration given the preferred // sample rates of the sources. class OutputRateCalculator { public: + virtual int CalculateOutputRateFromRange( + rtc::ArrayView preferred_sample_rates) { + // TODO(olka): Temporary workaround to reslove client dependencies. + std::vector sample_rates(preferred_sample_rates.cbegin(), + preferred_sample_rates.cend()); + return CalculateOutputRate(sample_rates); + } + + // TODO(olka) to be removed as soon as the clients are switched to + // CalculateOutputRateFromRange() virtual int CalculateOutputRate( - const std::vector& preferred_sample_rates) = 0; + const std::vector& preferred_sample_rates) { + return CalculateOutputRateFromRange(preferred_sample_rates); + } + virtual ~OutputRateCalculator() {} };