From 566d820e004700df9a152602cbda44fac839d7e7 Mon Sep 17 00:00:00 2001 From: michaelt Date: Thu, 12 Jan 2017 10:17:38 -0800 Subject: [PATCH] Update smoothed bitrate. BUG=webrtc:6443 Review-Url: https://codereview.webrtc.org/2546493002 Cr-Commit-Position: refs/heads/master@{#16036} --- webrtc/base/BUILD.gn | 31 ++++-- webrtc/media/BUILD.gn | 2 +- webrtc/modules/BUILD.gn | 4 +- webrtc/modules/audio_coding/BUILD.gn | 1 + .../audio_coding/acm2/audio_coding_module.cc | 3 +- .../audio_coding/codecs/audio_encoder.cc | 10 +- .../audio_coding/codecs/audio_encoder.h | 12 ++- .../codecs/cng/audio_encoder_cng.cc | 8 +- .../codecs/cng/audio_encoder_cng.h | 4 +- .../codecs/cng/audio_encoder_cng_unittest.cc | 5 +- .../codecs/mock/mock_audio_encoder.h | 5 +- .../codecs/opus/audio_encoder_opus.cc | 55 +++++++--- .../codecs/opus/audio_encoder_opus.h | 16 ++- .../opus/audio_encoder_opus_unittest.cc | 101 ++++++++++++------ .../codecs/red/audio_encoder_copy_red.cc | 8 +- .../codecs/red/audio_encoder_copy_red.h | 4 +- .../red/audio_encoder_copy_red_unittest.cc | 5 +- .../neteq/audio_decoder_unittest.cc | 2 +- webrtc/stats/BUILD.gn | 2 +- webrtc/voice_engine/channel.cc | 26 +---- webrtc/voice_engine/channel.h | 3 - 21 files changed, 199 insertions(+), 108 deletions(-) diff --git a/webrtc/base/BUILD.gn b/webrtc/base/BUILD.gn index c42f0564ea..ba31c3be2c 100644 --- a/webrtc/base/BUILD.gn +++ b/webrtc/base/BUILD.gn @@ -706,6 +706,26 @@ if (rtc_include_tests) { } } + rtc_source_set("rtc_base_tests_main") { + testonly = true + sources = [ + "unittest_main.cc", + ] + public_configs = [ ":rtc_base_tests_utils_exported_config" ] + deps = [ + ":rtc_base_tests_utils", + ] + public_deps = [ + "//testing/gmock", + "//testing/gtest", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + rtc_source_set("rtc_base_tests_utils") { testonly = true sources = [ @@ -731,7 +751,6 @@ if (rtc_include_tests) { "testechoserver.h", "testutils.h", "timedelta.h", - "unittest_main.cc", ] configs += [ ":rtc_base_tests_utils_warnings_config" ] public_configs = [ ":rtc_base_tests_utils_exported_config" ] @@ -763,7 +782,7 @@ if (rtc_include_tests) { ] deps = [ ":rtc_base", - ":rtc_base_tests_utils", + ":rtc_base_tests_main", "//testing/gtest", ] if (is_win) { @@ -818,7 +837,7 @@ if (rtc_include_tests) { ] deps = [ ":rtc_base_approved", - ":rtc_base_tests_utils", + ":rtc_base_tests_main", ] if (!build_with_chromium && is_clang) { # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). @@ -834,7 +853,7 @@ if (rtc_include_tests) { "weak_ptr_unittest.cc", ] deps = [ - ":rtc_base_tests_utils", + ":rtc_base_tests_main", ":rtc_task_queue", ] if (!build_with_chromium && is_clang) { @@ -851,7 +870,7 @@ if (rtc_include_tests) { ] deps = [ ":rtc_analytics", - ":rtc_base_tests_utils", + ":rtc_base_tests_main", ] } @@ -912,7 +931,7 @@ if (rtc_include_tests) { ] } deps = [ - ":rtc_base_tests_utils", + ":rtc_base_tests_main", ] public_deps = [ ":rtc_base", diff --git a/webrtc/media/BUILD.gn b/webrtc/media/BUILD.gn index 3aee5b7fe7..4e54d5aae4 100644 --- a/webrtc/media/BUILD.gn +++ b/webrtc/media/BUILD.gn @@ -271,7 +271,7 @@ if (rtc_include_tests) { } deps += [ - "../base:rtc_base_tests_utils", + "../base:rtc_base_tests_main", "//testing/gtest", ] public_deps += [ "//testing/gmock" ] diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 37b5b12ff3..f3b2ac9aaa 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -631,7 +631,7 @@ if (rtc_include_tests) { # TODO(jschuh): bugs.webrtc.org/1348: fix this warning. configs += [ "//build/config/compiler:no_size_t_to_int_warning" ] - if (!build_with_chromium && is_clang) { + if ((!build_with_chromium || is_win) && is_clang) { # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] } @@ -648,9 +648,11 @@ if (rtc_include_tests) { "..:webrtc_common", "../api:transport_api", "../base:rtc_base", # TODO(kjellander): Cleanup in bugs.webrtc.org/3806. + "../base:rtc_base_tests_utils", "../common_audio", "../common_video", "../system_wrappers", + "../system_wrappers:metrics_default", "../test:rtp_test_utils", "../test:test_common", "../test:test_main", diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn index b78ba7936a..c38cd4761f 100644 --- a/webrtc/modules/audio_coding/BUILD.gn +++ b/webrtc/modules/audio_coding/BUILD.gn @@ -839,6 +839,7 @@ rtc_static_library("webrtc_opus") { "../..:webrtc_common", "../../base:rtc_analytics", "../../base:rtc_base_approved", + "../../common_audio", "../../system_wrappers", ] public_deps = [ diff --git a/webrtc/modules/audio_coding/acm2/audio_coding_module.cc b/webrtc/modules/audio_coding/acm2/audio_coding_module.cc index fe3b252381..44450f2f3a 100644 --- a/webrtc/modules/audio_coding/acm2/audio_coding_module.cc +++ b/webrtc/modules/audio_coding/acm2/audio_coding_module.cc @@ -648,7 +648,8 @@ int AudioCodingModuleImpl::SendFrequency() const { void AudioCodingModuleImpl::SetBitRate(int bitrate_bps) { rtc::CritScope lock(&acm_crit_sect_); if (encoder_stack_) { - encoder_stack_->OnReceivedTargetAudioBitrate(bitrate_bps); + encoder_stack_->OnReceivedUplinkBandwidth(bitrate_bps, + rtc::Optional()); } } diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.cc b/webrtc/modules/audio_coding/codecs/audio_encoder.cc index 7b7325dc55..751f9cf6ff 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.cc @@ -72,12 +72,16 @@ bool AudioEncoder::EnableAudioNetworkAdaptor(const std::string& config_string, void AudioEncoder::DisableAudioNetworkAdaptor() {} -void AudioEncoder::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {} - void AudioEncoder::OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) {} -void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {} +void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) { + OnReceivedUplinkBandwidth(target_audio_bitrate_bps, rtc::Optional()); +} + +void AudioEncoder::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms) {} void AudioEncoder::OnReceivedRtt(int rtt_ms) {} diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.h b/webrtc/modules/audio_coding/codecs/audio_encoder.h index 5132c2ed8b..e1b3d82a5b 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.h +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.h @@ -17,6 +17,7 @@ #include "webrtc/base/array_view.h" #include "webrtc/base/buffer.h" #include "webrtc/base/deprecation.h" +#include "webrtc/base/optional.h" #include "webrtc/typedefs.h" namespace webrtc { @@ -167,16 +168,19 @@ class AudioEncoder { // Disables audio network adaptor. virtual void DisableAudioNetworkAdaptor(); - // Provides uplink bandwidth to this encoder to allow it to adapt. - virtual void OnReceivedUplinkBandwidth(int uplink_bandwidth_bps); - // Provides uplink packet loss fraction to this encoder to allow it to adapt. // |uplink_packet_loss_fraction| is in the range [0.0, 1.0]. virtual void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction); // Provides target audio bitrate to this encoder to allow it to adapt. - virtual void OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps); + virtual void OnReceivedTargetAudioBitrate(int target_bps); + + // Provides target audio bitrate and corresponding probing interval of + // the bandwidth estimator to this encoder to allow it to adapt. + virtual void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms); // Provides RTT to this encoder to allow it to adapt. virtual void OnReceivedRtt(int rtt_ms); diff --git a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc index 1beebd522d..bcb11eef8b 100644 --- a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc +++ b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc @@ -190,9 +190,11 @@ void AudioEncoderCng::OnReceivedUplinkPacketLossFraction( uplink_packet_loss_fraction); } -void AudioEncoderCng::OnReceivedTargetAudioBitrate( - int target_audio_bitrate_bps) { - speech_encoder_->OnReceivedTargetAudioBitrate(target_audio_bitrate_bps); +void AudioEncoderCng::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms) { + speech_encoder_->OnReceivedUplinkBandwidth(target_audio_bitrate_bps, + probing_interval_ms); } AudioEncoder::EncodedInfo AudioEncoderCng::EncodePassive( diff --git a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h index 8cec52c394..0075bd02fb 100644 --- a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h +++ b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h @@ -65,7 +65,9 @@ class AudioEncoderCng final : public AudioEncoder { override; void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) override; - void OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms) override; private: EncodedInfo EncodePassive(size_t frames_to_encode, diff --git a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc index db58b0dcc3..4f7b2fc61a 100644 --- a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng_unittest.cc @@ -219,8 +219,9 @@ TEST_F(AudioEncoderCngTest, CheckFrameSizePropagation) { TEST_F(AudioEncoderCngTest, CheckTargetAudioBitratePropagation) { CreateCng(MakeCngConfig()); - EXPECT_CALL(*mock_encoder_, OnReceivedTargetAudioBitrate(4711)); - cng_->OnReceivedTargetAudioBitrate(4711); + EXPECT_CALL(*mock_encoder_, + OnReceivedUplinkBandwidth(4711, rtc::Optional())); + cng_->OnReceivedUplinkBandwidth(4711, rtc::Optional()); } TEST_F(AudioEncoderCngTest, CheckPacketLossFractionPropagation) { diff --git a/webrtc/modules/audio_coding/codecs/mock/mock_audio_encoder.h b/webrtc/modules/audio_coding/codecs/mock/mock_audio_encoder.h index e277200f7e..4c1b5f7a27 100644 --- a/webrtc/modules/audio_coding/codecs/mock/mock_audio_encoder.h +++ b/webrtc/modules/audio_coding/codecs/mock/mock_audio_encoder.h @@ -41,8 +41,9 @@ class MockAudioEncoder : public AudioEncoder { MOCK_METHOD1(SetMaxPlaybackRate, void(int frequency_hz)); MOCK_METHOD1(SetMaxBitrate, void(int max_bps)); MOCK_METHOD1(SetMaxPayloadSize, void(int max_payload_size_bytes)); - MOCK_METHOD1(OnReceivedTargetAudioBitrate, - void(int target_audio_bitrate_bps)); + MOCK_METHOD2(OnReceivedUplinkBandwidth, + void(int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms)); MOCK_METHOD1(OnReceivedUplinkPacketLossFraction, void(float uplink_packet_loss_fraction)); diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index c6bac621c3..78f8a4b455 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -17,11 +17,11 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/safe_conversions.h" +#include "webrtc/base/timeutils.h" #include "webrtc/common_types.h" #include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" #include "webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h" #include "webrtc/modules/audio_coding/codecs/opus/opus_interface.h" -#include "webrtc/system_wrappers/include/clock.h" #include "webrtc/system_wrappers/include/field_trial.h" namespace webrtc { @@ -172,18 +172,23 @@ rtc::Optional AudioEncoderOpus::Config::GetNewComplexity() const { AudioEncoderOpus::AudioEncoderOpus( const Config& config, - AudioNetworkAdaptorCreator&& audio_network_adaptor_creator) + AudioNetworkAdaptorCreator&& audio_network_adaptor_creator, + std::unique_ptr bitrate_smoother) : packet_loss_rate_(0.0), inst_(nullptr), packet_loss_fraction_smoother_(new PacketLossFractionSmoother( - config.clock ? config.clock : Clock::GetRealTimeClock())), + config.clock)), audio_network_adaptor_creator_( audio_network_adaptor_creator ? std::move(audio_network_adaptor_creator) : [this](const std::string& config_string, const Clock* clock) { return DefaultAudioNetworkAdaptorCreator(config_string, clock); - }) { + }), + bitrate_smoother_(bitrate_smoother + ? std::move(bitrate_smoother) : std::unique_ptr( + // We choose 5sec as initial time constant due to empirical data. + new SmoothingFilterImpl(5000, config.clock))) { RTC_CHECK(RecreateEncoderInstance(config)); } @@ -272,13 +277,6 @@ void AudioEncoderOpus::DisableAudioNetworkAdaptor() { audio_network_adaptor_.reset(nullptr); } -void AudioEncoderOpus::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) { - if (!audio_network_adaptor_) - return; - audio_network_adaptor_->SetUplinkBandwidth(uplink_bandwidth_bps); - ApplyAudioNetworkAdaptor(); -} - void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) { if (!audio_network_adaptor_) { @@ -291,10 +289,26 @@ void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction( ApplyAudioNetworkAdaptor(); } -void AudioEncoderOpus::OnReceivedTargetAudioBitrate( - int target_audio_bitrate_bps) { +void AudioEncoderOpus::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms) { if (audio_network_adaptor_) { audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps); + // We give smoothed bitrate allocation to audio network adaptor as + // the uplink bandwidth. + // The probing spikes should not affect the bitrate smoother more than 25%. + // To simplify the calculations we use a step response as input signal. + // The step response of an exponential filter is + // u(t) = 1 - e^(-t / time_constant). + // In order to limit the affect of a BWE spike within 25% of its value + // before + // the next probing, we would choose a time constant that fulfills + // 1 - e^(-probing_interval_ms / time_constant) < 0.25 + // Then 4 * probing_interval_ms is a good choice. + if (probing_interval_ms) + bitrate_smoother_->SetTimeConstantMs(*probing_interval_ms * 4); + bitrate_smoother_->AddSample(target_audio_bitrate_bps); + ApplyAudioNetworkAdaptor(); } else if (webrtc::field_trial::FindFullName( "WebRTC-SendSideBwe-WithOverhead") == "Enabled") { @@ -354,6 +368,7 @@ AudioEncoder::EncodedInfo AudioEncoderOpus::EncodeImpl( uint32_t rtp_timestamp, rtc::ArrayView audio, rtc::Buffer* encoded) { + MaybeUpdateUplinkBandwidth(); if (input_buffer_.empty()) first_timestamp_in_buffer_ = rtp_timestamp; @@ -521,4 +536,18 @@ AudioEncoderOpus::DefaultAudioNetworkAdaptorCreator( GetTargetBitrate(), config_.fec_enabled, GetDtx(), clock))); } +void AudioEncoderOpus::MaybeUpdateUplinkBandwidth() { + if (audio_network_adaptor_) { + int64_t now_ms = rtc::TimeMillis(); + if (!bitrate_smoother_last_update_time_ || + now_ms - *bitrate_smoother_last_update_time_ >= + config_.uplink_bandwidth_update_interval_ms) { + rtc::Optional smoothed_bitrate = bitrate_smoother_->GetAverage(); + if (smoothed_bitrate) + audio_network_adaptor_->SetUplinkBandwidth(*smoothed_bitrate); + bitrate_smoother_last_update_time_ = rtc::Optional(now_ms); + } + } +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h index 0864c7bfb5..82595abc52 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -18,6 +18,7 @@ #include "webrtc/base/constructormagic.h" #include "webrtc/base/optional.h" +#include "webrtc/common_audio/smoothing_filter.h" #include "webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" #include "webrtc/modules/audio_coding/codecs/opus/opus_interface.h" #include "webrtc/modules/audio_coding/codecs/audio_encoder.h" @@ -62,7 +63,8 @@ class AudioEncoderOpus final : public AudioEncoder { int complexity_threshold_window_bps = 1500; bool dtx_enabled = false; std::vector supported_frame_lengths_ms; - const Clock* clock = nullptr; + const Clock* clock = Clock::GetRealTimeClock(); + int uplink_bandwidth_update_interval_ms = 200; private: #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_ARCH_ARM) @@ -79,7 +81,8 @@ class AudioEncoderOpus final : public AudioEncoder { const Clock*)>; AudioEncoderOpus( const Config& config, - AudioNetworkAdaptorCreator&& audio_network_adaptor_creator = nullptr); + AudioNetworkAdaptorCreator&& audio_network_adaptor_creator = nullptr, + std::unique_ptr bitrate_smoother = nullptr); explicit AudioEncoderOpus(const CodecInst& codec_inst); @@ -105,10 +108,11 @@ class AudioEncoderOpus final : public AudioEncoder { bool EnableAudioNetworkAdaptor(const std::string& config_string, const Clock* clock) override; void DisableAudioNetworkAdaptor() override; - void OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) override; void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) override; - void OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms) override; void OnReceivedRtt(int rtt_ms) override; void OnReceivedOverhead(size_t overhead_bytes_per_packet) override; void SetReceiverFrameLengthRange(int min_frame_length_ms, @@ -149,6 +153,8 @@ class AudioEncoderOpus final : public AudioEncoder { const std::string& config_string, const Clock* clock) const; + void MaybeUpdateUplinkBandwidth(); + Config config_; float packet_loss_rate_; std::vector input_buffer_; @@ -161,6 +167,8 @@ class AudioEncoderOpus final : public AudioEncoder { AudioNetworkAdaptorCreator audio_network_adaptor_creator_; std::unique_ptr audio_network_adaptor_; rtc::Optional overhead_bytes_per_packet_; + const std::unique_ptr bitrate_smoother_; + rtc::Optional bitrate_smoother_last_update_time_; RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderOpus); }; diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc index 5994a532ce..3e80595df2 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc @@ -8,9 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include #include #include "webrtc/base/checks.h" +#include "webrtc/base/fakeclock.h" +#include "webrtc/common_audio/mocks/mock_smoothing_filter.h" #include "webrtc/common_types.h" #include "webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h" #include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h" @@ -42,8 +45,10 @@ AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) { struct AudioEncoderOpusStates { std::shared_ptr mock_audio_network_adaptor; + MockSmoothingFilter* mock_bitrate_smoother; std::unique_ptr encoder; std::unique_ptr simulated_clock; + AudioEncoderOpus::Config config; }; AudioEncoderOpusStates CreateCodec(size_t num_channels) { @@ -68,11 +73,15 @@ AudioEncoderOpusStates CreateCodec(size_t num_channels) { CodecInst codec_inst = kDefaultOpusSettings; codec_inst.channels = num_channels; - auto config = CreateConfig(codec_inst); + states.config = CreateConfig(codec_inst); + std::unique_ptr bitrate_smoother( + new MockSmoothingFilter()); + states.mock_bitrate_smoother = bitrate_smoother.get(); states.simulated_clock.reset(new SimulatedClock(kInitialTimeUs)); - config.clock = states.simulated_clock.get(); + states.config.clock = states.simulated_clock.get(); - states.encoder.reset(new AudioEncoderOpus(config, std::move(creator))); + states.encoder.reset(new AudioEncoderOpus(states.config, std::move(creator), + std::move(bitrate_smoother))); return states; } @@ -153,26 +162,30 @@ TEST(AudioEncoderOpusTest, ToggleDtx) { } TEST(AudioEncoderOpusTest, - OnReceivedTargetAudioBitrateWithoutAudioNetworkAdaptor) { + OnReceivedUplinkBandwidthWithoutAudioNetworkAdaptor) { auto states = CreateCodec(1); // Constants are replicated from audio_states.encoderopus.cc. const int kMinBitrateBps = 500; const int kMaxBitrateBps = 512000; // Set a too low bitrate. - states.encoder->OnReceivedTargetAudioBitrate(kMinBitrateBps - 1); + states.encoder->OnReceivedUplinkBandwidth(kMinBitrateBps - 1, + rtc::Optional()); EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); // Set a too high bitrate. - states.encoder->OnReceivedTargetAudioBitrate(kMaxBitrateBps + 1); + states.encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps + 1, + rtc::Optional()); EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); // Set the minimum rate. - states.encoder->OnReceivedTargetAudioBitrate(kMinBitrateBps); + states.encoder->OnReceivedUplinkBandwidth(kMinBitrateBps, + rtc::Optional()); EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); // Set the maximum rate. - states.encoder->OnReceivedTargetAudioBitrate(kMaxBitrateBps); + states.encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps, + rtc::Optional()); EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); // Set rates from 1000 up to 32000 bps. for (int rate = 1000; rate <= 32000; rate += 1000) { - states.encoder->OnReceivedTargetAudioBitrate(rate); + states.encoder->OnReceivedUplinkBandwidth(rate, rtc::Optional()); EXPECT_EQ(rate, states.encoder->GetTargetBitrate()); } } @@ -250,23 +263,6 @@ TEST(AudioEncoderOpusTest, SetReceiverFrameLengthRange) { EXPECT_THAT(states.encoder->supported_frame_lengths_ms(), ElementsAre(20)); } -TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedUplinkBandwidth) { - auto states = CreateCodec(2); - states.encoder->EnableAudioNetworkAdaptor("", nullptr); - - auto config = CreateEncoderRuntimeConfig(); - EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) - .WillOnce(Return(config)); - - // Since using mock audio network adaptor, any bandwidth value is fine. - constexpr int kUplinkBandwidth = 50000; - EXPECT_CALL(**states.mock_audio_network_adaptor, - SetUplinkBandwidth(kUplinkBandwidth)); - states.encoder->OnReceivedUplinkBandwidth(kUplinkBandwidth); - - CheckEncoderRuntimeConfig(states.encoder.get(), config); -} - TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedUplinkPacketLossFraction) { auto states = CreateCodec(2); @@ -285,8 +281,7 @@ TEST(AudioEncoderOpusTest, CheckEncoderRuntimeConfig(states.encoder.get(), config); } -TEST(AudioEncoderOpusTest, - InvokeAudioNetworkAdaptorOnReceivedTargetAudioBitrate) { +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedUplinkBandwidth) { auto states = CreateCodec(2); states.encoder->EnableAudioNetworkAdaptor("", nullptr); @@ -296,9 +291,14 @@ TEST(AudioEncoderOpusTest, // Since using mock audio network adaptor, any target audio bitrate is fine. constexpr int kTargetAudioBitrate = 30000; + constexpr int64_t kProbingIntervalMs = 3000; EXPECT_CALL(**states.mock_audio_network_adaptor, SetTargetAudioBitrate(kTargetAudioBitrate)); - states.encoder->OnReceivedTargetAudioBitrate(kTargetAudioBitrate); + EXPECT_CALL(*states.mock_bitrate_smoother, + SetTimeConstantMs(kProbingIntervalMs * 4)); + EXPECT_CALL(*states.mock_bitrate_smoother, AddSample(kTargetAudioBitrate)); + states.encoder->OnReceivedUplinkBandwidth( + kTargetAudioBitrate, rtc::Optional(kProbingIntervalMs)); CheckEncoderRuntimeConfig(states.encoder.get(), config); } @@ -367,7 +367,8 @@ TEST(AudioEncoderOpusTest, DoNotInvokeSetTargetBitrateIfOverheadUnknown) { auto states = CreateCodec(2); - states.encoder->OnReceivedTargetAudioBitrate(kDefaultOpusSettings.rate * 2); + states.encoder->OnReceivedUplinkBandwidth(kDefaultOpusSettings.rate * 2, + rtc::Optional()); // Since |OnReceivedOverhead| has not been called, the codec bitrate should // not change. @@ -384,7 +385,8 @@ TEST(AudioEncoderOpusTest, OverheadRemovedFromTargetAudioBitrate) { states.encoder->OnReceivedOverhead(kOverheadBytesPerPacket); constexpr int kTargetBitrateBps = 40000; - states.encoder->OnReceivedTargetAudioBitrate(kTargetBitrateBps); + states.encoder->OnReceivedUplinkBandwidth(kTargetBitrateBps, + rtc::Optional()); int packet_rate = rtc::CheckedDivExact(48000, kDefaultOpusSettings.pacsize); EXPECT_EQ(kTargetBitrateBps - @@ -410,14 +412,16 @@ TEST(AudioEncoderOpusTest, BitrateBounded) { // subtracted. The eventual codec rate should be bounded by |kMinBitrateBps|. int target_bitrate = kOverheadBytesPerPacket * 8 * packet_rate + kMinBitrateBps - 1; - states.encoder->OnReceivedTargetAudioBitrate(target_bitrate); + states.encoder->OnReceivedUplinkBandwidth(target_bitrate, + rtc::Optional()); EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); // Set a target rate that is greater than |kMaxBitrateBps| when overhead is // subtracted. The eventual codec rate should be bounded by |kMaxBitrateBps|. target_bitrate = kOverheadBytesPerPacket * 8 * packet_rate + kMaxBitrateBps + 1; - states.encoder->OnReceivedTargetAudioBitrate(target_bitrate); + states.encoder->OnReceivedUplinkBandwidth(target_bitrate, + rtc::Optional()); EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); } @@ -464,4 +468,35 @@ TEST(AudioEncoderOpusTest, EmptyConfigDoesNotAffectEncoderSettings) { CheckEncoderRuntimeConfig(states.encoder.get(), config); } +TEST(AudioEncoderOpusTest, UpdateUplinkBandwidthInAudioNetworkAdaptor) { + rtc::ScopedFakeClock fake_clock; + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + std::array audio; + audio.fill(0); + rtc::Buffer encoded; + EXPECT_CALL(*states.mock_bitrate_smoother, GetAverage()) + .WillOnce(Return(rtc::Optional(50000))); + EXPECT_CALL(**states.mock_audio_network_adaptor, SetUplinkBandwidth(50000)); + states.encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + + // Repeat update uplink bandwidth tests. + for (int i = 0; i < 5; i++) { + // Don't update till it is time to update again. + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds( + states.config.uplink_bandwidth_update_interval_ms - 1)); + states.encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + + // Update when it is time to update. + EXPECT_CALL(*states.mock_bitrate_smoother, GetAverage()) + .WillOnce(Return(rtc::Optional(40000))); + EXPECT_CALL(**states.mock_audio_network_adaptor, SetUplinkBandwidth(40000)); + fake_clock.AdvanceTime(rtc::TimeDelta::FromMilliseconds(1)); + states.encoder->Encode( + 0, rtc::ArrayView(audio.data(), audio.size()), &encoded); + } +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc index 51b706490b..3faad68db4 100644 --- a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc +++ b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc @@ -126,9 +126,11 @@ void AudioEncoderCopyRed::OnReceivedUplinkPacketLossFraction( uplink_packet_loss_fraction); } -void AudioEncoderCopyRed::OnReceivedTargetAudioBitrate( - int target_audio_bitrate_bps) { - speech_encoder_->OnReceivedTargetAudioBitrate(target_audio_bitrate_bps); +void AudioEncoderCopyRed::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms) { + speech_encoder_->OnReceivedUplinkBandwidth(target_audio_bitrate_bps, + probing_interval_ms); } } // namespace webrtc diff --git a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h index d069cbef15..b3ec08595a 100644 --- a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h +++ b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h @@ -53,7 +53,9 @@ class AudioEncoderCopyRed final : public AudioEncoder { override; void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) override; - void OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + rtc::Optional probing_interval_ms) override; protected: EncodedInfo EncodeImpl(uint32_t rtp_timestamp, diff --git a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc index 8cf02f18d4..fff8ee303c 100644 --- a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc @@ -98,8 +98,9 @@ TEST_F(AudioEncoderCopyRedTest, CheckMaxFrameSizePropagation) { } TEST_F(AudioEncoderCopyRedTest, CheckTargetAudioBitratePropagation) { - EXPECT_CALL(*mock_encoder_, OnReceivedTargetAudioBitrate(4711)); - red_->OnReceivedTargetAudioBitrate(4711); + EXPECT_CALL(*mock_encoder_, + OnReceivedUplinkBandwidth(4711, rtc::Optional())); + red_->OnReceivedUplinkBandwidth(4711, rtc::Optional()); } TEST_F(AudioEncoderCopyRedTest, CheckPacketLossFractionPropagation) { diff --git a/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc b/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc index 0dd76dadf5..deb2a756f7 100644 --- a/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc @@ -469,7 +469,7 @@ TEST_F(AudioDecoderPcmUTest, EncodeDecode) { namespace { int SetAndGetTargetBitrate(AudioEncoder* audio_encoder, int rate) { - audio_encoder->OnReceivedTargetAudioBitrate(rate); + audio_encoder->OnReceivedUplinkBandwidth(rate, rtc::Optional()); return audio_encoder->GetTargetBitrate(); } void TestSetAndGetTargetBitratesWithFixedCodec(AudioEncoder* audio_encoder, diff --git a/webrtc/stats/BUILD.gn b/webrtc/stats/BUILD.gn index 501f9513ac..8fa80879a3 100644 --- a/webrtc/stats/BUILD.gn +++ b/webrtc/stats/BUILD.gn @@ -57,7 +57,7 @@ if (rtc_include_tests) { deps = [ ":rtc_stats", ":rtc_stats_test_utils", - "../base:rtc_base_tests_utils", + "../base:rtc_base_tests_main", "../system_wrappers:metrics_default", "//testing/gmock", ] diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc index 84d7a204b1..c12d87c5bf 100644 --- a/webrtc/voice_engine/channel.cc +++ b/webrtc/voice_engine/channel.cc @@ -926,10 +926,7 @@ Channel::Channel(int32_t channelId, rtp_packet_sender_proxy_(new RtpPacketSenderProxy()), retransmission_rate_limiter_(new RateLimiter(Clock::GetRealTimeClock(), kMaxRetransmissionWindowMs)), - decoder_factory_(config.acm_config.decoder_factory), - // Bitrate smoother can be initialized with arbitrary time constant - // (0 used here). The actual time constant will be set in SetBitRate. - bitrate_smoother_(0, Clock::GetRealTimeClock()) { + decoder_factory_(config.acm_config.decoder_factory) { WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId, _channelId), "Channel::Channel() - ctor"); AudioCodingModule::Config acm_config(config.acm_config); @@ -1332,30 +1329,13 @@ int32_t Channel::SetSendCodec(const CodecInst& codec) { void Channel::SetBitRate(int bitrate_bps, int64_t probing_interval_ms) { WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId), "Channel::SetBitRate(bitrate_bps=%d)", bitrate_bps); - audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { - if (*encoder) - (*encoder)->OnReceivedTargetAudioBitrate(bitrate_bps); - }); - retransmission_rate_limiter_->SetMaxRate(bitrate_bps); - - // We give smoothed bitrate allocation to audio network adaptor as - // the uplink bandwidth. - // The probing spikes should not affect the bitrate smoother more than 25%. - // To simplify the calculations we use a step response as input signal. - // The step response of an exponential filter is - // u(t) = 1 - e^(-t / time_constant). - // In order to limit the affect of a BWE spike within 25% of its value before - // the next probing, we would choose a time constant that fulfills - // 1 - e^(-probing_interval_ms / time_constant) < 0.25 - // Then 4 * probing_interval_ms is a good choice. - bitrate_smoother_.SetTimeConstantMs(probing_interval_ms * 4); - bitrate_smoother_.AddSample(bitrate_bps); audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { if (*encoder) { (*encoder)->OnReceivedUplinkBandwidth( - static_cast(*bitrate_smoother_.GetAverage())); + bitrate_bps, rtc::Optional(probing_interval_ms)); } }); + retransmission_rate_limiter_->SetMaxRate(bitrate_bps); } void Channel::OnIncomingFractionLoss(int fraction_lost) { diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h index fca682141d..d2b24a2ee0 100644 --- a/webrtc/voice_engine/channel.h +++ b/webrtc/voice_engine/channel.h @@ -17,7 +17,6 @@ #include "webrtc/api/call/audio_sink.h" #include "webrtc/base/criticalsection.h" #include "webrtc/base/optional.h" -#include "webrtc/common_audio/smoothing_filter.h" #include "webrtc/common_audio/resampler/include/push_resampler.h" #include "webrtc/common_types.h" #include "webrtc/modules/audio_coding/acm2/codec_manager.h" @@ -554,8 +553,6 @@ class Channel // TODO(ossu): Remove once GetAudioDecoderFactory() is no longer needed. rtc::scoped_refptr decoder_factory_; - - SmoothingFilterImpl bitrate_smoother_; }; } // namespace voe