diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.cc b/webrtc/modules/audio_coding/codecs/audio_encoder.cc index 1216484b4d..956c4e086f 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.cc @@ -77,9 +77,13 @@ void AudioEncoder::DisableAudioNetworkAdaptor() {} void AudioEncoder::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {} void AudioEncoder::OnReceivedUplinkPacketLossFraction( - float uplink_packet_loss_fraction) {} + float uplink_packet_loss_fraction) { + SetProjectedPacketLossRate(uplink_packet_loss_fraction); +} -void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {} +void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) { + SetTargetBitrate(target_audio_bitrate_bps); +} void AudioEncoder::OnReceivedRtt(int rtt_ms) {} diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index ae9dae2fbe..29f85ac470 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -13,6 +13,7 @@ #include #include "webrtc/base/checks.h" +#include "webrtc/base/exp_filter.h" #include "webrtc/base/safe_conversions.h" #include "webrtc/common_types.h" #include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" @@ -24,11 +25,15 @@ namespace webrtc { namespace { -const int kSampleRateHz = 48000; -const int kMinBitrateBps = 500; -const int kMaxBitrateBps = 512000; +constexpr int kSampleRateHz = 48000; +constexpr int kMinBitrateBps = 500; +constexpr int kMaxBitrateBps = 512000; constexpr int kSupportedFrameLengths[] = {20, 60}; +// PacketLossFractionSmoother uses an exponential filter with a time constant +// of -1.0 / ln(0.9999) = 10000 ms. +constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f; + AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) { AudioEncoderOpus::Config config; config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48); @@ -82,6 +87,35 @@ double OptimizePacketLossRate(double new_loss_rate, double old_loss_rate) { } // namespace +class AudioEncoderOpus::PacketLossFractionSmoother { + public: + explicit PacketLossFractionSmoother(const Clock* clock) + : clock_(clock), + last_sample_time_ms_(clock_->TimeInMilliseconds()), + smoother_(kAlphaForPacketLossFractionSmoother) {} + + // Gets the smoothed packet loss fraction. + float GetAverage() const { + float value = smoother_.filtered(); + return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value; + } + + // Add new observation to the packet loss fraction smoother. + void AddSample(float packet_loss_fraction) { + int64_t now_ms = clock_->TimeInMilliseconds(); + smoother_.Apply(static_cast(now_ms - last_sample_time_ms_), + packet_loss_fraction); + last_sample_time_ms_ = now_ms; + } + + private: + const Clock* const clock_; + int64_t last_sample_time_ms_; + + // An exponential filter is used to smooth the packet loss fraction. + rtc::ExpFilter smoother_; +}; + AudioEncoderOpus::Config::Config() = default; AudioEncoderOpus::Config::Config(const Config&) = default; AudioEncoderOpus::Config::~Config() = default; @@ -113,9 +147,11 @@ AudioEncoderOpus::AudioEncoderOpus( AudioNetworkAdaptorCreator&& audio_network_adaptor_creator) : packet_loss_rate_(0.0), inst_(nullptr), + packet_loss_fraction_smoother_(new PacketLossFractionSmoother( + config.clock ? config.clock : Clock::GetRealTimeClock())), audio_network_adaptor_creator_( audio_network_adaptor_creator - ? audio_network_adaptor_creator + ? std::move(audio_network_adaptor_creator) : [this](const std::string& config_string, const Clock* clock) { return DefaultAudioNetworkAdaptorCreator(config_string, clock); @@ -234,8 +270,11 @@ void AudioEncoderOpus::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) { void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) { - if (!audio_network_adaptor_) - return; + if (!audio_network_adaptor_) { + packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction); + float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage(); + return SetProjectedPacketLossRate(average_fraction_loss); + } audio_network_adaptor_->SetUplinkPacketLossFraction( uplink_packet_loss_fraction); ApplyAudioNetworkAdaptor(); @@ -244,7 +283,7 @@ void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction( void AudioEncoderOpus::OnReceivedTargetAudioBitrate( int target_audio_bitrate_bps) { if (!audio_network_adaptor_) - return; + return SetTargetBitrate(target_audio_bitrate_bps); audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps); ApplyAudioNetworkAdaptor(); } diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h index 150a84162c..342668e780 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -49,6 +49,7 @@ class AudioEncoderOpus final : public AudioEncoder { int max_playback_rate_hz = 48000; int complexity = kDefaultComplexity; bool dtx_enabled = false; + const Clock* clock = nullptr; private: #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_ARCH_ARM) @@ -115,6 +116,8 @@ class AudioEncoderOpus final : public AudioEncoder { rtc::Buffer* encoded) override; private: + class PacketLossFractionSmoother; + size_t Num10msFramesPerPacket() const; size_t SamplesPer10msFrame() const; size_t SufficientOutputBufferSize() const; @@ -133,6 +136,7 @@ class AudioEncoderOpus final : public AudioEncoder { uint32_t first_timestamp_in_buffer_; size_t num_channels_to_encode_; int next_frame_length_ms_; + std::unique_ptr packet_loss_fraction_smoother_; AudioNetworkAdaptorCreator audio_network_adaptor_creator_; std::unique_ptr audio_network_adaptor_; diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc index 3e0e1865ef..6a4c47c0e3 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc @@ -15,6 +15,7 @@ #include "webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h" #include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h" #include "webrtc/test/gtest.h" +#include "webrtc/system_wrappers/include/clock.h" namespace webrtc { using ::testing::NiceMock; @@ -23,6 +24,7 @@ using ::testing::Return; namespace { const CodecInst kDefaultOpusSettings = {105, "opus", 48000, 960, 1, 32000}; +constexpr int64_t kInitialTimeUs = 12345678; AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) { AudioEncoderOpus::Config config; @@ -38,6 +40,7 @@ AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) { struct AudioEncoderOpusStates { std::shared_ptr mock_audio_network_adaptor; std::unique_ptr encoder; + std::unique_ptr simulated_clock; }; AudioEncoderOpusStates CreateCodec(size_t num_channels) { @@ -63,6 +66,9 @@ AudioEncoderOpusStates CreateCodec(size_t num_channels) { CodecInst codec_inst = kDefaultOpusSettings; codec_inst.channels = num_channels; auto config = CreateConfig(codec_inst); + states.simulated_clock.reset(new SimulatedClock(kInitialTimeUs)); + config.clock = states.simulated_clock.get(); + states.encoder.reset(new AudioEncoderOpus(config, std::move(creator))); return states; } @@ -303,4 +309,30 @@ TEST(AudioEncoderOpusTest, CheckEncoderRuntimeConfig(states.encoder.get(), config); } +TEST(AudioEncoderOpusTest, + PacketLossFractionSmoothedOnSetUplinkPacketLossFraction) { + auto states = CreateCodec(2); + + // The values are carefully chosen so that if no smoothing is made, the test + // will fail. + constexpr float kPacketLossFraction_1 = 0.02f; + constexpr float kPacketLossFraction_2 = 0.198f; + // |kSecondSampleTimeMs| is chose to ease the calculation since + // 0.9999 ^ 6931 = 0.5. + constexpr float kSecondSampleTimeMs = 6931; + + // First time, no filtering. + states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1); + EXPECT_DOUBLE_EQ(0.01, states.encoder->packet_loss_rate()); + + states.simulated_clock->AdvanceTimeMilliseconds(kSecondSampleTimeMs); + states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2); + + // Now the output of packet loss fraction smoother should be + // (0.02 + 0.198) / 2 = 0.109, which reach the threshold for the optimized + // packet loss rate to increase to 0.05. If no smoothing has been made, the + // optimized packet loss rate should have been increase to 0.1. + EXPECT_DOUBLE_EQ(0.05, states.encoder->packet_loss_rate()); +} + } // namespace webrtc diff --git a/webrtc/modules/audio_coding/include/audio_coding_module.h b/webrtc/modules/audio_coding/include/audio_coding_module.h index 946ad1d82a..e62bbd70f9 100644 --- a/webrtc/modules/audio_coding/include/audio_coding_module.h +++ b/webrtc/modules/audio_coding/include/audio_coding_module.h @@ -252,6 +252,9 @@ class AudioCodingModule { /////////////////////////////////////////////////////////////////////////// // Sets the bitrate to the specified value in bits/sec. If the value is not // supported by the codec, it will choose another appropriate value. + // + // This is only used in test code that rely on old ACM APIs. + // TODO(minyue): Remove it when possible. virtual void SetBitRate(int bitrate_bps) = 0; // int32_t RegisterTransportCallback() @@ -371,6 +374,8 @@ class AudioCodingModule { // -1 if failed to set packet loss rate, // 0 if succeeded. // + // This is only used in test code that rely on old ACM APIs. + // TODO(minyue): Remove it when possible. virtual int SetPacketLossRate(int packet_loss_rate) = 0; /////////////////////////////////////////////////////////////////////////// diff --git a/webrtc/voice_engine/BUILD.gn b/webrtc/voice_engine/BUILD.gn index dd5546be13..0b6bb783b5 100644 --- a/webrtc/voice_engine/BUILD.gn +++ b/webrtc/voice_engine/BUILD.gn @@ -30,8 +30,6 @@ rtc_static_library("voice_engine") { "include/voe_volume_control.h", "monitor_module.cc", "monitor_module.h", - "network_predictor.cc", - "network_predictor.h", "output_mixer.cc", "output_mixer.h", "shared_data.cc", @@ -206,7 +204,6 @@ if (rtc_include_tests) { sources = [ "channel_unittest.cc", - "network_predictor_unittest.cc", "test/channel_transport/udp_socket_manager_unittest.cc", "test/channel_transport/udp_socket_wrapper_unittest.cc", "test/channel_transport/udp_transport_unittest.cc", diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc index 97132995ba..0965963fdd 100644 --- a/webrtc/voice_engine/channel.cc +++ b/webrtc/voice_engine/channel.cc @@ -899,7 +899,6 @@ Channel::Channel(int32_t channelId, _outputSpeechType(AudioFrame::kNormalSpeech), restored_packet_in_use_(false), rtcp_observer_(new VoERtcpObserver(this)), - network_predictor_(new NetworkPredictor(Clock::GetRealTimeClock())), associate_send_channel_(ChannelOwner(nullptr)), pacing_enabled_(config.enable_voice_pacing), feedback_observer_proxy_(new TransportFeedbackProxy()), @@ -1331,19 +1330,18 @@ int32_t Channel::SetSendCodec(const CodecInst& codec) { void Channel::SetBitRate(int bitrate_bps) { WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId), "Channel::SetBitRate(bitrate_bps=%d)", bitrate_bps); - audio_coding_->SetBitRate(bitrate_bps); + audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { + if (*encoder) + (*encoder)->OnReceivedTargetAudioBitrate(bitrate_bps); + }); retransmission_rate_limiter_->SetMaxRate(bitrate_bps); } void Channel::OnIncomingFractionLoss(int fraction_lost) { - network_predictor_->UpdatePacketLossRate(fraction_lost); - uint8_t average_fraction_loss = network_predictor_->GetLossRate(); - - // Normalizes rate to 0 - 100. - if (audio_coding_->SetPacketLossRate(100 * average_fraction_loss / 255) != - 0) { - assert(false); // This should not happen. - } + audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { + if (*encoder) + (*encoder)->OnReceivedUplinkPacketLossFraction(fraction_lost / 255.0f); + }); } int32_t Channel::SetVADStatus(bool enableVAD, @@ -1540,6 +1538,34 @@ int Channel::GetOpusDtx(bool* enabled) { return success; } +bool Channel::EnableAudioNetworkAdaptor(const std::string& config_string) { + bool success = false; + audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { + if (*encoder) { + success = (*encoder)->EnableAudioNetworkAdaptor( + config_string, Clock::GetRealTimeClock()); + } + }); + return success; +} + +void Channel::DisableAudioNetworkAdaptor() { + audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { + if (*encoder) + (*encoder)->DisableAudioNetworkAdaptor(); + }); +} + +void Channel::SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms) { + audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { + if (*encoder) { + (*encoder)->SetReceiverFrameLengthRange(min_frame_length_ms, + max_frame_length_ms); + } + }); +} + int32_t Channel::RegisterExternalTransport(Transport* transport) { WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId), "Channel::RegisterExternalTransport()"); @@ -1700,6 +1726,12 @@ int32_t Channel::ReceivedRTCPPacket(const uint8_t* data, size_t length) { } retransmission_rate_limiter_->SetWindowSize(nack_window_ms); + // Invoke audio encoders OnReceivedRtt(). + audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { + if (*encoder) + (*encoder)->OnReceivedRtt(rtt); + }); + uint32_t ntp_secs = 0; uint32_t ntp_frac = 0; uint32_t rtp_timestamp = 0; diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h index f2d48d8aad..fc123e384f 100644 --- a/webrtc/voice_engine/channel.h +++ b/webrtc/voice_engine/channel.h @@ -32,7 +32,6 @@ #include "webrtc/voice_engine/include/voe_base.h" #include "webrtc/voice_engine/include/voe_network.h" #include "webrtc/voice_engine/level_indicator.h" -#include "webrtc/voice_engine/network_predictor.h" #include "webrtc/voice_engine/shared_data.h" #include "webrtc/voice_engine/voice_engine_defines.h" @@ -209,6 +208,10 @@ class Channel int SetOpusMaxPlaybackRate(int frequency_hz); int SetOpusDtx(bool enable_dtx); int GetOpusDtx(bool* enabled); + bool EnableAudioNetworkAdaptor(const std::string& config_string); + void DisableAudioNetworkAdaptor(); + void SetReceiverFrameLengthRange(int min_frame_length_ms, + int max_frame_length_ms); // VoENetwork int32_t RegisterExternalTransport(Transport* transport); @@ -537,7 +540,6 @@ class Channel bool restored_packet_in_use_; // RtcpBandwidthObserver std::unique_ptr rtcp_observer_; - std::unique_ptr network_predictor_; // An associated send channel. rtc::CriticalSection assoc_send_channel_lock_; ChannelOwner associate_send_channel_ GUARDED_BY(assoc_send_channel_lock_); diff --git a/webrtc/voice_engine/network_predictor.cc b/webrtc/voice_engine/network_predictor.cc deleted file mode 100644 index f8fa2e315c..0000000000 --- a/webrtc/voice_engine/network_predictor.cc +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "webrtc/voice_engine/network_predictor.h" - -namespace webrtc { -namespace voe { - -NetworkPredictor::NetworkPredictor(Clock* clock) - : clock_(clock), - last_loss_rate_update_time_ms_(clock_->TimeInMilliseconds()), - loss_rate_filter_(new rtc::ExpFilter(0.9999f)) { -} - -uint8_t NetworkPredictor::GetLossRate() { - float value = loss_rate_filter_->filtered(); - return (value == rtc::ExpFilter::kValueUndefined) ? 0 : - static_cast(value + 0.5); -} - -void NetworkPredictor::UpdatePacketLossRate(uint8_t loss_rate) { - int64_t now_ms = clock_->TimeInMilliseconds(); - // Update the recursive average filter. - loss_rate_filter_->Apply( - static_cast(now_ms - last_loss_rate_update_time_ms_), - static_cast(loss_rate)); - last_loss_rate_update_time_ms_ = now_ms; -} - -} // namespace voe -} // namespace webrtc diff --git a/webrtc/voice_engine/network_predictor.h b/webrtc/voice_engine/network_predictor.h deleted file mode 100644 index bf08fe98ce..0000000000 --- a/webrtc/voice_engine/network_predictor.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_ -#define WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_ - -#include - -#include "webrtc/base/exp_filter.h" -#include "webrtc/system_wrappers/include/clock.h" - -namespace webrtc { - -namespace voe { - -// NetworkPredictor is to predict network conditions e.g., packet loss rate, for -// sender and/or receiver to cope with changes in the network condition. -class NetworkPredictor { - public: - explicit NetworkPredictor(Clock* clock); - ~NetworkPredictor() {} - - // Gets the predicted packet loss rate. - uint8_t GetLossRate(); - - // Updates the packet loss rate predictor, on receiving a new observation of - // packet loss rate from past. Input packet loss rate should be in the - // interval [0, 255]. - void UpdatePacketLossRate(uint8_t loss_rate); - - private: - Clock* clock_; - int64_t last_loss_rate_update_time_ms_; - - // An exponential filter is used to predict packet loss rate. - std::unique_ptr loss_rate_filter_; -}; - -} // namespace voe -} // namespace webrtc -#endif // WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_ diff --git a/webrtc/voice_engine/network_predictor_unittest.cc b/webrtc/voice_engine/network_predictor_unittest.cc deleted file mode 100644 index 9012379c75..0000000000 --- a/webrtc/voice_engine/network_predictor_unittest.cc +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include - -#include - -#include "webrtc/system_wrappers/include/clock.h" -#include "webrtc/test/gtest.h" -#include "webrtc/voice_engine/network_predictor.h" - -namespace webrtc { -namespace voe { - -class TestNetworkPredictor : public ::testing::Test { - protected: - TestNetworkPredictor() - : clock_(0), - network_predictor_(new NetworkPredictor(&clock_)) {} - SimulatedClock clock_; - std::unique_ptr network_predictor_; -}; - -TEST_F(TestNetworkPredictor, TestPacketLossRateFilter) { - // Test initial packet loss rate estimate is 0. - EXPECT_EQ(0, network_predictor_->GetLossRate()); - network_predictor_->UpdatePacketLossRate(32); - // First time, no filtering. - EXPECT_EQ(32, network_predictor_->GetLossRate()); - clock_.AdvanceTimeMilliseconds(1000); - network_predictor_->UpdatePacketLossRate(40); - float exp = pow(0.9999f, 1000); - float value = 32.0f * exp + (1 - exp) * 40.0f; - EXPECT_EQ(static_cast(value + 0.5f), - network_predictor_->GetLossRate()); -} -} // namespace voe -} // namespace webrtc diff --git a/webrtc/voice_engine/voice_engine.gyp b/webrtc/voice_engine/voice_engine.gyp index c264fe79cd..f27db2f9ec 100644 --- a/webrtc/voice_engine/voice_engine.gyp +++ b/webrtc/voice_engine/voice_engine.gyp @@ -56,8 +56,6 @@ 'channel_proxy.h', 'monitor_module.cc', 'monitor_module.h', - 'network_predictor.cc', - 'network_predictor.h', 'output_mixer.cc', 'output_mixer.h', 'shared_data.cc',