diff --git a/webrtc/audio/audio_send_stream.cc b/webrtc/audio/audio_send_stream.cc index 06c660e499..d16ae181af 100644 --- a/webrtc/audio/audio_send_stream.cc +++ b/webrtc/audio/audio_send_stream.cc @@ -277,17 +277,22 @@ void AudioSendStream::OnPacketFeedbackVector( // TODO(elad.alon): This fails in UT; fix and uncomment. // RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); rtc::Optional plr; + rtc::Optional rplr; { rtc::CritScope lock(&packet_loss_tracker_cs_); packet_loss_tracker_.OnPacketFeedbackVector(packet_feedback_vector); plr = packet_loss_tracker_.GetPacketLossRate(); + rplr = packet_loss_tracker_.GetRecoverablePacketLossRate(); } - // TODO(elad.alon): If PLR goes back to unknown, no indication is given that + // TODO(elad.alon): If R/PLR go back to unknown, no indication is given that // the previously sent value is no longer relevant. This will be taken care // of with some refactoring which is now being done. if (plr) { channel_proxy_->OnTwccBasedUplinkPacketLossRate(*plr); } + if (rplr) { + channel_proxy_->OnRecoverableUplinkPacketLossRate(*rplr); + } } const webrtc::AudioSendStream::Config& AudioSendStream::config() const { diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc b/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc index 612d2d35cc..e1952f4898 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc +++ b/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.cc @@ -68,6 +68,18 @@ void AudioNetworkAdaptorImpl::SetUplinkPacketLossFraction( UpdateNetworkMetrics(network_metrics); } +void AudioNetworkAdaptorImpl::SetUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) { + last_metrics_.uplink_recoverable_packet_loss_fraction = + rtc::Optional(uplink_recoverable_packet_loss_fraction); + DumpNetworkMetrics(); + + Controller::NetworkMetrics network_metrics; + network_metrics.uplink_recoverable_packet_loss_fraction = + rtc::Optional(uplink_recoverable_packet_loss_fraction); + UpdateNetworkMetrics(network_metrics); +} + void AudioNetworkAdaptorImpl::SetRtt(int rtt_ms) { last_metrics_.rtt_ms = rtc::Optional(rtt_ms); DumpNetworkMetrics(); diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h b/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h index 82062abc21..3713bdae6e 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h +++ b/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h @@ -45,6 +45,9 @@ class AudioNetworkAdaptorImpl final : public AudioNetworkAdaptor { void SetUplinkPacketLossFraction(float uplink_packet_loss_fraction) override; + void SetUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) override; + void SetRtt(int rtt_ms) override; void SetTargetAudioBitrate(int target_audio_bitrate_bps) override; diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc b/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc index db2a466c43..c434be35f5 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc +++ b/webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc @@ -36,7 +36,10 @@ MATCHER_P(NetworkMetricsIs, metric, "") { arg.target_audio_bitrate_bps == metric.target_audio_bitrate_bps && arg.rtt_ms == metric.rtt_ms && arg.overhead_bytes_per_packet == metric.overhead_bytes_per_packet && - arg.uplink_packet_loss_fraction == metric.uplink_packet_loss_fraction; + arg.uplink_packet_loss_fraction == + metric.uplink_packet_loss_fraction && + arg.uplink_recoverable_packet_loss_fraction == + metric.uplink_recoverable_packet_loss_fraction; } MATCHER_P(EncoderRuntimeConfigIs, config, "") { @@ -127,6 +130,18 @@ TEST(AudioNetworkAdaptorImplTest, states.audio_network_adaptor->SetUplinkPacketLossFraction(kPacketLoss); } +TEST(AudioNetworkAdaptorImplTest, + UpdateNetworkMetricsIsCalledOnSetUplinkRecoverablePacketLossFraction) { + auto states = CreateAudioNetworkAdaptor(); + constexpr float kRecoverablePacketLoss = 0.1f; + Controller::NetworkMetrics check; + check.uplink_recoverable_packet_loss_fraction = + rtc::Optional(kRecoverablePacketLoss); + SetExpectCallToUpdateNetworkMetrics(states.mock_controllers, check); + states.audio_network_adaptor->SetUplinkRecoverablePacketLossFraction( + kRecoverablePacketLoss); +} + TEST(AudioNetworkAdaptorImplTest, UpdateNetworkMetricsIsCalledOnSetRtt) { auto states = CreateAudioNetworkAdaptor(); constexpr int kRtt = 100; @@ -186,6 +201,7 @@ TEST(AudioNetworkAdaptorImplTest, constexpr int kBandwidth = 16000; constexpr float kPacketLoss = 0.7f; + const auto kRecoverablePacketLoss = 0.2f; constexpr int kRtt = 100; constexpr int kTargetAudioBitrate = 15000; constexpr size_t kOverhead = 64; @@ -205,6 +221,15 @@ TEST(AudioNetworkAdaptorImplTest, DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); states.audio_network_adaptor->SetUplinkPacketLossFraction(kPacketLoss); + states.simulated_clock->AdvanceTimeMilliseconds(50); + timestamp_check += 50; + check.uplink_recoverable_packet_loss_fraction = + rtc::Optional(kRecoverablePacketLoss); + EXPECT_CALL(*states.mock_debug_dump_writer, + DumpNetworkMetrics(NetworkMetricsIs(check), timestamp_check)); + states.audio_network_adaptor->SetUplinkRecoverablePacketLossFraction( + kRecoverablePacketLoss); + states.simulated_clock->AdvanceTimeMilliseconds(200); timestamp_check += 200; check.rtt_ms = rtc::Optional(kRtt); diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/controller.h b/webrtc/modules/audio_coding/audio_network_adaptor/controller.h index 7679a58734..0ed23c8d08 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/controller.h +++ b/webrtc/modules/audio_coding/audio_network_adaptor/controller.h @@ -23,6 +23,7 @@ class Controller { ~NetworkMetrics(); rtc::Optional uplink_bandwidth_bps; rtc::Optional uplink_packet_loss_fraction; + rtc::Optional uplink_recoverable_packet_loss_fraction; rtc::Optional target_audio_bitrate_bps; rtc::Optional rtt_ms; rtc::Optional overhead_bytes_per_packet; diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h b/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h index 338a3657e2..155b74913e 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h +++ b/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h @@ -81,6 +81,7 @@ class ControllerManagerImpl final : public ControllerManager { // Scoring point is a subset of NetworkMetrics that is used for comparing the // significance of controllers. struct ScoringPoint { + // TODO(elad.alon): Do we want to experiment with RPLR-based scoring? ScoringPoint(int uplink_bandwidth_bps, float uplink_packet_loss_fraction); // Calculate the normalized [0,1] distance between two scoring points. diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto b/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto index f425244998..3f9275a05c 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto +++ b/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump.proto @@ -7,14 +7,20 @@ message NetworkMetrics { optional float uplink_packet_loss_fraction = 2; optional int32 target_audio_bitrate_bps = 3; optional int32 rtt_ms = 4; + optional int32 uplink_recoverable_packet_loss_fraction = 5; } message EncoderRuntimeConfig { optional int32 bitrate_bps = 1; optional int32 frame_length_ms = 2; + // Note: This is what we tell the encoder. It doesn't have to reflect + // the actual NetworkMetrics; it's subject to our decision. optional float uplink_packet_loss_fraction = 3; optional bool enable_fec = 4; optional bool enable_dtx = 5; + // Some encoders can encode fewer channels than the actual input to make + // better use of the bandwidth. |num_channels| sets the number of channels + // to encode. optional uint32 num_channels = 6; } diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc b/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc index 7770e65a26..e0af3362b8 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc +++ b/webrtc/modules/audio_coding/audio_network_adaptor/debug_dump_writer.cc @@ -94,6 +94,11 @@ void DebugDumpWriterImpl::DumpNetworkMetrics( if (metrics.rtt_ms) dump_metrics->set_rtt_ms(*metrics.rtt_ms); + if (metrics.uplink_recoverable_packet_loss_fraction) { + dump_metrics->set_uplink_recoverable_packet_loss_fraction( + *metrics.uplink_recoverable_packet_loss_fraction); + } + DumpEventToFile(event, dump_file_.get()); #endif // WEBRTC_AUDIO_NETWORK_ADAPTOR_DEBUG_DUMP } diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h b/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h index 14ddbca0a7..0ad4a1e5f3 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h +++ b/webrtc/modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h @@ -26,6 +26,8 @@ class AudioNetworkAdaptor { ~EncoderRuntimeConfig(); rtc::Optional bitrate_bps; rtc::Optional frame_length_ms; + // Note: This is what we tell the encoder. It doesn't have to reflect + // the actual NetworkMetrics; it's subject to our decision. rtc::Optional uplink_packet_loss_fraction; rtc::Optional enable_fec; rtc::Optional enable_dtx; @@ -43,6 +45,9 @@ class AudioNetworkAdaptor { virtual void SetUplinkPacketLossFraction( float uplink_packet_loss_fraction) = 0; + virtual void SetUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) = 0; + virtual void SetRtt(int rtt_ms) = 0; virtual void SetTargetAudioBitrate(int target_audio_bitrate_bps) = 0; diff --git a/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h b/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h index a826911c2f..104dde6732 100644 --- a/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h +++ b/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h @@ -26,6 +26,9 @@ class MockAudioNetworkAdaptor : public AudioNetworkAdaptor { MOCK_METHOD1(SetUplinkPacketLossFraction, void(float uplink_packet_loss_fraction)); + MOCK_METHOD1(SetUplinkRecoverablePacketLossFraction, + void(float uplink_recoverable_packet_loss_fraction)); + MOCK_METHOD1(SetRtt, void(int rtt_ms)); MOCK_METHOD1(SetTargetAudioBitrate, void(int target_audio_bitrate_bps)); diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.cc b/webrtc/modules/audio_coding/codecs/audio_encoder.cc index 9e0cf4a510..c9d85f8605 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.cc @@ -76,6 +76,9 @@ void AudioEncoder::DisableAudioNetworkAdaptor() {} void AudioEncoder::OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) {} +void AudioEncoder::OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) {} + void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) { OnReceivedUplinkBandwidth(target_audio_bitrate_bps, rtc::Optional()); } diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.h b/webrtc/modules/audio_coding/codecs/audio_encoder.h index 47152f9eee..b58f964f52 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.h +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.h @@ -175,6 +175,12 @@ class AudioEncoder { virtual void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction); + // Provides 1st-order-FEC-recoverable uplink packet loss rate to this encoder + // to allow it to adapt. + // |uplink_recoverable_packet_loss_fraction| is in the range [0.0, 1.0]. + virtual void OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction); + // Provides target audio bitrate to this encoder to allow it to adapt. virtual void OnReceivedTargetAudioBitrate(int target_bps); diff --git a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc index bcb11eef8b..993fffea5a 100644 --- a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc +++ b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.cc @@ -190,6 +190,12 @@ void AudioEncoderCng::OnReceivedUplinkPacketLossFraction( uplink_packet_loss_fraction); } +void AudioEncoderCng::OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) { + speech_encoder_->OnReceivedUplinkRecoverablePacketLossFraction( + uplink_recoverable_packet_loss_fraction); +} + void AudioEncoderCng::OnReceivedUplinkBandwidth( int target_audio_bitrate_bps, rtc::Optional probing_interval_ms) { diff --git a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h index 0075bd02fb..827e463516 100644 --- a/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h +++ b/webrtc/modules/audio_coding/codecs/cng/audio_encoder_cng.h @@ -65,6 +65,8 @@ class AudioEncoderCng final : public AudioEncoder { override; void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) override; + void OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) override; void OnReceivedUplinkBandwidth( int target_audio_bitrate_bps, rtc::Optional probing_interval_ms) override; diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index bac963f4e8..103ec9b33f 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -304,6 +304,15 @@ void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction( ApplyAudioNetworkAdaptor(); } +void AudioEncoderOpus::OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) { + if (!audio_network_adaptor_) + return; + audio_network_adaptor_->SetUplinkRecoverablePacketLossFraction( + uplink_recoverable_packet_loss_fraction); + ApplyAudioNetworkAdaptor(); +} + void AudioEncoderOpus::OnReceivedUplinkBandwidth( int target_audio_bitrate_bps, rtc::Optional probing_interval_ms) { diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h index 3d0483b9ca..15ded47a91 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -114,6 +114,8 @@ class AudioEncoderOpus final : public AudioEncoder { void DisableAudioNetworkAdaptor() override; void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) override; + void OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) override; void OnReceivedUplinkBandwidth( int target_audio_bitrate_bps, rtc::Optional probing_interval_ms) override; diff --git a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc index 3faad68db4..251a1045dc 100644 --- a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc +++ b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc @@ -126,6 +126,12 @@ void AudioEncoderCopyRed::OnReceivedUplinkPacketLossFraction( uplink_packet_loss_fraction); } +void AudioEncoderCopyRed::OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) { + speech_encoder_->OnReceivedUplinkRecoverablePacketLossFraction( + uplink_recoverable_packet_loss_fraction); +} + void AudioEncoderCopyRed::OnReceivedUplinkBandwidth( int target_audio_bitrate_bps, rtc::Optional probing_interval_ms) { diff --git a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h index b3ec08595a..b9b46a8a1e 100644 --- a/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h +++ b/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.h @@ -53,6 +53,8 @@ class AudioEncoderCopyRed final : public AudioEncoder { override; void OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) override; + void OnReceivedUplinkRecoverablePacketLossFraction( + float uplink_recoverable_packet_loss_fraction) override; void OnReceivedUplinkBandwidth( int target_audio_bitrate_bps, rtc::Optional probing_interval_ms) override; diff --git a/webrtc/test/mock_voe_channel_proxy.h b/webrtc/test/mock_voe_channel_proxy.h index 24adcc285d..d704e1933b 100644 --- a/webrtc/test/mock_voe_channel_proxy.h +++ b/webrtc/test/mock_voe_channel_proxy.h @@ -88,6 +88,8 @@ class MockVoEChannelProxy : public voe::ChannelProxy { MOCK_METHOD2(SetSendCNPayloadType, bool(int type, PayloadFrequencies frequency)); MOCK_METHOD1(OnTwccBasedUplinkPacketLossRate, void(float packet_loss_rate)); + MOCK_METHOD1(OnRecoverableUplinkPacketLossRate, + void(float recoverable_packet_loss_rate)); }; } // namespace test } // namespace webrtc diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc index 7e49e321cf..d2b30ca8f7 100644 --- a/webrtc/voice_engine/channel.cc +++ b/webrtc/voice_engine/channel.cc @@ -1314,6 +1314,16 @@ void Channel::OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) { }); } +void Channel::OnRecoverableUplinkPacketLossRate( + float recoverable_packet_loss_rate) { + audio_coding_->ModifyEncoder([&](std::unique_ptr* encoder) { + if (*encoder) { + (*encoder)->OnReceivedUplinkRecoverablePacketLossFraction( + recoverable_packet_loss_rate); + } + }); +} + void Channel::OnUplinkPacketLossRate(float packet_loss_rate) { if (use_twcc_plr_for_ana_) return; diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h index d24eb5f183..0a12f21e92 100644 --- a/webrtc/voice_engine/channel.h +++ b/webrtc/voice_engine/channel.h @@ -384,6 +384,8 @@ class Channel // from RTCP-XR. void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate); + void OnRecoverableUplinkPacketLossRate(float recoverable_packet_loss_rate); + private: void OnUplinkPacketLossRate(float packet_loss_rate); diff --git a/webrtc/voice_engine/channel_proxy.cc b/webrtc/voice_engine/channel_proxy.cc index 5388b8d3e3..45cbf10819 100644 --- a/webrtc/voice_engine/channel_proxy.cc +++ b/webrtc/voice_engine/channel_proxy.cc @@ -372,6 +372,13 @@ void ChannelProxy::OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) { channel()->OnTwccBasedUplinkPacketLossRate(packet_loss_rate); } +void ChannelProxy::OnRecoverableUplinkPacketLossRate( + float recoverable_packet_loss_rate) { + // TODO(elad.alon): This fails in UT; fix and uncomment. + // RTC_DCHECK(worker_thread_checker_.CalledOnValidThread()); + channel()->OnRecoverableUplinkPacketLossRate(recoverable_packet_loss_rate); +} + Channel* ChannelProxy::channel() const { RTC_DCHECK(channel_owner_.channel()); return channel_owner_.channel(); diff --git a/webrtc/voice_engine/channel_proxy.h b/webrtc/voice_engine/channel_proxy.h index fd25378af5..685a168845 100644 --- a/webrtc/voice_engine/channel_proxy.h +++ b/webrtc/voice_engine/channel_proxy.h @@ -117,6 +117,8 @@ class ChannelProxy { virtual bool SetSendCodec(const CodecInst& codec_inst); virtual bool SetSendCNPayloadType(int type, PayloadFrequencies frequency); virtual void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate); + virtual void OnRecoverableUplinkPacketLossRate( + float recoverable_packet_loss_rate); private: Channel* channel() const;