From 602316c3cd8556cc78d44f3ea4cd5fc8e70d9417 Mon Sep 17 00:00:00 2001 From: pbos Date: Fri, 29 Apr 2016 16:10:27 -0700 Subject: [PATCH] Revert of Remove VCMQmRobustness. (patchset #1 id:1 of https://codereview.webrtc.org/1917083003/ ) Reason for revert: Speculative revert for perf regression. Original issue's description: > Remove VCMQmRobustness. > > Class contained a lot of not-really-wired-up functionality that ended up > being complicated ways of saying return 1; or return false;. This > removes this dependency that complicates code readability significantly. > > BUG=webrtc:5066 > R=marpan@google.com, marpan@webrtc.org > TBR=stefan@webrtc.org > > Committed: https://crrev.com/73894369791cb5eedc8788baf918ec07d11d351d > Cr-Commit-Position: refs/heads/master@{#12516} TBR=marpan@webrtc.org,stefan@webrtc.org,marpan@google.com # Not skipping CQ checks because original CL landed more than 1 days ago. BUG=webrtc:5066, chromium:607838 Review-Url: https://codereview.webrtc.org/1935753002 Cr-Commit-Position: refs/heads/master@{#12572} --- webrtc/modules/include/module_common_types.h | 1 + .../modules/rtp_rtcp/source/producer_fec.cc | 11 ++-- .../rtp_rtcp/source/producer_fec_unittest.cc | 6 +-- .../rtp_rtcp/source/rtp_sender_unittest.cc | 1 + webrtc/modules/video_coding/media_opt_util.cc | 29 ++++++++++- webrtc/modules/video_coding/media_opt_util.h | 4 ++ .../video_coding/media_optimization.cc | 10 ++++ webrtc/modules/video_coding/qm_select.cc | 52 +++++++++++++++++++ webrtc/modules/video_coding/qm_select.h | 30 +++++++++++ 9 files changed, 135 insertions(+), 9 deletions(-) diff --git a/webrtc/modules/include/module_common_types.h b/webrtc/modules/include/module_common_types.h index a2ab766ce5..82d87d5c5c 100644 --- a/webrtc/modules/include/module_common_types.h +++ b/webrtc/modules/include/module_common_types.h @@ -432,6 +432,7 @@ enum FecMaskType { // Struct containing forward error correction settings. struct FecProtectionParams { int fec_rate; + bool use_uep_protection; int max_fec_frames; FecMaskType fec_mask_type; }; diff --git a/webrtc/modules/rtp_rtcp/source/producer_fec.cc b/webrtc/modules/rtp_rtcp/source/producer_fec.cc index c7ea19db58..69a28ed4db 100644 --- a/webrtc/modules/rtp_rtcp/source/producer_fec.cc +++ b/webrtc/modules/rtp_rtcp/source/producer_fec.cc @@ -157,11 +157,12 @@ int ProducerFec::AddRtpPacketAndGenerateFec(const uint8_t* data_buffer, (ExcessOverheadBelowMax() && MinimumMediaPacketsReached()))) { assert(num_first_partition_ <= static_cast(ForwardErrorCorrection::kMaxMediaPackets)); - // TODO(pbos): Consider whether unequal protection should be enabled or not, - // it is currently always disabled. - int ret = fec_->GenerateFEC(media_packets_fec_, params_.fec_rate, - num_first_partition_, false, - params_.fec_mask_type, &fec_packets_); + int ret = fec_->GenerateFEC(media_packets_fec_, + params_.fec_rate, + num_first_partition_, + params_.use_uep_protection, + params_.fec_mask_type, + &fec_packets_); if (fec_packets_.empty()) { num_frames_ = 0; DeletePackets(); diff --git a/webrtc/modules/rtp_rtcp/source/producer_fec_unittest.cc b/webrtc/modules/rtp_rtcp/source/producer_fec_unittest.cc index ec5228afd5..deb9a15e03 100644 --- a/webrtc/modules/rtp_rtcp/source/producer_fec_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/producer_fec_unittest.cc @@ -79,7 +79,7 @@ TEST_F(ProducerFecTest, NoEmptyFecWithSeqNumGaps) { protected_packets.push_back({12, 3, 54, 0}); protected_packets.push_back({21, 0, 55, 0}); protected_packets.push_back({13, 3, 57, 1}); - FecProtectionParams params = {117, 3, kFecMaskBursty}; + FecProtectionParams params = {117, 0, 3, kFecMaskBursty}; producer_->SetFecParameters(¶ms, 0); uint8_t packet[28] = {0}; for (Packet p : protected_packets) { @@ -112,7 +112,7 @@ TEST_F(ProducerFecTest, OneFrameFec) { // of packets is within |kMaxExcessOverhead|, and (2) the total number of // media packets for 1 frame is at least |minimum_media_packets_fec_|. const int kNumPackets = 4; - FecProtectionParams params = {15, 3, kFecMaskRandom}; + FecProtectionParams params = {15, false, 3}; std::list rtp_packets; generator_->NewFrame(kNumPackets); producer_->SetFecParameters(¶ms, 0); // Expecting one FEC packet. @@ -153,7 +153,7 @@ TEST_F(ProducerFecTest, TwoFrameFec) { const int kNumPackets = 2; const int kNumFrames = 2; - FecProtectionParams params = {15, 3, kFecMaskRandom}; + FecProtectionParams params = {15, 0, 3}; std::list rtp_packets; producer_->SetFecParameters(¶ms, 0); // Expecting one FEC packet. uint32_t last_timestamp = 0; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc index f350effc21..cecad5d4ef 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -1293,6 +1293,7 @@ TEST_F(RtpSenderTestWithoutPacer, StreamDataCountersCallbacks) { fec_params.fec_mask_type = kFecMaskRandom; fec_params.fec_rate = 1; fec_params.max_fec_frames = 1; + fec_params.use_uep_protection = false; rtp_sender_->SetFecParameters(&fec_params, &fec_params); ASSERT_EQ(0, rtp_sender_->SendOutgoingData(kVideoFrameDelta, payload_type, 1234, 4321, payload, diff --git a/webrtc/modules/video_coding/media_opt_util.cc b/webrtc/modules/video_coding/media_opt_util.cc index 42db2facf1..69cf757f2b 100644 --- a/webrtc/modules/video_coding/media_opt_util.cc +++ b/webrtc/modules/video_coding/media_opt_util.cc @@ -34,10 +34,19 @@ VCMProtectionMethod::VCMProtectionMethod() _protectionFactorD(0), _scaleProtKey(2.0f), _maxPayloadSize(1460), + _qmRobustness(new VCMQmRobustness()), + _useUepProtectionK(false), + _useUepProtectionD(true), _corrFecCost(1.0), _type(kNone) {} -VCMProtectionMethod::~VCMProtectionMethod() {} +VCMProtectionMethod::~VCMProtectionMethod() { + delete _qmRobustness; +} +void VCMProtectionMethod::UpdateContentMetrics( + const VideoContentMetrics* contentMetrics) { + _qmRobustness->UpdateContent(contentMetrics); +} VCMNackFecMethod::VCMNackFecMethod(int64_t lowRttNackThresholdMs, int64_t highRttNackThresholdMs) @@ -324,6 +333,17 @@ bool VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) { codeRateDelta = kPacketLossMax - 1; } + float adjustFec = 1.0f; + // Avoid additional adjustments when layers are active. + // TODO(mikhal/marco): Update adjusmtent based on layer info. + if (parameters->numLayers == 1) { + adjustFec = _qmRobustness->AdjustFecFactor( + codeRateDelta, parameters->bitRate, parameters->frameRate, + parameters->rtt, packetLoss); + } + + codeRateDelta = static_cast(codeRateDelta * adjustFec); + // For Key frame: // Effectively at a higher rate, so we scale/boost the rate // The boost factor may depend on several factors: ratio of packet @@ -391,6 +411,13 @@ bool VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) { _corrFecCost = 0.0f; } + // TODO(marpan): Set the UEP protection on/off for Key and Delta frames + _useUepProtectionK = _qmRobustness->SetUepProtection( + codeRateKey, parameters->bitRate, packetLoss, 0); + + _useUepProtectionD = _qmRobustness->SetUepProtection( + codeRateDelta, parameters->bitRate, packetLoss, 1); + // DONE WITH FEC PROTECTION SETTINGS return true; } diff --git a/webrtc/modules/video_coding/media_opt_util.h b/webrtc/modules/video_coding/media_opt_util.h index 1501f72ef6..6b47e3b2d9 100644 --- a/webrtc/modules/video_coding/media_opt_util.h +++ b/webrtc/modules/video_coding/media_opt_util.h @@ -138,6 +138,9 @@ class VCMProtectionMethod { virtual int MaxFramesFec() const { return 1; } + // Updates content metrics + void UpdateContentMetrics(const VideoContentMetrics* contentMetrics); + protected: uint8_t _effectivePacketLoss; uint8_t _protectionFactorK; @@ -146,6 +149,7 @@ class VCMProtectionMethod { float _scaleProtKey; int32_t _maxPayloadSize; + VCMQmRobustness* _qmRobustness; bool _useUepProtectionK; bool _useUepProtectionD; float _corrFecCost; diff --git a/webrtc/modules/video_coding/media_optimization.cc b/webrtc/modules/video_coding/media_optimization.cc index f24637e0b2..a234a06f9b 100644 --- a/webrtc/modules/video_coding/media_optimization.cc +++ b/webrtc/modules/video_coding/media_optimization.cc @@ -33,6 +33,13 @@ void UpdateProtectionCallback( // Get the FEC code rate for Delta frames (set to 0 when NA). delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD(); + // Get the FEC-UEP protection status for Key frames: UEP on/off. + key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK(); + + // Get the FEC-UEP protection status for Delta frames: UEP on/off. + delta_fec_params.use_uep_protection = + selected_method->RequiredUepProtectionD(); + // The RTP module currently requires the same |max_fec_frames| for both // key and delta frames. delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); @@ -222,6 +229,9 @@ uint32_t MediaOptimization::SetTargetRates( // Update protection settings, when applicable. float sent_video_rate_kbps = 0.0f; if (loss_prot_logic_->SelectedType() != kNone) { + // Update protection method with content metrics. + selected_method->UpdateContentMetrics(content_->ShortTermAvgData()); + // Update method will compute the robustness settings for the given // protection method and the overhead cost // the protection method is set by the user via SetVideoProtection. diff --git a/webrtc/modules/video_coding/qm_select.cc b/webrtc/modules/video_coding/qm_select.cc index a090ba1e33..9da42bb33c 100644 --- a/webrtc/modules/video_coding/qm_select.cc +++ b/webrtc/modules/video_coding/qm_select.cc @@ -898,4 +898,56 @@ void VCMQmResolution::SelectSpatialDirectionMode(float transition_rate) { } } +// ROBUSTNESS CLASS + +VCMQmRobustness::VCMQmRobustness() { + Reset(); +} + +VCMQmRobustness::~VCMQmRobustness() {} + +void VCMQmRobustness::Reset() { + prev_total_rate_ = 0.0f; + prev_rtt_time_ = 0; + prev_packet_loss_ = 0; + prev_code_rate_delta_ = 0; + ResetQM(); +} + +// Adjust the FEC rate based on the content and the network state +// (packet loss rate, total rate/bandwidth, round trip time). +// Note that packetLoss here is the filtered loss value. +float VCMQmRobustness::AdjustFecFactor(uint8_t code_rate_delta, + float total_rate, + float framerate, + int64_t rtt_time, + uint8_t packet_loss) { + // Default: no adjustment + float adjust_fec = 1.0f; + if (content_metrics_ == NULL) { + return adjust_fec; + } + // Compute class state of the content. + ComputeMotionNFD(); + ComputeSpatial(); + + // TODO(marpan): Set FEC adjustment factor. + + // Keep track of previous values of network state: + // adjustment may be also based on pattern of changes in network state. + prev_total_rate_ = total_rate; + prev_rtt_time_ = rtt_time; + prev_packet_loss_ = packet_loss; + prev_code_rate_delta_ = code_rate_delta; + return adjust_fec; +} + +// Set the UEP (unequal-protection across packets) on/off for the FEC. +bool VCMQmRobustness::SetUepProtection(uint8_t code_rate_delta, + float total_rate, + uint8_t packet_loss, + bool frame_type) { + // Default. + return false; +} } // namespace webrtc diff --git a/webrtc/modules/video_coding/qm_select.h b/webrtc/modules/video_coding/qm_select.h index ae0463f499..764b5ed8e3 100644 --- a/webrtc/modules/video_coding/qm_select.h +++ b/webrtc/modules/video_coding/qm_select.h @@ -322,5 +322,35 @@ class VCMQmResolution : public VCMQmMethod { int num_layers_; }; +// Robustness settings class. + +class VCMQmRobustness : public VCMQmMethod { + public: + VCMQmRobustness(); + ~VCMQmRobustness(); + + virtual void Reset(); + + // Adjust FEC rate based on content: every ~1 sec from SetTargetRates. + // Returns an adjustment factor. + float AdjustFecFactor(uint8_t code_rate_delta, + float total_rate, + float framerate, + int64_t rtt_time, + uint8_t packet_loss); + + // Set the UEP protection on/off. + bool SetUepProtection(uint8_t code_rate_delta, + float total_rate, + uint8_t packet_loss, + bool frame_type); + + private: + // Previous state of network parameters. + float prev_total_rate_; + int64_t prev_rtt_time_; + uint8_t prev_packet_loss_; + uint8_t prev_code_rate_delta_; +}; } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_