From 022716b3af97bb3992f2ae56714551928da1673b Mon Sep 17 00:00:00 2001 From: "mikhal@google.com" Date: Wed, 20 Jul 2011 23:12:57 +0000 Subject: [PATCH] VCM/Media Opt Util: 1. Updating hybrid protection settings 2. Removing score 3. adding Fec as suffix to _residualPacketLoss for clarity. 4. Some clean up. Review URL: http://webrtc-codereview.appspot.com/88001 git-svn-id: http://webrtc.googlecode.com/svn/trunk@238 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/source/media_opt_util.cc | 211 +++++++----------- .../video_coding/main/source/media_opt_util.h | 73 +++--- 2 files changed, 118 insertions(+), 166 deletions(-) diff --git a/src/modules/video_coding/main/source/media_opt_util.cc b/src/modules/video_coding/main/source/media_opt_util.cc index 2808c67ad3..235cb8ed42 100644 --- a/src/modules/video_coding/main/source/media_opt_util.cc +++ b/src/modules/video_coding/main/source/media_opt_util.cc @@ -22,39 +22,15 @@ namespace webrtc { void -VCMProtectionMethod::UpdateContentMetrics( - const VideoContentMetrics* contentMetrics) +VCMProtectionMethod::UpdateContentMetrics(const + VideoContentMetrics* contentMetrics) { _qmRobustness->UpdateContent(contentMetrics); } bool -VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm) -{ - if (pm == NULL) - { - return true; - } - return pm->_score > _score; -} - -bool -VCMNackFecMethod::ProtectionFactor(const VCMProtectionParameters* /*parameters*/) -{ - // use FEC model with modification with RTT for now - return true; -} - -bool -VCMNackFecMethod::EffectivePacketLoss(const - VCMProtectionParameters* /*parameters*/) -{ - // use FEC model with modification with RTT for now - return true; -} - -bool -VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) +VCMNackFecMethod::ProtectionFactor(const + VCMProtectionParameters* parameters) { // Hybrid Nack FEC has three operational modes: // 1. Low RTT - Nack only (Set FEC rates to zero) @@ -68,97 +44,93 @@ VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) // Set the FEC parameters to 0 _protectionFactorK = 0; _protectionFactorD = 0; - - // assume packets will be restored via NACK - // TODO: relax this assumption? - _effectivePacketLoss = 0; - _score = _efficiency; return true; } - // otherwise: we count on FEC; if the RTT is below a threshold, then we can + + // Otherwise: we count on FEC; if the RTT is below a threshold, then we // nack the residual, based on a decision made in the JB. - // TODO(mikhal): adapt the FEC rate based on the RTT, i.e. the the level on - // which we will rely on NACK, e.g. less as we approach upper threshold. + VCMFecMethod fecMethod; - const WebRtc_UWord8 plossMax = 129; - - // Compute the protection factor + // Compute the protection factors fecMethod.ProtectionFactor(parameters); - // Compute the effective packet loss - fecMethod.EffectivePacketLoss(parameters); + _protectionFactorK = fecMethod._protectionFactorK; + _protectionFactorD = fecMethod._protectionFactorD; - WebRtc_UWord8 protFactorK = fecMethod._protectionFactorK; - WebRtc_UWord8 protFactorD = fecMethod._protectionFactorD; - WebRtc_UWord8 effPacketLoss = fecMethod._effectivePacketLoss; - float resPacketLoss = fecMethod._residualPacketLoss; - - // Correct FEC rates based on the RTT ( NACK effectiveness) - WebRtc_Word16 rttIndex= (WebRtc_UWord16) parameters->rtt; - float softnessRtt = 1.0; + // When in Hybrid mode, correct FEC rates based on the + // RTT (NACK effectiveness) if (parameters->rtt < kHighRttNackMs) { + WebRtc_UWord16 rttIndex = (WebRtc_UWord16) parameters->rtt; // TODO(mikhal): update table - softnessRtt = (float)VCMNackFecTable[rttIndex] / (float)4096.0; + float softnessRtt = (float)VCMNackFecTable[rttIndex] / (float)4096.0; - // soften ER with NACK on + // Soften FEC with NACK on (for delta frame only) // table depends on RTT relative to rttMax (NACK Threshold) - _effectivePacketLoss = (WebRtc_UWord8)(effPacketLoss * softnessRtt); - - // soften FEC with NACK on - // table depends on RTT relative to rttMax (NACK Threshold) - _protectionFactorK = (WebRtc_UWord8) (protFactorK * softnessRtt); - _protectionFactorD = (WebRtc_UWord8) (protFactorD * softnessRtt); + _protectionFactorD *= static_cast (softnessRtt); } - // else - NACK is disabled, rely on FEC only + return true; +} +bool +VCMNackFecMethod::EffectivePacketLoss(const + VCMProtectionParameters* parameters) +{ + // NACK only mode + if (parameters->rtt < kLowRttNackMs) + { + // Effective Packet Loss, NA in current version. + _effectivePacketLoss = 0; + // No FEC applied. + _residualPacketLossFec = 255 * parameters->lossPr; + return true; + } - // make sure I frame protection is at least larger than P frame protection, - // and at least as high as received loss - WebRtc_UWord8 packetLoss = (WebRtc_UWord8) (255 * parameters->lossPr); - _protectionFactorK = static_cast (VCM_MAX(packetLoss, - VCM_MAX(_scaleProtKey * protFactorD, protFactorK))); + // Set the effective packet loss for encoder (based on FEC code). + VCMFecMethod fecMethod; + // Compute the effective packet loss and residual packet loss due to FEC. + fecMethod.EffectivePacketLoss(parameters); + _effectivePacketLoss = fecMethod._effectivePacketLoss; + _residualPacketLossFec = fecMethod._residualPacketLossFec; - // check limit on amount of protection for I frame: 50% is max - if (_protectionFactorK >= plossMax) - _protectionFactorK = plossMax - 1; + return true; +} - // Bit cost for NackFec +bool +VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) +{ + ProtectionFactor(parameters); + EffectivePacketLoss(parameters); - // NACK cost: based on residual packet loss (since we should only NACK - // packets not recovered by FEC) - _efficiency = 0.0f; + // Efficiency computation is based on FEC and NACK + + // Add FEC cost: ignore I frames for now + float fecRate = static_cast (_protectionFactorD) / 255.0f; + _efficiency = parameters->bitRate * fecRate; + + // Add NACK cost, when applicable if (parameters->rtt < kHighRttNackMs) { - _efficiency = parameters->bitRate * resPacketLoss / - (1.0f + resPacketLoss); + // nackCost = (bitRate - nackCost) * (lossPr) + _efficiency += parameters->bitRate * _residualPacketLossFec / + (1.0f + _residualPacketLossFec); } - else - { - // efficiency based on FEC only - // add FEC cost: ignore I frames for now - float fecRate = static_cast (_protectionFactorD) / 255.0f; - if (fecRate >= 0.0f) - _efficiency += parameters->bitRate * fecRate; - } - _score = _efficiency; // Protection/fec rates obtained above are defined relative to total number // of packets (total rate: source + fec) FEC in RTP module assumes // protection factor is defined relative to source number of packets so we // should convert the factor to reduce mismatch between mediaOpt's rate and // the actual one - WebRtc_UWord8 codeRate = protFactorK; - _protectionFactorK = fecMethod.ConvertFECRate(codeRate); - codeRate = protFactorD; - _protectionFactorD = fecMethod.ConvertFECRate(codeRate); + VCMFecMethod fecMethod; + _protectionFactorK = fecMethod.ConvertFECRate(_protectionFactorK); + _protectionFactorD = fecMethod.ConvertFECRate(_protectionFactorK); return true; } bool -VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 effPacketLoss, +VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 packetLoss, WebRtc_UWord16 rttTime) { WebRtc_UWord16 rttMax = MaxRttNack(); @@ -167,9 +139,12 @@ VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 effPacketLoss, // packetLossEnc = 0 for RTT less than the NACK threshold if (rttTime < rttMax) { - effPacketLoss = 0; //may want a softer transition here + _effectivePacketLoss = 0; // may want a softer transition here + } + else + { + _effectivePacketLoss = packetLoss; } - _effectivePacketLoss = effPacketLoss; return true; } @@ -183,14 +158,14 @@ VCMNackMethod::UpdateParameters(const VCMProtectionParameters* parameters) EffectivePacketLoss(effPacketLoss, rttTime); // Compute the NACK bit cost - _efficiency = parameters->bitRate * parameters->lossPr / - (1.0f + parameters->lossPr); - _score = _efficiency; - if (parameters->rtt > _NACK_MAX_RTT) + if (rttTime > _NACK_MAX_RTT) { - _score = 0.0f; + _efficiency = 0.0f; return false; } + + _efficiency = parameters->bitRate * parameters->lossPr / + (1.0f + parameters->lossPr); return true; } @@ -199,7 +174,7 @@ VCMFecMethod::BoostCodeRateKey(WebRtc_UWord8 packetFrameDelta, WebRtc_UWord8 packetFrameKey) const { WebRtc_UWord8 boostRateKey = 2; - // default: ratio scales the FEC protection up for I frames + // Default: ratio scales the FEC protection up for I frames WebRtc_UWord8 ratio = 1; if (packetFrameDelta > 0) @@ -263,13 +238,13 @@ VCMFecMethod::AvgRecoveryFEC(const VCMProtectionParameters* parameters) const // index for ER tables: up to codeSizexcodeSize mask WebRtc_UWord16 codeIndexTable[codeSize * codeSize]; - WebRtc_UWord16 k = -1; + WebRtc_UWord16 k = 0; for (WebRtc_UWord8 i = 1; i <= codeSize; i++) { for (WebRtc_UWord8 j = 1; j <= i; j++) { - k += 1; codeIndexTable[(j - 1) * codeSize + i - 1] = k; + k += 1; } } @@ -426,7 +401,8 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) float adjustFec = _qmRobustness->AdjustFecFactor(codeRateDelta, bitRate, parameters->frameRate, - parameters->rtt, packetLoss); + parameters->rtt, + packetLoss); codeRateDelta = static_cast(codeRateDelta * adjustFec); @@ -480,7 +456,7 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) _useUepProtectionK = _qmRobustness->SetUepProtection(codeRateKey, bitRate, packetLoss, 0); - _useUepProtectionD = _qmRobustness->SetUepProtection(codeRateKey, bitRate, + _useUepProtectionD = _qmRobustness->SetUepProtection(codeRateDelta, bitRate, packetLoss, 1); // DONE WITH FEC PROTECTION SETTINGS @@ -490,39 +466,22 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) bool VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters) { - // ER SETTINGS: // Effective packet loss to encoder is based on RPL (residual packet loss) // this is a soft setting based on degree of FEC protection // RPL = received/input packet loss - average_FEC_recovery // note: received/input packet loss may be filtered based on FilteredLoss - // The input packet loss: - WebRtc_UWord8 effPacketLoss = (WebRtc_UWord8) (255 * parameters->lossPr); + // The packet loss: + WebRtc_UWord8 packetLoss = (WebRtc_UWord8) (255 * parameters->lossPr); - float scaleErRS = 0.5; - float scaleErXOR = 0.5; - float minErLevel = (float) 0.025; - // float scaleErRS = 1.0; - // float scaleErXOR = 1.0; - // float minErLevel = (float) 0.0; - - float avgFecRecov = 0.; - // Effective packet loss for ER: - float scaleEr = scaleErXOR; - avgFecRecov = AvgRecoveryFEC(parameters); + float avgFecRecov = AvgRecoveryFEC(parameters); // Residual Packet Loss: - _residualPacketLoss = (float) (effPacketLoss - avgFecRecov) / (float) 255.0; + _residualPacketLossFec = (float) (packetLoss - avgFecRecov) / 255.0f; - //Effective Packet Loss for encoder: + // Effective Packet Loss, NA in current version. _effectivePacketLoss = 0; - if (effPacketLoss > 0) { - _effectivePacketLoss = VCM_MAX((effPacketLoss - - (WebRtc_UWord8)(scaleEr * avgFecRecov)), - static_cast(minErLevel * 255)); - } - // DONE WITH ER SETTING return true; } @@ -553,7 +512,6 @@ VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) { _efficiency = 0.0f; } - _score = _efficiency; // Protection/fec rates obtained above is defined relative to total number // of packets (total rate: source+fec) FEC in RTP module assumes protection @@ -574,11 +532,9 @@ VCMIntraReqMethod::UpdateParameters(const VCMProtectionParameters* parameters) float lossRate = parameters->lossPr * packetRate; if (parameters->keyFrameSize <= 1e-3) { - _score = FLT_MAX; return false; } _efficiency = lossRate * parameters->keyFrameSize; - _score = _efficiency; if (parameters->lossPr >= 1.0f / parameters->keyFrameSize || parameters->rtt > _IREQ_MAX_RTT) { @@ -593,7 +549,6 @@ VCMPeriodicIntraMethod::UpdateParameters(const { // Periodic I-frames. The last thing we want to use. _efficiency = 0.0f; - _score = FLT_MAX; return true; } @@ -604,7 +559,6 @@ VCMMbIntraRefreshMethod::UpdateParameters(const // Assume optimal for now. _efficiency = parameters->bitRate * parameters->lossPr / (1.0f + parameters->lossPr); - _score = _efficiency; if (parameters->bitRate < _MBREF_MIN_BITRATE) { return false; @@ -730,7 +684,7 @@ VCMLossProtectionLogic::UpdateRtt(WebRtc_UWord32 rtt) void VCMLossProtectionLogic::UpdateResidualPacketLoss(float residualPacketLoss) { - _residualPacketLoss = residualPacketLoss; + _residualPacketLossFec = residualPacketLoss; } void @@ -893,7 +847,7 @@ VCMLossProtectionLogic::UpdateMethod(VCMProtectionMethod *newMethod /*=NULL */) _currentParameters.fecRateKey = _fecRateKey; _currentParameters.packetsPerFrame = _packetsPerFrame.Value(); _currentParameters.packetsPerFrameKey = _packetsPerFrameKey.Value(); - _currentParameters.residualPacketLoss = _residualPacketLoss; + _currentParameters.residualPacketLossFec = _residualPacketLossFec; _currentParameters.fecType = _fecType; _currentParameters.codecWidth = _codecWidth; _currentParameters.codecHeight = _codecHeight; @@ -943,7 +897,8 @@ VCMLossProtectionLogic::SelectedMethod() const return _selectedMethod; } -void VCMLossProtectionLogic::Reset() +void +VCMLossProtectionLogic::Reset() { const WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); _lastPrUpdateT = now; diff --git a/src/modules/video_coding/main/source/media_opt_util.h b/src/modules/video_coding/main/source/media_opt_util.h index d5ce7925a0..862538b955 100644 --- a/src/modules/video_coding/main/source/media_opt_util.h +++ b/src/modules/video_coding/main/source/media_opt_util.h @@ -44,10 +44,10 @@ enum HybridNackTH { struct VCMProtectionParameters { - VCMProtectionParameters() : rtt(0), lossPr(0), bitRate(0), packetsPerFrame(0), - frameRate(0), keyFrameSize(0), fecRateDelta(0), fecRateKey(0), - residualPacketLoss(0.0), fecType(kXORFec), codecWidth(0), - codecHeight(0) {} + VCMProtectionParameters() : rtt(0), lossPr(0), bitRate(0), + packetsPerFrame(0), frameRate(0), keyFrameSize(0), fecRateDelta(0), + fecRateKey(0), residualPacketLossFec(0.0), fecType(kXORFec), + codecWidth(0), codecHeight(0) {} WebRtc_UWord32 rtt; float lossPr; @@ -58,7 +58,7 @@ struct VCMProtectionParameters float keyFrameSize; WebRtc_UWord8 fecRateDelta; WebRtc_UWord8 fecRateKey; - float residualPacketLoss; + float residualPacketLossFec; VCMFecTypes fecType; WebRtc_UWord16 codecWidth; WebRtc_UWord16 codecHeight; @@ -97,8 +97,8 @@ class VCMProtectionMethod public: //friend VCMProtectionMethod; VCMProtectionMethod(VCMProtectionMethodEnum type) : _protectionFactorK(0), - _protectionFactorD(0), _residualPacketLoss(0.0), _scaleProtKey(2.0), - _maxPayloadSize(1460), _efficiency(0), _score(0), _type(type), + _protectionFactorD(0), _residualPacketLossFec(0.0), _scaleProtKey(2.0), + _maxPayloadSize(1460), _efficiency(0), _type(type), _useUepProtectionK(false), _useUepProtectionD(true) {_qmRobustness = new VCMQmRobustness();} virtual ~VCMProtectionMethod() { delete _qmRobustness;} @@ -117,13 +117,6 @@ public: // Return value : The protection type enum VCMProtectionMethodEnum Type() const { return _type; } - // Evaluates if this protection method is considered - // better than the provided method. - // - // Input: - // - pm : The protection method to compare with - bool BetterThan(VCMProtectionMethod *pm); - // Returns the bit rate required by this protection method // during these conditions. // @@ -161,7 +154,8 @@ public: WebRtc_UWord8 _effectivePacketLoss; WebRtc_UWord8 _protectionFactorK; WebRtc_UWord8 _protectionFactorD; - float _residualPacketLoss; + // Estimation of residual loss after the FEC + float _residualPacketLossFec; float _scaleProtKey; WebRtc_Word32 _maxPayloadSize; @@ -171,7 +165,6 @@ public: protected: float _efficiency; - float _score; private: const enum VCMProtectionMethodEnum _type; @@ -184,9 +177,9 @@ public: VCMNackMethod() : VCMProtectionMethod(kNACK), _NACK_MAX_RTT(200) {} virtual ~VCMNackMethod() {} virtual bool UpdateParameters(const VCMProtectionParameters* parameters); - //get the effective packet loss for ER - bool EffectivePacketLoss(WebRtc_UWord8 effPacketLoss, WebRtc_UWord16 rttTime); - //get the threshold for NACK + // Get the effective packet loss for ER + bool EffectivePacketLoss(WebRtc_UWord8 packetLoss, WebRtc_UWord16 rttTime); + // Get the threshold for NACK WebRtc_UWord16 MaxRttNack() const; private: const WebRtc_UWord16 _NACK_MAX_RTT; @@ -198,16 +191,16 @@ public: VCMFecMethod() : VCMProtectionMethod(kFEC) {} virtual ~VCMFecMethod() {} virtual bool UpdateParameters(const VCMProtectionParameters* parameters); - //get the effective packet loss for ER + // Get the effective packet loss for ER bool EffectivePacketLoss(const VCMProtectionParameters* parameters); - //get the FEC protection factors + // Get the FEC protection factors bool ProtectionFactor(const VCMProtectionParameters* parameters); - //get the boost for key frame protection + // Get the boost for key frame protection WebRtc_UWord8 BoostCodeRateKey(WebRtc_UWord8 packetFrameDelta, WebRtc_UWord8 packetFrameKey) const; - //convert the rates: defined relative to total# packets or source# packets + // Convert the rates: defined relative to total# packets or source# packets WebRtc_UWord8 ConvertFECRate(WebRtc_UWord8 codeRate) const; - //get the average effective recovery from FEC: for random loss model + // Get the average effective recovery from FEC: for random loss model float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const; }; @@ -217,9 +210,9 @@ class VCMNackFecMethod : public VCMProtectionMethod public: VCMNackFecMethod() : VCMProtectionMethod(kNackFec) {} virtual bool UpdateParameters(const VCMProtectionParameters* parameters); - //get the effective packet loss for ER + // Get the effective packet loss for ER bool EffectivePacketLoss(const VCMProtectionParameters* parameters); - //get the FEC protection factors + // Get the FEC protection factors bool ProtectionFactor(const VCMProtectionParameters* parameters); }; @@ -228,7 +221,8 @@ public: class VCMIntraReqMethod : public VCMProtectionMethod { public: - VCMIntraReqMethod() : VCMProtectionMethod(kIntraRequest), _IREQ_MAX_RTT(150) {} + VCMIntraReqMethod() : VCMProtectionMethod(kIntraRequest), _IREQ_MAX_RTT(150) + {} virtual bool UpdateParameters(const VCMProtectionParameters* parameters); private: const WebRtc_UWord32 _IREQ_MAX_RTT; @@ -256,10 +250,11 @@ class VCMLossProtectionLogic { public: VCMLossProtectionLogic() : _availableMethods(), _selectedMethod(NULL), - _bestNotOkMethod(NULL), _rtt(0), _lossPr(0.0f), _bitRate(0.0f), _frameRate(0.0f), - _keyFrameSize(0.0f), _fecRateKey(0), _fecRateDelta(0), _lastPrUpdateT(0), - _lossPr255(0.9999f), _lossPrHistory(), _shortMaxLossPr255(0), - _packetsPerFrame(0.9999f), _packetsPerFrameKey(0.9999f), _residualPacketLoss(0), + _bestNotOkMethod(NULL), _rtt(0), _lossPr(0.0f), _bitRate(0.0f), + _frameRate(0.0f), _keyFrameSize(0.0f), _fecRateKey(0), _fecRateDelta(0), + _lastPrUpdateT(0), _lossPr255(0.9999f), _lossPrHistory(), + _shortMaxLossPr255(0), _packetsPerFrame(0.9999f), + _packetsPerFrameKey(0.9999f), _residualPacketLossFec(0), _boostRateKey(2), _codecWidth(0), _codecHeight(0) { Reset(); } @@ -280,8 +275,9 @@ public: // Update residual packet loss // // Input: - // - residualPacketLoss : residual packet loss: effective loss after FEC recovery - void UpdateResidualPacketLoss(float residualPacketLoss); + // - residualPacketLoss : residual packet loss: + // effective loss after FEC recovery + void UpdateResidualPacketLoss(float _residualPacketLoss); // Update fecType // @@ -292,14 +288,15 @@ public: // Update the loss probability. // // Input: - // - lossPr255 : The packet loss probability in the interval [0, 255], + // - lossPr255 : The packet loss probability [0, 255], // reported by RTCP. void UpdateLossPr(WebRtc_UWord8 lossPr255); // Update the filtered packet loss. // // Input: - // - packetLossEnc : The reported packet loss filtered (max window or average) + // - packetLossEnc : The reported packet loss filtered + // (max window or average) void UpdateFilteredLossPr(WebRtc_UWord8 packetLossEnc); // Update the current target bit rate. @@ -311,13 +308,13 @@ public: // Update the number of packets per frame estimate, for delta frames // // Input: - // - nPackets : Number of packets used to send the latest frame. + // - nPackets : Number of packets in the latest sent frame. void UpdatePacketsPerFrame(float nPackets); // Update the number of packets per frame estimate, for key frames // // Input: - // - nPackets : Number of packets used to send the latest frame. + // - nPackets : umber of packets in the latest sent frame. void UpdatePacketsPerFrameKey(float nPackets); // Update the keyFrameSize estimate @@ -400,7 +397,7 @@ private: WebRtc_UWord8 _shortMaxLossPr255; VCMExpFilter _packetsPerFrame; VCMExpFilter _packetsPerFrameKey; - float _residualPacketLoss; + float _residualPacketLossFec; WebRtc_UWord8 _boostRateKey; VCMFecTypes _fecType; WebRtc_UWord16 _codecWidth;