diff --git a/src/modules/video_coding/main/source/media_opt_util.cc b/src/modules/video_coding/main/source/media_opt_util.cc index c3766bed1d..f042bf0a9f 100644 --- a/src/modules/video_coding/main/source/media_opt_util.cc +++ b/src/modules/video_coding/main/source/media_opt_util.cc @@ -91,7 +91,7 @@ VCMNackFecMethod::EffectivePacketLoss(const // Effective Packet Loss, NA in current version. _effectivePacketLoss = 0; // No FEC applied. - _residualPacketLossFec = 255 * parameters->lossPr; + _residualPacketLossFec = parameters->lossPr; return true; } @@ -130,27 +130,16 @@ VCMNackFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) // should convert the factor to reduce mismatch between mediaOpt's rate and // the actual one _protectionFactorK = _fecMethod->ConvertFECRate(_protectionFactorK); - _protectionFactorD = _fecMethod->ConvertFECRate(_protectionFactorK); + _protectionFactorD = _fecMethod->ConvertFECRate(_protectionFactorD); return true; } bool -VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 packetLoss, - WebRtc_UWord16 rttTime) +VCMNackMethod::EffectivePacketLoss(const VCMProtectionParameters* parameter) { - WebRtc_UWord16 rttMax = MaxRttNack(); - - // For large RTT, we should rely on some Error Resilience, so we set - // packetLossEnc = 0 for RTT less than the NACK threshold - if (rttTime < rttMax) - { - _effectivePacketLoss = 0; // may want a softer transition here - } - else - { - _effectivePacketLoss = packetLoss; - } + // Effective Packet Loss, NA in current version. + _effectivePacketLoss = 0; return true; } @@ -158,18 +147,10 @@ VCMNackMethod::EffectivePacketLoss(WebRtc_UWord8 packetLoss, bool VCMNackMethod::UpdateParameters(const VCMProtectionParameters* parameters) { - // Compute the effective packet loss for ER - WebRtc_UWord8 effPacketLoss = (WebRtc_UWord8) (255 * parameters->lossPr); - WebRtc_UWord16 rttTime = (WebRtc_UWord16) parameters->rtt; - EffectivePacketLoss(effPacketLoss, rttTime); - - // Compute the NACK bit cost - if (rttTime > _NACK_MAX_RTT) - { - _efficiency = 0.0f; - return false; - } + // Compute the effective packet loss + EffectivePacketLoss(parameters); + // nackCost = (bitRate - nackCost) * (lossPr) _efficiency = parameters->bitRate * parameters->lossPr / (1.0f + parameters->lossPr); return true; @@ -564,12 +545,6 @@ VCMMbIntraRefreshMethod::UpdateParameters(const return true; } -WebRtc_UWord16 -VCMNackMethod::MaxRttNack() const -{ - return _NACK_MAX_RTT; -} - VCMLossProtectionLogic::~VCMLossProtectionLogic() { ClearLossProtections(); @@ -579,14 +554,15 @@ void VCMLossProtectionLogic::ClearLossProtections() { ListItem *item; - while ((item = _availableMethods.First()) != 0) { - VCMProtectionMethod *method = static_cast + while ((item = _availableMethods.First()) != 0) + { + VCMProtectionMethod *method = static_cast (item->GetItem()); - if (method != NULL) - { - delete method; - } - _availableMethods.PopFront(); + if (method != NULL) + { + delete method; + } + _availableMethods.PopFront(); } _selectedMethod = NULL; } @@ -685,12 +661,6 @@ VCMLossProtectionLogic::UpdateResidualPacketLoss(float residualPacketLoss) _residualPacketLossFec = residualPacketLoss; } -void -VCMLossProtectionLogic::UpdateFecType(VCMFecTypes fecType) -{ - _fecType = fecType; -} - void VCMLossProtectionLogic::UpdateLossPr(WebRtc_UWord8 lossPr255) { @@ -846,13 +816,11 @@ VCMLossProtectionLogic::UpdateMethod(VCMProtectionMethod *newMethod /*=NULL */) _currentParameters.packetsPerFrame = _packetsPerFrame.Value(); _currentParameters.packetsPerFrameKey = _packetsPerFrameKey.Value(); _currentParameters.residualPacketLossFec = _residualPacketLossFec; - _currentParameters.fecType = _fecType; _currentParameters.codecWidth = _codecWidth; _currentParameters.codecHeight = _codecHeight; if (newMethod == NULL) { - //_selectedMethod = _bestNotOkMethod = NULL; VCMProtectionMethod *method; ListItem *item; for (item = _availableMethods.First(); item != NULL; diff --git a/src/modules/video_coding/main/source/media_opt_util.h b/src/modules/video_coding/main/source/media_opt_util.h index 59d3960f84..e3419f058e 100644 --- a/src/modules/video_coding/main/source/media_opt_util.h +++ b/src/modules/video_coding/main/source/media_opt_util.h @@ -28,12 +28,8 @@ namespace webrtc class ListWrapper; enum { kLossPrHistorySize = 30 }; // 30 time periods -enum { kLossPrShortFilterWinMs = 1000 }; // 1000 ms, total filter length is 30 000 ms - -enum VCMFecTypes -{ - kXORFec -}; +// 1000 ms, total filter length is 30 000 ms +enum { kLossPrShortFilterWinMs = 1000 }; // Thresholds for hybrid NACK/FEC // common to media optimization and the jitter buffer. @@ -46,8 +42,8 @@ struct VCMProtectionParameters { VCMProtectionParameters() : rtt(0), lossPr(0), bitRate(0), packetsPerFrame(0), frameRate(0), keyFrameSize(0), fecRateDelta(0), - fecRateKey(0), residualPacketLossFec(0.0), fecType(kXORFec), - codecWidth(0), codecHeight(0) {} + fecRateKey(0), residualPacketLossFec(0.0), codecWidth(0), codecHeight(0) + {} WebRtc_UWord32 rtt; float lossPr; @@ -59,7 +55,6 @@ struct VCMProtectionParameters WebRtc_UWord8 fecRateDelta; WebRtc_UWord8 fecRateKey; float residualPacketLossFec; - VCMFecTypes fecType; WebRtc_UWord16 codecWidth; WebRtc_UWord16 codecHeight; @@ -91,16 +86,16 @@ public: }; - class VCMProtectionMethod { public: - //friend VCMProtectionMethod; - VCMProtectionMethod(VCMProtectionMethodEnum type) : _protectionFactorK(0), - _protectionFactorD(0), _residualPacketLossFec(0.0), _scaleProtKey(2.0), - _maxPayloadSize(1460), _efficiency(0), _type(type), - _useUepProtectionK(false), _useUepProtectionD(true) + VCMProtectionMethod(VCMProtectionMethodEnum type) : _effectivePacketLoss(0), + _protectionFactorK(0), _protectionFactorD(0), + _residualPacketLossFec(0.0), _scaleProtKey(2.0), _maxPayloadSize(1460), + _useUepProtectionK(false), _useUepProtectionD(true), _efficiency(0), + _type(type) {_qmRobustness = new VCMQmRobustness();} + virtual ~VCMProtectionMethod() { delete _qmRobustness;} // Updates the efficiency of the method using the parameters provided @@ -149,7 +144,7 @@ public: virtual WebRtc_UWord8 RequiredUepProtectionD() { return _useUepProtectionD; } // Updates content metrics - void UpdateContentMetrics(const VideoContentMetrics* contentMetrics); + void UpdateContentMetrics(const VideoContentMetrics* contentMetrics); WebRtc_UWord8 _effectivePacketLoss; WebRtc_UWord8 _protectionFactorK; @@ -174,15 +169,11 @@ private: class VCMNackMethod : public VCMProtectionMethod { public: - VCMNackMethod() : VCMProtectionMethod(kNACK), _NACK_MAX_RTT(200) {} + VCMNackMethod() : VCMProtectionMethod(kNACK) {} virtual ~VCMNackMethod() {} virtual bool UpdateParameters(const VCMProtectionParameters* parameters); - // Get the effective packet loss for ER - bool EffectivePacketLoss(WebRtc_UWord8 packetLoss, WebRtc_UWord16 rttTime); - // Get the threshold for NACK - WebRtc_UWord16 MaxRttNack() const; -private: - const WebRtc_UWord16 _NACK_MAX_RTT; + // Get the effective packet loss + bool EffectivePacketLoss(const VCMProtectionParameters* parameter); }; class VCMFecMethod : public VCMProtectionMethod @@ -253,10 +244,10 @@ class VCMLossProtectionLogic { public: VCMLossProtectionLogic() : _availableMethods(), _selectedMethod(NULL), - _bestNotOkMethod(NULL), _rtt(0), _lossPr(0.0f), _bitRate(0.0f), - _frameRate(0.0f), _keyFrameSize(0.0f), _fecRateKey(0), _fecRateDelta(0), - _lastPrUpdateT(0), _lossPr255(0.9999f), _lossPrHistory(), - _shortMaxLossPr255(0), _packetsPerFrame(0.9999f), + _bestNotOkMethod(NULL), _currentParameters(), _rtt(0), _lossPr(0.0f), + _bitRate(0.0f), _frameRate(0.0f), _keyFrameSize(0.0f), _fecRateKey(0), + _fecRateDelta(0), _lastPrUpdateT(0), _lossPr255(0.9999f), + _lossPrHistory(), _shortMaxLossPr255(0), _packetsPerFrame(0.9999f), _packetsPerFrameKey(0.9999f), _residualPacketLossFec(0), _boostRateKey(2), _codecWidth(0), _codecHeight(0) { Reset(); } @@ -282,12 +273,6 @@ public: // effective loss after FEC recovery void UpdateResidualPacketLoss(float _residualPacketLoss); - // Update fecType - // - // Input: - // - fecType : kXORFec for generic XOR FEC - void UpdateFecType(VCMFecTypes fecType); - // Update the loss probability. // // Input: @@ -342,22 +327,22 @@ public: // The amount of packet loss to cover for with FEC. // // Input: - // - fecRateKey : Packet loss to cover for with FEC when sending key frames. - // - fecRateDelta : Packet loss to cover for with FEC when sending delta frames. + // - fecRateKey : Packet loss to cover for with FEC when + // sending key frames. + // - fecRateDelta : Packet loss to cover for with FEC when + // sending delta frames. void UpdateFECRates(WebRtc_UWord8 fecRateKey, WebRtc_UWord8 fecRateDelta) - { _fecRateKey = fecRateKey; _fecRateDelta = fecRateDelta; } + { _fecRateKey = fecRateKey; + _fecRateDelta = fecRateDelta; } - // Update the protection methods with the current VCMProtectionParameters and - // choose the best method available. The update involves computing the robustness settings - // for the protection method. + // Update the protection methods with the current VCMProtectionParameters + // and choose the best method available. The update involves computing the + // robustness settings for the protection method. // // Input: - // - newMethod : If not NULL, this is method will be selected by force. + // - newMethod : If not NULL, this method will be selected. // - // Return value : True if the selected method is recommended using these settings, - // false if it's the best method, but still not recommended to be used. - // E.g. if NACK is the best available, but the RTT is too large, false - // will be returned. + // Return value : Returns true on update bool UpdateMethod(VCMProtectionMethod *newMethod = NULL); // Returns the method currently selected. @@ -370,11 +355,6 @@ public: // Return value : The filtered loss probability WebRtc_UWord8 FilteredLoss() const; - // Get constraint on NACK - // - // return value : RTT threshold for using NACK - WebRtc_UWord8 GetNackThreshold() const; - void Reset(); private: @@ -402,7 +382,6 @@ private: VCMExpFilter _packetsPerFrameKey; float _residualPacketLossFec; WebRtc_UWord8 _boostRateKey; - VCMFecTypes _fecType; WebRtc_UWord16 _codecWidth; WebRtc_UWord16 _codecHeight; }; diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index 5119254784..6ab61922d9 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -101,9 +101,6 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, _lossProtLogic->UpdateRtt(roundTripTimeMs); _lossProtLogic->UpdateResidualPacketLoss(static_cast(fractionLost)); - VCMFecTypes fecType = kXORFec; // generic FEC - _lossProtLogic->UpdateFecType(fecType); - // Get frame rate for encoder: this is the actual/sent frame rate float actualFrameRate = SentFrameRate(); @@ -125,92 +122,51 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, // low/high loss rates, etc. WebRtc_UWord8 packetLossEnc = _lossProtLogic->FilteredLoss(); - //For now use the filtered loss for computing the robustness settings + // For now use the filtered loss for computing the robustness settings _lossProtLogic->UpdateFilteredLossPr(packetLossEnc); // Rate cost of the protection methods _lossProtOverhead = 0; - if (selectedMethod && (selectedMethod->Type() == kFEC || - selectedMethod->Type() == kNackFec )) + // Update protection settings, when applicable + if (selectedMethod) { - // Update protection method with content metrics selectedMethod->UpdateContentMetrics(_content->ShortTermAvgData()); // Update method will compute the robustness settings for the given // protection method and the overhead cost // the protection method is set by the user via SetVideoProtection. - // The robustness settings are: the effective packet loss for ER and the - // FEC protection settings _lossProtLogic->UpdateMethod(); - // Get the FEC code rate for Key frames - const WebRtc_UWord8 codeRateKeyRTP = selectedMethod->RequiredProtectionFactorK(); + // Update protection callback with protection settings + UpdateProtectionCallback(selectedMethod); - // Get the FEC code rate for Delta frames - const WebRtc_UWord8 codeRateDeltaRTP = selectedMethod->RequiredProtectionFactorD(); + // Get the bit cost of protection method + _lossProtOverhead = static_cast + (_lossProtLogic->HighestOverhead() + 0.5f); - // Get the FEC-UEP protection status for Key frames: UEP on/off - const bool useUepProtectionKeyRTP = selectedMethod->RequiredUepProtectionK(); - - // Get the FEC-UEP protection status for Delta frames: UEP on/off - const bool useUepProtectionDeltaRTP = selectedMethod->RequiredUepProtectionD(); - - // Get the effective packet loss for ER + // Get the effective packet loss for encoder ER + // when applicable, should be passed to encoder via fractionLost packetLossEnc = selectedMethod->RequiredPacketLossER(); - - // NACK is on for NACK and NackFec protection method: off for FEC method - bool nackStatus = (selectedMethod->Type() == kNackFec || - selectedMethod->Type() == kNACK); - - if(_videoProtectionCallback) - { - _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP, - codeRateKeyRTP, - useUepProtectionDeltaRTP, - useUepProtectionKeyRTP, - nackStatus); - } - } - - // Get the bit cost of protection method - _lossProtOverhead = static_cast(_lossProtLogic->HighestOverhead() + 0.5f); - - // Update effective packet loss for encoder: note: fractionLost was passed as reference - fractionLost = packetLossEnc; - - WebRtc_UWord32 nackBitRate=0; - if(selectedMethod && _lossProtLogic->FindMethod(kNACK) != NULL) - { - // TODO(mikhal): update frame dropper with bit rate including both nack and fec - // Make sure we don't over-use the channel momentarily. This is - // necessary for NACK since it can be very bursty. - nackBitRate = (_lastBitRate * fractionLost) / 255; - if (nackBitRate > _targetBitRate) - { - nackBitRate = _targetBitRate; - } - _frameDropper->SetRates(static_cast(bitRate - nackBitRate), 0); - } - else - { - _frameDropper->SetRates(static_cast(bitRate - _lossProtOverhead), 0); } + // Update encoding rates following protection settings + _frameDropper->SetRates(static_cast(bitRate - + _lossProtOverhead), 0); // This may be used for UpdateEncoderBitRate: lastBitRate is total rate, // before compensation _lastBitRate = _targetBitRate; - //Source coding rate: total rate - protection overhead + // Source coding rate: total rate - protection overhead _targetBitRate = bitRate - _lossProtOverhead; if (_enableQm) { - //Update QM with rates + // Update QM with rates _qmResolution->UpdateRates((float)_targetBitRate, _avgSentBitRateBps, _incomingFrameRate, _fractionLost); - //Check for QM selection + // Check for QM selection bool selectQM = checkStatusForQMchange(); if (selectQM) { @@ -223,6 +179,40 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, return _targetBitRate; } +WebRtc_UWord32 +VCMMediaOptimization::UpdateProtectionCallback(VCMProtectionMethod + *selectedMethod) +{ + if (!_videoProtectionCallback) + { + return VCM_OK; + } + // Get the FEC code rate for Key frames (set to 0 when NA) + const WebRtc_UWord8 + codeRateKeyRTP = selectedMethod->RequiredProtectionFactorK(); + + // Get the FEC code rate for Delta frames (set to 0 when NA) + const WebRtc_UWord8 + codeRateDeltaRTP = selectedMethod->RequiredProtectionFactorD(); + + // Get the FEC-UEP protection status for Key frames: UEP on/off + const bool + useUepProtectionKeyRTP = selectedMethod->RequiredUepProtectionK(); + + // Get the FEC-UEP protection status for Delta frames: UEP on/off + const bool + useUepProtectionDeltaRTP = selectedMethod->RequiredUepProtectionD(); + + // NACK is on for NACK and NackFec protection method: off for FEC method + bool nackStatus = (selectedMethod->Type() == kNackFec || + selectedMethod->Type() == kNACK); + + return _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP, + codeRateKeyRTP, + useUepProtectionDeltaRTP, + useUepProtectionKeyRTP, + nackStatus); +} bool VCMMediaOptimization::DropFrame() @@ -245,9 +235,10 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3 WebRtc_UWord32 frameRate, WebRtc_UWord32 bitRate, WebRtc_UWord16 width, WebRtc_UWord16 height) { - // Everything codec specific should be reset here since this means the codec has changed. - // If native dimension values have changed, then either user initiated change, or QM - // initiated change. Will be able to determine only after the processing of the first frame + // Everything codec specific should be reset here since this means the codec + // has changed. If native dimension values have changed, then either user + // initiated change, or QM initiated change. Will be able to determine only + // after the processing of the first frame. _lastChangeTime = VCMTickTime::MillisecondTimestamp(); _content->Reset(); _content->UpdateFrameRate(frameRate); @@ -259,7 +250,8 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3 _lossProtLogic->UpdateFrameRate(static_cast(frameRate)); _lossProtLogic->UpdateFrameSize(width, height); _frameDropper->Reset(); - _frameDropper->SetRates(static_cast(bitRate), static_cast(frameRate)); + _frameDropper->SetRates(static_cast(bitRate), + static_cast(frameRate)); _userFrameRate = static_cast(frameRate); _codecWidth = width; _codecHeight = height; @@ -270,7 +262,8 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3 } WebRtc_Word32 -VCMMediaOptimization::RegisterProtectionCallback(VCMProtectionCallback* protectionCallback) +VCMMediaOptimization::RegisterProtectionCallback(VCMProtectionCallback* + protectionCallback) { _videoProtectionCallback = protectionCallback; return VCM_OK; @@ -383,9 +376,10 @@ VCMMediaOptimization::SetMtu(WebRtc_Word32 mtu) float VCMMediaOptimization::SentFrameRate() { - if(_frameDropper) + if (_frameDropper) { - return _frameDropper->ActualFrameRate((WebRtc_UWord32)(InputFrameRate() + 0.5f)); + return _frameDropper->ActualFrameRate((WebRtc_UWord32)(InputFrameRate() + + 0.5f)); } return VCM_CODEC_ERROR; @@ -442,9 +436,12 @@ VCMMediaOptimization::UpdateWithEncodedData(WebRtc_Word32 encodedLength, } // updating counters - if (deltaFrame){ + if (deltaFrame) + { _deltaFrameCnt++; - } else { + } + else + { _keyFrameCnt++; } @@ -469,9 +466,11 @@ void VCMMediaOptimization::UpdateBitRateEstimate(WebRtc_Word64 encodedLength, // Found empty slot break; } - if (nowMs - _encodedFrameSamples[i]._timeCompleteMs < kBitrateAverageWinMs) + if (nowMs - _encodedFrameSamples[i]._timeCompleteMs < + kBitrateAverageWinMs) { - frameSizeSum += static_cast(_encodedFrameSamples[i]._sizeBytes); + frameSizeSum += static_cast + (_encodedFrameSamples[i]._sizeBytes); if (timeOldest == -1) { timeOldest = _encodedFrameSamples[i]._timeCompleteMs; @@ -515,7 +514,8 @@ void VCMMediaOptimization::UpdateBitRateEstimate(WebRtc_Word64 encodedLength, WebRtc_Word32 -VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback *videoQMSettings) +VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback* + videoQMSettings) { _videoQMSettingsCallback = videoQMSettings; // Callback setting controls QM @@ -531,12 +531,13 @@ VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback *videoQMSett } void -VCMMediaOptimization::updateContentData(const VideoContentMetrics *contentMetrics) +VCMMediaOptimization::updateContentData(const VideoContentMetrics* + contentMetrics) { - //Updating content metrics + // Updating content metrics if (contentMetrics == NULL) { - //No QM if metrics are NULL + // Disable QM if metrics are NULL _enableQm = false; _qmResolution->Reset(); } @@ -664,7 +665,7 @@ void VCMMediaOptimization::UpdateIncomingFrameRate() { WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); - if(_incomingFrameTimes[0] == 0) + if (_incomingFrameTimes[0] == 0) { // first no shift } else diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h index eecf270745..4902de3ffa 100644 --- a/src/modules/video_coding/main/source/media_optimization.h +++ b/src/modules/video_coding/main/source/media_optimization.h @@ -154,6 +154,11 @@ public: private: + /* + * Update protection callback with protection settings + */ + WebRtc_UWord32 UpdateProtectionCallback(VCMProtectionMethod *selectedMethod); + void UpdateBitRateEstimate(WebRtc_Word64 encodedLength, WebRtc_Word64 nowMs); /* * verify if QM settings differ from default, i.e. if an update is required