diff --git a/modules/video_coding/main/source/content_metrics_processing.cc b/modules/video_coding/main/source/content_metrics_processing.cc index 8a214820f7..5206da00c4 100644 --- a/modules/video_coding/main/source/content_metrics_processing.cc +++ b/modules/video_coding/main/source/content_metrics_processing.cc @@ -24,46 +24,79 @@ namespace webrtc { VCMContentMetricsProcessing::VCMContentMetricsProcessing(): _frameRate(0), _recAvgFactor(1 / 150.0f), // matched to 30fps -_frameCnt(0), -_prevAvgSizeZeroMotion(0), -_avgSizeZeroMotion(0), -_prevAvgSpatialPredErr(0), -_avgSpatialPredErr(0), -_frameCntForCC(0), -_lastCCpdateTime(0) +_frameCntRecursiveAvg(0), +_frameCntUniformAvg(0), +_avgMotionLevel(0.0f), +_avgSpatialLevel(0.0f) { - _globalRecursiveAvg = new VideoContentMetrics(); + _recursiveAvg = new VideoContentMetrics(); + _uniformAvg = new VideoContentMetrics(); } VCMContentMetricsProcessing::~VCMContentMetricsProcessing() { - delete _globalRecursiveAvg; + delete _recursiveAvg; + delete _uniformAvg; } WebRtc_Word32 VCMContentMetricsProcessing::Reset() { - _globalRecursiveAvg->Reset(); - _frameCnt = 0; + _recursiveAvg->Reset(); + _uniformAvg->Reset(); _frameRate = 0; - //_recAvgFactor = 1 / 150.0f; // matched to 30 fps - _prevAvgSizeZeroMotion = 0; - _avgSizeZeroMotion = 0; - _prevAvgSpatialPredErr = 0; - _avgSpatialPredErr = 0; - _frameCntForCC = 0; - + _frameCntRecursiveAvg = 0; + _frameCntUniformAvg = 0; + _avgMotionLevel = 0.0f; + _avgSpatialLevel = 0.0f; return VCM_OK; } + void VCMContentMetricsProcessing::UpdateFrameRate(WebRtc_UWord32 frameRate) { _frameRate = frameRate; - //Update recursive avg factor - _recAvgFactor = (float) 1000 / ((float)(_frameRate * kQmMinIntervalMs)); + // Update factor for recursive averaging. + _recAvgFactor = (float) 1000.0f / ((float)(_frameRate * kQmMinIntervalMs)); } +VideoContentMetrics* +VCMContentMetricsProcessing::LongTermAvgData() +{ + if (_frameCntRecursiveAvg == 0) + { + return NULL; + } + return _recursiveAvg; +} + +VideoContentMetrics* +VCMContentMetricsProcessing::ShortTermAvgData() +{ + if (_frameCntUniformAvg == 0) + { + return NULL; + } + + // Two metrics are used: motion and spatial level. + _uniformAvg->motionMagnitudeNZ = _avgMotionLevel / + (float)(_frameCntUniformAvg); + _uniformAvg->spatialPredErr = _avgSpatialLevel / + (float)(_frameCntUniformAvg); + + return _uniformAvg; +} + +void +VCMContentMetricsProcessing::ResetShortTermAvgData() +{ + // Reset + _avgMotionLevel = 0.0f; + _avgSpatialLevel = 0.0f; + _frameCntUniformAvg = 0; +} + WebRtc_Word32 VCMContentMetricsProcessing::UpdateContentData(const VideoContentMetrics *contentMetrics) { @@ -75,151 +108,106 @@ VCMContentMetricsProcessing::UpdateContentData(const VideoContentMetrics *conten } -VideoContentMetrics* -VCMContentMetricsProcessing::Data() -{ - if (_frameCnt == 0) - { - return NULL; - } - return _globalRecursiveAvg; -} - WebRtc_UWord32 VCMContentMetricsProcessing::ProcessContent(const VideoContentMetrics *contentMetrics) { - // update global metric - UpdateGlobalMetric(contentMetrics); + // Update the recursive averaged metrics + // average is over longer window of time: over QmMinIntervalMs ms. + UpdateRecursiveAvg(contentMetrics); - //Update metrics over local window for content change (CC) detection: - //two metrics are used for CC detection: size of zero motion, and spatial prediction error - //Not currently used: - //UpdateLocalMetricCC(contentMetrics->sizeZeroMotion, contentMetrics->spatialPredErr); + // Update the uniform averaged metrics: + // average is over shorter window of time: based on ~RTCP reports. + UpdateUniformAvg(contentMetrics); return VCM_OK; } -bool -VCMContentMetricsProcessing::ContentChangeCheck() -{ - bool result = false; - - // Thresholds for bitrate and content change detection - float qmContentChangePercMotion = 0.4f; - float qmContentChangePercSpatial = 0.4f; - - WebRtc_Word64 now = VCMTickTime::MillisecondTimestamp(); - if ( (now - _lastCCpdateTime) < kCcMinIntervalMs) - { - //keep averaging - return result; - } - else //check for detection and reset - { - //normalize - _avgSizeZeroMotion = _avgSizeZeroMotion / (float)(_frameCntForCC); - _prevAvgSpatialPredErr = _prevAvgSpatialPredErr / (float)(_frameCntForCC); - - //check for content change - float diffMotion = fabs(_avgSizeZeroMotion - _prevAvgSizeZeroMotion); - float diffSpatial = fabs(_avgSpatialPredErr -_prevAvgSpatialPredErr); - if ((diffMotion > (_avgSizeZeroMotion * qmContentChangePercMotion)) || - (diffSpatial > (_prevAvgSpatialPredErr * qmContentChangePercSpatial))) - { - result = true; - } - - //copy to previous - _prevAvgSizeZeroMotion = _avgSizeZeroMotion; - _prevAvgSpatialPredErr = _avgSpatialPredErr; - - //reset - _avgSizeZeroMotion = 0.; - _avgSpatialPredErr = 0.; - _frameCntForCC = 0; - - _lastCCpdateTime = now; - - } - - return result; -} - -//update metrics for content change detection: update is uniform average over soem time window -void VCMContentMetricsProcessing::UpdateLocalMetricCC(float motionVal, float spatialVal) +void +VCMContentMetricsProcessing::UpdateUniformAvg(const VideoContentMetrics *contentMetrics) { - _frameCntForCC += 1; - _avgSizeZeroMotion += motionVal; - _avgSpatialPredErr += spatialVal; + // Update frame counter + _frameCntUniformAvg += 1; + + // Update averaged metrics: motion and spatial level are used. + _avgMotionLevel += contentMetrics->motionMagnitudeNZ; + _avgSpatialLevel += contentMetrics->spatialPredErr; return; } void -VCMContentMetricsProcessing::UpdateGlobalMetric(const VideoContentMetrics *contentMetrics) +VCMContentMetricsProcessing::UpdateRecursiveAvg(const VideoContentMetrics *contentMetrics) { - // Threshold for size of zero motion cluster: for updating 3 metrics: - // motion magnitude, cluster distortion, and horizontalness + // Threshold for size of zero motion cluster: + // Use for updating 3 motion vector derived metrics: + // motion magnitude, cluster distortion, and horizontalness. float nonZeroMvThr = 0.1f; - // first zero and one: take value as is (no motion search in frame zero). float tmpRecAvgFactor = _recAvgFactor; - if (_frameCnt < 1) + + // Take value as is for first frame (no motion search in frame zero). + if (_frameCntRecursiveAvg < 1) { - _recAvgFactor = 1; + tmpRecAvgFactor = 1; } - _globalRecursiveAvg->motionPredErr = (1 - _recAvgFactor) * _globalRecursiveAvg->motionPredErr + - _recAvgFactor * contentMetrics->motionPredErr; + _recursiveAvg->motionPredErr = (1 - tmpRecAvgFactor) * + _recursiveAvg->motionPredErr + + tmpRecAvgFactor * contentMetrics->motionPredErr; - _globalRecursiveAvg->sizeZeroMotion = (1 - _recAvgFactor) * _globalRecursiveAvg->sizeZeroMotion + - _recAvgFactor * contentMetrics->sizeZeroMotion; + _recursiveAvg->sizeZeroMotion = (1 - tmpRecAvgFactor) * + _recursiveAvg->sizeZeroMotion + + tmpRecAvgFactor * contentMetrics->sizeZeroMotion; - _globalRecursiveAvg->spatialPredErr = (1 - _recAvgFactor) * _globalRecursiveAvg->spatialPredErr + - _recAvgFactor * contentMetrics->spatialPredErr; + _recursiveAvg->spatialPredErr = (1 - tmpRecAvgFactor) * + _recursiveAvg->spatialPredErr + + tmpRecAvgFactor * contentMetrics->spatialPredErr; - _globalRecursiveAvg->spatialPredErrH = (1 - _recAvgFactor) * _globalRecursiveAvg->spatialPredErrH + - _recAvgFactor * contentMetrics->spatialPredErrH; + _recursiveAvg->spatialPredErrH = (1 - tmpRecAvgFactor) * + _recursiveAvg->spatialPredErrH + + tmpRecAvgFactor * contentMetrics->spatialPredErrH; - _globalRecursiveAvg->spatialPredErrV = (1 - _recAvgFactor) * _globalRecursiveAvg->spatialPredErrV + - _recAvgFactor * contentMetrics->spatialPredErrV; + _recursiveAvg->spatialPredErrV = (1 - tmpRecAvgFactor) * + _recursiveAvg->spatialPredErrV + + tmpRecAvgFactor * contentMetrics->spatialPredErrV; - //motionMag metric is derived from NFD (normalized frame difference) + // motionMag metric is derived from NFD (normalized frame difference). if (kNfdMetric == 1) { - _globalRecursiveAvg->motionMagnitudeNZ = (1 - _recAvgFactor) * _globalRecursiveAvg->motionMagnitudeNZ + - _recAvgFactor * contentMetrics->motionMagnitudeNZ; + _recursiveAvg->motionMagnitudeNZ = (1 - tmpRecAvgFactor) * + _recursiveAvg->motionMagnitudeNZ + + tmpRecAvgFactor * contentMetrics->motionMagnitudeNZ; } if (contentMetrics->sizeZeroMotion > nonZeroMvThr) { - _globalRecursiveAvg->motionClusterDistortion = (1 - _recAvgFactor) * _globalRecursiveAvg->motionClusterDistortion + - _recAvgFactor *contentMetrics->motionClusterDistortion; + _recursiveAvg->motionClusterDistortion = (1 - tmpRecAvgFactor) * + _recursiveAvg->motionClusterDistortion + + tmpRecAvgFactor *contentMetrics->motionClusterDistortion; - _globalRecursiveAvg->motionHorizontalness = (1 - _recAvgFactor) * _globalRecursiveAvg->motionHorizontalness + - _recAvgFactor * contentMetrics->motionHorizontalness; + _recursiveAvg->motionHorizontalness = (1 - _recAvgFactor) * + _recursiveAvg->motionHorizontalness + + tmpRecAvgFactor * contentMetrics->motionHorizontalness; - //motionMag metric is derived from motion vectors + // motionMag metric is derived from motion vectors. if (kNfdMetric == 0) { - _globalRecursiveAvg->motionMagnitudeNZ = (1 - _recAvgFactor) * _globalRecursiveAvg->motionMagnitudeNZ + - _recAvgFactor * contentMetrics->motionMagnitudeNZ; + _recursiveAvg->motionMagnitudeNZ = (1 - tmpRecAvgFactor) * + _recursiveAvg->motionMagnitudeNZ + + tmpRecAvgFactor * contentMetrics->motionMagnitudeNZ; } } - // update native values: - _globalRecursiveAvg->nativeHeight = contentMetrics->nativeHeight; - _globalRecursiveAvg->nativeWidth = contentMetrics->nativeWidth; - _globalRecursiveAvg->nativeFrameRate = contentMetrics->nativeFrameRate; + // Update native values: + // TODO (marpan): we don't need to update this every frame. + _recursiveAvg->nativeHeight = contentMetrics->nativeHeight; + _recursiveAvg->nativeWidth = contentMetrics->nativeWidth; + _recursiveAvg->nativeFrameRate = contentMetrics->nativeFrameRate; + + _frameCntRecursiveAvg++; - if (_frameCnt < 1) - { - _recAvgFactor = tmpRecAvgFactor; - } - _frameCnt++; return; } - -} +} //end of namespace diff --git a/modules/video_coding/main/source/content_metrics_processing.h b/modules/video_coding/main/source/content_metrics_processing.h index 8295230248..155c4adc61 100644 --- a/modules/video_coding/main/source/content_metrics_processing.h +++ b/modules/video_coding/main/source/content_metrics_processing.h @@ -18,12 +18,11 @@ namespace webrtc struct VideoContentMetrics; -// QM interval time +// QM interval time (in ms) enum { kQmMinIntervalMs = 10000 }; -enum { kCcMinIntervalMs = 5000 }; -//Flag for NFD metric vs motion metric -enum { kNfdMetric = 1 }; +// Flag for NFD metric vs motion metric +enum { kNfdMetric = 1 }; /**********************************/ /* Content Metrics Processing */ @@ -37,39 +36,41 @@ public: // Update class with latest metrics WebRtc_Word32 UpdateContentData(const VideoContentMetrics *contentMetrics); - // Check for content change detection - bool ContentChangeCheck(); + // Reset the short-term averaged content data + void ResetShortTermAvgData(); - //Initialize to + // Initialize to WebRtc_Word32 Reset(); // Inform class of current frame rate void UpdateFrameRate(WebRtc_UWord32 frameRate); - // Get working (avg) value - VideoContentMetrics* Data(); + // Returns the long-term averaged content data: + // recursive average over longer time scale + VideoContentMetrics* LongTermAvgData(); + + // Returns the short-term averaged content data: + // uniform average over shorter time scale + VideoContentMetrics* ShortTermAvgData(); private: // Compute working avg WebRtc_UWord32 ProcessContent(const VideoContentMetrics *contentMetrics); - // Computation of global metric - void UpdateGlobalMetric(const VideoContentMetrics *contentMetrics); + // Update the recursive averaged metrics: longer time average (~5/10 secs). + void UpdateRecursiveAvg(const VideoContentMetrics *contentMetrics); - // Compute local average of certain metrics for content change detection - void UpdateLocalMetricCC(float motionVal, float spatialVal); + // Update the uniform averaged metrics: shorter time average (~RTCP reports). + void UpdateUniformAvg(const VideoContentMetrics *contentMetrics); - VideoContentMetrics* _globalRecursiveAvg; + VideoContentMetrics* _recursiveAvg; + VideoContentMetrics* _uniformAvg; WebRtc_UWord32 _frameRate; float _recAvgFactor; - WebRtc_UWord32 _frameCnt; - - float _prevAvgSizeZeroMotion; - float _avgSizeZeroMotion; - float _prevAvgSpatialPredErr; - float _avgSpatialPredErr; - WebRtc_UWord32 _frameCntForCC; - WebRtc_UWord64 _lastCCpdateTime; + WebRtc_UWord32 _frameCntRecursiveAvg; + WebRtc_UWord32 _frameCntUniformAvg; + float _avgMotionLevel; + float _avgSpatialLevel; }; } // namespace webrtc diff --git a/modules/video_coding/main/source/media_optimization.cc b/modules/video_coding/main/source/media_optimization.cc index 0b388bf1bb..e879720bae 100644 --- a/modules/video_coding/main/source/media_optimization.cc +++ b/modules/video_coding/main/source/media_optimization.cc @@ -204,6 +204,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, { SelectQuality(); } + // Reset the short-term averaged content data. + _content->ResetShortTermAvgData(); } return _targetBitRate; @@ -537,7 +539,7 @@ VCMMediaOptimization::SelectQuality() // Select quality mode VCMQualityMode* qm = NULL; - WebRtc_Word32 ret = _qms->SelectQuality(_content->Data(), &qm); + WebRtc_Word32 ret = _qms->SelectQuality(_content->LongTermAvgData(), &qm); if (ret < 0) { return ret; @@ -596,7 +598,7 @@ VCMMediaOptimization::QMUpdate(VCMQualityMode* qm) } // Content metrics hold native values - VideoContentMetrics* cm = _content->Data(); + VideoContentMetrics* cm = _content->LongTermAvgData(); // Temporal WebRtc_UWord32 frameRate = static_cast(_incomingFrameRate + 0.5f);