From bd5648f2db2c143584cc2c55df99d11453e13818 Mon Sep 17 00:00:00 2001 From: "marpan@webrtc.org" Date: Fri, 17 Feb 2012 23:16:58 +0000 Subject: [PATCH] Reverting 1718: failed linux video test. TBR=stefan, andrew, marpan. Review URL: https://webrtc-codereview.appspot.com/392018 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1719 4adac7df-926f-26a2-2b94-8c16560cd09d --- src/modules/interface/module_common_types.h | 40 ++-- .../main/source/content_metrics_processing.cc | 113 +++++++++--- .../main/source/content_metrics_processing.h | 6 +- .../main/source/media_optimization.cc | 66 +++---- .../main/source/media_optimization.h | 2 - .../video_coding/main/source/qm_select.cc | 173 +++++++++++++++--- .../video_coding/main/source/qm_select.h | 15 +- .../main/source/content_analysis.cc | 15 +- .../main/source/content_analysis.h | 7 +- .../main/source/frame_preprocessor.cc | 11 +- .../main/source/frame_preprocessor.h | 4 +- 11 files changed, 330 insertions(+), 122 deletions(-) diff --git a/src/modules/interface/module_common_types.h b/src/modules/interface/module_common_types.h index fb20725aa4..6d571a4f45 100644 --- a/src/modules/interface/module_common_types.h +++ b/src/modules/interface/module_common_types.h @@ -394,23 +394,29 @@ public: }; // Video Content Metrics -struct VideoContentMetrics { - VideoContentMetrics() - : motionMagnitudeNZ(0.0f), - spatialPredErr(0.0f), - spatialPredErrH(0.0f), - spatialPredErrV(0.0f) { - } - void Reset() { - motionMagnitudeNZ = 0.0f; - spatialPredErr = 0.0f; - spatialPredErrH = 0.0f; - spatialPredErrV = 0.0f; - } - float motionMagnitudeNZ; - float spatialPredErr; - float spatialPredErrH; - float spatialPredErrV; +struct VideoContentMetrics +{ + VideoContentMetrics(): motionMagnitudeNZ(0), sizeZeroMotion(0), spatialPredErr(0), + spatialPredErrH(0), spatialPredErrV(0), motionPredErr(0), + motionHorizontalness(0), motionClusterDistortion(0), + nativeWidth(0), nativeHeight(0), contentChange(false) { } + void Reset(){ motionMagnitudeNZ = 0; sizeZeroMotion = 0; spatialPredErr = 0; + spatialPredErrH = 0; spatialPredErrV = 0; motionPredErr = 0; + motionHorizontalness = 0; motionClusterDistortion = 0; + nativeWidth = 0; nativeHeight = 0; contentChange = false; } + + float motionMagnitudeNZ; + float sizeZeroMotion; + float spatialPredErr; + float spatialPredErrH; + float spatialPredErrV; + float motionPredErr; + float motionHorizontalness; + float motionClusterDistortion; + WebRtc_UWord32 nativeWidth; + WebRtc_UWord32 nativeHeight; + WebRtc_UWord32 nativeFrameRate; + bool contentChange; }; /************************************************* diff --git a/src/modules/video_coding/main/source/content_metrics_processing.cc b/src/modules/video_coding/main/source/content_metrics_processing.cc index 02fc190ac2..0805178c58 100644 --- a/src/modules/video_coding/main/source/content_metrics_processing.cc +++ b/src/modules/video_coding/main/source/content_metrics_processing.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -21,7 +21,9 @@ namespace webrtc { ////////////////////////////////// VCMContentMetricsProcessing::VCMContentMetricsProcessing(): +_frameRate(0), _recAvgFactor(1 / 150.0f), // matched to 30fps +_frameCntRecursiveAvg(0), _frameCntUniformAvg(0), _avgMotionLevel(0.0f), _avgSpatialLevel(0.0f) @@ -41,6 +43,8 @@ VCMContentMetricsProcessing::Reset() { _recursiveAvg->Reset(); _uniformAvg->Reset(); + _frameRate = 0; + _frameCntRecursiveAvg = 0; _frameCntUniformAvg = 0; _avgMotionLevel = 0.0f; _avgSpatialLevel = 0.0f; @@ -50,20 +54,30 @@ VCMContentMetricsProcessing::Reset() void VCMContentMetricsProcessing::UpdateFrameRate(WebRtc_UWord32 frameRate) { + _frameRate = frameRate; // Update factor for recursive averaging. - _recAvgFactor = (float) 1000.0f / ((float)(frameRate * kQmMinIntervalMs)); + _recAvgFactor = (float) 1000.0f / ((float)(_frameRate * kQmMinIntervalMs)); } VideoContentMetrics* VCMContentMetricsProcessing::LongTermAvgData() { + if (_frameCntRecursiveAvg == 0) + { + return NULL; + } return _recursiveAvg; } VideoContentMetrics* VCMContentMetricsProcessing::ShortTermAvgData() { + if (_frameCntUniformAvg == 0) + { + return NULL; + } + // Two metrics are used: motion and spatial level. _uniformAvg->motionMagnitudeNZ = _avgMotionLevel / (float)(_frameCntUniformAvg); @@ -83,8 +97,7 @@ VCMContentMetricsProcessing::ResetShortTermAvgData() } WebRtc_Word32 -VCMContentMetricsProcessing::UpdateContentData( - const VideoContentMetrics *contentMetrics) +VCMContentMetricsProcessing::UpdateContentData(const VideoContentMetrics *contentMetrics) { if (contentMetrics == NULL) { @@ -95,8 +108,7 @@ VCMContentMetricsProcessing::UpdateContentData( } WebRtc_UWord32 -VCMContentMetricsProcessing::ProcessContent( - const VideoContentMetrics *contentMetrics) +VCMContentMetricsProcessing::ProcessContent(const VideoContentMetrics *contentMetrics) { // Update the recursive averaged metrics // average is over longer window of time: over QmMinIntervalMs ms. @@ -110,9 +122,9 @@ VCMContentMetricsProcessing::ProcessContent( } void -VCMContentMetricsProcessing::UpdateUniformAvg( - const VideoContentMetrics *contentMetrics) +VCMContentMetricsProcessing::UpdateUniformAvg(const VideoContentMetrics *contentMetrics) { + // Update frame counter _frameCntUniformAvg += 1; @@ -123,27 +135,78 @@ VCMContentMetricsProcessing::UpdateUniformAvg( return; } -void VCMContentMetricsProcessing::UpdateRecursiveAvg( - const VideoContentMetrics *contentMetrics) { +void +VCMContentMetricsProcessing::UpdateRecursiveAvg(const VideoContentMetrics *contentMetrics) +{ - // Spatial metrics: 2x2, 1x2(H), 2x1(V). - _recursiveAvg->spatialPredErr = (1 - _recAvgFactor) * - _recursiveAvg->spatialPredErr + - _recAvgFactor * contentMetrics->spatialPredErr; + // Threshold for size of zero motion cluster: + // Use for updating 3 motion vector derived metrics: + // motion magnitude, cluster distortion, and horizontalness. + float nonZeroMvThr = 0.1f; - _recursiveAvg->spatialPredErrH = (1 - _recAvgFactor) * - _recursiveAvg->spatialPredErrH + - _recAvgFactor * contentMetrics->spatialPredErrH; + float tmpRecAvgFactor = _recAvgFactor; - _recursiveAvg->spatialPredErrV = (1 - _recAvgFactor) * - _recursiveAvg->spatialPredErrV + - _recAvgFactor * contentMetrics->spatialPredErrV; + // Take value as is for first frame (no motion search in frame zero). + if (_frameCntRecursiveAvg < 1) + { + tmpRecAvgFactor = 1; + } - // Motion metric: Derived from NFD (normalized frame difference) - _recursiveAvg->motionMagnitudeNZ = (1 - _recAvgFactor) * - _recursiveAvg->motionMagnitudeNZ + - _recAvgFactor * contentMetrics->motionMagnitudeNZ; + _recursiveAvg->motionPredErr = (1 - tmpRecAvgFactor) * + _recursiveAvg->motionPredErr + + tmpRecAvgFactor * contentMetrics->motionPredErr; - return; + _recursiveAvg->sizeZeroMotion = (1 - tmpRecAvgFactor) * + _recursiveAvg->sizeZeroMotion + + tmpRecAvgFactor * contentMetrics->sizeZeroMotion; + + _recursiveAvg->spatialPredErr = (1 - tmpRecAvgFactor) * + _recursiveAvg->spatialPredErr + + tmpRecAvgFactor * contentMetrics->spatialPredErr; + + _recursiveAvg->spatialPredErrH = (1 - tmpRecAvgFactor) * + _recursiveAvg->spatialPredErrH + + tmpRecAvgFactor * contentMetrics->spatialPredErrH; + + _recursiveAvg->spatialPredErrV = (1 - tmpRecAvgFactor) * + _recursiveAvg->spatialPredErrV + + tmpRecAvgFactor * contentMetrics->spatialPredErrV; + + // motionMag metric is derived from NFD (normalized frame difference). + if (kNfdMetric == 1) + { + _recursiveAvg->motionMagnitudeNZ = (1 - tmpRecAvgFactor) * + _recursiveAvg->motionMagnitudeNZ + + tmpRecAvgFactor * contentMetrics->motionMagnitudeNZ; + } + + if (contentMetrics->sizeZeroMotion > nonZeroMvThr) + { + _recursiveAvg->motionClusterDistortion = (1 - tmpRecAvgFactor) * + _recursiveAvg->motionClusterDistortion + + tmpRecAvgFactor *contentMetrics->motionClusterDistortion; + + _recursiveAvg->motionHorizontalness = (1 - _recAvgFactor) * + _recursiveAvg->motionHorizontalness + + tmpRecAvgFactor * contentMetrics->motionHorizontalness; + + // motionMag metric is derived from motion vectors. + if (kNfdMetric == 0) + { + _recursiveAvg->motionMagnitudeNZ = (1 - tmpRecAvgFactor) * + _recursiveAvg->motionMagnitudeNZ + + tmpRecAvgFactor * contentMetrics->motionMagnitudeNZ; + } + } + + // Update native values: + // TODO (marpan): we don't need to update this every frame. + _recursiveAvg->nativeHeight = contentMetrics->nativeHeight; + _recursiveAvg->nativeWidth = contentMetrics->nativeWidth; + _recursiveAvg->nativeFrameRate = contentMetrics->nativeFrameRate; + + _frameCntRecursiveAvg++; + + return; } } //end of namespace diff --git a/src/modules/video_coding/main/source/content_metrics_processing.h b/src/modules/video_coding/main/source/content_metrics_processing.h index 72666fd9ba..155c4adc61 100644 --- a/src/modules/video_coding/main/source/content_metrics_processing.h +++ b/src/modules/video_coding/main/source/content_metrics_processing.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -60,12 +60,14 @@ private: // Update the recursive averaged metrics: longer time average (~5/10 secs). void UpdateRecursiveAvg(const VideoContentMetrics *contentMetrics); - // Update the uniform averaged metrics: shorter time average (~RTCP report). + // Update the uniform averaged metrics: shorter time average (~RTCP reports). void UpdateUniformAvg(const VideoContentMetrics *contentMetrics); VideoContentMetrics* _recursiveAvg; VideoContentMetrics* _uniformAvg; + WebRtc_UWord32 _frameRate; float _recAvgFactor; + WebRtc_UWord32 _frameCntRecursiveAvg; WebRtc_UWord32 _frameCntUniformAvg; float _avgMotionLevel; float _avgSpatialLevel; diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index 334385fc14..e2c4329803 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -24,8 +24,6 @@ _maxBitRate(0), _sendCodecType(kVideoCodecUnknown), _codecWidth(0), _codecHeight(0), -_initCodecWidth(0), -_initCodecHeight(0), _userFrameRate(0), _packetLossEnc(0), _fractionLost(0), @@ -66,7 +64,7 @@ WebRtc_Word32 VCMMediaOptimization::Reset() { memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); - _incomingFrameRate = 0.0; + InputFrameRate(); // Resets _incomingFrameRate _frameDropper->Reset(); _lossProtLogic->Reset(_clock->MillisecondTimestamp()); _frameDropper->SetRates(0, 0); @@ -284,8 +282,6 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, _userFrameRate = static_cast(frameRate); _codecWidth = width; _codecHeight = height; - _initCodecWidth = width; - _initCodecHeight = height; _numLayers = (numLayers <= 1) ? 1 : numLayers; // Can also be zero. WebRtc_Word32 ret = VCM_OK; ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate, @@ -579,50 +575,43 @@ VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) // Check for no change if (qm->spatialHeightFact == 1 && qm->spatialWidthFact == 1 && - qm->temporalFact == 1) { + qm->temporalFact == 1) + { return false; } + // Content metrics hold native values + VideoContentMetrics* cm = _content->LongTermAvgData(); + // Temporal WebRtc_UWord32 frameRate = static_cast (_incomingFrameRate + 0.5f); // Check if go back up in temporal resolution - if (qm->temporalFact == 0) { - // Currently only allow for 1/2 frame rate reduction per action. - // TODO (marpan): allow for 2/3 reduction. - frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; + if (qm->temporalFact == 0) + { + frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; } // go down in temporal resolution - else { - frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); - } - // Reset _incomingFrameRate if temporal action was selected. - if (qm->temporalFact != 1) { - memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); - _incomingFrameRate = frameRate; + else + { + frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); } // Spatial WebRtc_UWord32 height = _codecHeight; WebRtc_UWord32 width = _codecWidth; - // Check if go back up in spatial resolution, and update frame sizes. - // Currently only allow for 2x2 spatial down-sampling. - // TODO (marpan): allow for 1x2, 2x1, and 4/3x4/3 (or 3/2x3/2). - if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) { - width = _codecWidth * 2; - height = _codecHeight * 2; - } else { - width = _codecWidth / qm->spatialWidthFact; - height = _codecHeight / qm->spatialHeightFact; + // Check if go back up in spatial resolution + if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) + { + height = cm->nativeHeight; + width = cm->nativeWidth; + } + else + { + height = _codecHeight / qm->spatialHeightFact; + width = _codecWidth / qm->spatialWidthFact; } - _codecWidth = width; - _codecHeight = height; - - // New frame sizes should never exceed the original sizes - // from SetEncodingData(). - assert(_codecWidth <= _initCodecWidth); - assert(_codecHeight <= _initCodecHeight); WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, _id, "Quality Mode Update: W = %d, H = %d, FR = %f", @@ -631,12 +620,11 @@ VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) // Update VPM with new target frame rate and size _videoQMSettingsCallback->SetVideoQMSettings(frameRate, width, height); - _content->UpdateFrameRate(frameRate); - _qmResolution->UpdateCodecFrameSize(width, height); - return true; } + + void VCMMediaOptimization::UpdateIncomingFrameRate() { @@ -683,6 +671,10 @@ VCMMediaOptimization::ProcessIncomingFrameRate(WebRtc_Word64 now) _incomingFrameRate = nrOfFrames * 1000.0f / static_cast(diff); } } + else + { + _incomingFrameRate = static_cast(nrOfFrames); + } } WebRtc_UWord32 diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h index 14e5d1a2d7..7d87a6d041 100644 --- a/src/modules/video_coding/main/source/media_optimization.h +++ b/src/modules/video_coding/main/source/media_optimization.h @@ -168,8 +168,6 @@ private: VideoCodecType _sendCodecType; WebRtc_UWord16 _codecWidth; WebRtc_UWord16 _codecHeight; - WebRtc_UWord16 _initCodecWidth; - WebRtc_UWord16 _initCodecHeight; float _userFrameRate; VCMFrameDropper* _frameDropper; diff --git a/src/modules/video_coding/main/source/qm_select.cc b/src/modules/video_coding/main/source/qm_select.cc index a03fc358a4..99fc89d1b1 100644 --- a/src/modules/video_coding/main/source/qm_select.cc +++ b/src/modules/video_coding/main/source/qm_select.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -28,6 +28,7 @@ VCMQmMethod::VCMQmMethod() _height(0), _nativeWidth(0), _nativeHeight(0), + _nativeFrameRate(0), _init(false) { ResetQM(); } @@ -42,6 +43,8 @@ VCMQmMethod::ResetQM() { _motion.Reset(); _spatial.Reset(); + _coherence.Reset(); + _stationaryMotion = 0; _aspectRatio = 1; _imageType = 2; return; @@ -74,6 +77,40 @@ VCMQmMethod::MotionNFD() } +void +VCMQmMethod::Motion() +{ + + float sizeZeroMotion = _contentMetrics->sizeZeroMotion; + float motionMagNZ = _contentMetrics->motionMagnitudeNZ; + + // Take product of size and magnitude with equal weight + _motion.value = (1.0f - sizeZeroMotion) * motionMagNZ; + + // Stabilize: motionMagNZ could be large when only a + // few motion blocks are non-zero + _stationaryMotion = false; + if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE) + { + _motion.value = 0.0f; + _stationaryMotion = true; + } + // Determine motion level + if (_motion.value < LOW_MOTION) + { + _motion.level = kLow; + } + else if (_motion.value > HIGH_MOTION) + { + _motion.level = kHigh; + } + else + { + _motion.level = kDefault; + } +} + + void VCMQmMethod::Spatial() { @@ -104,6 +141,31 @@ VCMQmMethod::Spatial() } } +void +VCMQmMethod::Coherence() +{ + float horizNZ = _contentMetrics->motionHorizontalness; + float distortionNZ = _contentMetrics->motionClusterDistortion; + + // Coherence measure: combine horizontalness with cluster distortion + _coherence.value = COH_MAX; + if (distortionNZ > 0.) + { + _coherence.value = horizNZ / distortionNZ; + } + _coherence.value = VCM_MIN(COH_MAX, _coherence.value); + + if (_coherence.value < COHERENCE_THR) + { + _coherence.level = kLow; + } + else + { + _coherence.level = kHigh; + } + +} + WebRtc_Word8 VCMQmMethod::GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height) { @@ -165,6 +227,7 @@ VCMQmResolution::ResetRates() _sumEncodedBytes = 0; _sumTargetRate = 0.0f; _sumIncomingFrameRate = 0.0f; + _sumFrameRateMM = 0.0f; _sumSeqRateMM = 0.0f; _sumPacketLoss = 0.0f; _frameCnt = 0; @@ -190,32 +253,48 @@ VCMQmResolution::Reset() } // Initialize rate control quantities after reset of encoder -WebRtc_Word32 VCMQmResolution::Initialize(float bitRate, - float userFrameRate, - WebRtc_UWord32 width, - WebRtc_UWord32 height) { - if (userFrameRate == 0.0f || width == 0 || height == 0) { - return VCM_PARAMETER_ERROR; - } - _targetBitRate = bitRate; - _userFrameRate = userFrameRate; - // Native width and height. - _nativeWidth = width; - _nativeHeight = height; - UpdateCodecFrameSize(width, height); - // Initial buffer level. - _bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate; - // Per-frame bandwidth. - _perFrameBandwidth = _targetBitRate / _userFrameRate; - _init = true; - return VCM_OK; -} +WebRtc_Word32 +VCMQmResolution::Initialize(float bitRate, float userFrameRate, + WebRtc_UWord32 width, WebRtc_UWord32 height) +{ + if (userFrameRate == 0.0f || width == 0 || height == 0) + { + return VCM_PARAMETER_ERROR; + } + _targetBitRate = bitRate; + _userFrameRate = userFrameRate; -void VCMQmResolution::UpdateCodecFrameSize(uint32_t width, uint32_t height) { - _width = width; - _height = height; - // Set the imageType for the encoder width/height. - _imageType = GetImageType(_width, _height); + // Encoder width and height + _width = width; + _height = height; + + // Aspect ratio: used for selection of 1x2,2x1,2x2 + _aspectRatio = static_cast(_width) / static_cast(_height); + + // Set the imageType for the encoder width/height. + _imageType = GetImageType(_width, _height); + + // Initial buffer level + _bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate; + + // Per-frame bandwidth + if ( _incomingFrameRate == 0 ) + { + _perFrameBandwidth = _targetBitRate / _userFrameRate; + _incomingFrameRate = _userFrameRate; + } + else + { + // Take average: this is due to delay in update of new encoder frame rate: + // userFrameRate is the new one, + // incomingFrameRate is the old one (based on previous ~ 1sec/RTCP report) + _perFrameBandwidth = 0.5 *( _targetBitRate / _userFrameRate + + _targetBitRate / _incomingFrameRate ); + } + _init = true; + + + return VCM_OK; } // Update after every encoded frame @@ -234,15 +313,34 @@ VCMQmResolution::UpdateEncodedSize(WebRtc_Word64 encodedSize, // per_frame_BW is updated when encoder is updated, every RTCP reports _bufferLevel += _perFrameBandwidth - encodedSizeKbits; + // Mismatch here is based on difference of actual encoded frame size and + // per-frame bandwidth, for delta frames + // This is a much stronger condition on rate mismatch than sumSeqRateMM + // Note: not used in this version + /* + const bool deltaFrame = (encodedFrameType != kVideoFrameKey && + encodedFrameType != kVideoFrameGolden); + + // Sum the frame mismatch: + if (deltaFrame) + { + _frameCntDelta++; + if (encodedSizeKbits > 0) + _sumFrameRateMM += + (float) (fabs(encodedSizeKbits - _perFrameBandwidth) / + encodedSizeKbits); + } + */ + // Counter for occurrences of low buffer level if (_bufferLevel <= PERC_BUFFER_THR * OPT_BUFFER_LEVEL * _targetBitRate) { _lowBufferCnt++; } + } // Update various quantities after SetTargetRates in MediaOpt -// TODO (marpan): use sent_video_rate_bps from mediaOPt for avgSentBitRate. void VCMQmResolution::UpdateRates(float targetBitRate, float avgSentBitRate, float incomingFrameRate, WebRtc_UWord8 packetLoss) @@ -284,11 +382,11 @@ VCMQmResolution::UpdateRates(float targetBitRate, float avgSentBitRate, { _perFrameBandwidth = _targetBitRate / _incomingFrameRate; } + } // Select the resolution factors: frame size and frame rate change: (QM modes) // Selection is for going back up in resolution, or going down in. -// TODO (marpan): break up into small routines/clean-up and update. WebRtc_Word32 VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { @@ -308,6 +406,11 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) _qm->spatialHeightFact = 1; _qm->temporalFact = 1; + // Update native values + _nativeWidth = _contentMetrics->nativeWidth; + _nativeHeight = _contentMetrics->nativeHeight; + _nativeFrameRate = _contentMetrics->nativeFrameRate; + float avgTargetRate = 0.0f; float avgIncomingFrameRate = 0.0f; float ratioBufferLow = 0.0f; @@ -321,6 +424,8 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { // Use seq-rate mismatch for now rateMisMatch = (float)_sumSeqRateMM / (float)_updateRateCnt; + //rateMisMatch = (float)_sumFrameRateMM / (float)_frameCntDelta; + // Average target and incoming frame rates avgTargetRate = (float)_sumTargetRate / (float)_updateRateCnt; avgIncomingFrameRate = (float)_sumIncomingFrameRate / @@ -399,7 +504,7 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) if (_stateDecFactorSpatial > 1) { // Check conditions on buffer level and rate_mismatch - if ((avgTargetRate > estimatedTransRateUpS) && + if ( (avgTargetRate > estimatedTransRateUpS) && (ratioBufferLow < MAX_BUFFER_LOW) && (rateMisMatch < MAX_RATE_MM)) { // width/height scaled back up: @@ -459,6 +564,7 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { estimatedTransRateDown = LOSS_RATE_FAC * estimatedTransRateDown; } + if ((avgTargetRate < estimatedTransRateDown ) || (ratioBufferLow > MAX_BUFFER_LOW) || (rateMisMatch > MAX_RATE_MM)) @@ -551,13 +657,14 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) *qm = _qm; return VCM_OK; + + } WebRtc_Word32 VCMQmResolution::SelectSpatialDirectionMode(float transRate) { // Default is 1x2 (H) - _aspectRatio = static_cast(_width) / static_cast(_height); // For bit rates well below transitional rate, we select 2x2 if ( _targetBitRate < transRate * RATE_RED_SPATIAL_2X2 ) @@ -568,6 +675,7 @@ VCMQmResolution::SelectSpatialDirectionMode(float transRate) } // Otherwise check prediction errors, aspect ratio, horizontalness + float spatialErr = _contentMetrics->spatialPredErr; float spatialErrH = _contentMetrics->spatialPredErrH; float spatialErrV = _contentMetrics->spatialPredErrV; @@ -599,6 +707,7 @@ VCMQmResolution::SelectSpatialDirectionMode(float transRate) _qm->spatialHeightFact = 2; return VCM_OK; } + return VCM_OK; } @@ -655,9 +764,11 @@ VCMQmRobustness::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate, _prevTotalRate = totalRate; _prevRttTime = rttTime; _prevPacketLoss = packetLoss; + _prevCodeRateDelta = codeRateDelta; return adjustFec; + } // Set the UEP (unequal-protection) on/off for the FEC @@ -673,6 +784,8 @@ VCMQmRobustness::SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate, return uepProtection; } + return uepProtection; } + } // end of namespace diff --git a/src/modules/video_coding/main/source/qm_select.h b/src/modules/video_coding/main/source/qm_select.h index 761d40af3d..3fb9040361 100644 --- a/src/modules/video_coding/main/source/qm_select.h +++ b/src/modules/video_coding/main/source/qm_select.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -71,9 +71,15 @@ public: // Compute spatial texture magnitude and level void Spatial(); + // Compute motion magnitude and level + void Motion(); + // Compute motion magnitude and level for NFD metric void MotionNFD(); + // Compute coherence magnitude and level + void Coherence(); + // Get the imageType (CIF, VGA, HD, etc) for the system width/height WebRtc_Word8 GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height); @@ -85,6 +91,7 @@ public: WebRtc_UWord32 _height; WebRtc_UWord32 _nativeWidth; WebRtc_UWord32 _nativeHeight; + WebRtc_UWord32 _nativeFrameRate; float _aspectRatio; // Image type for the current encoder system size. WebRtc_UWord8 _imageType; @@ -92,6 +99,8 @@ public: // Content L/M/H values. stationary flag VCMContFeature _motion; VCMContFeature _spatial; + VCMContFeature _coherence; + bool _stationaryMotion; bool _init; }; @@ -114,9 +123,6 @@ public: WebRtc_Word32 Initialize(float bitRate, float userFrameRate, WebRtc_UWord32 width, WebRtc_UWord32 height); - // Update the encoder frame size. - void UpdateCodecFrameSize(uint32_t width, uint32_t height); - // Update QM with actual bit rate (size of the latest encoded frame) // and frame type, after every encoded frame. void UpdateEncodedSize(WebRtc_Word64 encodedSize, @@ -146,6 +152,7 @@ private: float _sumTargetRate; float _sumIncomingFrameRate; float _sumSeqRateMM; + float _sumFrameRateMM; float _sumPacketLoss; WebRtc_Word64 _sumEncodedBytes; diff --git a/src/modules/video_processing/main/source/content_analysis.cc b/src/modules/video_processing/main/source/content_analysis.cc index 767dbbf7d0..32ee09a18f 100644 --- a/src/modules/video_processing/main/source/content_analysis.cc +++ b/src/modules/video_processing/main/source/content_analysis.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -27,6 +27,10 @@ _motionMagnitudeNZ(0.0f), _spatialPredErr(0.0f), _spatialPredErrH(0.0f), _spatialPredErrV(0.0f), +_sizeZeroMotion(0.0f), +_motionPredErr(0.0f), +_motionHorizontalness(0.0f), +_motionClusterDistortion(0.0f), _firstFrame(true), _CAInit(false), _cMetrics(NULL) @@ -325,12 +329,19 @@ VPMContentAnalysis::ContentMetrics() return NULL; } + _cMetrics->spatialPredErr = _spatialPredErr; _cMetrics->spatialPredErrH = _spatialPredErrH; _cMetrics->spatialPredErrV = _spatialPredErrV; - // Motion metric: normalized temporal difference (MAD) + // normalized temporal difference (MAD) _cMetrics->motionMagnitudeNZ = _motionMagnitudeNZ; + // Set to zero: not computed + _cMetrics->motionPredErr = _motionPredErr; + _cMetrics->sizeZeroMotion = _sizeZeroMotion; + _cMetrics->motionHorizontalness = _motionHorizontalness; + _cMetrics->motionClusterDistortion = _motionClusterDistortion; + return _cMetrics; } diff --git a/src/modules/video_processing/main/source/content_analysis.h b/src/modules/video_processing/main/source/content_analysis.h index a279edcf86..5051650291 100644 --- a/src/modules/video_processing/main/source/content_analysis.h +++ b/src/modules/video_processing/main/source/content_analysis.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -78,6 +78,11 @@ private: float _spatialPredErr; // spatial class float _spatialPredErrH; // spatial class float _spatialPredErrV; // spatial class + float _sizeZeroMotion; // motion class + float _motionPredErr; // complexity class: + float _motionHorizontalness; // coherence class + float _motionClusterDistortion; // coherence class + bool _firstFrame; bool _CAInit; diff --git a/src/modules/video_processing/main/source/frame_preprocessor.cc b/src/modules/video_processing/main/source/frame_preprocessor.cc index 76fdac8384..fa13b2df7e 100644 --- a/src/modules/video_processing/main/source/frame_preprocessor.cc +++ b/src/modules/video_processing/main/source/frame_preprocessor.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -16,6 +16,8 @@ namespace webrtc { VPMFramePreprocessor::VPMFramePreprocessor(): _id(0), _contentMetrics(NULL), +_nativeHeight(0), +_nativeWidth(0), _maxFrameRate(0), _resampledFrame(), _enableCA(false) @@ -44,6 +46,8 @@ VPMFramePreprocessor::ChangeUniqueId(const WebRtc_Word32 id) void VPMFramePreprocessor::Reset() { + _nativeWidth = 0; + _nativeHeight = 0; _ca->Release(); _vd->Reset(); _contentMetrics = NULL; @@ -168,6 +172,11 @@ VPMFramePreprocessor::PreprocessFrame(const VideoFrame* frame, VideoFrame** proc } else { _contentMetrics = _ca->ComputeContentMetrics(&_resampledFrame); } + // Update native values: + _contentMetrics->nativeHeight = frame->Height(); + _contentMetrics->nativeWidth = frame->Width(); + // Max value as set by user + _contentMetrics->nativeFrameRate = _maxFrameRate; } return VPM_OK; } diff --git a/src/modules/video_processing/main/source/frame_preprocessor.h b/src/modules/video_processing/main/source/frame_preprocessor.h index 3c07a47e54..cb5f97d4bb 100644 --- a/src/modules/video_processing/main/source/frame_preprocessor.h +++ b/src/modules/video_processing/main/source/frame_preprocessor.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -66,6 +66,8 @@ private: WebRtc_Word32 _id; VideoContentMetrics* _contentMetrics; + WebRtc_UWord32 _nativeHeight; + WebRtc_UWord32 _nativeWidth; WebRtc_UWord32 _maxFrameRate; VideoFrame _resampledFrame; VPMSpatialResampler* _spatialResampler;