From f4944d49cfb0742de0685c686c950675318e7a82 Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Mon, 18 Mar 2013 17:04:52 +0000 Subject: [PATCH] Fix framerate sent to account for actually sent frames. TESTS=trybots BUG=1481 Review URL: https://webrtc-codereview.appspot.com/1195005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@3682 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/source/generic_encoder.cc | 3 +- .../main/source/media_optimization.cc | 146 ++++++++---------- .../main/source/media_optimization.h | 38 +++-- .../main/source/video_coding_impl.cc | 8 +- 4 files changed, 95 insertions(+), 100 deletions(-) diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc index 3006a57e32..b53851efe1 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc @@ -221,7 +221,8 @@ VCMEncodedFrameCallback::Encoded( } _encodedBytes = encodedBytes; if (_mediaOpt != NULL) { - _mediaOpt->UpdateWithEncodedData(_encodedBytes, frameType); + _mediaOpt->UpdateWithEncodedData(_encodedBytes, encodedImage._timeStamp, + frameType); if (_internalSource) { return _mediaOpt->DropFrame(); // Signal to encoder to drop next frame diff --git a/webrtc/modules/video_coding/main/source/media_optimization.cc b/webrtc/modules/video_coding/main/source/media_optimization.cc index b5f35689e6..2d0798d8ff 100644 --- a/webrtc/modules/video_coding/main/source/media_optimization.cc +++ b/webrtc/modules/video_coding/main/source/media_optimization.cc @@ -36,7 +36,8 @@ _enableQm(false), _videoProtectionCallback(NULL), _videoQMSettingsCallback(NULL), _encodedFrameSamples(), -_avgSentBitRateBps(0.0f), +_avgSentBitRateBps(0), +_avgSentFramerate(0), _keyFrameCnt(0), _deltaFrameCnt(0), _lastQMUpdateTime(0), @@ -82,12 +83,8 @@ VCMMediaOptimization::Reset() _deltaFrameCnt = 0; _lastQMUpdateTime = 0; _lastChangeTime = 0; - for (WebRtc_Word32 i = 0; i < kBitrateMaxFrameSamples; i++) - { - _encodedFrameSamples[i]._sizeBytes = -1; - _encodedFrameSamples[i]._timeCompleteMs = -1; - } - _avgSentBitRateBps = 0.0f; + _encodedFrameSamples.clear(); + _avgSentBitRateBps = 0; _numLayers = 1; return VCM_OK; } @@ -187,7 +184,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 target_bitrate, _qmResolution->UpdateRates(target_bitrate_kbps, sent_video_rate_kbps, _incomingFrameRate, _fractionLost); // Check for QM selection - bool selectQM = checkStatusForQMchange(); + bool selectQM = CheckStatusForQMchange(); if (selectQM) { SelectQuality(); @@ -346,22 +343,20 @@ VCMMediaOptimization::SetMtu(WebRtc_Word32 mtu) _maxPayloadSize = mtu; } -float +uint32_t VCMMediaOptimization::SentFrameRate() { - if (_frameDropper) - { - return _frameDropper->ActualFrameRate((WebRtc_UWord32)(InputFrameRate() - + 0.5f)); - } - - return VCM_CODEC_ERROR; + PurgeOldFrameSamples(_clock->TimeInMilliseconds()); + UpdateSentFramerate(); + return _avgSentFramerate; } -float +uint32_t VCMMediaOptimization::SentBitRate() { - UpdateBitRateEstimate(-1, _clock->TimeInMilliseconds()); + const int64_t now_ms = _clock->TimeInMilliseconds(); + PurgeOldFrameSamples(now_ms); + UpdateSentBitrate(now_ms); return _avgSentBitRateBps; } @@ -372,11 +367,16 @@ VCMMediaOptimization::MaxBitRate() } WebRtc_Word32 -VCMMediaOptimization::UpdateWithEncodedData(WebRtc_Word32 encodedLength, +VCMMediaOptimization::UpdateWithEncodedData(int encodedLength, + uint32_t timestamp, FrameType encodedFrameType) { - // look into the ViE version - debug mode - needs also number of layers. - UpdateBitRateEstimate(encodedLength, _clock->TimeInMilliseconds()); + const int64_t now_ms = _clock->TimeInMilliseconds(); + PurgeOldFrameSamples(now_ms); + _encodedFrameSamples.push_back(VCMEncodedFrameSample( + encodedLength, timestamp, now_ms)); + UpdateSentBitrate(now_ms); + UpdateSentFramerate(); if(encodedLength > 0) { const bool deltaFrame = (encodedFrameType != kVideoFrameKey && @@ -426,67 +426,51 @@ VCMMediaOptimization::UpdateWithEncodedData(WebRtc_Word32 encodedLength, } -void VCMMediaOptimization::UpdateBitRateEstimate(WebRtc_Word64 encodedLength, - WebRtc_Word64 nowMs) -{ - int i = kBitrateMaxFrameSamples - 1; - WebRtc_UWord32 frameSizeSum = 0; - WebRtc_Word64 timeOldest = -1; - // Find an empty slot for storing the new sample and at the same time - // accumulate the history. - for (; i >= 0; i--) - { - if (_encodedFrameSamples[i]._sizeBytes == -1) - { - // Found empty slot - break; - } - if (nowMs - _encodedFrameSamples[i]._timeCompleteMs < - kBitrateAverageWinMs) - { - frameSizeSum += static_cast - (_encodedFrameSamples[i]._sizeBytes); - if (timeOldest == -1) - { - timeOldest = _encodedFrameSamples[i]._timeCompleteMs; - } - } - } - if (encodedLength > 0) - { - if (i < 0) - { - // No empty slot, shift - for (i = kBitrateMaxFrameSamples - 2; i >= 0; i--) - { - _encodedFrameSamples[i + 1] = _encodedFrameSamples[i]; - } - i++; - } - // Insert new sample - _encodedFrameSamples[i]._sizeBytes = encodedLength; - _encodedFrameSamples[i]._timeCompleteMs = nowMs; - } - if (timeOldest > -1) - { - // Update average bit rate - float denom = static_cast(nowMs - timeOldest); - if (denom < 1.0) - { - denom = 1.0; - } - _avgSentBitRateBps = (frameSizeSum + encodedLength) * 8 * 1000 / denom; - } - else if (encodedLength > 0) - { - _avgSentBitRateBps = static_cast(encodedLength * 8); - } - else - { - _avgSentBitRateBps = 0; +void VCMMediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { + while (!_encodedFrameSamples.empty()) { + if (now_ms - _encodedFrameSamples.front().time_complete_ms > + kBitrateAverageWinMs) { + _encodedFrameSamples.pop_front(); + } else { + break; } + } } +void VCMMediaOptimization::UpdateSentBitrate(int64_t now_ms) { + if (_encodedFrameSamples.empty()) { + _avgSentBitRateBps = 0; + return; + } + int framesize_sum = 0; + for (FrameSampleList::iterator it = _encodedFrameSamples.begin(); + it != _encodedFrameSamples.end(); ++it) { + framesize_sum += it->size_bytes; + } + float denom = static_cast( + now_ms - _encodedFrameSamples.back().time_complete_ms); + if (denom >= 1.0f) { + _avgSentBitRateBps = static_cast(framesize_sum * 8 * 1000 / + denom + 0.5f); + } else { + _avgSentBitRateBps = framesize_sum * 8; + } +} + +void VCMMediaOptimization::UpdateSentFramerate() { + if (_encodedFrameSamples.size() <= 1) { + _avgSentFramerate = _encodedFrameSamples.size(); + return; + } + int denom = _encodedFrameSamples.back().timestamp - + _encodedFrameSamples.front().timestamp; + if (denom > 0) { + _avgSentFramerate = (90000 * (_encodedFrameSamples.size() - 1) + denom / 2) + / denom; + } else { + _avgSentFramerate = _encodedFrameSamples.size(); + } +} WebRtc_Word32 VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback* @@ -506,7 +490,7 @@ VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback* } void -VCMMediaOptimization::updateContentData(const VideoContentMetrics* +VCMMediaOptimization::UpdateContentData(const VideoContentMetrics* contentMetrics) { // Updating content metrics @@ -560,7 +544,7 @@ VCMMediaOptimization::SelectQuality() // (2) target bit rate bool -VCMMediaOptimization::checkStatusForQMchange() +VCMMediaOptimization::CheckStatusForQMchange() { bool status = true; diff --git a/webrtc/modules/video_coding/main/source/media_optimization.h b/webrtc/modules/video_coding/main/source/media_optimization.h index 39bd9af519..e2b0c3d475 100644 --- a/webrtc/modules/video_coding/main/source/media_optimization.h +++ b/webrtc/modules/video_coding/main/source/media_optimization.h @@ -17,6 +17,8 @@ #include "media_opt_util.h" #include "qm_select.h" +#include + namespace webrtc { class Clock; @@ -28,12 +30,16 @@ namespace media_optimization { enum { kBitrateMaxFrameSamples = 60 }; enum { kBitrateAverageWinMs = 1000 }; -struct VCMEncodedFrameSample -{ - VCMEncodedFrameSample() : _sizeBytes(-1), _timeCompleteMs(-1) {} +struct VCMEncodedFrameSample { + VCMEncodedFrameSample(int size_bytes, uint32_t timestamp, + int64_t time_complete_ms) + : size_bytes(size_bytes), + timestamp(timestamp), + time_complete_ms(time_complete_ms) {} - WebRtc_Word64 _sizeBytes; - WebRtc_Word64 _timeCompleteMs; + uint32_t size_bytes; + uint32_t timestamp; + int64_t time_complete_ms; }; class VCMMediaOptimization @@ -87,11 +93,11 @@ public: /* * Get actual sent frame rate */ - float SentFrameRate(); + uint32_t SentFrameRate(); /* * Get actual sent bit rate */ - float SentBitRate(); + uint32_t SentBitRate(); /* * Get maximum allowed bit rate */ @@ -99,7 +105,8 @@ public: /* * Inform Media Optimization of encoding output: Length and frame type */ - WebRtc_Word32 UpdateWithEncodedData(WebRtc_Word32 encodedLength, + WebRtc_Word32 UpdateWithEncodedData(int encodedLength, + uint32_t timestamp, FrameType encodedFrameType); /* * Register a protection callback to be used to inform the user about the @@ -128,7 +135,7 @@ public: /** * Update content metric Data */ - void updateContentData(const VideoContentMetrics* contentMetrics); + void UpdateContentData(const VideoContentMetrics* contentMetrics); /** * Compute new Quality Mode @@ -136,6 +143,7 @@ public: WebRtc_Word32 SelectQuality(); private: + typedef std::list FrameSampleList; /* * Update protection callback with protection settings @@ -145,7 +153,10 @@ private: uint32_t* nack_overhead_rate_bps, uint32_t* fec_overhead_rate_bps); - void UpdateBitRateEstimate(WebRtc_Word64 encodedLength, WebRtc_Word64 nowMs); + void PurgeOldFrameSamples(int64_t now_ms); + void UpdateSentBitrate(int64_t nowMs); + void UpdateSentFramerate(); + /* * verify if QM settings differ from default, i.e. if an update is required * Compute actual values, as will be sent to the encoder @@ -155,7 +166,7 @@ private: * check if we should make a QM change * will return 1 if yes, 0 otherwise */ - bool checkStatusForQMchange(); + bool CheckStatusForQMchange(); void ProcessIncomingFrameRate(WebRtc_Word64 now); @@ -188,8 +199,9 @@ private: VCMProtectionCallback* _videoProtectionCallback; VCMQMSettingsCallback* _videoQMSettingsCallback; - VCMEncodedFrameSample _encodedFrameSamples[kBitrateMaxFrameSamples]; - float _avgSentBitRateBps; + std::list _encodedFrameSamples; + uint32_t _avgSentBitRateBps; + uint32_t _avgSentFramerate; WebRtc_UWord32 _keyFrameCnt; WebRtc_UWord32 _deltaFrameCnt; diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.cc b/webrtc/modules/video_coding/main/source/video_coding_impl.cc index 7c484fcb55..440796cc71 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.cc +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.cc @@ -167,10 +167,8 @@ VideoCodingModuleImpl::Process() WebRtc_UWord32 frameRate; { CriticalSectionScoped cs(_sendCritSect); - bitRate = static_cast( - _mediaOpt.SentBitRate() + 0.5f); - frameRate = static_cast( - _mediaOpt.SentFrameRate() + 0.5f); + bitRate = _mediaOpt.SentBitRate(); + frameRate = _mediaOpt.SentFrameRate(); } _sendStatsCallback->SendStatistics(bitRate, frameRate); } @@ -700,7 +698,7 @@ VideoCodingModuleImpl::AddVideoFrame(const I420VideoFrame& videoFrame, } else { - _mediaOpt.updateContentData(contentMetrics); + _mediaOpt.UpdateContentData(contentMetrics); WebRtc_Word32 ret = _encoder->Encode(videoFrame, codecSpecificInfo, _nextFrameTypes);