From 19b4fecf08e3fe215e431a260fb673553c15e569 Mon Sep 17 00:00:00 2001 From: phoglund Date: Tue, 12 Apr 2016 09:05:58 -0700 Subject: [PATCH] Revert of Make QualityScaler more responsive to downgrades. (patchset #3 id:40001 of https://codereview.webrtc.org/1830593003/ ) Reason for revert: Speculative revert: want to see if this causes the regression in https://crbug.com/602621 Original issue's description: > Make QualityScaler more responsive to downgrades. > > Permits going from HD to QVGA in 6 seconds instead of 10. Also adds > windows for going up quickly in the beginning of a call (before any > downscaling happens due to bad quality). > > BUG=webrtc:5678 > R=glaznev@webrtc.org, stefan@webrtc.org > > Committed: https://crrev.com/85829fd90cc4e7a91c9857921b19e8fc126aeb60 > Cr-Commit-Position: refs/heads/master@{#12219} TBR=glaznev@webrtc.org,stefan@webrtc.org,pbos@webrtc.org # Not skipping CQ checks because original CL landed more than 1 days ago. BUG=webrtc:5678 NOTRY=true Review URL: https://codereview.webrtc.org/1880103002 Cr-Commit-Position: refs/heads/master@{#12331} --- .../api/java/jni/androidmediaencoder_jni.cc | 11 +- .../video_coding/codecs/vp8/vp8_impl.cc | 4 +- .../video_coding/utility/quality_scaler.cc | 49 +++----- .../video_coding/utility/quality_scaler.h | 12 +- .../utility/quality_scaler_unittest.cc | 113 +++--------------- 5 files changed, 43 insertions(+), 146 deletions(-) diff --git a/webrtc/api/java/jni/androidmediaencoder_jni.cc b/webrtc/api/java/jni/androidmediaencoder_jni.cc index 99419e0f5a..dba71acac9 100644 --- a/webrtc/api/java/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/java/jni/androidmediaencoder_jni.cc @@ -380,17 +380,15 @@ int32_t MediaCodecVideoEncoder::InitEncode( const int kLowQpThreshold = 32; const int kBadQpThreshold = 92; quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false, - codec_settings->startBitrate, codec_settings->width, - codec_settings->height, - codec_settings->maxFramerate); + codec_settings->startBitrate, + codec_settings->width, codec_settings->height); } else if (codecType_ == kVideoCodecH264) { // H264 QP is in the range [0, 51]. const int kLowQpThreshold = 21; const int kBadQpThreshold = 36; quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false, - codec_settings->startBitrate, codec_settings->width, - codec_settings->height, - codec_settings->maxFramerate); + codec_settings->startBitrate, + codec_settings->width, codec_settings->height); } else { // When adding codec support to additional hardware codecs, also configure // their QP thresholds for scaling. @@ -398,6 +396,7 @@ int32_t MediaCodecVideoEncoder::InitEncode( scale_ = false; } quality_scaler_.SetMinResolution(kMinDimension, kMinDimension); + quality_scaler_.ReportFramerate(codec_settings->maxFramerate); QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); init_width = std::max(res.width, kMinDimension); init_height = std::max(res.height, kMinDimension); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index b34288632c..a62bd41c6b 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -604,8 +604,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, // TODO(glaznev/sprang): consider passing codec initial bitrate to quality // scaler to avoid starting with HD for low initial bitrates. quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator, - kDisabledBadQpThreshold, false, 0, 0, 0, - codec_.maxFramerate); + kDisabledBadQpThreshold, false, 0, 0, 0); + quality_scaler_.ReportFramerate(codec_.maxFramerate); // Only apply scaling to improve for single-layer streams. The scaling metrics // use frame drops as a signal and is only applicable when we drop frames. diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index c6e5669731..68ae9218ea 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -11,11 +11,8 @@ namespace webrtc { -static const int kMinFps = 5; -static const int kMeasureSecondsDownscale = 3; -// Threshold constant used until first downscale (to permit fast rampup). -static const int kMeasureSecondsFastUpscale = 2; -static const int kMeasureSecondsUpscale = 5; +static const int kMinFps = 10; +static const int kMeasureSeconds = 5; static const int kFramedropPercentThreshold = 60; static const int kHdResolutionThreshold = 700 * 500; static const int kHdBitrateThresholdKbps = 500; @@ -26,7 +23,9 @@ const int QualityScaler::kDefaultLowQpDenominator = 3; const int QualityScaler::kDefaultMinDownscaleDimension = 90; QualityScaler::QualityScaler() - : low_qp_threshold_(-1), + : num_samples_(0), + low_qp_threshold_(-1), + downscale_shift_(0), framerate_down_(false), min_width_(kDefaultMinDownscaleDimension), min_height_(kDefaultMinDownscaleDimension) {} @@ -36,17 +35,12 @@ void QualityScaler::Init(int low_qp_threshold, bool use_framerate_reduction, int initial_bitrate_kbps, int width, - int height, - int fps) { + int height) { ClearSamples(); low_qp_threshold_ = low_qp_threshold; high_qp_threshold_ = high_qp_threshold; use_framerate_reduction_ = use_framerate_reduction; downscale_shift_ = 0; - // Use a faster window for upscaling initially (but be more graceful later). - // This enables faster initial rampups without risking strong up-down - // behavior later. - measure_seconds_upscale_ = kMeasureSecondsFastUpscale; const int init_width = width; const int init_height = height; // TODO(glaznev): Investigate using thresholds for other resolutions @@ -61,7 +55,6 @@ void QualityScaler::Init(int low_qp_threshold, } } UpdateTargetResolution(init_width, init_height); - ReportFramerate(fps); target_framerate_ = -1; } @@ -72,14 +65,14 @@ void QualityScaler::SetMinResolution(int min_width, int min_height) { // Report framerate(fps) to estimate # of samples. void QualityScaler::ReportFramerate(int framerate) { + num_samples_ = static_cast( + kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate)); framerate_ = framerate; - UpdateSampleCounts(); } void QualityScaler::ReportQP(int qp) { framedrop_percent_.AddSample(0); - average_qp_downscale_.AddSample(qp); - average_qp_upscale_.AddSample(qp); + average_qp_.AddSample(qp); } void QualityScaler::ReportDroppedFrame() { @@ -89,8 +82,7 @@ void QualityScaler::ReportDroppedFrame() { void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { // Should be set through InitEncode -> Should be set by now. assert(low_qp_threshold_ >= 0); - assert(num_samples_upscale_ > 0); - assert(num_samples_downscale_ > 0); + assert(num_samples_ > 0); // Update scale factor. int avg_drop = 0; @@ -98,9 +90,9 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { // When encoder consistently overshoots, framerate reduction and spatial // resizing will be triggered to get a smoother video. - if ((framedrop_percent_.GetAverage(num_samples_downscale_, &avg_drop) && + if ((framedrop_percent_.GetAverage(num_samples_, &avg_drop) && avg_drop >= kFramedropPercentThreshold) || - (average_qp_downscale_.GetAverage(num_samples_downscale_, &avg_qp) && + (average_qp_.GetAverage(num_samples_, &avg_qp) && avg_qp > high_qp_threshold_)) { // Reducing frame rate before spatial resolution change. // Reduce frame rate only when it is above a certain number. @@ -115,7 +107,7 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { } else { AdjustScale(false); } - } else if (average_qp_upscale_.GetAverage(num_samples_upscale_, &avg_qp) && + } else if (average_qp_.GetAverage(num_samples_, &avg_qp) && avg_qp <= low_qp_threshold_) { if (use_framerate_reduction_ && framerate_down_) { target_framerate_ = -1; @@ -168,26 +160,13 @@ void QualityScaler::UpdateTargetResolution(int frame_width, int frame_height) { void QualityScaler::ClearSamples() { framedrop_percent_.Reset(); - average_qp_downscale_.Reset(); - average_qp_upscale_.Reset(); -} - -void QualityScaler::UpdateSampleCounts() { - num_samples_downscale_ = static_cast( - kMeasureSecondsDownscale * (framerate_ < kMinFps ? kMinFps : framerate_)); - num_samples_upscale_ = static_cast( - measure_seconds_upscale_ * (framerate_ < kMinFps ? kMinFps : framerate_)); + average_qp_.Reset(); } void QualityScaler::AdjustScale(bool up) { downscale_shift_ += up ? -1 : 1; if (downscale_shift_ < 0) downscale_shift_ = 0; - if (!up) { - // Hit first downscale, start using a slower threshold for going up. - measure_seconds_upscale_ = kMeasureSecondsUpscale; - UpdateSampleCounts(); - } ClearSamples(); } diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index 34dda0e9f3..ebceb06893 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -30,8 +30,7 @@ class QualityScaler { bool use_framerate_reduction, int initial_bitrate_kbps, int width, - int height, - int fps); + int height); void SetMinResolution(int min_width, int min_height); void ReportFramerate(int framerate); void ReportQP(int qp); @@ -47,22 +46,17 @@ class QualityScaler { void AdjustScale(bool up); void UpdateTargetResolution(int frame_width, int frame_height); void ClearSamples(); - void UpdateSampleCounts(); Scaler scaler_; VideoFrame scaled_frame_; - size_t num_samples_downscale_; - size_t num_samples_upscale_; - int measure_seconds_upscale_; - MovingAverage average_qp_upscale_; - MovingAverage average_qp_downscale_; - + size_t num_samples_; int framerate_; int target_framerate_; int low_qp_threshold_; int high_qp_threshold_; MovingAverage framedrop_percent_; + MovingAverage average_qp_; Resolution res_; int downscale_shift_; diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index 72e9db405e..a5e3219f70 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -27,11 +27,6 @@ static const int kHighQp = 40; static const int kMaxQp = 56; static const int kDisabledBadQpThreshold = kMaxQp + 1; static const int kLowInitialBitrateKbps = 300; -// These values need to be in sync with corresponding constants -// in quality_scaler.cc -static const int kMeasureSecondsDownscale = 3; -static const int kMeasureSecondsFastUpscale = 2; -static const int kMeasureSecondsUpscale = 5; } // namespace class QualityScalerTest : public ::testing::Test { @@ -56,7 +51,8 @@ class QualityScalerTest : public ::testing::Test { input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth, kHalfWidth); qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, - 0, 0, 0, kFramerate); + 0, 0, 0); + qs_.ReportFramerate(kFramerate); qs_.OnEncodeFrame(input_frame_); } @@ -108,8 +104,7 @@ class QualityScalerTest : public ::testing::Test { int initial_framerate); void VerifyQualityAdaptation(int initial_framerate, - int seconds_downscale, - int seconds_upscale, + int seconds, bool expect_spatial_resize, bool expect_framerate_reduction); @@ -303,19 +298,18 @@ QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange( void QualityScalerTest::VerifyQualityAdaptation( int initial_framerate, - int seconds_downscale, - int seconds_upscale, + int seconds, bool expect_spatial_resize, bool expect_framerate_reduction) { qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, - kDisabledBadQpThreshold, true, 0, 0, 0, initial_framerate); + kDisabledBadQpThreshold, true, 0, 0, 0); qs_.OnEncodeFrame(input_frame_); int init_width = qs_.GetScaledResolution().width; int init_height = qs_.GetScaledResolution().height; // Test reducing framerate by dropping frame continuously. QualityScalerTest::Resolution res = - TriggerResolutionChange(kDropFrame, seconds_downscale, initial_framerate); + TriggerResolutionChange(kDropFrame, seconds, initial_framerate); if (expect_framerate_reduction) { EXPECT_LT(res.framerate, initial_framerate); @@ -333,40 +327,35 @@ void QualityScalerTest::VerifyQualityAdaptation( } // The "seconds * 1.5" is to ensure spatial resolution to recover. - // For example, in 6 seconds test, framerate reduction happens in the first - // 3 seconds from 30fps to 15fps and causes the buffer size to be half of the - // original one. Then it will take only 45 samples to downscale (twice in 90 + // For example, in 10 seconds test, framerate reduction happens in the first + // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the + // original one. Then it will take only 75 samples to downscale (twice in 150 // samples). So to recover the resolution changes, we need more than 10 - // seconds (i.e, seconds_upscale * 1.5). This is because the framerate - // increases before spatial size recovers, so it will take 150 samples to - // recover spatial size (300 for twice). - res = TriggerResolutionChange(kReportLowQP, seconds_upscale * 1.5, - initial_framerate); + // seconds (i.e, seconds * 1.5). This is because the framerate increases + // before spatial size recovers, so it will take 150 samples to recover + // spatial size (300 for twice). + res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate); EXPECT_EQ(-1, res.framerate); EXPECT_EQ(init_width, res.width); EXPECT_EQ(init_height, res.height); } -// In 3 seconds test, only framerate adjusting should happen and 5 second -// upscaling duration, only a framerate adjusting should happen. +// In 5 seconds test, only framerate adjusting should happen. TEST_F(QualityScalerTest, ChangeFramerateOnly) { - VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale, - kMeasureSecondsUpscale, false, true); + VerifyQualityAdaptation(kFramerate, 5, false, true); } -// In 6 seconds test, framerate adjusting and scaling are both +// In 10 seconds test, framerate adjusting and scaling are both // triggered, it shows that scaling would happen after framerate // adjusting. TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) { - VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale * 2, - kMeasureSecondsUpscale * 2, true, true); + VerifyQualityAdaptation(kFramerate, 10, true, true); } // When starting from a low framerate, only spatial size will be changed. TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) { qs_.ReportFramerate(kFramerate >> 1); - VerifyQualityAdaptation(kFramerate >> 1, kMeasureSecondsDownscale * 2, - kMeasureSecondsUpscale * 2, true, false); + VerifyQualityAdaptation(kFramerate >> 1, 10, true, false); } TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) { @@ -382,7 +371,7 @@ TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) { TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) { qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kDisabledBadQpThreshold, true, - kLowInitialBitrateKbps, kWidth, kHeight, kFramerate); + kLowInitialBitrateKbps, kWidth, kHeight); qs_.OnEncodeFrame(input_frame_); int init_width = qs_.GetScaledResolution().width; int init_height = qs_.GetScaledResolution().height; @@ -390,70 +379,6 @@ TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) { EXPECT_LE(init_height, kHeightVga); } -TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) { - QualityScalerTest::Resolution initial_res; - qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, 0, - kWidth, kHeight, kFramerate); - qs_.OnEncodeFrame(input_frame_); - initial_res.width = qs_.GetScaledResolution().width; - initial_res.height = qs_.GetScaledResolution().height; - - // Should not downscale if less than kMeasureSecondsDownscale seconds passed. - for (int i = 0; i < kFramerate * kMeasureSecondsDownscale - 1; ++i) { - qs_.ReportQP(kHighQp + 1); - qs_.OnEncodeFrame(input_frame_); - } - EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); - - // Should downscale if more than kMeasureSecondsDownscale seconds passed (add - // last frame). - qs_.ReportQP(kHighQp + 1); - qs_.OnEncodeFrame(input_frame_); - EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height); - - // Should not upscale if less than kMeasureSecondsUpscale seconds passed since - // we saw issues initially (have already gone down). - for (int i = 0; i < kFramerate * kMeasureSecondsUpscale - 1; ++i) { - qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_); - } - EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height); - - // Should upscale (back to initial) if kMeasureSecondsUpscale seconds passed - // (add last frame). - qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_); - EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); -} - -TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) { - QualityScalerTest::Resolution initial_res; - qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, - kLowInitialBitrateKbps, kWidth, kHeight, kFramerate); - qs_.OnEncodeFrame(input_frame_); - initial_res.width = qs_.GetScaledResolution().width; - initial_res.height = qs_.GetScaledResolution().height; - - // Should not upscale if less than kMeasureSecondsFastUpscale seconds passed. - for (int i = 0; i < kFramerate * kMeasureSecondsFastUpscale - 1; ++i) { - qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_); - } - EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); - - // Should upscale if kMeasureSecondsFastUpscale seconds passed (add last - // frame). - qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_); - EXPECT_LT(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_LT(initial_res.height, qs_.GetScaledResolution().height); -} - void QualityScalerTest::DownscaleEndsAt(int input_width, int input_height, int end_width,