From 17417707428816ffb88d9c71dcc8a5d492cf9fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Bostr=C3=B6m?= Date: Fri, 25 Sep 2015 17:03:26 +0200 Subject: [PATCH] Implement a high-QP threshold for Android H.264. Android hardware H.264 seems to keep a steady high-QP flow instead of dropping frames, so framedrops aren't sufficient to detect a bad state where downscaling would be beneficial. BUG=webrtc:4968 R=magjed@webrtc.org, stefan@webrtc.org Review URL: https://codereview.webrtc.org/1364253002 . Cr-Commit-Position: refs/heads/master@{#10078} --- .../java/jni/androidmediaencoder_jni.cc | 11 ++++-- .../video_coding/codecs/vp8/vp8_impl.cc | 5 ++- .../utility/include/quality_scaler.h | 5 ++- .../video_coding/utility/quality_scaler.cc | 11 ++++-- .../utility/quality_scaler_unittest.cc | 36 +++++++++++++++++-- 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc index f6a3b650a6..a25a3cc0ef 100644 --- a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc +++ b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc @@ -306,11 +306,18 @@ int32_t MediaCodecVideoEncoder::InitEncode( // always = 127. Note that in SW, QP is that of the user-level range [0, // 63]. const int kMaxQp = 127; - quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, true); + // TODO(pbos): Investigate whether high-QP thresholds make sense for VP8. + // This effectively disables high QP as VP8 QP can't go above this + // threshold. + const int kDisabledBadQpThreshold = kMaxQp + 1; + quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, + kDisabledBadQpThreshold, true); } else if (codecType_ == kVideoCodecH264) { // H264 QP is in the range [0, 51]. const int kMaxQp = 51; - quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, true); + const int kBadQpThreshold = 40; + quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, + false); } else { // When adding codec support to additional hardware codecs, also configure // their QP thresholds for scaling. diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 48ed02ae35..651406abf9 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -579,8 +579,11 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, } rps_.Init(); + // Disable both high-QP limits and framedropping. Both are handled by libvpx + // internally. + const int kDisabledBadQpThreshold = 64; quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator, - false); + kDisabledBadQpThreshold, false); quality_scaler_.ReportFramerate(codec_.maxFramerate); return InitAndSetControlSettings(); diff --git a/webrtc/modules/video_coding/utility/include/quality_scaler.h b/webrtc/modules/video_coding/utility/include/quality_scaler.h index 8535171f65..e92c55ddc0 100644 --- a/webrtc/modules/video_coding/utility/include/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/include/quality_scaler.h @@ -25,7 +25,9 @@ class QualityScaler { }; QualityScaler(); - void Init(int low_qp_threshold, bool use_framerate_reduction); + void Init(int low_qp_threshold, + int high_qp_threshold, + bool use_framerate_reduction); void SetMinResolution(int min_width, int min_height); void ReportFramerate(int framerate); void ReportQP(int qp); @@ -47,6 +49,7 @@ class QualityScaler { int framerate_; int target_framerate_; int low_qp_threshold_; + int high_qp_threshold_; MovingAverage framedrop_percent_; MovingAverage average_qp_; Resolution res_; diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index f7d4651511..ec7715230e 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -29,9 +29,12 @@ QualityScaler::QualityScaler() min_height_(kDefaultMinDownscaleDimension) { } -void QualityScaler::Init(int low_qp_threshold, bool use_framerate_reduction) { +void QualityScaler::Init(int low_qp_threshold, + int high_qp_threshold, + bool use_framerate_reduction) { ClearSamples(); low_qp_threshold_ = low_qp_threshold; + high_qp_threshold_ = high_qp_threshold; use_framerate_reduction_ = use_framerate_reduction; target_framerate_ = -1; } @@ -70,8 +73,10 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { // When encoder consistently overshoots, framerate reduction and spatial // resizing will be triggered to get a smoother video. - if (framedrop_percent_.GetAverage(num_samples_, &avg_drop) && - avg_drop >= kFramedropPercentThreshold) { + if ((framedrop_percent_.GetAverage(num_samples_, &avg_drop) && + avg_drop >= kFramedropPercentThreshold) || + (average_qp_.GetAverage(num_samples_, &avg_qp) && + avg_qp > high_qp_threshold_)) { // Reducing frame rate before spatial resolution change. // Reduce frame rate only when it is above a certain number. // Only one reduction is allowed for now. diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index d1710b6dd7..2ce1107472 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -21,6 +21,7 @@ static const int kHeight = 1080; static const int kFramerate = 30; static const int kLowQp = 15; static const int kNormalQp = 30; +static const int kHighQp = 40; static const int kMaxQp = 56; } // namespace @@ -33,13 +34,18 @@ class QualityScalerTest : public ::testing::Test { int height; }; protected: - enum ScaleDirection { kScaleDown, kScaleUp }; + enum ScaleDirection { + kKeepScaleAtHighQp, + kScaleDown, + kScaleDownAboveHighQp, + kScaleUp + }; enum BadQualityMetric { kDropFrame, kReportLowQP }; QualityScalerTest() { input_frame_.CreateEmptyFrame( kWidth, kHeight, kWidth, kHalfWidth, kHalfWidth); - qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, false); + qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false); qs_.ReportFramerate(kFramerate); qs_.OnEncodeFrame(input_frame_); } @@ -55,6 +61,12 @@ class QualityScalerTest : public ::testing::Test { case kScaleDown: qs_.ReportDroppedFrame(); break; + case kKeepScaleAtHighQp: + qs_.ReportQP(kHighQp); + break; + case kScaleDownAboveHighQp: + qs_.ReportQP(kHighQp + 1); + break; } qs_.OnEncodeFrame(input_frame_); if (qs_.GetScaledResolution().width != initial_width) @@ -117,6 +129,22 @@ TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) { EXPECT_LT(res.height, input_frame_.height()); } +TEST_F(QualityScalerTest, KeepsScaleAtHighQp) { + EXPECT_FALSE(TriggerScale(kKeepScaleAtHighQp)) + << "Downscale at high threshold which should keep scale."; + QualityScaler::Resolution res = qs_.GetScaledResolution(); + EXPECT_EQ(res.width, input_frame_.width()); + EXPECT_EQ(res.height, input_frame_.height()); +} + +TEST_F(QualityScalerTest, DownscalesAboveHighQp) { + EXPECT_TRUE(TriggerScale(kScaleDownAboveHighQp)) + << "No downscale within " << kNumSeconds << " seconds."; + QualityScaler::Resolution res = qs_.GetScaledResolution(); + EXPECT_LT(res.width, input_frame_.width()); + EXPECT_LT(res.height, input_frame_.height()); +} + TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) { qs_.ReportQP(kNormalQp); @@ -262,7 +290,9 @@ QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange( void QualityScalerTest::VerifyQualityAdaptation( int initial_framerate, int seconds, bool expect_spatial_resize, bool expect_framerate_reduction) { - qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, true); + const int kDisabledBadQpThreshold = kMaxQp + 1; + qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, + kDisabledBadQpThreshold, true); qs_.OnEncodeFrame(input_frame_); int init_width = qs_.GetScaledResolution().width; int init_height = qs_.GetScaledResolution().height;