From a9d08929463e915f078860c206f2706c0284cba6 Mon Sep 17 00:00:00 2001 From: Alex Glaznev Date: Fri, 19 Feb 2016 15:24:06 -0800 Subject: [PATCH] Add initial bitrate and frame resolution parameters to quality scaler. - Scale down to VGA immediately if call starts with HD resolution and bitrate below 500 kbps. - Adjust QP threshold for HW VP8 encoder to scale down faster. BUG=b/26504665 R=mflodman@webrtc.org, pbos@webrtc.org, sprang@google.com, stefan@webrtc.org Review URL: https://codereview.webrtc.org/1672173002 . Cr-Commit-Position: refs/heads/master@{#11692} --- .../api/java/jni/androidmediaencoder_jni.cc | 36 ++++++++++++------- .../video_coding/codecs/vp8/vp8_impl.cc | 4 ++- .../video_coding/utility/quality_scaler.cc | 20 ++++++++++- .../video_coding/utility/quality_scaler.h | 5 ++- .../utility/quality_scaler_unittest.cc | 21 +++++++++-- 5 files changed, 68 insertions(+), 18 deletions(-) diff --git a/webrtc/api/java/jni/androidmediaencoder_jni.cc b/webrtc/api/java/jni/androidmediaencoder_jni.cc index 7fa93de149..71e297c948 100644 --- a/webrtc/api/java/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/java/jni/androidmediaencoder_jni.cc @@ -45,6 +45,7 @@ using webrtc::VideoCodecType; using webrtc::kVideoCodecH264; using webrtc::kVideoCodecVP8; using webrtc::kVideoCodecVP9; +using webrtc::QualityScaler; namespace webrtc_jni { @@ -242,7 +243,7 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, bool drop_next_input_frame_; // Global references; must be deleted in Release(). std::vector input_buffers_; - webrtc::QualityScaler quality_scaler_; + QualityScaler quality_scaler_; // Dynamic resolution change, off by default. bool scale_; @@ -355,7 +356,6 @@ int32_t MediaCodecVideoEncoder::InitEncode( size_t /* max_payload_size */) { const int kMinWidth = 320; const int kMinHeight = 180; - const int kLowQpThresholdDenominator = 3; if (codec_settings == NULL) { ALOGE << "NULL VideoCodec instance"; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -365,40 +365,52 @@ int32_t MediaCodecVideoEncoder::InitEncode( << "Unsupported codec " << codec_settings->codecType << " for " << codecType_; - ALOGD << "InitEncode request"; codec_mode_ = codec_settings->mode; + int init_width = codec_settings->width; + int init_height = codec_settings->height; scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName( "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled"); + + ALOGD << "InitEncode request: " << init_width << " x " << init_height; ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); + if (scale_) { if (codecType_ == kVideoCodecVP8) { // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is // always = 127. Note that in SW, QP is that of the user-level range [0, // 63]. - const int kMaxQp = 127; - const int kBadQpThreshold = 95; - quality_scaler_.Init( - kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false); + const int kLowQpThreshold = 32; + const int kBadQpThreshold = 92; + quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false, + codec_settings->startBitrate, + codec_settings->width, codec_settings->height); } else if (codecType_ == kVideoCodecH264) { // H264 QP is in the range [0, 51]. - const int kMaxQp = 51; + const int kLowQpThreshold = 17; const int kBadQpThreshold = 40; - quality_scaler_.Init( - kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false); + quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false, + codec_settings->startBitrate, + codec_settings->width, codec_settings->height); } else { // When adding codec support to additional hardware codecs, also configure // their QP thresholds for scaling. RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds."; + scale_ = false; } quality_scaler_.SetMinResolution(kMinWidth, kMinHeight); quality_scaler_.ReportFramerate(codec_settings->maxFramerate); + QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); + init_width = std::max(res.width, kMinWidth); + init_height = std::max(res.height, kMinHeight); + ALOGD << "Scaled resolution: " << init_width << " x " << init_height; } + return codec_thread_->Invoke( Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, this, - codec_settings->width, - codec_settings->height, + init_width, + init_height, codec_settings->startBitrate, codec_settings->maxFramerate, false /* use_surface */)); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 5d22d72144..eac0afd983 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -600,8 +600,10 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, // Disable both high-QP limits and framedropping. Both are handled by libvpx // internally. const int kDisabledBadQpThreshold = 64; + // TODO(glaznev/sprang): consider passing codec initial bitrate to quality + // scaler to avoid starting with HD for low initial bitrates. quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator, - kDisabledBadQpThreshold, false); + kDisabledBadQpThreshold, false, 0, 0, 0); quality_scaler_.ReportFramerate(codec_.maxFramerate); // Only apply scaling to improve for single-layer streams. The scaling metrics diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index 76bf9f5b03..a46e8502cb 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -14,6 +14,8 @@ namespace webrtc { static const int kMinFps = 10; static const int kMeasureSeconds = 5; static const int kFramedropPercentThreshold = 60; +static const int kHdResolutionThreshold = 700 * 500; +static const int kHdBitrateThresholdKbps = 500; const int QualityScaler::kDefaultLowQpDenominator = 3; // Note that this is the same for width and height to permit 120x90 in both @@ -30,11 +32,27 @@ QualityScaler::QualityScaler() void QualityScaler::Init(int low_qp_threshold, int high_qp_threshold, - bool use_framerate_reduction) { + bool use_framerate_reduction, + int initial_bitrate_kbps, + int width, + int height) { ClearSamples(); low_qp_threshold_ = low_qp_threshold; high_qp_threshold_ = high_qp_threshold; use_framerate_reduction_ = use_framerate_reduction; + // TODO(glaznev): Investigate using thresholds for other resolutions + // or threshold tables. + if (initial_bitrate_kbps > 0 && + initial_bitrate_kbps < kHdBitrateThresholdKbps) { + // Start scaling to roughly VGA. + while (width * height > kHdResolutionThreshold) { + ++downscale_shift_; + width /= 2; + height /= 2; + } + } + res_.width = width; + res_.height = height; target_framerate_ = -1; } diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index a1233cca51..4a2c207af6 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -27,7 +27,10 @@ class QualityScaler { QualityScaler(); void Init(int low_qp_threshold, int high_qp_threshold, - bool use_framerate_reduction); + bool use_framerate_reduction, + int initial_bitrate_kbps, + int width, + int height); void SetMinResolution(int min_width, int min_height); void ReportFramerate(int framerate); void ReportQP(int qp); diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index bad73a748c..a5e3219f70 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -16,13 +16,17 @@ namespace webrtc { namespace { static const int kNumSeconds = 10; static const int kWidth = 1920; +static const int kWidthVga = 640; static const int kHalfWidth = kWidth / 2; static const int kHeight = 1080; +static const int kHeightVga = 480; static const int kFramerate = 30; static const int kLowQp = 15; static const int kNormalQp = 30; static const int kHighQp = 40; static const int kMaxQp = 56; +static const int kDisabledBadQpThreshold = kMaxQp + 1; +static const int kLowInitialBitrateKbps = 300; } // namespace class QualityScalerTest : public ::testing::Test { @@ -46,7 +50,8 @@ class QualityScalerTest : public ::testing::Test { QualityScalerTest() { input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth, kHalfWidth); - qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false); + qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, + 0, 0, 0); qs_.ReportFramerate(kFramerate); qs_.OnEncodeFrame(input_frame_); } @@ -296,9 +301,8 @@ void QualityScalerTest::VerifyQualityAdaptation( int seconds, bool expect_spatial_resize, bool expect_framerate_reduction) { - const int kDisabledBadQpThreshold = kMaxQp + 1; qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, - kDisabledBadQpThreshold, true); + kDisabledBadQpThreshold, true, 0, 0, 0); qs_.OnEncodeFrame(input_frame_); int init_width = qs_.GetScaledResolution().width; int init_height = qs_.GetScaledResolution().height; @@ -364,6 +368,17 @@ TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) { 1000, 2 * QualityScaler::kDefaultMinDownscaleDimension - 1); } +TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) { + qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, + kDisabledBadQpThreshold, true, + kLowInitialBitrateKbps, kWidth, kHeight); + qs_.OnEncodeFrame(input_frame_); + int init_width = qs_.GetScaledResolution().width; + int init_height = qs_.GetScaledResolution().height; + EXPECT_LE(init_width, kWidthVga); + EXPECT_LE(init_height, kHeightVga); +} + void QualityScalerTest::DownscaleEndsAt(int input_width, int input_height, int end_width,