From 478681e1e6e6c5d66366672614d58b10ad7df083 Mon Sep 17 00:00:00 2001 From: kthelgason Date: Wed, 28 Sep 2016 08:17:43 -0700 Subject: [PATCH] Move the QP scaling thresholds to the relevant encoders. Also provide a new set of thresholds for the VideoToolbox encoder. The new thresholds were experimentally determined to work well on the iPhone 6S, and also adequately on the iPhone 5S. BUG=webrtc:5678 Review-Url: https://codereview.webrtc.org/2309743002 Cr-Commit-Position: refs/heads/master@{#14420} --- .../android/jni/androidmediaencoder_jni.cc | 13 ++--- .../codecs/h264/h264_encoder_impl.cc | 7 ++- .../codecs/h264/h264_video_toolbox_encoder.mm | 8 ++- .../video_coding/codecs/vp8/vp8_impl.cc | 5 +- .../video_coding/utility/quality_scaler.cc | 50 ++++++++++++------- .../video_coding/utility/quality_scaler.h | 15 +++--- 6 files changed, 52 insertions(+), 46 deletions(-) diff --git a/webrtc/api/android/jni/androidmediaencoder_jni.cc b/webrtc/api/android/jni/androidmediaencoder_jni.cc index 8d0d3b54bd..e84b850287 100644 --- a/webrtc/api/android/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/android/jni/androidmediaencoder_jni.cc @@ -395,16 +395,9 @@ int32_t MediaCodecVideoEncoder::InitEncode( ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); if (scale_) { - if (codecType_ == kVideoCodecVP8) { - quality_scaler_.Init( - QualityScaler::kLowVp8QpThreshold, QualityScaler::kBadVp8QpThreshold, - codec_settings->startBitrate, codec_settings->width, - codec_settings->height, codec_settings->maxFramerate); - } else if (codecType_ == kVideoCodecH264) { - quality_scaler_.Init(QualityScaler::kLowH264QpThreshold, - QualityScaler::kBadH264QpThreshold, - codec_settings->startBitrate, codec_settings->width, - codec_settings->height, + if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecH264) { + quality_scaler_.Init(codecType_, codec_settings->startBitrate, + codec_settings->width, codec_settings->height, codec_settings->maxFramerate); } else { // When adding codec support to additional hardware codecs, also configure diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc index bba30a3ca8..3dfa0b559d 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -213,10 +213,9 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings, return WEBRTC_VIDEO_CODEC_ERROR; } // TODO(pbos): Base init params on these values before submitting. - quality_scaler_.Init(QualityScaler::kLowH264QpThreshold, - QualityScaler::kBadH264QpThreshold, - codec_settings_.startBitrate, codec_settings_.width, - codec_settings_.height, codec_settings_.maxFramerate); + quality_scaler_.Init(codec_settings_.codecType, codec_settings_.startBitrate, + codec_settings_.width, codec_settings_.height, + codec_settings_.maxFramerate); int video_format = EVideoFormatType::videoFormatI420; openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format); diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm index 2d1e2a1416..88687136ed 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm @@ -33,6 +33,10 @@ namespace internal { // kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher // than the average bit rate to avoid undershooting the target. const float kLimitToAverageBitRateFactor = 1.5f; +// These thresholds deviate from the default h264 QP thresholds, as they +// have been found to work better on devices that support VideoToolbox +const int kLowH264QpThreshold = 28; +const int kHighH264QpThreshold = 39; // Convenience function for creating a dictionary. inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, @@ -232,8 +236,8 @@ int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); { rtc::CritScope lock(&quality_scaler_crit_); - quality_scaler_.Init(QualityScaler::kLowH264QpThreshold, - QualityScaler::kBadH264QpThreshold, + quality_scaler_.Init(internal::kLowH264QpThreshold, + internal::kHighH264QpThreshold, codec_settings->startBitrate, codec_settings->width, codec_settings->height, codec_settings->maxFramerate); QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 73a7eb659b..01b3385fdd 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -562,9 +562,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, } rps_.Init(); - quality_scaler_.Init(QualityScaler::kLowVp8QpThreshold, - QualityScaler::kBadVp8QpThreshold, codec_.startBitrate, - codec_.width, codec_.height, codec_.maxFramerate); + quality_scaler_.Init(codec_.codecType, codec_.startBitrate, codec_.width, + codec_.height, codec_.maxFramerate); // Only apply scaling to improve for single-layer streams. The scaling metrics // use frame drops as a signal and is only applicable when we drop frames. diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index 7ef7c575ba..55cb274209 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -14,6 +14,8 @@ #include +#include "webrtc/base/checks.h" + // TODO(kthelgason): Some versions of Android have issues with log2. // See https://code.google.com/p/android/issues/detail?id=212634 for details #if defined(WEBRTC_ANDROID) @@ -38,42 +40,54 @@ static const int kVgaBitrateThresholdKbps = 500; static const int kVgaNumPixels = 700 * 500; // 640x480 static const int kQvgaBitrateThresholdKbps = 250; static const int kQvgaNumPixels = 400 * 300; // 320x240 + +// QP scaling threshold defaults: +static const int kLowH264QpThreshold = 24; +static const int kHighH264QpThreshold = 37; +// QP is obtained from VP8-bitstream for HW, so the QP corresponds to the +// bitstream range of [0, 127] and not the user-level range of [0,63]. +static const int kLowVp8QpThreshold = 29; +static const int kHighVp8QpThreshold = 95; } // namespace -// QP thresholds are chosen to be high enough to be hit in practice when quality -// is good, but also low enough to not cause a flip-flop behavior (e.g. going up -// in resolution shouldn't give so bad quality that we should go back down). - -const int QualityScaler::kLowVp8QpThreshold = 29; -const int QualityScaler::kBadVp8QpThreshold = 95; - -#if defined(WEBRTC_IOS) -const int QualityScaler::kLowH264QpThreshold = 32; -const int QualityScaler::kBadH264QpThreshold = 42; -#else -const int QualityScaler::kLowH264QpThreshold = 24; -const int QualityScaler::kBadH264QpThreshold = 37; -#endif - // Default values. Should immediately get set to something more sensible. QualityScaler::QualityScaler() : average_qp_(kMeasureSecondsUpscale * 30), framedrop_percent_(kMeasureSecondsUpscale * 30), low_qp_threshold_(-1) {} +void QualityScaler::Init(VideoCodecType codec_type, + int initial_bitrate_kbps, + int width, + int height, + int fps) { + int low = -1, high = -1; + switch (codec_type) { + case kVideoCodecH264: + low = kLowH264QpThreshold; + high = kHighH264QpThreshold; + break; + case kVideoCodecVP8: + low = kLowVp8QpThreshold; + high = kHighVp8QpThreshold; + break; + default: + RTC_NOTREACHED() << "Invalid codec type for QualityScaler."; + } + Init(low, high, initial_bitrate_kbps, width, height, fps); +} + void QualityScaler::Init(int low_qp_threshold, int high_qp_threshold, int initial_bitrate_kbps, int width, int height, int fps) { + ClearSamples(); low_qp_threshold_ = low_qp_threshold; high_qp_threshold_ = high_qp_threshold; downscale_shift_ = 0; - fast_rampup_ = true; - - ClearSamples(); ReportFramerate(fps); const int init_width = width; diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index c1ae50b06e..c0f94409ce 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -11,6 +11,7 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ +#include "webrtc/common_types.h" #include "webrtc/common_video/include/i420_buffer_pool.h" #include "webrtc/modules/video_coding/utility/moving_average.h" @@ -23,6 +24,11 @@ class QualityScaler { }; QualityScaler(); + void Init(VideoCodecType codec_type, + int initial_bitrate_kbps, + int width, + int height, + int fps); void Init(int low_qp_threshold, int high_qp_threshold, int initial_bitrate_kbps, @@ -38,15 +44,6 @@ class QualityScaler { const rtc::scoped_refptr& frame); int downscale_shift() const { return downscale_shift_; } - // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the - // bitstream range of [0, 127] and not the user-level range of [0,63]. - static const int kLowVp8QpThreshold; - static const int kBadVp8QpThreshold; - - // H264 QP is in the range [0, 51]. - static const int kLowH264QpThreshold; - static const int kBadH264QpThreshold; - private: void ClearSamples(); void ScaleUp();