diff --git a/webrtc/api/java/jni/androidmediaencoder_jni.cc b/webrtc/api/java/jni/androidmediaencoder_jni.cc index f4d8b9d238..d83c4314c6 100644 --- a/webrtc/api/java/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/java/jni/androidmediaencoder_jni.cc @@ -380,15 +380,17 @@ int32_t MediaCodecVideoEncoder::InitEncode( const int kLowQpThreshold = 29; const int kBadQpThreshold = 90; quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false, - codec_settings->startBitrate, - codec_settings->width, codec_settings->height); + codec_settings->startBitrate, codec_settings->width, + codec_settings->height, + codec_settings->maxFramerate); } else if (codecType_ == kVideoCodecH264) { // H264 QP is in the range [0, 51]. const int kLowQpThreshold = 21; const int kBadQpThreshold = 33; quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false, - codec_settings->startBitrate, - codec_settings->width, codec_settings->height); + codec_settings->startBitrate, codec_settings->width, + codec_settings->height, + codec_settings->maxFramerate); } else { // When adding codec support to additional hardware codecs, also configure // their QP thresholds for scaling. @@ -396,7 +398,6 @@ int32_t MediaCodecVideoEncoder::InitEncode( scale_ = false; } quality_scaler_.SetMinResolution(kMinDimension, kMinDimension); - quality_scaler_.ReportFramerate(codec_settings->maxFramerate); QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); init_width = std::max(res.width, kMinDimension); init_height = std::max(res.height, kMinDimension); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index a62bd41c6b..b34288632c 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -604,8 +604,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, // TODO(glaznev/sprang): consider passing codec initial bitrate to quality // scaler to avoid starting with HD for low initial bitrates. quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator, - kDisabledBadQpThreshold, false, 0, 0, 0); - quality_scaler_.ReportFramerate(codec_.maxFramerate); + kDisabledBadQpThreshold, false, 0, 0, 0, + codec_.maxFramerate); // Only apply scaling to improve for single-layer streams. The scaling metrics // use frame drops as a signal and is only applicable when we drop frames. diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index 68ae9218ea..c6e5669731 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -11,8 +11,11 @@ namespace webrtc { -static const int kMinFps = 10; -static const int kMeasureSeconds = 5; +static const int kMinFps = 5; +static const int kMeasureSecondsDownscale = 3; +// Threshold constant used until first downscale (to permit fast rampup). +static const int kMeasureSecondsFastUpscale = 2; +static const int kMeasureSecondsUpscale = 5; static const int kFramedropPercentThreshold = 60; static const int kHdResolutionThreshold = 700 * 500; static const int kHdBitrateThresholdKbps = 500; @@ -23,9 +26,7 @@ const int QualityScaler::kDefaultLowQpDenominator = 3; const int QualityScaler::kDefaultMinDownscaleDimension = 90; QualityScaler::QualityScaler() - : num_samples_(0), - low_qp_threshold_(-1), - downscale_shift_(0), + : low_qp_threshold_(-1), framerate_down_(false), min_width_(kDefaultMinDownscaleDimension), min_height_(kDefaultMinDownscaleDimension) {} @@ -35,12 +36,17 @@ void QualityScaler::Init(int low_qp_threshold, bool use_framerate_reduction, int initial_bitrate_kbps, int width, - int height) { + int height, + int fps) { ClearSamples(); low_qp_threshold_ = low_qp_threshold; high_qp_threshold_ = high_qp_threshold; use_framerate_reduction_ = use_framerate_reduction; downscale_shift_ = 0; + // Use a faster window for upscaling initially (but be more graceful later). + // This enables faster initial rampups without risking strong up-down + // behavior later. + measure_seconds_upscale_ = kMeasureSecondsFastUpscale; const int init_width = width; const int init_height = height; // TODO(glaznev): Investigate using thresholds for other resolutions @@ -55,6 +61,7 @@ void QualityScaler::Init(int low_qp_threshold, } } UpdateTargetResolution(init_width, init_height); + ReportFramerate(fps); target_framerate_ = -1; } @@ -65,14 +72,14 @@ void QualityScaler::SetMinResolution(int min_width, int min_height) { // Report framerate(fps) to estimate # of samples. void QualityScaler::ReportFramerate(int framerate) { - num_samples_ = static_cast( - kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate)); framerate_ = framerate; + UpdateSampleCounts(); } void QualityScaler::ReportQP(int qp) { framedrop_percent_.AddSample(0); - average_qp_.AddSample(qp); + average_qp_downscale_.AddSample(qp); + average_qp_upscale_.AddSample(qp); } void QualityScaler::ReportDroppedFrame() { @@ -82,7 +89,8 @@ void QualityScaler::ReportDroppedFrame() { void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { // Should be set through InitEncode -> Should be set by now. assert(low_qp_threshold_ >= 0); - assert(num_samples_ > 0); + assert(num_samples_upscale_ > 0); + assert(num_samples_downscale_ > 0); // Update scale factor. int avg_drop = 0; @@ -90,9 +98,9 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { // When encoder consistently overshoots, framerate reduction and spatial // resizing will be triggered to get a smoother video. - if ((framedrop_percent_.GetAverage(num_samples_, &avg_drop) && + if ((framedrop_percent_.GetAverage(num_samples_downscale_, &avg_drop) && avg_drop >= kFramedropPercentThreshold) || - (average_qp_.GetAverage(num_samples_, &avg_qp) && + (average_qp_downscale_.GetAverage(num_samples_downscale_, &avg_qp) && avg_qp > high_qp_threshold_)) { // Reducing frame rate before spatial resolution change. // Reduce frame rate only when it is above a certain number. @@ -107,7 +115,7 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { } else { AdjustScale(false); } - } else if (average_qp_.GetAverage(num_samples_, &avg_qp) && + } else if (average_qp_upscale_.GetAverage(num_samples_upscale_, &avg_qp) && avg_qp <= low_qp_threshold_) { if (use_framerate_reduction_ && framerate_down_) { target_framerate_ = -1; @@ -160,13 +168,26 @@ void QualityScaler::UpdateTargetResolution(int frame_width, int frame_height) { void QualityScaler::ClearSamples() { framedrop_percent_.Reset(); - average_qp_.Reset(); + average_qp_downscale_.Reset(); + average_qp_upscale_.Reset(); +} + +void QualityScaler::UpdateSampleCounts() { + num_samples_downscale_ = static_cast( + kMeasureSecondsDownscale * (framerate_ < kMinFps ? kMinFps : framerate_)); + num_samples_upscale_ = static_cast( + measure_seconds_upscale_ * (framerate_ < kMinFps ? kMinFps : framerate_)); } void QualityScaler::AdjustScale(bool up) { downscale_shift_ += up ? -1 : 1; if (downscale_shift_ < 0) downscale_shift_ = 0; + if (!up) { + // Hit first downscale, start using a slower threshold for going up. + measure_seconds_upscale_ = kMeasureSecondsUpscale; + UpdateSampleCounts(); + } ClearSamples(); } diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index ebceb06893..34dda0e9f3 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -30,7 +30,8 @@ class QualityScaler { bool use_framerate_reduction, int initial_bitrate_kbps, int width, - int height); + int height, + int fps); void SetMinResolution(int min_width, int min_height); void ReportFramerate(int framerate); void ReportQP(int qp); @@ -46,17 +47,22 @@ class QualityScaler { void AdjustScale(bool up); void UpdateTargetResolution(int frame_width, int frame_height); void ClearSamples(); + void UpdateSampleCounts(); Scaler scaler_; VideoFrame scaled_frame_; - size_t num_samples_; + size_t num_samples_downscale_; + size_t num_samples_upscale_; + int measure_seconds_upscale_; + MovingAverage average_qp_upscale_; + MovingAverage average_qp_downscale_; + int framerate_; int target_framerate_; int low_qp_threshold_; int high_qp_threshold_; MovingAverage framedrop_percent_; - MovingAverage average_qp_; Resolution res_; int downscale_shift_; diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index a5e3219f70..72e9db405e 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -27,6 +27,11 @@ static const int kHighQp = 40; static const int kMaxQp = 56; static const int kDisabledBadQpThreshold = kMaxQp + 1; static const int kLowInitialBitrateKbps = 300; +// These values need to be in sync with corresponding constants +// in quality_scaler.cc +static const int kMeasureSecondsDownscale = 3; +static const int kMeasureSecondsFastUpscale = 2; +static const int kMeasureSecondsUpscale = 5; } // namespace class QualityScalerTest : public ::testing::Test { @@ -51,8 +56,7 @@ class QualityScalerTest : public ::testing::Test { input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth, kHalfWidth); qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, - 0, 0, 0); - qs_.ReportFramerate(kFramerate); + 0, 0, 0, kFramerate); qs_.OnEncodeFrame(input_frame_); } @@ -104,7 +108,8 @@ class QualityScalerTest : public ::testing::Test { int initial_framerate); void VerifyQualityAdaptation(int initial_framerate, - int seconds, + int seconds_downscale, + int seconds_upscale, bool expect_spatial_resize, bool expect_framerate_reduction); @@ -298,18 +303,19 @@ QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange( void QualityScalerTest::VerifyQualityAdaptation( int initial_framerate, - int seconds, + int seconds_downscale, + int seconds_upscale, bool expect_spatial_resize, bool expect_framerate_reduction) { qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, - kDisabledBadQpThreshold, true, 0, 0, 0); + kDisabledBadQpThreshold, true, 0, 0, 0, initial_framerate); qs_.OnEncodeFrame(input_frame_); int init_width = qs_.GetScaledResolution().width; int init_height = qs_.GetScaledResolution().height; // Test reducing framerate by dropping frame continuously. QualityScalerTest::Resolution res = - TriggerResolutionChange(kDropFrame, seconds, initial_framerate); + TriggerResolutionChange(kDropFrame, seconds_downscale, initial_framerate); if (expect_framerate_reduction) { EXPECT_LT(res.framerate, initial_framerate); @@ -327,35 +333,40 @@ void QualityScalerTest::VerifyQualityAdaptation( } // The "seconds * 1.5" is to ensure spatial resolution to recover. - // For example, in 10 seconds test, framerate reduction happens in the first - // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the - // original one. Then it will take only 75 samples to downscale (twice in 150 + // For example, in 6 seconds test, framerate reduction happens in the first + // 3 seconds from 30fps to 15fps and causes the buffer size to be half of the + // original one. Then it will take only 45 samples to downscale (twice in 90 // samples). So to recover the resolution changes, we need more than 10 - // seconds (i.e, seconds * 1.5). This is because the framerate increases - // before spatial size recovers, so it will take 150 samples to recover - // spatial size (300 for twice). - res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate); + // seconds (i.e, seconds_upscale * 1.5). This is because the framerate + // increases before spatial size recovers, so it will take 150 samples to + // recover spatial size (300 for twice). + res = TriggerResolutionChange(kReportLowQP, seconds_upscale * 1.5, + initial_framerate); EXPECT_EQ(-1, res.framerate); EXPECT_EQ(init_width, res.width); EXPECT_EQ(init_height, res.height); } -// In 5 seconds test, only framerate adjusting should happen. +// In 3 seconds test, only framerate adjusting should happen and 5 second +// upscaling duration, only a framerate adjusting should happen. TEST_F(QualityScalerTest, ChangeFramerateOnly) { - VerifyQualityAdaptation(kFramerate, 5, false, true); + VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale, + kMeasureSecondsUpscale, false, true); } -// In 10 seconds test, framerate adjusting and scaling are both +// In 6 seconds test, framerate adjusting and scaling are both // triggered, it shows that scaling would happen after framerate // adjusting. TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) { - VerifyQualityAdaptation(kFramerate, 10, true, true); + VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale * 2, + kMeasureSecondsUpscale * 2, true, true); } // When starting from a low framerate, only spatial size will be changed. TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) { qs_.ReportFramerate(kFramerate >> 1); - VerifyQualityAdaptation(kFramerate >> 1, 10, true, false); + VerifyQualityAdaptation(kFramerate >> 1, kMeasureSecondsDownscale * 2, + kMeasureSecondsUpscale * 2, true, false); } TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) { @@ -371,7 +382,7 @@ TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) { TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) { qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kDisabledBadQpThreshold, true, - kLowInitialBitrateKbps, kWidth, kHeight); + kLowInitialBitrateKbps, kWidth, kHeight, kFramerate); qs_.OnEncodeFrame(input_frame_); int init_width = qs_.GetScaledResolution().width; int init_height = qs_.GetScaledResolution().height; @@ -379,6 +390,70 @@ TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) { EXPECT_LE(init_height, kHeightVga); } +TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) { + QualityScalerTest::Resolution initial_res; + qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, 0, + kWidth, kHeight, kFramerate); + qs_.OnEncodeFrame(input_frame_); + initial_res.width = qs_.GetScaledResolution().width; + initial_res.height = qs_.GetScaledResolution().height; + + // Should not downscale if less than kMeasureSecondsDownscale seconds passed. + for (int i = 0; i < kFramerate * kMeasureSecondsDownscale - 1; ++i) { + qs_.ReportQP(kHighQp + 1); + qs_.OnEncodeFrame(input_frame_); + } + EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); + EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); + + // Should downscale if more than kMeasureSecondsDownscale seconds passed (add + // last frame). + qs_.ReportQP(kHighQp + 1); + qs_.OnEncodeFrame(input_frame_); + EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width); + EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height); + + // Should not upscale if less than kMeasureSecondsUpscale seconds passed since + // we saw issues initially (have already gone down). + for (int i = 0; i < kFramerate * kMeasureSecondsUpscale - 1; ++i) { + qs_.ReportQP(kLowQp); + qs_.OnEncodeFrame(input_frame_); + } + EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width); + EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height); + + // Should upscale (back to initial) if kMeasureSecondsUpscale seconds passed + // (add last frame). + qs_.ReportQP(kLowQp); + qs_.OnEncodeFrame(input_frame_); + EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); + EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); +} + +TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) { + QualityScalerTest::Resolution initial_res; + qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, + kLowInitialBitrateKbps, kWidth, kHeight, kFramerate); + qs_.OnEncodeFrame(input_frame_); + initial_res.width = qs_.GetScaledResolution().width; + initial_res.height = qs_.GetScaledResolution().height; + + // Should not upscale if less than kMeasureSecondsFastUpscale seconds passed. + for (int i = 0; i < kFramerate * kMeasureSecondsFastUpscale - 1; ++i) { + qs_.ReportQP(kLowQp); + qs_.OnEncodeFrame(input_frame_); + } + EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); + EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); + + // Should upscale if kMeasureSecondsFastUpscale seconds passed (add last + // frame). + qs_.ReportQP(kLowQp); + qs_.OnEncodeFrame(input_frame_); + EXPECT_LT(initial_res.width, qs_.GetScaledResolution().width); + EXPECT_LT(initial_res.height, qs_.GetScaledResolution().height); +} + void QualityScalerTest::DownscaleEndsAt(int input_width, int input_height, int end_width,