Reland of Make QualityScaler more responsive to downgrades. (patchset #1 id:1 of https://codereview.webrtc.org/1880103002/ )

Reason for revert:
Regressed behavior is actually desirable (go down to 360p instead of producing super-bad 720p).

Original issue's description:
> Revert of Make QualityScaler more responsive to downgrades. (patchset #3 id:40001 of https://codereview.webrtc.org/1830593003/ )
>
> Reason for revert:
> Speculative revert: want to see if this causes the regression in https://crbug.com/602621
>
> Original issue's description:
> > Make QualityScaler more responsive to downgrades.
> >
> > Permits going from HD to QVGA in 6 seconds instead of 10. Also adds
> > windows for going up quickly in the beginning of a call (before any
> > downscaling happens due to bad quality).
> >
> > BUG=webrtc:5678
> > R=glaznev@webrtc.org, stefan@webrtc.org
> >
> > Committed: https://crrev.com/85829fd90cc4e7a91c9857921b19e8fc126aeb60
> > Cr-Commit-Position: refs/heads/master@{#12219}
>
> TBR=glaznev@webrtc.org,stefan@webrtc.org,pbos@webrtc.org
> # Not skipping CQ checks because original CL landed more than 1 days ago.
> BUG=webrtc:5678
> NOTRY=true
>
> Committed: https://crrev.com/19b4fecf08e3fe215e431a260fb673553c15e569
> Cr-Commit-Position: refs/heads/master@{#12331}

TBR=glaznev@webrtc.org,stefan@webrtc.org,phoglund@webrtc.org
# Skipping CQ checks because original CL landed less than 1 days ago.
NOPRESUBMIT=true
NOTREECHECKS=true
NOTRY=true
BUG=chromium:602621, webrtc:5678

Review URL: https://codereview.webrtc.org/1887493003

Cr-Commit-Position: refs/heads/master@{#12341}

webrtc/api/java/jni/androidmediaencoder_jni.cc

@@ -380,15 +380,17 @@ int32_t MediaCodecVideoEncoder::InitEncode(
       const int kLowQpThreshold = 29;
       const int kBadQpThreshold = 90;
       quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false,
-          codec_settings->startBitrate,
+                           codec_settings->startBitrate, codec_settings->width,
-          codec_settings->width, codec_settings->height);
+                           codec_settings->height,
+                           codec_settings->maxFramerate);
     } else if (codecType_ == kVideoCodecH264) {
       // H264 QP is in the range [0, 51].
       const int kLowQpThreshold = 21;
       const int kBadQpThreshold = 33;
       quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false,
-          codec_settings->startBitrate,
+                           codec_settings->startBitrate, codec_settings->width,
-          codec_settings->width, codec_settings->height);
+                           codec_settings->height,
+                           codec_settings->maxFramerate);
     } else {
       // When adding codec support to additional hardware codecs, also configure
       // their QP thresholds for scaling.
@@ -396,7 +398,6 @@ int32_t MediaCodecVideoEncoder::InitEncode(
       scale_ = false;
     }
     quality_scaler_.SetMinResolution(kMinDimension, kMinDimension);
-    quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
     QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
     init_width = std::max(res.width, kMinDimension);
     init_height = std::max(res.height, kMinDimension);

webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc

@@ -604,8 +604,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
   // TODO(glaznev/sprang): consider passing codec initial bitrate to quality
   // scaler to avoid starting with HD for low initial bitrates.
   quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator,
-                       kDisabledBadQpThreshold, false, 0, 0, 0);
+                       kDisabledBadQpThreshold, false, 0, 0, 0,
-  quality_scaler_.ReportFramerate(codec_.maxFramerate);
+                       codec_.maxFramerate);
 
   // Only apply scaling to improve for single-layer streams. The scaling metrics
   // use frame drops as a signal and is only applicable when we drop frames.

webrtc/modules/video_coding/utility/quality_scaler.cc

@@ -11,8 +11,11 @@
 
 namespace webrtc {
 
-static const int kMinFps = 10;
+static const int kMinFps = 5;
-static const int kMeasureSeconds = 5;
+static const int kMeasureSecondsDownscale = 3;
+// Threshold constant used until first downscale (to permit fast rampup).
+static const int kMeasureSecondsFastUpscale = 2;
+static const int kMeasureSecondsUpscale = 5;
 static const int kFramedropPercentThreshold = 60;
 static const int kHdResolutionThreshold = 700 * 500;
 static const int kHdBitrateThresholdKbps = 500;
@@ -23,9 +26,7 @@ const int QualityScaler::kDefaultLowQpDenominator = 3;
 const int QualityScaler::kDefaultMinDownscaleDimension = 90;
 
 QualityScaler::QualityScaler()
-    : num_samples_(0),
+    : low_qp_threshold_(-1),
-      low_qp_threshold_(-1),
-      downscale_shift_(0),
       framerate_down_(false),
       min_width_(kDefaultMinDownscaleDimension),
       min_height_(kDefaultMinDownscaleDimension) {}
@@ -35,12 +36,17 @@ void QualityScaler::Init(int low_qp_threshold,
                          bool use_framerate_reduction,
                          int initial_bitrate_kbps,
                          int width,
-                         int height) {
+                         int height,
+                         int fps) {
   ClearSamples();
   low_qp_threshold_ = low_qp_threshold;
   high_qp_threshold_ = high_qp_threshold;
   use_framerate_reduction_ = use_framerate_reduction;
   downscale_shift_ = 0;
+  // Use a faster window for upscaling initially (but be more graceful later).
+  // This enables faster initial rampups without risking strong up-down
+  // behavior later.
+  measure_seconds_upscale_ = kMeasureSecondsFastUpscale;
   const int init_width = width;
   const int init_height = height;
   // TODO(glaznev): Investigate using thresholds for other resolutions
@@ -55,6 +61,7 @@ void QualityScaler::Init(int low_qp_threshold,
     }
   }
   UpdateTargetResolution(init_width, init_height);
+  ReportFramerate(fps);
   target_framerate_ = -1;
 }
 
@@ -65,14 +72,14 @@ void QualityScaler::SetMinResolution(int min_width, int min_height) {
 
 // Report framerate(fps) to estimate # of samples.
 void QualityScaler::ReportFramerate(int framerate) {
-  num_samples_ = static_cast<size_t>(
-      kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate));
   framerate_ = framerate;
+  UpdateSampleCounts();
 }
 
 void QualityScaler::ReportQP(int qp) {
   framedrop_percent_.AddSample(0);
-  average_qp_.AddSample(qp);
+  average_qp_downscale_.AddSample(qp);
+  average_qp_upscale_.AddSample(qp);
 }
 
 void QualityScaler::ReportDroppedFrame() {
@@ -82,7 +89,8 @@ void QualityScaler::ReportDroppedFrame() {
 void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
   // Should be set through InitEncode -> Should be set by now.
   assert(low_qp_threshold_ >= 0);
-  assert(num_samples_ > 0);
+  assert(num_samples_upscale_ > 0);
+  assert(num_samples_downscale_ > 0);
 
   // Update scale factor.
   int avg_drop = 0;
@@ -90,9 +98,9 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
 
   // When encoder consistently overshoots, framerate reduction and spatial
   // resizing will be triggered to get a smoother video.
-  if ((framedrop_percent_.GetAverage(num_samples_, &avg_drop) &&
+  if ((framedrop_percent_.GetAverage(num_samples_downscale_, &avg_drop) &&
        avg_drop >= kFramedropPercentThreshold) ||
-      (average_qp_.GetAverage(num_samples_, &avg_qp) &&
+      (average_qp_downscale_.GetAverage(num_samples_downscale_, &avg_qp) &&
        avg_qp > high_qp_threshold_)) {
     // Reducing frame rate before spatial resolution change.
     // Reduce frame rate only when it is above a certain number.
@@ -107,7 +115,7 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
     } else {
       AdjustScale(false);
     }
-  } else if (average_qp_.GetAverage(num_samples_, &avg_qp) &&
+  } else if (average_qp_upscale_.GetAverage(num_samples_upscale_, &avg_qp) &&
              avg_qp <= low_qp_threshold_) {
     if (use_framerate_reduction_ && framerate_down_) {
       target_framerate_ = -1;
@@ -160,13 +168,26 @@ void QualityScaler::UpdateTargetResolution(int frame_width, int frame_height) {
 
 void QualityScaler::ClearSamples() {
   framedrop_percent_.Reset();
-  average_qp_.Reset();
+  average_qp_downscale_.Reset();
+  average_qp_upscale_.Reset();
+}
+
+void QualityScaler::UpdateSampleCounts() {
+  num_samples_downscale_ = static_cast<size_t>(
+      kMeasureSecondsDownscale * (framerate_ < kMinFps ? kMinFps : framerate_));
+  num_samples_upscale_ = static_cast<size_t>(
+      measure_seconds_upscale_ * (framerate_ < kMinFps ? kMinFps : framerate_));
 }
 
 void QualityScaler::AdjustScale(bool up) {
   downscale_shift_ += up ? -1 : 1;
   if (downscale_shift_ < 0)
     downscale_shift_ = 0;
+  if (!up) {
+    // Hit first downscale, start using a slower threshold for going up.
+    measure_seconds_upscale_ = kMeasureSecondsUpscale;
+    UpdateSampleCounts();
+  }
   ClearSamples();
 }
 

webrtc/modules/video_coding/utility/quality_scaler.h

@@ -30,7 +30,8 @@ class QualityScaler {
             bool use_framerate_reduction,
             int initial_bitrate_kbps,
             int width,
-            int height);
+            int height,
+            int fps);
   void SetMinResolution(int min_width, int min_height);
   void ReportFramerate(int framerate);
   void ReportQP(int qp);
@@ -46,17 +47,22 @@ class QualityScaler {
   void AdjustScale(bool up);
   void UpdateTargetResolution(int frame_width, int frame_height);
   void ClearSamples();
+  void UpdateSampleCounts();
 
   Scaler scaler_;
   VideoFrame scaled_frame_;
 
-  size_t num_samples_;
+  size_t num_samples_downscale_;
+  size_t num_samples_upscale_;
+  int measure_seconds_upscale_;
+  MovingAverage<int> average_qp_upscale_;
+  MovingAverage<int> average_qp_downscale_;
+
   int framerate_;
   int target_framerate_;
   int low_qp_threshold_;
   int high_qp_threshold_;
   MovingAverage<int> framedrop_percent_;
-  MovingAverage<int> average_qp_;
   Resolution res_;
 
   int downscale_shift_;

webrtc/modules/video_coding/utility/quality_scaler_unittest.cc

@@ -27,6 +27,11 @@ static const int kHighQp = 40;
 static const int kMaxQp = 56;
 static const int kDisabledBadQpThreshold = kMaxQp + 1;
 static const int kLowInitialBitrateKbps = 300;
+// These values need to be in sync with corresponding constants
+// in quality_scaler.cc
+static const int kMeasureSecondsDownscale = 3;
+static const int kMeasureSecondsFastUpscale = 2;
+static const int kMeasureSecondsUpscale = 5;
 }  // namespace
 
 class QualityScalerTest : public ::testing::Test {
@@ -51,8 +56,7 @@ class QualityScalerTest : public ::testing::Test {
     input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth,
                                   kHalfWidth);
     qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false,
-             0, 0, 0);
+             0, 0, 0, kFramerate);
-    qs_.ReportFramerate(kFramerate);
     qs_.OnEncodeFrame(input_frame_);
   }
 
@@ -104,7 +108,8 @@ class QualityScalerTest : public ::testing::Test {
                                      int initial_framerate);
 
   void VerifyQualityAdaptation(int initial_framerate,
-                               int seconds,
+                               int seconds_downscale,
+                               int seconds_upscale,
                                bool expect_spatial_resize,
                                bool expect_framerate_reduction);
 
@@ -298,18 +303,19 @@ QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange(
 
 void QualityScalerTest::VerifyQualityAdaptation(
     int initial_framerate,
-    int seconds,
+    int seconds_downscale,
+    int seconds_upscale,
     bool expect_spatial_resize,
     bool expect_framerate_reduction) {
   qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
-           kDisabledBadQpThreshold, true, 0, 0, 0);
+           kDisabledBadQpThreshold, true, 0, 0, 0, initial_framerate);
   qs_.OnEncodeFrame(input_frame_);
   int init_width = qs_.GetScaledResolution().width;
   int init_height = qs_.GetScaledResolution().height;
 
   // Test reducing framerate by dropping frame continuously.
   QualityScalerTest::Resolution res =
-      TriggerResolutionChange(kDropFrame, seconds, initial_framerate);
+      TriggerResolutionChange(kDropFrame, seconds_downscale, initial_framerate);
 
   if (expect_framerate_reduction) {
     EXPECT_LT(res.framerate, initial_framerate);
@@ -327,35 +333,40 @@ void QualityScalerTest::VerifyQualityAdaptation(
   }
 
   // The "seconds * 1.5" is to ensure spatial resolution to recover.
-  // For example, in 10 seconds test, framerate reduction happens in the first
+  // For example, in 6 seconds test, framerate reduction happens in the first
-  // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the
+  // 3 seconds from 30fps to 15fps and causes the buffer size to be half of the
-  // original one. Then it will take only 75 samples to downscale (twice in 150
+  // original one. Then it will take only 45 samples to downscale (twice in 90
   // samples). So to recover the resolution changes, we need more than 10
-  // seconds (i.e, seconds * 1.5). This is because the framerate increases
+  // seconds (i.e, seconds_upscale * 1.5). This is because the framerate
-  // before spatial size recovers, so it will take 150 samples to recover
+  // increases before spatial size recovers, so it will take 150 samples to
-  // spatial size (300 for twice).
+  // recover spatial size (300 for twice).
-  res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate);
+  res = TriggerResolutionChange(kReportLowQP, seconds_upscale * 1.5,
+                                initial_framerate);
   EXPECT_EQ(-1, res.framerate);
   EXPECT_EQ(init_width, res.width);
   EXPECT_EQ(init_height, res.height);
 }
 
-// In 5 seconds test, only framerate adjusting should happen.
+// In 3 seconds test, only framerate adjusting should happen and 5 second
+// upscaling duration, only a framerate adjusting should happen.
 TEST_F(QualityScalerTest, ChangeFramerateOnly) {
-  VerifyQualityAdaptation(kFramerate, 5, false, true);
+  VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale,
+                          kMeasureSecondsUpscale, false, true);
 }
 
-// In 10 seconds test, framerate adjusting and scaling are both
+// In 6 seconds test, framerate adjusting and scaling are both
 // triggered, it shows that scaling would happen after framerate
 // adjusting.
 TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
-  VerifyQualityAdaptation(kFramerate, 10, true, true);
+  VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale * 2,
+                          kMeasureSecondsUpscale * 2, true, true);
 }
 
 // When starting from a low framerate, only spatial size will be changed.
 TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
   qs_.ReportFramerate(kFramerate >> 1);
-  VerifyQualityAdaptation(kFramerate >> 1, 10, true, false);
+  VerifyQualityAdaptation(kFramerate >> 1, kMeasureSecondsDownscale * 2,
+                          kMeasureSecondsUpscale * 2, true, false);
 }
 
 TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
@@ -371,7 +382,7 @@ TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) {
 TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) {
   qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
            kDisabledBadQpThreshold, true,
-           kLowInitialBitrateKbps, kWidth, kHeight);
+           kLowInitialBitrateKbps, kWidth, kHeight, kFramerate);
   qs_.OnEncodeFrame(input_frame_);
   int init_width = qs_.GetScaledResolution().width;
   int init_height = qs_.GetScaledResolution().height;
@@ -379,6 +390,70 @@ TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) {
   EXPECT_LE(init_height, kHeightVga);
 }
 
+TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) {
+  QualityScalerTest::Resolution initial_res;
+  qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, 0,
+           kWidth, kHeight, kFramerate);
+  qs_.OnEncodeFrame(input_frame_);
+  initial_res.width = qs_.GetScaledResolution().width;
+  initial_res.height = qs_.GetScaledResolution().height;
+
+  // Should not downscale if less than kMeasureSecondsDownscale seconds passed.
+  for (int i = 0; i < kFramerate * kMeasureSecondsDownscale - 1; ++i) {
+    qs_.ReportQP(kHighQp + 1);
+    qs_.OnEncodeFrame(input_frame_);
+  }
+  EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
+  EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
+
+  // Should downscale if more than kMeasureSecondsDownscale seconds passed (add
+  // last frame).
+  qs_.ReportQP(kHighQp + 1);
+  qs_.OnEncodeFrame(input_frame_);
+  EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width);
+  EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height);
+
+  // Should not upscale if less than kMeasureSecondsUpscale seconds passed since
+  // we saw issues initially (have already gone down).
+  for (int i = 0; i < kFramerate * kMeasureSecondsUpscale - 1; ++i) {
+    qs_.ReportQP(kLowQp);
+    qs_.OnEncodeFrame(input_frame_);
+  }
+  EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width);
+  EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height);
+
+  // Should upscale (back to initial) if kMeasureSecondsUpscale seconds passed
+  // (add last frame).
+  qs_.ReportQP(kLowQp);
+  qs_.OnEncodeFrame(input_frame_);
+  EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
+  EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
+}
+
+TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) {
+  QualityScalerTest::Resolution initial_res;
+  qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false,
+           kLowInitialBitrateKbps, kWidth, kHeight, kFramerate);
+  qs_.OnEncodeFrame(input_frame_);
+  initial_res.width = qs_.GetScaledResolution().width;
+  initial_res.height = qs_.GetScaledResolution().height;
+
+  // Should not upscale if less than kMeasureSecondsFastUpscale seconds passed.
+  for (int i = 0; i < kFramerate * kMeasureSecondsFastUpscale - 1; ++i) {
+    qs_.ReportQP(kLowQp);
+    qs_.OnEncodeFrame(input_frame_);
+  }
+  EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
+  EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
+
+  // Should upscale if kMeasureSecondsFastUpscale seconds passed (add last
+  // frame).
+  qs_.ReportQP(kLowQp);
+  qs_.OnEncodeFrame(input_frame_);
+  EXPECT_LT(initial_res.width, qs_.GetScaledResolution().width);
+  EXPECT_LT(initial_res.height, qs_.GetScaledResolution().height);
+}
+
 void QualityScalerTest::DownscaleEndsAt(int input_width,
                                         int input_height,
                                         int end_width,