diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc
index 5722d06c1e..343cd4c8f0 100644
--- a/webrtc/video/vie_encoder.cc
+++ b/webrtc/video/vie_encoder.cc
@@ -32,6 +32,8 @@ namespace webrtc {
 namespace {
 // Time interval for logging frame counts.
 const int64_t kFrameLogIntervalMs = 60000;
+// We will never ask for a resolution lower than this.
+const int kMinPixelsPerFrame = 120 * 90;
 
 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
 // pipelining encoders better (multiple input frames before something comes
@@ -176,7 +178,10 @@ class ViEEncoder::VideoSourceProxy {
     // The input video frame size will have a resolution with less than or
     // equal to |max_pixel_count| depending on how the source can scale the
     // input frame size.
-    sink_wants_.max_pixel_count = rtc::Optional<int>((pixel_count * 3) / 5);
+    const int pixels_wanted = (pixel_count * 3) / 5;
+    if (pixels_wanted < kMinPixelsPerFrame)
+      return;
+    sink_wants_.max_pixel_count = rtc::Optional<int>(pixels_wanted);
     sink_wants_.max_pixel_count_step_up = rtc::Optional<int>();
     if (source_)
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
@@ -681,8 +686,6 @@ void ViEEncoder::ScaleDown(ScaleReason reason) {
     return;
   switch (reason) {
     case kQuality:
-      if (scale_counter_[reason] >= kMaxQualityDowngrades)
-        return;
       stats_proxy_->OnQualityRestrictedResolutionChanged(true);
       break;
     case kCpu:
diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h
index ae9f0bae5b..7884bbe297 100644
--- a/webrtc/video/vie_encoder.h
+++ b/webrtc/video/vie_encoder.h
@@ -62,8 +62,6 @@ class ViEEncoder : public rtc::VideoSinkInterface<VideoFrame>,
 
   // Downscale resolution at most 2 times for CPU reasons.
   static const int kMaxCpuDowngrades = 2;
-  // Downscale resolution at most 2 times for low-quality reasons.
-  static const int kMaxQualityDowngrades = 2;
 
   ViEEncoder(uint32_t number_of_cores,
              SendStatisticsProxy* stats_proxy,
diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc
index 6cc25cfb80..1bb904be7c 100644
--- a/webrtc/video/vie_encoder_unittest.cc
+++ b/webrtc/video/vie_encoder_unittest.cc
@@ -585,12 +585,15 @@ TEST_F(ViEEncoderTest,
   int frame_width = 1280;
   int frame_height = 720;
 
+  video_source_.IncomingCapturedFrame(
+      CreateFrame(1, frame_width, frame_height));
+  sink_.WaitForEncodedFrame(1);
   // Trigger CPU overuse.
   vie_encoder_->TriggerCpuOveruse();
 
   video_source_.IncomingCapturedFrame(
-      CreateFrame(1, frame_width, frame_height));
-  sink_.WaitForEncodedFrame(1);
+      CreateFrame(2, frame_width, frame_height));
+  sink_.WaitForEncodedFrame(2);
   EXPECT_LT(video_source_.sink_wants().max_pixel_count.value_or(
                 std::numeric_limits<int>::max()),
             frame_width * frame_height);
@@ -606,8 +609,8 @@ TEST_F(ViEEncoderTest,
   EXPECT_FALSE(new_video_source.sink_wants().max_pixel_count_step_up);
 
   new_video_source.IncomingCapturedFrame(
-      CreateFrame(2, frame_width, frame_height));
-  sink_.WaitForEncodedFrame(2);
+      CreateFrame(3, frame_width, frame_height));
+  sink_.WaitForEncodedFrame(3);
   EXPECT_FALSE(new_video_source.sink_wants().max_pixel_count);
   EXPECT_FALSE(new_video_source.sink_wants().max_pixel_count_step_up);
 
@@ -874,12 +877,17 @@ TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) {
   EXPECT_FALSE(video_source_.sink_wants().max_pixel_count);
   EXPECT_FALSE(video_source_.sink_wants().max_pixel_count_step_up);
 
-  // Trigger scale down
-  vie_encoder_->TriggerQualityLow();
   video_source_.IncomingCapturedFrame(
       CreateFrame(1, frame_width, frame_height));
   sink_.WaitForEncodedFrame(1);
 
+  // Trigger scale down
+  vie_encoder_->TriggerQualityLow();
+
+  video_source_.IncomingCapturedFrame(
+      CreateFrame(2, frame_width, frame_height));
+  sink_.WaitForEncodedFrame(2);
+
   // Expect a scale down.
   EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
   EXPECT_LT(*video_source_.sink_wants().max_pixel_count,
@@ -894,8 +902,8 @@ TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) {
   // Trigger scale down
   vie_encoder_->TriggerQualityLow();
   new_video_source.IncomingCapturedFrame(
-      CreateFrame(2, frame_width, frame_height));
-  sink_.WaitForEncodedFrame(2);
+      CreateFrame(3, frame_width, frame_height));
+  sink_.WaitForEncodedFrame(3);
 
   // Expect no scaling
   EXPECT_FALSE(new_video_source.sink_wants().max_pixel_count);
@@ -903,8 +911,8 @@ TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) {
   // Trigger scale up
   vie_encoder_->TriggerQualityHigh();
   new_video_source.IncomingCapturedFrame(
-      CreateFrame(3, frame_width, frame_height));
-  sink_.WaitForEncodedFrame(3);
+      CreateFrame(4, frame_width, frame_height));
+  sink_.WaitForEncodedFrame(4);
 
   // Expect nothing to change, still no scaling
   EXPECT_FALSE(new_video_source.sink_wants().max_pixel_count);
@@ -912,6 +920,26 @@ TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) {
   vie_encoder_->Stop();
 }
 
+TEST_F(ViEEncoderTest, DoesNotScaleBelowSetLimit) {
+  const int kTargetBitrateBps = 100000;
+  int frame_width = 1280;
+  int frame_height = 720;
+  // from vie_encoder.cc
+  const int kMinPixelsPerFrame = 120 * 90;
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+  for (size_t i = 1; i <= 10; i++) {
+    video_source_.IncomingCapturedFrame(
+        CreateFrame(i, frame_width, frame_height));
+    sink_.WaitForEncodedFrame(i);
+    // Trigger scale down
+    vie_encoder_->TriggerQualityLow();
+    EXPECT_GE(*video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
+  }
+
+  vie_encoder_->Stop();
+}
+
 TEST_F(ViEEncoderTest, UMACpuLimitedResolutionInPercent) {
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);