Get min bitrate from spatial layers for AV1 (instead of bitrate limits).

Bitrate limits should have been applied to the spatial layers in ApplySpatialLayerBitrateLimits (and usage is restricted to a single active stream/layer).

Bug: b/299588022
Change-Id: Iaae4ece28b8a95eea7d4bacba292847ba5b4000b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/355841
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42588}

modules/video_coding/video_codec_initializer.cc

@@ -345,7 +345,8 @@ VideoCodec VideoCodecInitializer::SetupCodec(
         rtc::saturated_cast<int>(experimental_min_bitrate->kbps());
     video_codec.minBitrate = experimental_min_bitrate_kbps;
     video_codec.simulcastStream[0].minBitrate = experimental_min_bitrate_kbps;
-    if (video_codec.codecType == kVideoCodecVP9) {
+    if (video_codec.codecType == kVideoCodecVP9 ||
+        video_codec.codecType == kVideoCodecAV1) {
       video_codec.spatialLayers[0].minBitrate = experimental_min_bitrate_kbps;
     }
   }

video/encoder_bitrate_adjuster.cc

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include "api/field_trials_view.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
 #include "rtc_base/experiments/rate_control_settings.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/time_utils.h"
@@ -51,21 +52,30 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster(
                                       .BitrateAdjusterCanUseNetworkHeadroom()),
       frames_since_layout_change_(0),
       min_bitrates_bps_{},
-      frame_size_pixels_{},
       codec_(codec_settings.codecType),
       codec_mode_(codec_settings.mode) {
   // TODO(https://crbug.com/webrtc/14891): If we want to support simulcast of
   // SVC streams, EncoderBitrateAdjuster needs to be updated to care about both
   // `simulcastStream` and `spatialLayers` at the same time.
-  if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9 &&
+  if (codec_settings.codecType == VideoCodecType::kVideoCodecAV1 &&
+      codec_settings.numberOfSimulcastStreams <= 1 &&
+      codec_settings.GetScalabilityMode().has_value()) {
+    for (int si = 0; si < ScalabilityModeToNumSpatialLayers(
+                              *(codec_settings.GetScalabilityMode()));
+         ++si) {
+      if (codec_settings.spatialLayers[si].active) {
+        min_bitrates_bps_[si] =
+            std::max(codec_settings.minBitrate * 1000,
+                     codec_settings.spatialLayers[si].minBitrate * 1000);
+      }
+    }
+  } else if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9 &&
              codec_settings.numberOfSimulcastStreams <= 1) {
     for (size_t si = 0; si < codec_settings.VP9().numberOfSpatialLayers; ++si) {
       if (codec_settings.spatialLayers[si].active) {
         min_bitrates_bps_[si] =
             std::max(codec_settings.minBitrate * 1000,
                      codec_settings.spatialLayers[si].minBitrate * 1000);
-        frame_size_pixels_[si] = codec_settings.spatialLayers[si].width *
-                                 codec_settings.spatialLayers[si].height;
       }
     }
   } else {
@@ -74,8 +84,6 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster(
         min_bitrates_bps_[si] =
             std::max(codec_settings.minBitrate * 1000,
                      codec_settings.simulcastStream[si].minBitrate * 1000);
-        frame_size_pixels_[si] = codec_settings.spatialLayers[si].width *
-                                 codec_settings.spatialLayers[si].height;
       }
     }
   }
@@ -322,12 +330,6 @@ void EncoderBitrateAdjuster::OnEncoderInfo(
   // Copy allocation into current state and re-allocate.
   for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
     current_fps_allocation_[si] = encoder_info.fps_allocation[si];
-    if (frame_size_pixels_[si] > 0) {
-      if (auto bwlimit = encoder_info.GetEncoderBitrateLimitsForResolution(
-              frame_size_pixels_[si])) {
-        min_bitrates_bps_[si] = bwlimit->min_bitrate_bps;
-      }
-    }
   }
 
   // Trigger re-allocation so that overshoot detectors have correct targets.

video/encoder_bitrate_adjuster.h

@@ -76,9 +76,6 @@ class EncoderBitrateAdjuster {
   // Minimum bitrates allowed, per spatial layer.
   uint32_t min_bitrates_bps_[kMaxSpatialLayers];
 
-  // Size in pixels of each spatial layer.
-  uint32_t frame_size_pixels_[kMaxSpatialLayers];
-
   // Codec type used for encoding.
   VideoCodecType codec_;
 

video/encoder_bitrate_adjuster_unittest.cc

@@ -41,38 +41,10 @@ class EncoderBitrateAdjusterTest : public ::testing::Test {
         sequence_idx_{} {}
 
  protected:
-  void SetUpAdjuster(size_t num_spatial_layers,
+  void SetUpAdjusterWithCodec(size_t num_spatial_layers,
                               size_t num_temporal_layers,
-                     bool vp9_svc) {
+                              const VideoCodec& codec) {
-    // Initialize some default VideoCodec instance with the given number of
+    codec_ = codec;
-    // layers.
-    if (vp9_svc) {
-      codec_.codecType = VideoCodecType::kVideoCodecVP9;
-      codec_.numberOfSimulcastStreams = 1;
-      codec_.VP9()->numberOfSpatialLayers = num_spatial_layers;
-      codec_.VP9()->numberOfTemporalLayers = num_temporal_layers;
-      for (size_t si = 0; si < num_spatial_layers; ++si) {
-        codec_.spatialLayers[si].minBitrate = 100 * (1 << si);
-        codec_.spatialLayers[si].targetBitrate = 200 * (1 << si);
-        codec_.spatialLayers[si].maxBitrate = 300 * (1 << si);
-        codec_.spatialLayers[si].active = true;
-        codec_.spatialLayers[si].numberOfTemporalLayers = num_temporal_layers;
-      }
-    } else {
-      codec_.codecType = VideoCodecType::kVideoCodecVP8;
-      codec_.numberOfSimulcastStreams = num_spatial_layers;
-      codec_.VP8()->numberOfTemporalLayers = num_temporal_layers;
-      for (size_t si = 0; si < num_spatial_layers; ++si) {
-        codec_.simulcastStream[si].minBitrate = 100 * (1 << si);
-        codec_.simulcastStream[si].targetBitrate = 200 * (1 << si);
-        codec_.simulcastStream[si].maxBitrate = 300 * (1 << si);
-        codec_.simulcastStream[si].active = true;
-        codec_.simulcastStream[si].numberOfTemporalLayers = num_temporal_layers;
-        codec_.spatialLayers[si].width = 320 * (1<<si);
-        codec_.spatialLayers[si].height = 180 * (1<<si);
-      }
-    }
-
     for (size_t si = 0; si < num_spatial_layers; ++si) {
       encoder_info_.fps_allocation[si].resize(num_temporal_layers);
       double fraction = 1.0;
@@ -91,6 +63,39 @@ class EncoderBitrateAdjusterTest : public ::testing::Test {
             current_input_allocation_, target_framerate_fps_));
   }
 
+  void SetUpAdjuster(size_t num_spatial_layers,
+                     size_t num_temporal_layers,
+                     bool vp9_svc) {
+    // Initialize some default VideoCodec instance with the given number of
+    // layers.
+    VideoCodec codec;
+    if (vp9_svc) {
+      codec.codecType = VideoCodecType::kVideoCodecVP9;
+      codec.numberOfSimulcastStreams = 1;
+      codec.VP9()->numberOfSpatialLayers = num_spatial_layers;
+      codec.VP9()->numberOfTemporalLayers = num_temporal_layers;
+      for (size_t si = 0; si < num_spatial_layers; ++si) {
+        codec.spatialLayers[si].minBitrate = 100 * (1 << si);
+        codec.spatialLayers[si].targetBitrate = 200 * (1 << si);
+        codec.spatialLayers[si].maxBitrate = 300 * (1 << si);
+        codec.spatialLayers[si].active = true;
+        codec.spatialLayers[si].numberOfTemporalLayers = num_temporal_layers;
+      }
+    } else {
+      codec.codecType = VideoCodecType::kVideoCodecVP8;
+      codec.numberOfSimulcastStreams = num_spatial_layers;
+      codec.VP8()->numberOfTemporalLayers = num_temporal_layers;
+      for (size_t si = 0; si < num_spatial_layers; ++si) {
+        codec.simulcastStream[si].minBitrate = 100 * (1 << si);
+        codec.simulcastStream[si].targetBitrate = 200 * (1 << si);
+        codec.simulcastStream[si].maxBitrate = 300 * (1 << si);
+        codec.simulcastStream[si].active = true;
+        codec.simulcastStream[si].numberOfTemporalLayers = num_temporal_layers;
+      }
+    }
+    SetUpAdjusterWithCodec(num_spatial_layers, num_temporal_layers, codec);
+  }
+
   void InsertFrames(std::vector<std::vector<double>> media_utilization_factors,
                     int64_t duration_ms) {
     InsertFrames(media_utilization_factors, media_utilization_factors,
@@ -509,23 +514,30 @@ TEST_F(EncoderBitrateAdjusterTest, DontExceedMediaRateEvenWithHeadroom) {
   }
 }
 
-TEST_F(EncoderBitrateAdjusterTest, HonorMinBitrateSettingFromEncoderInfo) {
+TEST_F(EncoderBitrateAdjusterTest, HonorsMinBitrateWithAv1) {
   // Single layer, well behaved encoder.
-  const int high_bitrate = 20000;
+  const DataRate kHighBitrate = DataRate::KilobitsPerSec(20);
-  const int a_lower_min_bitrate = 12000;
+  const DataRate kALowerMinBitrate = DataRate::KilobitsPerSec(15);
-  current_input_allocation_.SetBitrate(0, 0, high_bitrate);
+
+  current_input_allocation_.SetBitrate(0, 0, kHighBitrate.bps());
+
   VideoBitrateAllocation expected_input_allocation;
-  expected_input_allocation.SetBitrate(0, 0, a_lower_min_bitrate);
+  expected_input_allocation.SetBitrate(0, 0, kALowerMinBitrate.bps());
 
   target_framerate_fps_ = 30;
 
-  SetUpAdjuster(1, 1, false);
+  VideoCodec codec;
+  codec.codecType = VideoCodecType::kVideoCodecAV1;
+  codec.numberOfSimulcastStreams = 1;
+  codec.SetScalabilityMode(ScalabilityMode::kL1T1);
+  codec.spatialLayers[0].minBitrate = kALowerMinBitrate.kbps();
+  codec.spatialLayers[0].targetBitrate = 500;
+  codec.spatialLayers[0].maxBitrate = 1000;
+  codec.spatialLayers[0].active = true;
+  codec.spatialLayers[0].numberOfTemporalLayers = 1;
 
-  auto new_resolution_limit = VideoEncoder::ResolutionBitrateLimits(
+  SetUpAdjusterWithCodec(/*num_spatial_layers=*/1, /*num_temporal_layers=*/1,
-      codec_.spatialLayers[0].width * codec_.spatialLayers[0].height, 15000,
+                         codec);
-      a_lower_min_bitrate, 2000000);
-  encoder_info_.resolution_bitrate_limits.push_back(new_resolution_limit);
-  adjuster_->OnEncoderInfo(encoder_info_);
 
   InsertFrames({{2.0}}, kWindowSizeMs);