Update H.265 single-cast bitrate limits.

Update to use similar bitrate limits as VP9, depending on whether QP
from encoder is trusted or not.

Bug: chromium:392060821
Change-Id: I305182fecf7acfe84dbd2e049f9ce712a7a30ede
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/376762
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#43898}

modules/video_coding/utility/bandwidth_quality_scaler.cc

@@ -92,10 +92,12 @@ void BandwidthQualityScaler::ReportEncodeInfo(int frame_size_bytes,
 
 void BandwidthQualityScaler::SetResolutionBitrateLimits(
     const std::vector<VideoEncoder::ResolutionBitrateLimits>&
-        resolution_bitrate_limits) {
+        resolution_bitrate_limits,
+    VideoCodecType codec_type) {
   if (resolution_bitrate_limits.empty()) {
-    resolution_bitrate_limits_ = EncoderInfoSettings::
+    resolution_bitrate_limits_ =
-        GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted();
+        EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(
+            codec_type);
   } else {
     resolution_bitrate_limits_ = resolution_bitrate_limits;
   }

modules/video_coding/utility/bandwidth_quality_scaler.h

@@ -58,12 +58,13 @@ class BandwidthQualityScaler {
                         uint32_t encoded_width,
                         uint32_t encoded_height);
 
-  // We prioritise to using the |resolution_bitrate_limits| provided by the
+  // We prioritize the |resolution_bitrate_limits| provided by the
-  // current decoder. If not provided, we will use the default data by
+  // current encoder. If not provided, we will use the default data by
   // GetDefaultResolutionBitrateLimits().
   void SetResolutionBitrateLimits(
       const std::vector<VideoEncoder::ResolutionBitrateLimits>&
-          resolution_bitrate_limits);
+          resolution_bitrate_limits,
+      VideoCodecType codec_type);
 
  private:
   enum class CheckBitrateResult {

modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc

@@ -14,6 +14,7 @@
 #include <string>
 
 #include "api/units/time_delta.h"
+#include "api/video/video_codec_type.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/event.h"
 #include "rtc_base/experiments/encoder_info_settings.h"
@@ -80,17 +81,19 @@ class BandwidthQualityScalerTest : public ::testing::Test {
     int actual_height;
   };
 
-  BandwidthQualityScalerTest()
+  explicit BandwidthQualityScalerTest(VideoCodecType codec_type)
       : task_queue_(time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
             "BandwidthQualityScalerTestQueue",
             TaskQueueFactory::Priority::NORMAL)),
-        handler_(std::make_unique<FakeBandwidthQualityScalerHandler>()) {
+        handler_(std::make_unique<FakeBandwidthQualityScalerHandler>()),
+        codec_type_(codec_type) {
     task_queue_.SendTask([this] {
       bandwidth_quality_scaler_ =
           std::make_unique<BandwidthQualityScaler>(handler_.get());
       bandwidth_quality_scaler_->SetResolutionBitrateLimits(
           EncoderInfoSettings::
-              GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+              GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(codec_type_),
+          codec_type_);
       // Only for testing. Set first_timestamp_ in RateStatistics to 0.
       bandwidth_quality_scaler_->ReportEncodeInfo(0, 0, 0, 0);
     });
@@ -131,7 +134,8 @@ class BandwidthQualityScalerTest : public ::testing::Test {
         GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
             frame_size_pixels,
             EncoderInfoSettings::
-                GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+                GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(
+                    codec_type_));
   }
 
   void TriggerBandwidthQualityScalerTest(
@@ -178,9 +182,17 @@ class BandwidthQualityScalerTest : public ::testing::Test {
   TaskQueueForTest task_queue_;
   std::unique_ptr<BandwidthQualityScaler> bandwidth_quality_scaler_;
   std::unique_ptr<FakeBandwidthQualityScalerHandler> handler_;
+  VideoCodecType codec_type_;
 };
 
-TEST_F(BandwidthQualityScalerTest, AllNormalFrame_640x360) {
+class BandwidthQualityScalerTests
+    : public BandwidthQualityScalerTest,
+      public ::testing::WithParamInterface<VideoCodecType> {
+ protected:
+  BandwidthQualityScalerTests() : BandwidthQualityScalerTest(GetParam()) {}
+};
+
+TEST_P(BandwidthQualityScalerTests, AllNormalFrame_640x360) {
   const std::vector<FrameConfig> frame_configs{
       FrameConfig(150, FrameType::kNormalFrame, 640, 360)};
   TriggerBandwidthQualityScalerTest(frame_configs);
@@ -193,7 +205,7 @@ TEST_F(BandwidthQualityScalerTest, AllNormalFrame_640x360) {
   EXPECT_EQ(0, handler_->adapt_up_event_count_);
 }
 
-TEST_F(BandwidthQualityScalerTest, AllNormalFrame_AboveMaxBandwidth_640x360) {
+TEST_P(BandwidthQualityScalerTests, AllNormalFrame_AboveMaxBandwidth_640x360) {
   const std::vector<FrameConfig> frame_configs{
       FrameConfig(150, FrameType::kNormalFrame_Overuse, 640, 360)};
   TriggerBandwidthQualityScalerTest(frame_configs);
@@ -206,7 +218,7 @@ TEST_F(BandwidthQualityScalerTest, AllNormalFrame_AboveMaxBandwidth_640x360) {
   EXPECT_EQ(1, handler_->adapt_up_event_count_);
 }
 
-TEST_F(BandwidthQualityScalerTest, AllNormalFrame_Underuse_640x360) {
+TEST_P(BandwidthQualityScalerTests, AllNormalFrame_Underuse_640x360) {
   const std::vector<FrameConfig> frame_configs{
       FrameConfig(150, FrameType::kNormalFrame_Underuse, 640, 360)};
   TriggerBandwidthQualityScalerTest(frame_configs);
@@ -219,7 +231,7 @@ TEST_F(BandwidthQualityScalerTest, AllNormalFrame_Underuse_640x360) {
   EXPECT_EQ(0, handler_->adapt_up_event_count_);
 }
 
-TEST_F(BandwidthQualityScalerTest, FixedFrameTypeTest1_640x360) {
+TEST_P(BandwidthQualityScalerTests, FixedFrameTypeTest1_640x360) {
   const std::vector<FrameConfig> frame_configs{
       FrameConfig(5, FrameType::kNormalFrame_Underuse, 640, 360),
       FrameConfig(110, FrameType::kNormalFrame, 640, 360),
@@ -236,7 +248,7 @@ TEST_F(BandwidthQualityScalerTest, FixedFrameTypeTest1_640x360) {
   EXPECT_EQ(1, handler_->adapt_up_event_count_);
 }
 
-TEST_F(BandwidthQualityScalerTest, FixedFrameTypeTest2_640x360) {
+TEST_P(BandwidthQualityScalerTests, FixedFrameTypeTest2_640x360) {
   const std::vector<FrameConfig> frame_configs{
       FrameConfig(10, FrameType::kNormalFrame_Underuse, 640, 360),
       FrameConfig(50, FrameType::kNormalFrame, 640, 360),
@@ -253,4 +265,8 @@ TEST_F(BandwidthQualityScalerTest, FixedFrameTypeTest2_640x360) {
   EXPECT_EQ(1, handler_->adapt_up_event_count_);
 }
 
+INSTANTIATE_TEST_SUITE_P(AllCodecs,
+                         BandwidthQualityScalerTests,
+                         ::testing::Values(kVideoCodecH264, kVideoCodecH265));
+
 }  // namespace webrtc

modules/video_coding/video_codec_initializer.cc

@@ -359,6 +359,10 @@ VideoCodec VideoCodecInitializer::SetupCodec(
           break;
         }
       }
+      video_codec.spatialLayers[0].minBitrate = video_codec.minBitrate;
+      video_codec.spatialLayers[0].targetBitrate = video_codec.maxBitrate;
+      video_codec.spatialLayers[0].maxBitrate = video_codec.maxBitrate;
+      video_codec.spatialLayers[0].active = codec_active;
       break;
     default:
       // TODO(pbos): Support encoder_settings codec-agnostically.

rtc_base/experiments/encoder_info_settings.cc

@@ -53,11 +53,13 @@ EncoderInfoSettings::GetDefaultSinglecastBitrateLimits(
             {1280 * 720, 576000, 0, 1536000}};
   }
 
-  if (codec_type == kVideoCodecVP9) {
+  if (codec_type == kVideoCodecVP9 || codec_type == kVideoCodecH265) {
     // VP9 singlecast bitrate limits are derived ~directly from VP9 SVC bitrate
     // limits. The current max limits are unnecessarily too strict for
     // singlecast, where BWE is known end-to-end, especially for low
     // resolutions.
+    // TODO(crbugs.com/39206082): Consider fine-tuning H.265 to have its own
+    // bitrate settings separate from VP9.
     return {{320 * 180, 0, 30000, 150000},
             {480 * 270, 120000, 30000, 300000},
             {640 * 360, 190000, 30000, 420000},
@@ -85,18 +87,28 @@ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
 
 // Return the suitable bitrate limits for specified resolution when qp is
 // untrusted, they are experimental values.
-// TODO(bugs.webrtc.org/12942): Maybe we need to add other codecs(VP8/VP9)
-// experimental values.
 std::vector<VideoEncoder::ResolutionBitrateLimits>
-EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted() {
+EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(
-  // Specific limits for H264/AVC
+    VideoCodecType codec_type) {
-  return {{0 * 0, 0, 0, 0},
+  if (codec_type == kVideoCodecH265) {
-          {320 * 180, 0, 30000, 300000},
+    // Similar settings from the simulcast bitate limits for H.265.
-          {480 * 270, 300000, 30000, 500000},
+    return {{0 * 0, 0, 0, 0},
-          {640 * 360, 500000, 30000, 800000},
+            {320 * 180, 0, 30000, 150000},
-          {960 * 540, 800000, 30000, 1500000},
+            {480 * 270, 150000, 30000, 300000},
-          {1280 * 720, 1500000, 30000, 2500000},
+            {640 * 360, 300000, 30000, 420000},
-          {1920 * 1080, 2500000, 30000, 4000000}};
+            {960 * 540, 420000, 30000, 1000000},
+            {1280 * 720, 1000000, 30000, 1500000},
+            {1920 * 1080, 1500000, 30000, 3300000}};
+  } else {
+    // Settings for H.264. Other codecs will not work in QP-untrusted mode.
+    return {{0 * 0, 0, 0, 0},
+            {320 * 180, 0, 30000, 300000},
+            {480 * 270, 300000, 30000, 500000},
+            {640 * 360, 500000, 30000, 800000},
+            {960 * 540, 800000, 30000, 1500000},
+            {1280 * 720, 1500000, 30000, 2500000},
+            {1920 * 1080, 2500000, 30000, 4000000}};
+  }
 }
 
 // Through linear interpolation, return the bitrate limit corresponding to the

rtc_base/experiments/encoder_info_settings.h

@@ -51,7 +51,7 @@ class EncoderInfoSettings {
                                                  int frame_size_pixels);
 
   static std::vector<VideoEncoder::ResolutionBitrateLimits>
-  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted();
+  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(VideoCodecType codec_type);
 
   static std::optional<VideoEncoder::ResolutionBitrateLimits>
   GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(

video/adaptation/bandwidth_quality_scaler_resource.cc

@@ -40,7 +40,8 @@ bool BandwidthQualityScalerResource::is_started() const {
 
 void BandwidthQualityScalerResource::StartCheckForOveruse(
     const std::vector<VideoEncoder::ResolutionBitrateLimits>&
-        resolution_bitrate_limits) {
+        resolution_bitrate_limits,
+    VideoCodecType codec_type) {
   RTC_DCHECK_RUN_ON(encoder_queue());
   RTC_DCHECK(!is_started());
   bandwidth_quality_scaler_ = std::make_unique<BandwidthQualityScaler>(this);
@@ -48,7 +49,7 @@ void BandwidthQualityScalerResource::StartCheckForOveruse(
   // If the configuration parameters more than one, we should define and
   // declare the function BandwidthQualityScaler::Initialize() and call it.
   bandwidth_quality_scaler_->SetResolutionBitrateLimits(
-      resolution_bitrate_limits);
+      resolution_bitrate_limits, codec_type);
 }
 
 void BandwidthQualityScalerResource::StopCheckForOveruse() {

video/adaptation/bandwidth_quality_scaler_resource.h

@@ -45,7 +45,8 @@ class BandwidthQualityScalerResource
 
   void StartCheckForOveruse(
       const std::vector<VideoEncoder::ResolutionBitrateLimits>&
-          resolution_bitrate_limits);
+          resolution_bitrate_limits,
+      VideoCodecType codec_type);
   void StopCheckForOveruse();
 
   // BandwidthScalerQpUsageHandlerInterface implementation.

video/adaptation/video_stream_encoder_resource_manager.cc

@@ -538,7 +538,8 @@ void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings(
 void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings(
     bool bandwidth_quality_scaling_allowed,
     const std::vector<VideoEncoder::ResolutionBitrateLimits>&
-        resolution_bitrate_limits) {
+        resolution_bitrate_limits,
+    VideoCodecType codec_type) {
   RTC_DCHECK_RUN_ON(encoder_queue_);
 
   if (!bandwidth_quality_scaling_allowed) {
@@ -553,7 +554,7 @@ void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings(
       AddResource(bandwidth_quality_scaler_resource_,
                   webrtc::VideoAdaptationReason::kQuality);
       bandwidth_quality_scaler_resource_->StartCheckForOveruse(
-          resolution_bitrate_limits);
+          resolution_bitrate_limits, codec_type);
     }
   }
 }
@@ -612,8 +613,9 @@ void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler(
        encoder_settings_->encoder_config().is_quality_scaling_allowed) &&
       !encoder_info.is_qp_trusted.value_or(true);
 
-  UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed,
+  UpdateBandwidthQualityScalerSettings(
-                                       encoder_info.resolution_bitrate_limits);
+      bandwidth_quality_scaling_allowed, encoder_info.resolution_bitrate_limits,
+      GetVideoCodecTypeOrGeneric(encoder_settings_));
   UpdateStatsAdaptationSettings();
 }
 

video/adaptation/video_stream_encoder_resource_manager.h

@@ -171,7 +171,8 @@ class VideoStreamEncoderResourceManager
   void UpdateBandwidthQualityScalerSettings(
       bool bandwidth_quality_scaling_allowed,
       const std::vector<VideoEncoder::ResolutionBitrateLimits>&
-          resolution_bitrate_limits);
+          resolution_bitrate_limits,
+      VideoCodecType codec_type);
 
   void UpdateStatsAdaptationSettings() const;
 

video/video_stream_encoder.cc

@@ -386,10 +386,24 @@ VideoEncoder::EncoderInfo GetEncoderInfoWithBitrateLimitUpdate(
     const VideoEncoder::EncoderInfo& info,
     const VideoEncoderConfig& encoder_config,
     bool default_limits_allowed) {
-  if (!default_limits_allowed || !info.resolution_bitrate_limits.empty() ||
+  bool are_all_bitrate_limits_zero = true;
+  // Hardware encoders commonly only report resolution limits, while reporting
+  // the bitrate limits as 0. In such case, we should not use them for setting
+  // bitrate limits.
+  if (!info.resolution_bitrate_limits.empty()) {
+    are_all_bitrate_limits_zero = std::all_of(
+        info.resolution_bitrate_limits.begin(),
+        info.resolution_bitrate_limits.end(),
+        [](const VideoEncoder::ResolutionBitrateLimits& limit) {
+          return limit.max_bitrate_bps == 0 && limit.min_bitrate_bps == 0;
+        });
+  }
+
+  if (!default_limits_allowed || !are_all_bitrate_limits_zero ||
       encoder_config.simulcast_layers.size() <= 1) {
     return info;
   }
+
   // Bitrate limits are not configured and more than one layer is used, use
   // the default limits (bitrate limits are not used for simulcast).
   VideoEncoder::EncoderInfo new_info = info;
@@ -1070,7 +1084,8 @@ void VideoStreamEncoder::ReconfigureEncoder() {
     const std::vector<VideoEncoder::ResolutionBitrateLimits>& bitrate_limits =
         encoder_->GetEncoderInfo().resolution_bitrate_limits.empty()
             ? EncoderInfoSettings::
-                  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()
+                  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(
+                      encoder_config_.codec_type)
             : encoder_->GetEncoderInfo().resolution_bitrate_limits;
 
     // For BandwidthQualityScaler, its implement based on a certain pixel_count

video/video_stream_encoder_unittest.cc

@@ -172,6 +172,11 @@ const uint8_t kCodedFrameVp8Qp25[] = {
     0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
     0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
 
+#ifdef RTC_ENABLE_H265
+// Default value from encoder_info_settings.cc
+const DataRate kDefaultH265Bitrate180p = DataRate::KilobitsPerSec(150);
+#endif
+
 VideoFrame CreateSimpleNV12Frame() {
   return VideoFrame::Builder()
       .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
@@ -2273,6 +2278,43 @@ TEST_F(VideoStreamEncoderTest,
   video_stream_encoder_->Stop();
 }
 
+// ReconfigureEncoder checks RTC_ENABLE_H265 flag, and we want to test specific
+// behavior of H265, when bitrate limits reported from hardware encoders are 0,
+// expecting default target bitrate to be used in this case.
+#ifdef RTC_ENABLE_H265
+TEST_F(VideoStreamEncoderTest,
+       ApplyDefaultBitrateLimitsWhenEncoderInfoResolutionBitrateLimitsAreZero) {
+  video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+      kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+  const uint32_t kMinEncBitrateKbps = 100;
+  const uint32_t kMaxEncBitrateKbps = 1000;
+  const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_180p(
+      320 * 180,
+      /*min_start_bitrate_bps=*/0,
+      /*min_bitrate_bps=*/0,
+      /*max_bitrate_bps=*/0);
+  fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits_180p});
+
+  VideoEncoderConfig video_encoder_config;
+  test::FillEncoderConfiguration(kVideoCodecH265, 1, &video_encoder_config);
+  video_encoder_config.max_bitrate_bps = kMaxEncBitrateKbps * 1000;
+  video_encoder_config.simulcast_layers[0].min_bitrate_bps =
+      kMinEncBitrateKbps * 1000;
+  video_encoder_config.simulcast_layers[0].width = 320;
+  video_encoder_config.simulcast_layers[0].height = 180;
+  video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+                                          kMaxPayloadLength);
+
+  video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+  WaitForEncodedFrame(1);
+  EXPECT_EQ(bitrate_allocator_factory_.codec_config()
+                .simulcastStream[0]
+                .targetBitrate,
+            kDefaultH265Bitrate180p.kbps());
+  video_stream_encoder_->Stop();
+}
+#endif
+
 TEST_F(VideoStreamEncoderTest,
        EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
   video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
@@ -8951,7 +8993,8 @@ TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
           GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
               codec_width_ * codec_height_,
               EncoderInfoSettings::
-                  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+                  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(
+                      kVideoCodecH264));
   EXPECT_TRUE(suitable_bitrate_limit.has_value());
 
   const int max_encoder_bitrate = suitable_bitrate_limit->max_bitrate_bps;
@@ -8992,7 +9035,8 @@ TEST_F(VideoStreamEncoderTest,
           GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
               codec_width_ * codec_height_,
               EncoderInfoSettings::
-                  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+                  GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(
+                      kVideoCodecH264));
   EXPECT_TRUE(suitable_bitrate_limit.has_value());
 
   const int max_encoder_bitrate = suitable_bitrate_limit->max_bitrate_bps;