Add ability to specify encoder bitrate multiplier in PC level tests

Bug: webrtc:10138
Change-Id: I40b42e83ccec7b08226606d2770f3afa80e3fcc6
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/130241
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Peter Slatala <psla@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27440}

api/test/peerconnection_quality_test_fixture.h

@@ -174,10 +174,20 @@ class PeerConnectionE2EQualityTestFixture {
   // Contains parameters, that describe how long framework should run quality
   // test.
   struct RunParams {
+    explicit RunParams(TimeDelta run_duration) : run_duration(run_duration) {}
+
     // Specifies how long the test should be run. This time shows how long
     // the media should flow after connection was established and before
     // it will be shut downed.
     TimeDelta run_duration;
+
+    // Specifies how much video encoder target bitrate should be different than
+    // target bitrate, provided by WebRTC stack. Must be greater then 0. Can be
+    // used to emulate overshooting of video encoders. This multiplier will
+    // be applied for all video encoder on both sides for all layers. Bitrate
+    // estimated by WebRTC stack will be multiplied on this multiplier and then
+    // provided into VideoEncoder::SetRateAllocation(...).
+    double video_encoder_bitrate_multiplier = 1.0;
   };
 
   virtual ~PeerConnectionE2EQualityTestFixture() = default;

test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc

@@ -10,6 +10,7 @@
 
 #include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h"
 
+#include <cmath>
 #include <utility>
 
 #include "absl/memory/memory.h"
@@ -23,17 +24,43 @@ namespace webrtc_pc_e2e {
 namespace {
 
 constexpr size_t kMaxFrameInPipelineCount = 1000;
+constexpr double kNoMultiplier = 1.0;
+constexpr double kEps = 1e-6;
+
+std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec,
+                                                   size_t spatial_idx) {
+  uint32_t min_bitrate = codec.minBitrate;
+  uint32_t max_bitrate = codec.maxBitrate;
+  if (spatial_idx < codec.numberOfSimulcastStreams &&
+      codec.codecType != VideoCodecType::kVideoCodecVP9) {
+    min_bitrate =
+        std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate);
+    max_bitrate =
+        std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate);
+  }
+  if (codec.codecType == VideoCodecType::kVideoCodecVP9 &&
+      spatial_idx < codec.VP9().numberOfSpatialLayers) {
+    min_bitrate =
+        std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate);
+    max_bitrate =
+        std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate);
+  }
+  RTC_DCHECK_GT(max_bitrate, min_bitrate);
+  return {min_bitrate * 1000, max_bitrate * 1000};
+}
 
 }  // namespace
 
 QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder(
     int id,
     std::unique_ptr<VideoEncoder> delegate,
+    double bitrate_multiplier,
     std::map<std::string, absl::optional<int>> stream_required_spatial_index,
     EncodedImageDataInjector* injector,
     VideoQualityAnalyzerInterface* analyzer)
     : id_(id),
       delegate_(std::move(delegate)),
+      bitrate_multiplier_(bitrate_multiplier),
       stream_required_spatial_index_(std::move(stream_required_spatial_index)),
       injector_(injector),
       analyzer_(analyzer) {}
@@ -44,6 +71,7 @@ int32_t QualityAnalyzingVideoEncoder::InitEncode(
     int32_t number_of_cores,
     size_t max_payload_size) {
   rtc::CritScope crit(&lock_);
+  codec_settings_ = *codec_settings;
   mode_ = SimulcastMode::kNormal;
   if (codec_settings->codecType == kVideoCodecVP9) {
     if (codec_settings->VP9().numberOfSpatialLayers > 1) {
@@ -127,7 +155,46 @@ int32_t QualityAnalyzingVideoEncoder::SetRates(uint32_t bitrate,
 int32_t QualityAnalyzingVideoEncoder::SetRateAllocation(
     const VideoBitrateAllocation& allocation,
     uint32_t framerate) {
-  return delegate_->SetRateAllocation(allocation, framerate);
+  RTC_DCHECK_GT(bitrate_multiplier_, 0.0);
+  if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) {
+    return delegate_->SetRateAllocation(allocation, framerate);
+  }
+
+  // Simulating encoder overshooting target bitrate, by configuring actual
+  // encoder too high. Take care not to adjust past limits of config,
+  // otherwise encoders may crash on DCHECK.
+  VideoBitrateAllocation multiplied_allocation;
+  for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+    const uint32_t spatial_layer_bitrate_bps =
+        allocation.GetSpatialLayerSum(si);
+    if (spatial_layer_bitrate_bps == 0) {
+      continue;
+    }
+
+    uint32_t min_bitrate_bps;
+    uint32_t max_bitrate_bps;
+    std::tie(min_bitrate_bps, max_bitrate_bps) =
+        GetMinMaxBitratesBps(codec_settings_, si);
+    double bitrate_multiplier = bitrate_multiplier_;
+    const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>(
+        bitrate_multiplier * spatial_layer_bitrate_bps);
+    if (corrected_bitrate < min_bitrate_bps) {
+      bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps;
+    } else if (corrected_bitrate > max_bitrate_bps) {
+      bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps;
+    }
+
+    for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+      if (allocation.HasBitrate(si, ti)) {
+        multiplied_allocation.SetBitrate(
+            si, ti,
+            rtc::checked_cast<uint32_t>(bitrate_multiplier *
+                                        allocation.GetBitrate(si, ti)));
+      }
+    }
+  }
+
+  return delegate_->SetRateAllocation(multiplied_allocation, framerate);
 }
 
 VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const {
@@ -259,11 +326,13 @@ bool QualityAnalyzingVideoEncoder::ShouldDiscard(
 
 QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory(
     std::unique_ptr<VideoEncoderFactory> delegate,
+    double bitrate_multiplier,
     std::map<std::string, absl::optional<int>> stream_required_spatial_index,
     IdGenerator<int>* id_generator,
     EncodedImageDataInjector* injector,
     VideoQualityAnalyzerInterface* analyzer)
     : delegate_(std::move(delegate)),
+      bitrate_multiplier_(bitrate_multiplier),
       stream_required_spatial_index_(std::move(stream_required_spatial_index)),
       id_generator_(id_generator),
       injector_(injector),
@@ -287,7 +356,8 @@ QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder(
     const SdpVideoFormat& format) {
   return absl::make_unique<QualityAnalyzingVideoEncoder>(
       id_generator_->GetNextId(), delegate_->CreateVideoEncoder(format),
-      stream_required_spatial_index_, injector_, analyzer_);
+      bitrate_multiplier_, stream_required_spatial_index_, injector_,
+      analyzer_);
 }
 
 }  // namespace webrtc_pc_e2e

test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h

@@ -55,6 +55,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
   QualityAnalyzingVideoEncoder(
       int id,
       std::unique_ptr<VideoEncoder> delegate,
+      double bitrate_multiplier,
       std::map<std::string, absl::optional<int>> stream_required_spatial_index,
       EncodedImageDataInjector* injector,
       VideoQualityAnalyzerInterface* analyzer);
@@ -135,6 +136,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
 
   const int id_;
   std::unique_ptr<VideoEncoder> delegate_;
+  const double bitrate_multiplier_;
   std::map<std::string, absl::optional<int>> stream_required_spatial_index_;
   EncodedImageDataInjector* const injector_;
   VideoQualityAnalyzerInterface* const analyzer_;
@@ -144,6 +146,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
   // from received VideoFrame to resulted EncodedImage.
   rtc::CriticalSection lock_;
 
+  VideoCodec codec_settings_;
   SimulcastMode mode_ RTC_GUARDED_BY(lock_);
   EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_);
   std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_
@@ -157,6 +160,7 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory {
  public:
   QualityAnalyzingVideoEncoderFactory(
       std::unique_ptr<VideoEncoderFactory> delegate,
+      double bitrate_multiplier,
       std::map<std::string, absl::optional<int>> stream_required_spatial_index,
       IdGenerator<int>* id_generator,
       EncodedImageDataInjector* injector,
@@ -172,6 +176,7 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory {
 
  private:
   std::unique_ptr<VideoEncoderFactory> delegate_;
+  const double bitrate_multiplier_;
   std::map<std::string, absl::optional<int>> stream_required_spatial_index_;
   IdGenerator<int>* const id_generator_;
   EncodedImageDataInjector* const injector_;

test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc

@@ -103,10 +103,12 @@ VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() =
 std::unique_ptr<VideoEncoderFactory>
 VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory(
     std::unique_ptr<VideoEncoderFactory> delegate,
+    double bitrate_multiplier,
     std::map<std::string, absl::optional<int>> stream_required_spatial_index)
     const {
   return absl::make_unique<QualityAnalyzingVideoEncoderFactory>(
-      std::move(delegate), std::move(stream_required_spatial_index),
+      std::move(delegate), bitrate_multiplier,
+      std::move(stream_required_spatial_index),
       encoding_entities_id_generator_.get(), injector_, analyzer_.get());
 }
 

test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h

@@ -44,6 +44,7 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface {
   // before encoding and encoded images after.
   std::unique_ptr<VideoEncoderFactory> WrapVideoEncoderFactory(
       std::unique_ptr<VideoEncoderFactory> delegate,
+      double bitrate_multiplier,
       std::map<std::string, absl::optional<int>> stream_required_spatial_index)
       const;
   // Wraps video decoder factory to give video quality analyzer access to

test/pc/e2e/peer_connection_e2e_smoke_test.cc

@@ -97,7 +97,9 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) {
                      bob->SetAudioConfig(AudioConfig());
                    });
 
-  fixture->Run(RunParams{TimeDelta::seconds(5)});
+  RunParams run_params(TimeDelta::seconds(5));
+  run_params.video_encoder_bitrate_multiplier = 1.1;
+  fixture->Run(run_params);
 
   for (auto stream_label : video_analyzer_ptr->GetKnownVideoStreams()) {
     FrameCounters stream_conters =

test/pc/e2e/peer_connection_quality_test.cc

@@ -219,7 +219,7 @@ void PeerConnectionE2EQualityTest::Run(
   peer_configurations_.clear();
 
   SetDefaultValuesForMissingParams({alice_params.get(), bob_params.get()});
-  ValidateParams({alice_params.get(), bob_params.get()});
+  ValidateParams(run_params, {alice_params.get(), bob_params.get()});
 
   // Print test summary
   RTC_LOG(INFO)
@@ -258,7 +258,8 @@ void PeerConnectionE2EQualityTest::Run(
           },
           [this]() { StartVideo(alice_video_sources_); }),
       video_quality_analyzer_injection_helper_.get(), signaling_thread.get(),
-      alice_audio_output_dump_file_name);
+      alice_audio_output_dump_file_name,
+      run_params.video_encoder_bitrate_multiplier);
   bob_ = TestPeer::CreateTestPeer(
       std::move(bob_components), std::move(bob_params),
       absl::make_unique<FixturePeerConnectionObserver>(
@@ -268,7 +269,8 @@ void PeerConnectionE2EQualityTest::Run(
           },
           [this]() { StartVideo(bob_video_sources_); }),
       video_quality_analyzer_injection_helper_.get(), signaling_thread.get(),
-      bob_audio_output_dump_file_name);
+      bob_audio_output_dump_file_name,
+      run_params.video_encoder_bitrate_multiplier);
 
   int num_cores = CpuInfo::DetectNumberOfCores();
   RTC_DCHECK_GE(num_cores, 1);
@@ -400,7 +402,10 @@ void PeerConnectionE2EQualityTest::SetDefaultValuesForMissingParams(
   }
 }
 
-void PeerConnectionE2EQualityTest::ValidateParams(std::vector<Params*> params) {
+void PeerConnectionE2EQualityTest::ValidateParams(const RunParams& run_params,
+                                                  std::vector<Params*> params) {
+  RTC_CHECK_GT(run_params.video_encoder_bitrate_multiplier, 0.0);
+
   std::set<std::string> video_labels;
   std::set<std::string> audio_labels;
   int media_streams_count = 0;

test/pc/e2e/peer_connection_quality_test.h

@@ -190,7 +190,7 @@ class PeerConnectionE2EQualityTest
   void SetDefaultValuesForMissingParams(std::vector<Params*> params);
   // Validate peer's parameters, also ensure uniqueness of all video stream
   // labels.
-  void ValidateParams(std::vector<Params*> params);
+  void ValidateParams(const RunParams& run_params, std::vector<Params*> params);
   void SetupVideoSink(rtc::scoped_refptr<RtpTransceiverInterface> transceiver,
                       std::vector<VideoConfig> remote_video_configs);
   // Have to be run on the signaling thread.

test/pc/e2e/test_peer.cc

@@ -104,6 +104,7 @@ rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(
 std::unique_ptr<VideoEncoderFactory> CreateVideoEncoderFactory(
     PeerConnectionFactoryComponents* pcf_dependencies,
     VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
+    double bitrate_multiplier,
     std::map<std::string, absl::optional<int>> stream_required_spatial_index) {
   std::unique_ptr<VideoEncoderFactory> video_encoder_factory;
   if (pcf_dependencies->video_encoder_factory != nullptr) {
@@ -112,7 +113,7 @@ std::unique_ptr<VideoEncoderFactory> CreateVideoEncoderFactory(
     video_encoder_factory = CreateBuiltinVideoEncoderFactory();
   }
   return video_analyzer_helper->WrapVideoEncoderFactory(
-      std::move(video_encoder_factory),
+      std::move(video_encoder_factory), bitrate_multiplier,
       std::move(stream_required_spatial_index));
 }
 
@@ -132,6 +133,7 @@ std::unique_ptr<VideoDecoderFactory> CreateVideoDecoderFactory(
 std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine(
     PeerConnectionFactoryComponents* pcf_dependencies,
     absl::optional<AudioConfig> audio_config,
+    double bitrate_multiplier,
     std::map<std::string, absl::optional<int>> stream_required_spatial_index,
     VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
     absl::optional<std::string> audio_output_file_name) {
@@ -140,6 +142,7 @@ std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine(
 
   std::unique_ptr<VideoEncoderFactory> video_encoder_factory =
       CreateVideoEncoderFactory(pcf_dependencies, video_analyzer_helper,
+                                bitrate_multiplier,
                                 std::move(stream_required_spatial_index));
   std::unique_ptr<VideoDecoderFactory> video_decoder_factory =
       CreateVideoDecoderFactory(pcf_dependencies, video_analyzer_helper);
@@ -158,6 +161,7 @@ std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine(
 PeerConnectionFactoryDependencies CreatePCFDependencies(
     std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies,
     absl::optional<AudioConfig> audio_config,
+    double bitrate_multiplier,
     std::map<std::string, absl::optional<int>> stream_required_spatial_index,
     VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
     rtc::Thread* network_thread,
@@ -167,7 +171,7 @@ PeerConnectionFactoryDependencies CreatePCFDependencies(
   pcf_deps.network_thread = network_thread;
   pcf_deps.signaling_thread = signaling_thread;
   pcf_deps.media_engine = CreateMediaEngine(
-      pcf_dependencies.get(), std::move(audio_config),
+      pcf_dependencies.get(), std::move(audio_config), bitrate_multiplier,
       std::move(stream_required_spatial_index), video_analyzer_helper,
       std::move(audio_output_file_name));
 
@@ -227,7 +231,8 @@ std::unique_ptr<TestPeer> TestPeer::CreateTestPeer(
     std::unique_ptr<MockPeerConnectionObserver> observer,
     VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
     rtc::Thread* signaling_thread,
-    absl::optional<std::string> audio_output_file_name) {
+    absl::optional<std::string> audio_output_file_name,
+    double bitrate_multiplier) {
   RTC_DCHECK(components);
   RTC_DCHECK(params);
   SetMandatoryEntities(components.get());
@@ -248,8 +253,8 @@ std::unique_ptr<TestPeer> TestPeer::CreateTestPeer(
   // Create peer connection factory.
   PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies(
       std::move(components->pcf_dependencies), params->audio_config,
-      std::move(stream_required_spatial_index), video_analyzer_helper,
-      components->network_thread, signaling_thread,
+      bitrate_multiplier, std::move(stream_required_spatial_index),
+      video_analyzer_helper, components->network_thread, signaling_thread,
       std::move(audio_output_file_name));
   rtc::scoped_refptr<PeerConnectionFactoryInterface> pcf =
       CreateModularPeerConnectionFactory(std::move(pcf_deps));

test/pc/e2e/test_peer.h

@@ -53,7 +53,8 @@ class TestPeer final : public PeerConnectionWrapper {
       std::unique_ptr<MockPeerConnectionObserver> observer,
       VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
       rtc::Thread* signaling_thread,
-      absl::optional<std::string> audio_output_file_name);
+      absl::optional<std::string> audio_output_file_name,
+      double bitrate_multiplier);
 
   Params* params() const { return params_.get(); }