diff --git a/api/test/peerconnection_quality_test_fixture.h b/api/test/peerconnection_quality_test_fixture.h index 65e1f349bc..b22b392f1b 100644 --- a/api/test/peerconnection_quality_test_fixture.h +++ b/api/test/peerconnection_quality_test_fixture.h @@ -174,10 +174,20 @@ class PeerConnectionE2EQualityTestFixture { // Contains parameters, that describe how long framework should run quality // test. struct RunParams { + explicit RunParams(TimeDelta run_duration) : run_duration(run_duration) {} + // Specifies how long the test should be run. This time shows how long // the media should flow after connection was established and before // it will be shut downed. TimeDelta run_duration; + + // Specifies how much video encoder target bitrate should be different than + // target bitrate, provided by WebRTC stack. Must be greater then 0. Can be + // used to emulate overshooting of video encoders. This multiplier will + // be applied for all video encoder on both sides for all layers. Bitrate + // estimated by WebRTC stack will be multiplied on this multiplier and then + // provided into VideoEncoder::SetRateAllocation(...). + double video_encoder_bitrate_multiplier = 1.0; }; virtual ~PeerConnectionE2EQualityTestFixture() = default; diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc index 2e863753ef..39d5701774 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -10,6 +10,7 @@ #include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" +#include #include #include "absl/memory/memory.h" @@ -23,17 +24,43 @@ namespace webrtc_pc_e2e { namespace { constexpr size_t kMaxFrameInPipelineCount = 1000; +constexpr double kNoMultiplier = 1.0; +constexpr double kEps = 1e-6; + +std::pair GetMinMaxBitratesBps(const VideoCodec& codec, + size_t spatial_idx) { + uint32_t min_bitrate = codec.minBitrate; + uint32_t max_bitrate = codec.maxBitrate; + if (spatial_idx < codec.numberOfSimulcastStreams && + codec.codecType != VideoCodecType::kVideoCodecVP9) { + min_bitrate = + std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate); + } + if (codec.codecType == VideoCodecType::kVideoCodecVP9 && + spatial_idx < codec.VP9().numberOfSpatialLayers) { + min_bitrate = + std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate); + } + RTC_DCHECK_GT(max_bitrate, min_bitrate); + return {min_bitrate * 1000, max_bitrate * 1000}; +} } // namespace QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( int id, std::unique_ptr delegate, + double bitrate_multiplier, std::map> stream_required_spatial_index, EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer) : id_(id), delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), stream_required_spatial_index_(std::move(stream_required_spatial_index)), injector_(injector), analyzer_(analyzer) {} @@ -44,6 +71,7 @@ int32_t QualityAnalyzingVideoEncoder::InitEncode( int32_t number_of_cores, size_t max_payload_size) { rtc::CritScope crit(&lock_); + codec_settings_ = *codec_settings; mode_ = SimulcastMode::kNormal; if (codec_settings->codecType == kVideoCodecVP9) { if (codec_settings->VP9().numberOfSpatialLayers > 1) { @@ -127,7 +155,46 @@ int32_t QualityAnalyzingVideoEncoder::SetRates(uint32_t bitrate, int32_t QualityAnalyzingVideoEncoder::SetRateAllocation( const VideoBitrateAllocation& allocation, uint32_t framerate) { - return delegate_->SetRateAllocation(allocation, framerate); + RTC_DCHECK_GT(bitrate_multiplier_, 0.0); + if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) { + return delegate_->SetRateAllocation(allocation, framerate); + } + + // Simulating encoder overshooting target bitrate, by configuring actual + // encoder too high. Take care not to adjust past limits of config, + // otherwise encoders may crash on DCHECK. + VideoBitrateAllocation multiplied_allocation; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + const uint32_t spatial_layer_bitrate_bps = + allocation.GetSpatialLayerSum(si); + if (spatial_layer_bitrate_bps == 0) { + continue; + } + + uint32_t min_bitrate_bps; + uint32_t max_bitrate_bps; + std::tie(min_bitrate_bps, max_bitrate_bps) = + GetMinMaxBitratesBps(codec_settings_, si); + double bitrate_multiplier = bitrate_multiplier_; + const uint32_t corrected_bitrate = rtc::checked_cast( + bitrate_multiplier * spatial_layer_bitrate_bps); + if (corrected_bitrate < min_bitrate_bps) { + bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps; + } else if (corrected_bitrate > max_bitrate_bps) { + bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps; + } + + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (allocation.HasBitrate(si, ti)) { + multiplied_allocation.SetBitrate( + si, ti, + rtc::checked_cast(bitrate_multiplier * + allocation.GetBitrate(si, ti))); + } + } + } + + return delegate_->SetRateAllocation(multiplied_allocation, framerate); } VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { @@ -259,11 +326,13 @@ bool QualityAnalyzingVideoEncoder::ShouldDiscard( QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( std::unique_ptr delegate, + double bitrate_multiplier, std::map> stream_required_spatial_index, IdGenerator* id_generator, EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer) : delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), stream_required_spatial_index_(std::move(stream_required_spatial_index)), id_generator_(id_generator), injector_(injector), @@ -287,7 +356,8 @@ QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( const SdpVideoFormat& format) { return absl::make_unique( id_generator_->GetNextId(), delegate_->CreateVideoEncoder(format), - stream_required_spatial_index_, injector_, analyzer_); + bitrate_multiplier_, stream_required_spatial_index_, injector_, + analyzer_); } } // namespace webrtc_pc_e2e diff --git a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h index fcfc8f058e..d7dd5d1da3 100644 --- a/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h +++ b/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -55,6 +55,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, QualityAnalyzingVideoEncoder( int id, std::unique_ptr delegate, + double bitrate_multiplier, std::map> stream_required_spatial_index, EncodedImageDataInjector* injector, VideoQualityAnalyzerInterface* analyzer); @@ -135,6 +136,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, const int id_; std::unique_ptr delegate_; + const double bitrate_multiplier_; std::map> stream_required_spatial_index_; EncodedImageDataInjector* const injector_; VideoQualityAnalyzerInterface* const analyzer_; @@ -144,6 +146,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder, // from received VideoFrame to resulted EncodedImage. rtc::CriticalSection lock_; + VideoCodec codec_settings_; SimulcastMode mode_ RTC_GUARDED_BY(lock_); EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_); std::list> timestamp_to_frame_id_list_ @@ -157,6 +160,7 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { public: QualityAnalyzingVideoEncoderFactory( std::unique_ptr delegate, + double bitrate_multiplier, std::map> stream_required_spatial_index, IdGenerator* id_generator, EncodedImageDataInjector* injector, @@ -172,6 +176,7 @@ class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { private: std::unique_ptr delegate_; + const double bitrate_multiplier_; std::map> stream_required_spatial_index_; IdGenerator* const id_generator_; EncodedImageDataInjector* const injector_; diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc index 939f594936..7b71de0769 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -103,10 +103,12 @@ VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() = std::unique_ptr VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory( std::unique_ptr delegate, + double bitrate_multiplier, std::map> stream_required_spatial_index) const { return absl::make_unique( - std::move(delegate), std::move(stream_required_spatial_index), + std::move(delegate), bitrate_multiplier, + std::move(stream_required_spatial_index), encoding_entities_id_generator_.get(), injector_, analyzer_.get()); } diff --git a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h index ee86aa7398..c831b845ab 100644 --- a/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h +++ b/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -44,6 +44,7 @@ class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { // before encoding and encoded images after. std::unique_ptr WrapVideoEncoderFactory( std::unique_ptr delegate, + double bitrate_multiplier, std::map> stream_required_spatial_index) const; // Wraps video decoder factory to give video quality analyzer access to diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index 6c566e4082..8b19cd50ad 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -97,7 +97,9 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) { bob->SetAudioConfig(AudioConfig()); }); - fixture->Run(RunParams{TimeDelta::seconds(5)}); + RunParams run_params(TimeDelta::seconds(5)); + run_params.video_encoder_bitrate_multiplier = 1.1; + fixture->Run(run_params); for (auto stream_label : video_analyzer_ptr->GetKnownVideoStreams()) { FrameCounters stream_conters = diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index 008842977d..738892d315 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -219,7 +219,7 @@ void PeerConnectionE2EQualityTest::Run( peer_configurations_.clear(); SetDefaultValuesForMissingParams({alice_params.get(), bob_params.get()}); - ValidateParams({alice_params.get(), bob_params.get()}); + ValidateParams(run_params, {alice_params.get(), bob_params.get()}); // Print test summary RTC_LOG(INFO) @@ -258,7 +258,8 @@ void PeerConnectionE2EQualityTest::Run( }, [this]() { StartVideo(alice_video_sources_); }), video_quality_analyzer_injection_helper_.get(), signaling_thread.get(), - alice_audio_output_dump_file_name); + alice_audio_output_dump_file_name, + run_params.video_encoder_bitrate_multiplier); bob_ = TestPeer::CreateTestPeer( std::move(bob_components), std::move(bob_params), absl::make_unique( @@ -268,7 +269,8 @@ void PeerConnectionE2EQualityTest::Run( }, [this]() { StartVideo(bob_video_sources_); }), video_quality_analyzer_injection_helper_.get(), signaling_thread.get(), - bob_audio_output_dump_file_name); + bob_audio_output_dump_file_name, + run_params.video_encoder_bitrate_multiplier); int num_cores = CpuInfo::DetectNumberOfCores(); RTC_DCHECK_GE(num_cores, 1); @@ -400,7 +402,10 @@ void PeerConnectionE2EQualityTest::SetDefaultValuesForMissingParams( } } -void PeerConnectionE2EQualityTest::ValidateParams(std::vector params) { +void PeerConnectionE2EQualityTest::ValidateParams(const RunParams& run_params, + std::vector params) { + RTC_CHECK_GT(run_params.video_encoder_bitrate_multiplier, 0.0); + std::set video_labels; std::set audio_labels; int media_streams_count = 0; diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index a8846705c9..762bdb4613 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -190,7 +190,7 @@ class PeerConnectionE2EQualityTest void SetDefaultValuesForMissingParams(std::vector params); // Validate peer's parameters, also ensure uniqueness of all video stream // labels. - void ValidateParams(std::vector params); + void ValidateParams(const RunParams& run_params, std::vector params); void SetupVideoSink(rtc::scoped_refptr transceiver, std::vector remote_video_configs); // Have to be run on the signaling thread. diff --git a/test/pc/e2e/test_peer.cc b/test/pc/e2e/test_peer.cc index 6491b94cb9..1726fe6cc9 100644 --- a/test/pc/e2e/test_peer.cc +++ b/test/pc/e2e/test_peer.cc @@ -104,6 +104,7 @@ rtc::scoped_refptr CreateAudioDeviceModule( std::unique_ptr CreateVideoEncoderFactory( PeerConnectionFactoryComponents* pcf_dependencies, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, + double bitrate_multiplier, std::map> stream_required_spatial_index) { std::unique_ptr video_encoder_factory; if (pcf_dependencies->video_encoder_factory != nullptr) { @@ -112,7 +113,7 @@ std::unique_ptr CreateVideoEncoderFactory( video_encoder_factory = CreateBuiltinVideoEncoderFactory(); } return video_analyzer_helper->WrapVideoEncoderFactory( - std::move(video_encoder_factory), + std::move(video_encoder_factory), bitrate_multiplier, std::move(stream_required_spatial_index)); } @@ -132,6 +133,7 @@ std::unique_ptr CreateVideoDecoderFactory( std::unique_ptr CreateMediaEngine( PeerConnectionFactoryComponents* pcf_dependencies, absl::optional audio_config, + double bitrate_multiplier, std::map> stream_required_spatial_index, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, absl::optional audio_output_file_name) { @@ -140,6 +142,7 @@ std::unique_ptr CreateMediaEngine( std::unique_ptr video_encoder_factory = CreateVideoEncoderFactory(pcf_dependencies, video_analyzer_helper, + bitrate_multiplier, std::move(stream_required_spatial_index)); std::unique_ptr video_decoder_factory = CreateVideoDecoderFactory(pcf_dependencies, video_analyzer_helper); @@ -158,6 +161,7 @@ std::unique_ptr CreateMediaEngine( PeerConnectionFactoryDependencies CreatePCFDependencies( std::unique_ptr pcf_dependencies, absl::optional audio_config, + double bitrate_multiplier, std::map> stream_required_spatial_index, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, rtc::Thread* network_thread, @@ -167,7 +171,7 @@ PeerConnectionFactoryDependencies CreatePCFDependencies( pcf_deps.network_thread = network_thread; pcf_deps.signaling_thread = signaling_thread; pcf_deps.media_engine = CreateMediaEngine( - pcf_dependencies.get(), std::move(audio_config), + pcf_dependencies.get(), std::move(audio_config), bitrate_multiplier, std::move(stream_required_spatial_index), video_analyzer_helper, std::move(audio_output_file_name)); @@ -227,7 +231,8 @@ std::unique_ptr TestPeer::CreateTestPeer( std::unique_ptr observer, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, rtc::Thread* signaling_thread, - absl::optional audio_output_file_name) { + absl::optional audio_output_file_name, + double bitrate_multiplier) { RTC_DCHECK(components); RTC_DCHECK(params); SetMandatoryEntities(components.get()); @@ -248,8 +253,8 @@ std::unique_ptr TestPeer::CreateTestPeer( // Create peer connection factory. PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies( std::move(components->pcf_dependencies), params->audio_config, - std::move(stream_required_spatial_index), video_analyzer_helper, - components->network_thread, signaling_thread, + bitrate_multiplier, std::move(stream_required_spatial_index), + video_analyzer_helper, components->network_thread, signaling_thread, std::move(audio_output_file_name)); rtc::scoped_refptr pcf = CreateModularPeerConnectionFactory(std::move(pcf_deps)); diff --git a/test/pc/e2e/test_peer.h b/test/pc/e2e/test_peer.h index a72faa8bd1..0cc93892bb 100644 --- a/test/pc/e2e/test_peer.h +++ b/test/pc/e2e/test_peer.h @@ -53,7 +53,8 @@ class TestPeer final : public PeerConnectionWrapper { std::unique_ptr observer, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, rtc::Thread* signaling_thread, - absl::optional audio_output_file_name); + absl::optional audio_output_file_name, + double bitrate_multiplier); Params* params() const { return params_.get(); }