From c252a40b47920ac16ab632dc1ec1b3a2c2d5e091 Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Thu, 10 Aug 2023 13:16:56 +0200 Subject: [PATCH] Use layer/encode target resolution in DropDueToSize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It used input frame resolution before this change which caused unnecessary resolution adaptations when resolution scaling is used. Found that initial frame dropping was always enabled for AV1 SVC. After fixing DropDueToSize the AV1 SVC tests [1] started to fail ("number of encoded temporal layers is less than expected") on bots. The tests encode 1850x1110 in L3T3 for 5s using the default 300kbps start bitrate. Before the fix the initial frame dropping kicked in and reduced the resolution to a level that let encoder to generate all temporal layers. After the fix the resolution stayed at 1850x1110 and encoder dropped all T1 and T2 layer frames. Mitigated this by increasing test duration from 5 to 10s. This gives enough time for BWE to ramp up and for encoder to generate (stop dropping) all temporal layers. [1] https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/pc/test/svc_e2e_tests.cc;l=460;bpv=1 Bug: chromium:1466809 Change-Id: I16802689e234f8fc16f891f024d5f644985de01c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/315142 Reviewed-by: Florent Castelli Commit-Queue: Sergey Silkin Reviewed-by: Åsa Persson Cr-Commit-Position: refs/heads/main@{#40536} --- pc/test/svc_e2e_tests.cc | 2 +- video/video_stream_encoder.cc | 20 ++++---------- video/video_stream_encoder_unittest.cc | 38 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/pc/test/svc_e2e_tests.cc b/pc/test/svc_e2e_tests.cc index 4aced7f160..ae35c7f676 100644 --- a/pc/test/svc_e2e_tests.cc +++ b/pc/test/svc_e2e_tests.cc @@ -344,7 +344,7 @@ TEST_P(SvcTest, ScalabilityModeSupported) { alice->SetVideoCodecs({video_codec_config}); }, [](PeerConfigurer* bob) {}, std::move(analyzer)); - fixture->Run(RunParams(TimeDelta::Seconds(5))); + fixture->Run(RunParams(TimeDelta::Seconds(10))); EXPECT_THAT(analyzer_ptr->encoder_layers_seen(), HasSpatialAndTemporalLayers( SvcTestParameters().expected_spatial_layers, diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 1e2409372a..1fd82bb012 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -2365,25 +2365,15 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, } } -bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { +bool VideoStreamEncoder::DropDueToSize(uint32_t source_pixel_count) const { if (!encoder_ || !stream_resource_manager_.DropInitialFrames() || - !encoder_target_bitrate_bps_.has_value()) { + !encoder_target_bitrate_bps_ || + !stream_resource_manager_.SingleActiveStreamPixels()) { return false; } - bool simulcast_or_svc = - (send_codec_.codecType == VideoCodecType::kVideoCodecVP9 && - send_codec_.VP9().numberOfSpatialLayers > 1) || - (send_codec_.numberOfSimulcastStreams > 1 || - encoder_config_.simulcast_layers.size() > 1); - - if (simulcast_or_svc) { - if (stream_resource_manager_.SingleActiveStreamPixels()) { - pixel_count = stream_resource_manager_.SingleActiveStreamPixels().value(); - } else { - return false; - } - } + int pixel_count = std::min( + source_pixel_count, *stream_resource_manager_.SingleActiveStreamPixels()); uint32_t bitrate_bps = stream_resource_manager_.UseBandwidthAllocationBps().value_or( diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b9d376d92b..44fc53f90d 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -5636,6 +5636,44 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, InitialFrameDropAccountsForResolutionScaling) { + VideoEncoderConfig video_encoder_config; + webrtc::VideoEncoder::EncoderInfo encoder_info; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 1, + &video_encoder_config); + video_encoder_config.video_stream_factory = + rtc::make_ref_counted( + "VP8", /*max qp*/ 56, /*screencast*/ false, + /*screenshare enabled*/ false, encoder_info); + for (auto& layer : video_encoder_config.simulcast_layers) { + layer.num_temporal_layers = 1; + layer.max_framerate = kDefaultFramerate; + } + video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps(); + video_encoder_config.content_type = + VideoEncoderConfig::ContentType::kRealtimeVideo; + + video_encoder_config.simulcast_layers[0].active = true; + video_encoder_config.simulcast_layers[0].scale_resolution_down_by = 4; + + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + // Bitrate is not enough for 720p. + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::KilobitsPerSec(30), DataRate::KilobitsPerSec(30), + DataRate::KilobitsPerSec(30), 0, 0, 0); + + // Pass 720p frame. Resolution scaling factor is set to 4 which means that + // the target encode resolution is 180p. The default initial frame dropping + // should not be active for 180p no matter of available bitrate. + video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) { const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400); // Set simulcast.