From 84bc34841be9b65599425fac773ec34de10332d3 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Thu, 26 Nov 2020 16:58:47 +0100 Subject: [PATCH] Reset initial frame dropper if the stream changes for external reasons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit External reasons here are simulcast configuration and source resolution change. Initial frame dropper should be enabled in these cases because the client can request way too big resolution for available bitrate and usual quality scaling would take too long. Bug: none Change-Id: I02fbbd3c15b53b39672c083c2a1f9a780256c507 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195004 Reviewed-by: Åsa Persson Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#32707} --- .../video_stream_encoder_resource_manager.cc | 60 ++++++- .../video_stream_encoder_resource_manager.h | 3 + video/video_stream_encoder_unittest.cc | 151 ++++++++++++++++++ 3 files changed, 209 insertions(+), 5 deletions(-) diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 697ebbe027..7247850d3a 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -10,6 +10,7 @@ #include "video/adaptation/video_stream_encoder_resource_manager.h" +#include #include #include #include @@ -72,6 +73,21 @@ absl::optional GetSingleActiveStreamPixels(const VideoCodec& codec) { return pixels; } +std::vector GetActiveLayersFlags(const VideoCodec& codec) { + const int num_streams = codec.numberOfSimulcastStreams; + std::vector flags(num_streams); + for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { + flags[i] = codec.simulcastStream[i].active; + } + return flags; +} + +bool EqualFlags(const std::vector& a, const std::vector& b) { + if (a.size() != b.size()) + return false; + return std::equal(a.begin(), a.end(), b.begin()); +} + } // namespace class VideoStreamEncoderResourceManager::InitialFrameDropper { @@ -83,7 +99,9 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { has_seen_first_bwe_drop_(false), set_start_bitrate_(DataRate::Zero()), set_start_bitrate_time_ms_(0), - initial_framedrop_(0) { + initial_framedrop_(0), + last_input_width_(0), + last_input_height_(0) { RTC_DCHECK(quality_scaler_resource_); } @@ -122,13 +140,38 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { } } - void OnEncoderSettingsUpdated(const VideoCodec& codec) { + void OnEncoderSettingsUpdated( + const VideoCodec& codec, + const VideoAdaptationCounters& adaptation_counters) { + std::vector active_flags = GetActiveLayersFlags(codec); + // Check if the source resolution has changed for the external reasons, + // i.e. without any adaptation from WebRTC. + const bool source_resolution_changed = + (last_input_width_ != codec.width || + last_input_height_ != codec.height) && + adaptation_counters.resolution_adaptations == + last_adaptation_counters_.resolution_adaptations; + if (!EqualFlags(active_flags, last_active_flags_) || + source_resolution_changed) { + // Streams configuration has changed. + // Initial frame drop must be enabled because BWE might be way too low + // for the selected resolution. + if (quality_scaler_resource_->is_started()) { + RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed " + "stream parameters"; + initial_framedrop_ = 0; + } + } + last_adaptation_counters_ = adaptation_counters; + last_active_flags_ = active_flags; + last_input_width_ = codec.width; + last_input_height_ = codec.height; single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec); } void OnFrameDroppedDueToSize() { ++initial_framedrop_; } - void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; } + void Disable() { initial_framedrop_ = kMaxInitialFramedrop; } void OnQualityScalerSettingsUpdated() { if (quality_scaler_resource_->is_started()) { @@ -153,6 +196,11 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { // Counts how many frames we've dropped in the initial framedrop phase. int initial_framedrop_; absl::optional single_active_stream_pixels_; + + std::vector last_active_flags_; + VideoAdaptationCounters last_adaptation_counters_; + int last_input_width_; + int last_input_height_; }; VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( @@ -283,7 +331,7 @@ void VideoStreamEncoderResourceManager::SetEncoderSettings( encoder_settings_ = std::move(encoder_settings); bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_); initial_frame_dropper_->OnEncoderSettingsUpdated( - encoder_settings_->video_codec()); + encoder_settings_->video_codec(), current_adaptation_counters_); MaybeUpdateTargetFrameRate(); } @@ -372,7 +420,7 @@ VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const { void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() { RTC_DCHECK_RUN_ON(encoder_queue_); - initial_frame_dropper_->OnMaybeEncodeFrame(); + initial_frame_dropper_->Disable(); if (quality_rampup_experiment_ && quality_scaler_resource_->is_started()) { DataRate bandwidth = encoder_rates_.has_value() ? encoder_rates_->bandwidth_allocation @@ -488,6 +536,8 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( rtc::scoped_refptr reason, const VideoSourceRestrictions& unfiltered_restrictions) { RTC_DCHECK_RUN_ON(encoder_queue_); + current_adaptation_counters_ = adaptation_counters; + // TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback. if (!reason && adaptation_counters.Total() == 0) { // Adaptation was manually reset - clear the per-reason counters too. diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 5190373f1b..623d17adc3 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -184,6 +184,9 @@ class VideoStreamEncoderResourceManager VideoSourceRestrictions video_source_restrictions_ RTC_GUARDED_BY(encoder_queue_); + VideoAdaptationCounters current_adaptation_counters_ + RTC_GUARDED_BY(encoder_queue_); + const BalancedDegradationSettings balanced_settings_; Clock* clock_ RTC_GUARDED_BY(encoder_queue_); const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 1e7dabb8bf..4501e9f6a7 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -4560,6 +4560,157 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) { + const int kLowTargetBitrateBps = 400000; + // Set simulcast. + ResetEncoder("VP8", 3, 1, 1, false); + fake_encoder_.SetQualityScaling(true); + const int kWidth = 1280; + const int kHeight = 720; + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); + // Frame should not be dropped. + WaitForEncodedFrame(1); + + // Trigger QVGA "singlecast" + // Update the config. + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3, + &video_encoder_config); + for (auto& layer : video_encoder_config.simulcast_layers) { + layer.num_temporal_layers = 1; + layer.max_framerate = kDefaultFramerate; + } + video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps; + video_encoder_config.content_type = + VideoEncoderConfig::ContentType::kRealtimeVideo; + + video_encoder_config.simulcast_layers[0].active = true; + video_encoder_config.simulcast_layers[1].active = false; + video_encoder_config.simulcast_layers[2].active = false; + + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); + // Frame should not be dropped. + WaitForEncodedFrame(2); + + // Trigger HD "singlecast" + video_encoder_config.simulcast_layers[0].active = false; + video_encoder_config.simulcast_layers[1].active = false; + video_encoder_config.simulcast_layers[2].active = true; + + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight)); + // Frame should be dropped because of initial frame drop. + ExpectDroppedFrame(); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_TRUE_WAIT( + video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000); + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + InitialFrameDropActivatesWhenResolutionIncreases) { + const int kWidth = 640; + const int kHeight = 360; + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2)); + // Frame should not be dropped. + WaitForEncodedFrame(1); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2)); + // Frame should not be dropped, bitrate not too low for frame. + WaitForEncodedFrame(2); + + // Incoming resolution increases. + video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight)); + // Expect to drop this frame, bitrate too low for frame. + ExpectDroppedFrame(); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_TRUE_WAIT( + video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000); + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) { + const int kWidth = 640; + const int kHeight = 360; + // So that quality scaling doesn't happen by itself. + fake_encoder_.SetQp(kQpHigh); + + AdaptingFrameForwarder source(&time_controller_); + source.set_adaptation_enabled(true); + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + + int timestamp = 1; + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight)); + WaitForEncodedFrame(timestamp); + timestamp += 9000; + // Long pause to disable all first BWE drop logic. + AdvanceTime(TimeDelta::Millis(1000)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); + source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight)); + // Not dropped frame, as initial frame drop is disabled by now. + WaitForEncodedFrame(timestamp); + timestamp += 9000; + AdvanceTime(TimeDelta::Millis(100)); + + // Quality adaptation down. + video_stream_encoder_->TriggerQualityLow(); + + // Adaptation has an effect. + EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight, + 5000); + + // Frame isn't dropped as initial frame dropper is disabled. + source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight)); + WaitForEncodedFrame(timestamp); + timestamp += 9000; + AdvanceTime(TimeDelta::Millis(100)); + + // Quality adaptation up. + video_stream_encoder_->TriggerQualityHigh(); + + // Adaptation has an effect. + EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight, + 5000); + + source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight)); + // Frame should not be dropped, as initial framedropper is off. + WaitForEncodedFrame(timestamp); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) { webrtc::test::ScopedFieldTrials field_trials( "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");