From cde4a9f66990ae8e78157dc24f876fc6418b1d3b Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Fri, 27 Nov 2020 14:06:08 +0100 Subject: [PATCH] Enable initial frame drop for SVC 'singlecast' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: none Change-Id: Ideda726f4f7df5e92556048a199cda06261e76b4 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195542 Reviewed-by: Åsa Persson Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#32714} --- .../video_stream_encoder_resource_manager.cc | 37 ++++++++--- video/video_stream_encoder.cc | 24 +++---- video/video_stream_encoder_unittest.cc | 65 +++++++++++++++++++ 3 files changed, 105 insertions(+), 21 deletions(-) diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 7247850d3a..340b2e8508 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -62,22 +62,39 @@ std::string ToString(VideoAdaptationReason reason) { absl::optional GetSingleActiveStreamPixels(const VideoCodec& codec) { int num_active = 0; absl::optional pixels; - for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { - if (codec.simulcastStream[i].active) { - ++num_active; - pixels = codec.simulcastStream[i].width * codec.simulcastStream[i].height; + if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { + if (codec.spatialLayers[i].active) { + ++num_active; + pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height; + } + } + } else { + for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { + if (codec.simulcastStream[i].active) { + ++num_active; + pixels = + codec.simulcastStream[i].width * codec.simulcastStream[i].height; + } } - if (num_active > 1) - return absl::nullopt; } + if (num_active > 1) + return absl::nullopt; return pixels; } std::vector GetActiveLayersFlags(const VideoCodec& codec) { - const int num_streams = codec.numberOfSimulcastStreams; - std::vector flags(num_streams); - for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { - flags[i] = codec.simulcastStream[i].active; + std::vector flags; + if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + flags.resize(codec.VP9().numberOfSpatialLayers); + for (size_t i = 0; i < flags.size(); ++i) { + flags[i] = codec.spatialLayers[i].active; + } + } else { + flags.resize(codec.numberOfSimulcastStreams); + for (size_t i = 0; i < flags.size(); ++i) { + flags[i] = codec.simulcastStream[i].active; + } } return flags; } diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 5502139eb9..14f356623d 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1847,21 +1847,23 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, } bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { - bool simulcast_or_svc = - (send_codec_.codecType == VideoCodecType::kVideoCodecVP9 && - send_codec_.VP9().numberOfSpatialLayers > 1) || - ((send_codec_.numberOfSimulcastStreams > 1 || - encoder_config_.simulcast_layers.size() > 1) && - !stream_resource_manager_.SingleActiveStreamPixels()); - - if (simulcast_or_svc || !stream_resource_manager_.DropInitialFrames() || + if (!stream_resource_manager_.DropInitialFrames() || !encoder_target_bitrate_bps_.has_value()) { return false; } - if (send_codec_.numberOfSimulcastStreams > 1 && - stream_resource_manager_.SingleActiveStreamPixels()) { - pixel_count = stream_resource_manager_.SingleActiveStreamPixels().value(); + bool simulcast_or_svc = + (send_codec_.codecType == VideoCodecType::kVideoCodecVP9 && + send_codec_.VP9().numberOfSpatialLayers > 1) || + (send_codec_.numberOfSimulcastStreams > 1 || + encoder_config_.simulcast_layers.size() > 1); + + if (simulcast_or_svc) { + if (stream_resource_manager_.SingleActiveStreamPixels()) { + pixel_count = stream_resource_manager_.SingleActiveStreamPixels().value(); + } else { + return false; + } } absl::optional encoder_bitrate_limits = diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 4501e9f6a7..267169a2f2 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -4619,6 +4619,71 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) { + const int kLowTargetBitrateBps = 400000; + // Set simulcast. + ResetEncoder("VP9", 1, 1, 3, false); + fake_encoder_.SetQualityScaling(true); + const int kWidth = 1280; + const int kHeight = 720; + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), + DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); + // Frame should not be dropped. + WaitForEncodedFrame(1); + + // Trigger QVGA "singlecast" + // Update the config. + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1, + &video_encoder_config); + VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings(); + vp9_settings.numberOfSpatialLayers = 3; + // Since only one layer is active - automatic resize should be enabled. + vp9_settings.automaticResizeOn = true; + video_encoder_config.encoder_specific_settings = + new rtc::RefCountedObject( + vp9_settings); + video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps; + video_encoder_config.content_type = + VideoEncoderConfig::ContentType::kRealtimeVideo; + // Currently simulcast layers |active| flags are used to inidicate + // which SVC layers are active. + video_encoder_config.simulcast_layers.resize(3); + + video_encoder_config.simulcast_layers[0].active = true; + video_encoder_config.simulcast_layers[1].active = false; + video_encoder_config.simulcast_layers[2].active = false; + + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); + // Frame should not be dropped. + WaitForEncodedFrame(2); + + // Trigger HD "singlecast" + video_encoder_config.simulcast_layers[0].active = false; + video_encoder_config.simulcast_layers[1].active = false; + video_encoder_config.simulcast_layers[2].active = true; + + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight)); + // Frame should be dropped because of initial frame drop. + ExpectDroppedFrame(); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_TRUE_WAIT( + video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000); + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenResolutionIncreases) { const int kWidth = 640;