From f46723c8aadc8f789a7b4dd7bac7557689892469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Fri, 20 Nov 2020 15:45:44 +0100 Subject: [PATCH] Enable initial frame drop for one active simulcast layer. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:12216 Change-Id: Ib2ac2fab45e560ba3eae30a926ce72667a257b07 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/193840 Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Åsa Persson Cr-Commit-Position: refs/heads/master@{#32652} --- .../video_stream_encoder_resource_manager.cc | 33 ++++++- .../video_stream_encoder_resource_manager.h | 5 +- video/quality_scaling_tests.cc | 86 ++++++++++++++++--- video/video_stream_encoder.cc | 11 ++- 4 files changed, 119 insertions(+), 16 deletions(-) diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index c7ca4bccf1..697ebbe027 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -58,6 +58,20 @@ std::string ToString(VideoAdaptationReason reason) { RTC_CHECK_NOTREACHED(); } +absl::optional GetSingleActiveStreamPixels(const VideoCodec& codec) { + int num_active = 0; + absl::optional pixels; + for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { + if (codec.simulcastStream[i].active) { + ++num_active; + pixels = codec.simulcastStream[i].width * codec.simulcastStream[i].height; + } + if (num_active > 1) + return absl::nullopt; + } + return pixels; +} + } // namespace class VideoStreamEncoderResourceManager::InitialFrameDropper { @@ -78,6 +92,10 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { return initial_framedrop_ < kMaxInitialFramedrop; } + absl::optional single_active_stream_pixels() const { + return single_active_stream_pixels_; + } + // Input signals. void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) { set_start_bitrate_ = start_bitrate; @@ -104,6 +122,10 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { } } + void OnEncoderSettingsUpdated(const VideoCodec& codec) { + single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec); + } + void OnFrameDroppedDueToSize() { ++initial_framedrop_; } void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; } @@ -130,6 +152,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper { int64_t set_start_bitrate_time_ms_; // Counts how many frames we've dropped in the initial framedrop phase. int initial_framedrop_; + absl::optional single_active_stream_pixels_; }; VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( @@ -230,7 +253,7 @@ void VideoStreamEncoderResourceManager::AddResource( RTC_DCHECK(resource); bool inserted; std::tie(std::ignore, inserted) = resources_.emplace(resource, reason); - RTC_DCHECK(inserted) << "Resurce " << resource->Name() + RTC_DCHECK(inserted) << "Resource " << resource->Name() << " already was inserted"; adaptation_processor_->AddResource(resource); } @@ -259,6 +282,8 @@ void VideoStreamEncoderResourceManager::SetEncoderSettings( RTC_DCHECK_RUN_ON(encoder_queue_); encoder_settings_ = std::move(encoder_settings); bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_); + initial_frame_dropper_->OnEncoderSettingsUpdated( + encoder_settings_->video_codec()); MaybeUpdateTargetFrameRate(); } @@ -339,6 +364,12 @@ bool VideoStreamEncoderResourceManager::DropInitialFrames() const { return initial_frame_dropper_->DropInitialFrames(); } +absl::optional +VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return initial_frame_dropper_->single_active_stream_pixels(); +} + void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() { RTC_DCHECK_RUN_ON(encoder_queue_); initial_frame_dropper_->OnMaybeEncodeFrame(); diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 932d90c209..5190373f1b 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -121,9 +121,10 @@ class VideoStreamEncoderResourceManager VideoAdaptationReason reason); void RemoveResource(rtc::scoped_refptr resource); std::vector AdaptationConstraints() const; - // If true, the VideoStreamEncoder should eexecute its logic to maybe drop - // frames baseed on size and bitrate. + // If true, the VideoStreamEncoder should execute its logic to maybe drop + // frames based on size and bitrate. bool DropInitialFrames() const; + absl::optional SingleActiveStreamPixels() const; // VideoSourceRestrictionsListener implementation. // Updates |video_source_restrictions_|. diff --git a/video/quality_scaling_tests.cc b/video/quality_scaling_tests.cc index 65a23dbbcc..b72b25b86b 100644 --- a/video/quality_scaling_tests.cc +++ b/video/quality_scaling_tests.cc @@ -56,6 +56,7 @@ class QualityScalingTest : public test::CallTest { protected: void RunTest(VideoEncoderFactory* encoder_factory, const std::string& payload_name, + const std::vector& streams_active, int start_bps, bool automatic_resize, bool frame_dropping, @@ -67,6 +68,7 @@ class QualityScalingTest : public test::CallTest { void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory, const std::string& payload_name, + const std::vector& streams_active, int start_bps, bool automatic_resize, bool frame_dropping, @@ -77,6 +79,7 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory, public: ScalingObserver(VideoEncoderFactory* encoder_factory, const std::string& payload_name, + const std::vector& streams_active, int start_bps, bool automatic_resize, bool frame_dropping, @@ -84,6 +87,7 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory, : SendTest(expect_adaptation ? kDefaultTimeoutMs : kTimeoutMs), encoder_factory_(encoder_factory), payload_name_(payload_name), + streams_active_(streams_active), start_bps_(start_bps), automatic_resize_(automatic_resize), frame_dropping_(frame_dropping), @@ -108,6 +112,10 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory, bitrate_config->start_bitrate_bps = start_bps_; } + size_t GetNumVideoStreams() const override { + return streams_active_.size(); + } + void ModifyVideoConfigs( VideoSendStream::Config* send_config, std::vector* receive_configs, @@ -117,7 +125,15 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory, send_config->rtp.payload_type = kVideoSendPayloadType; const VideoCodecType codec_type = PayloadStringToCodecType(payload_name_); encoder_config->codec_type = codec_type; - encoder_config->max_bitrate_bps = start_bps_; + encoder_config->max_bitrate_bps = + std::max(start_bps_, encoder_config->max_bitrate_bps); + double scale_factor = 1.0; + for (int i = streams_active_.size() - 1; i >= 0; --i) { + VideoStream& stream = encoder_config->simulcast_layers[i]; + stream.active = streams_active_[i]; + stream.scale_resolution_down_by = scale_factor; + scale_factor *= 2.0; + } SetEncoderSpecific(encoder_config, codec_type, automatic_resize_, frame_dropping_); } @@ -129,12 +145,13 @@ void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory, VideoEncoderFactory* const encoder_factory_; const std::string payload_name_; + const std::vector streams_active_; const int start_bps_; const bool automatic_resize_; const bool frame_dropping_; const bool expect_adaptation_; - } test(encoder_factory, payload_name, start_bps, automatic_resize, - frame_dropping, expect_adaptation); + } test(encoder_factory, payload_name, streams_active, start_bps, + automatic_resize, frame_dropping, expect_adaptation); RunBaseTest(&test); } @@ -150,7 +167,7 @@ TEST_F(QualityScalingTest, AdaptsDownForHighQp_Vp8) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); - RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize, + RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize, kFrameDropping, kExpectAdapt); } @@ -165,7 +182,7 @@ TEST_F(QualityScalingTest, NoAdaptDownForHighQpWithResizeOff_Vp8) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); - RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize, + RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize, kFrameDropping, kExpectAdapt); } @@ -182,7 +199,7 @@ TEST_F(QualityScalingTest, test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); - RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize, + RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize, kFrameDropping, kExpectAdapt); } @@ -197,7 +214,7 @@ TEST_F(QualityScalingTest, NoAdaptDownForNormalQp_Vp8) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); - RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize, + RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize, kFrameDropping, kExpectAdapt); } @@ -212,10 +229,57 @@ TEST_F(QualityScalingTest, AdaptsDownForLowStartBitrate) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); - RunTest(&encoder_factory, "VP8", kLowStartBps, kAutomaticResize, + RunTest(&encoder_factory, "VP8", {true}, kLowStartBps, kAutomaticResize, kFrameDropping, kExpectAdapt); } +TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrate_Simulcast) { + // VP8 QP thresholds, low:1, high:127 -> normal QP. + test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd); + + // QualityScaler disabled. + const bool kAutomaticResize = false; + const bool kFrameDropping = true; + const bool kExpectAdapt = false; + + test::FunctionVideoEncoderFactory encoder_factory( + []() { return VP8Encoder::Create(); }); + RunTest(&encoder_factory, "VP8", {true, true}, kLowStartBps, kAutomaticResize, + kFrameDropping, kExpectAdapt); +} + +TEST_F(QualityScalingTest, + AdaptsDownForLowStartBitrate_SimulcastOneActiveHighRes) { + // VP8 QP thresholds, low:1, high:127 -> normal QP. + test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd); + + // QualityScaler enabled. + const bool kAutomaticResize = true; + const bool kFrameDropping = true; + const bool kExpectAdapt = true; + + test::FunctionVideoEncoderFactory encoder_factory( + []() { return VP8Encoder::Create(); }); + RunTest(&encoder_factory, "VP8", {false, false, true}, kLowStartBps, + kAutomaticResize, kFrameDropping, kExpectAdapt); +} + +TEST_F(QualityScalingTest, + NoAdaptDownForLowStartBitrate_SimulcastOneActiveLowRes) { + // VP8 QP thresholds, low:1, high:127 -> normal QP. + test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd); + + // QualityScaler enabled. + const bool kAutomaticResize = true; + const bool kFrameDropping = true; + const bool kExpectAdapt = false; + + test::FunctionVideoEncoderFactory encoder_factory( + []() { return VP8Encoder::Create(); }); + RunTest(&encoder_factory, "VP8", {true, false, false}, kLowStartBps, + kAutomaticResize, kFrameDropping, kExpectAdapt); +} + TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrateWithScalingOff) { // VP8 QP thresholds, low:1, high:127 -> normal QP. test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd); @@ -227,7 +291,7 @@ TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrateWithScalingOff) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP8Encoder::Create(); }); - RunTest(&encoder_factory, "VP8", kLowStartBps, kAutomaticResize, + RunTest(&encoder_factory, "VP8", {true}, kLowStartBps, kAutomaticResize, kFrameDropping, kExpectAdapt); } @@ -243,7 +307,7 @@ TEST_F(QualityScalingTest, NoAdaptDownForHighQp_Vp9) { test::FunctionVideoEncoderFactory encoder_factory( []() { return VP9Encoder::Create(); }); - RunTest(&encoder_factory, "VP9", kHighStartBps, kAutomaticResize, + RunTest(&encoder_factory, "VP9", {true}, kHighStartBps, kAutomaticResize, kFrameDropping, kExpectAdapt); } @@ -259,7 +323,7 @@ TEST_F(QualityScalingTest, AdaptsDownForHighQp_H264) { test::FunctionVideoEncoderFactory encoder_factory( []() { return H264Encoder::Create(cricket::VideoCodec("H264")); }); - RunTest(&encoder_factory, "H264", kHighStartBps, kAutomaticResize, + RunTest(&encoder_factory, "H264", {true}, kHighStartBps, kAutomaticResize, kFrameDropping, kExpectAdapt); } #endif // defined(WEBRTC_USE_H264) diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 836b20c67e..c197d234dd 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -565,6 +565,7 @@ void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { encoder_queue_.PostTask([this, start_bitrate_bps] { RTC_DCHECK_RUN_ON(&encoder_queue_); + RTC_LOG(LS_INFO) << "SetStartBitrate " << start_bitrate_bps; encoder_target_bitrate_bps_ = start_bitrate_bps != 0 ? absl::optional(start_bitrate_bps) : absl::nullopt; @@ -1848,14 +1849,20 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { bool simulcast_or_svc = (send_codec_.codecType == VideoCodecType::kVideoCodecVP9 && send_codec_.VP9().numberOfSpatialLayers > 1) || - send_codec_.numberOfSimulcastStreams > 1 || - encoder_config_.simulcast_layers.size() > 1; + ((send_codec_.numberOfSimulcastStreams > 1 || + encoder_config_.simulcast_layers.size() > 1) && + !stream_resource_manager_.SingleActiveStreamPixels()); if (simulcast_or_svc || !stream_resource_manager_.DropInitialFrames() || !encoder_target_bitrate_bps_.has_value()) { return false; } + if (send_codec_.numberOfSimulcastStreams > 1 && + stream_resource_manager_.SingleActiveStreamPixels()) { + pixel_count = stream_resource_manager_.SingleActiveStreamPixels().value(); + } + absl::optional encoder_bitrate_limits = encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( pixel_count);