From 37132e10fdd282f278567fbe6ae6624f93275c13 Mon Sep 17 00:00:00 2001 From: Jonas Oreland Date: Thu, 6 Oct 2022 10:56:07 +0200 Subject: [PATCH] RtpEncodingParameters::request_resolution patch 3 This cl/ adds resource adapation to the requested_resolution feature. The restrictions that are sent to the video source are also saved inside video_stream_encoder and used when determining layer resolution. Anticipated further patches 4) Let VideoSource do adaption if possible Bug: webrtc:14451 Change-Id: Ia9b990a6b92b76af7ff6665a562f84585f79c35b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/277580 Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Jonas Oreland Reviewed-by: Rasmus Brandt Cr-Commit-Position: refs/heads/main@{#38306} --- call/adaptation/video_source_restrictions.cc | 25 ++++++ call/adaptation/video_source_restrictions.h | 3 + .../video_source_restrictions_unittest.cc | 17 ++++ video/BUILD.gn | 1 + video/config/BUILD.gn | 7 +- video/config/encoder_stream_factory.cc | 15 +++- video/config/encoder_stream_factory.h | 6 +- .../config/encoder_stream_factory_unittest.cc | 79 +++++++++++++++++++ video/video_stream_encoder.cc | 39 ++++++++- video/video_stream_encoder.h | 14 ++++ 10 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 video/config/encoder_stream_factory_unittest.cc diff --git a/call/adaptation/video_source_restrictions.cc b/call/adaptation/video_source_restrictions.cc index e9d6c26137..719bc53278 100644 --- a/call/adaptation/video_source_restrictions.cc +++ b/call/adaptation/video_source_restrictions.cc @@ -10,6 +10,7 @@ #include "call/adaptation/video_source_restrictions.h" +#include #include #include "rtc_base/checks.h" @@ -79,6 +80,30 @@ void VideoSourceRestrictions::set_max_frame_rate( max_frame_rate_ = std::move(max_frame_rate); } +void VideoSourceRestrictions::UpdateMin(const VideoSourceRestrictions& other) { + if (max_pixels_per_frame_.has_value()) { + max_pixels_per_frame_ = std::min(*max_pixels_per_frame_, + other.max_pixels_per_frame().value_or( + std::numeric_limits::max())); + } else { + max_pixels_per_frame_ = other.max_pixels_per_frame(); + } + if (target_pixels_per_frame_.has_value()) { + target_pixels_per_frame_ = std::min( + *target_pixels_per_frame_, other.target_pixels_per_frame().value_or( + std::numeric_limits::max())); + } else { + target_pixels_per_frame_ = other.target_pixels_per_frame(); + } + if (max_frame_rate_.has_value()) { + max_frame_rate_ = std::min( + *max_frame_rate_, + other.max_frame_rate().value_or(std::numeric_limits::max())); + } else { + max_frame_rate_ = other.max_frame_rate(); + } +} + bool DidRestrictionsIncrease(VideoSourceRestrictions before, VideoSourceRestrictions after) { bool decreased_resolution = DidDecreaseResolution(before, after); diff --git a/call/adaptation/video_source_restrictions.h b/call/adaptation/video_source_restrictions.h index 004cc09055..be8520a385 100644 --- a/call/adaptation/video_source_restrictions.h +++ b/call/adaptation/video_source_restrictions.h @@ -60,6 +60,9 @@ class VideoSourceRestrictions { absl::optional target_pixels_per_frame); void set_max_frame_rate(absl::optional max_frame_rate); + // Update `this` with min(`this`, `other`). + void UpdateMin(const VideoSourceRestrictions& other); + private: // These map to rtc::VideoSinkWants's `max_pixel_count` and // `target_pixel_count`. diff --git a/call/adaptation/video_source_restrictions_unittest.cc b/call/adaptation/video_source_restrictions_unittest.cc index 92e34f96f3..8c1ae4c896 100644 --- a/call/adaptation/video_source_restrictions_unittest.cc +++ b/call/adaptation/video_source_restrictions_unittest.cc @@ -126,4 +126,21 @@ TEST(VideoSourceRestrictions, EXPECT_FALSE(DidRestrictionsDecrease(kHd, k15fps)); } +TEST(VideoSourceRestrictions, UpdateMin) { + VideoSourceRestrictions one(kHdPixels / 2, kHdPixels, 7.0); + VideoSourceRestrictions two(kHdPixels, kHdPixels / 3, 15.0); + + one.UpdateMin(two); + + EXPECT_EQ(one.max_pixels_per_frame(), kHdPixels / 2); + EXPECT_EQ(one.target_pixels_per_frame(), kHdPixels / 3); + EXPECT_EQ(one.max_frame_rate(), 7.0); + + two.UpdateMin(one); + + EXPECT_EQ(two.max_pixels_per_frame(), kHdPixels / 2); + EXPECT_EQ(two.target_pixels_per_frame(), kHdPixels / 3); + EXPECT_EQ(two.max_frame_rate(), 7.0); +} + } // namespace webrtc diff --git a/video/BUILD.gn b/video/BUILD.gn index 425e7540cd..6b5eb2a19a 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -938,6 +938,7 @@ if (rtc_include_tests) { "adaptation:video_adaptation", "config:encoder_config", "config:streams_config", + "config:video_config_tests", ] absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", diff --git a/video/config/BUILD.gn b/video/config/BUILD.gn index 11871b8cbc..8a5f03b074 100644 --- a/video/config/BUILD.gn +++ b/video/config/BUILD.gn @@ -23,6 +23,7 @@ rtc_library("streams_config") { "../../api/units:data_rate", "../../api/video:video_codec_constants", "../../api/video_codecs:video_codecs_api", + "../../call/adaptation:resource_adaptation", "../../media:rtc_media_base", "../../modules/video_coding:video_coding_utility", "../../modules/video_coding:webrtc_vp9_helpers", @@ -69,10 +70,14 @@ if (rtc_include_tests) { testonly = true defines = [] - sources = [ "simulcast_unittest.cc" ] + sources = [ + "encoder_stream_factory_unittest.cc", + "simulcast_unittest.cc", + ] deps = [ ":streams_config", "../../api/transport:field_trial_based_config", + "../../call/adaptation:resource_adaptation", "../../test:field_trial", "../../test:test_support", ] diff --git a/video/config/encoder_stream_factory.cc b/video/config/encoder_stream_factory.cc index c43922da17..a413e4ac42 100644 --- a/video/config/encoder_stream_factory.cc +++ b/video/config/encoder_stream_factory.cc @@ -10,6 +10,7 @@ #include "video/config/encoder_stream_factory.h" #include +#include #include #include #include @@ -115,6 +116,7 @@ EncoderStreamFactory::EncoderStreamFactory( bool is_screenshare, bool conference_mode, const webrtc::VideoEncoder::EncoderInfo& encoder_info, + absl::optional restrictions, const webrtc::FieldTrialsView* trials) : codec_name_(codec_name), max_qp_(max_qp), @@ -122,7 +124,8 @@ EncoderStreamFactory::EncoderStreamFactory( conference_mode_(conference_mode), trials_(trials ? *trials : fallback_trials_), encoder_info_requested_resolution_alignment_( - encoder_info.requested_resolution_alignment) {} + encoder_info.requested_resolution_alignment), + restrictions_(restrictions) {} std::vector EncoderStreamFactory::CreateEncoderStreams( int frame_width, @@ -434,6 +437,16 @@ EncoderStreamFactory::GetLayerResolutionFromRequestedResolution( adapter.OnOutputFormatRequest(requested_resolution.ToPair(), requested_resolution.PixelCount(), absl::nullopt); + if (restrictions_) { + rtc::VideoSinkWants wants; + wants.is_active = true; + wants.target_pixel_count = restrictions_->target_pixels_per_frame(); + wants.max_pixel_count = + rtc::dchecked_cast(restrictions_->max_pixels_per_frame().value_or( + std::numeric_limits::max())); + wants.resolution_alignment = encoder_info_requested_resolution_alignment_; + adapter.OnSinkWants(wants); + } int cropped_width, cropped_height; int out_width = 0, out_height = 0; if (!adapter.AdaptFrameResolution(frame_width, frame_height, 0, diff --git a/video/config/encoder_stream_factory.h b/video/config/encoder_stream_factory.h index 11910eb1fe..37abb93876 100644 --- a/video/config/encoder_stream_factory.h +++ b/video/config/encoder_stream_factory.h @@ -16,6 +16,7 @@ #include "api/transport/field_trial_based_config.h" #include "api/units/data_rate.h" #include "api/video_codecs/video_encoder.h" +#include "call/adaptation/video_source_restrictions.h" #include "video/config/video_encoder_config.h" namespace cricket { @@ -34,14 +35,16 @@ class EncoderStreamFactory bool is_screenshare, bool conference_mode, const webrtc::VideoEncoder::EncoderInfo& encoder_info, + absl::optional + restrictions = absl::nullopt, const webrtc::FieldTrialsView* trials = nullptr); - private: std::vector CreateEncoderStreams( int width, int height, const webrtc::VideoEncoderConfig& encoder_config) override; + private: std::vector CreateDefaultVideoStreams( int width, int height, @@ -69,6 +72,7 @@ class EncoderStreamFactory const webrtc::FieldTrialBasedConfig fallback_trials_; const webrtc::FieldTrialsView& trials_; const int encoder_info_requested_resolution_alignment_; + const absl::optional restrictions_; }; } // namespace cricket diff --git a/video/config/encoder_stream_factory_unittest.cc b/video/config/encoder_stream_factory_unittest.cc new file mode 100644 index 0000000000..7214f40216 --- /dev/null +++ b/video/config/encoder_stream_factory_unittest.cc @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/config/encoder_stream_factory.h" + +#include "call/adaptation/video_source_restrictions.h" +#include "test/gtest.h" + +namespace webrtc { + +using cricket::EncoderStreamFactory; +constexpr int kMaxQp = 48; + +namespace { + +std::vector GetStreamResolutions( + const std::vector& streams) { + std::vector res; + for (const auto& s : streams) { + if (s.active) { + res.push_back( + {rtc::checked_cast(s.width), rtc::checked_cast(s.height)}); + } + } + return res; +} + +VideoStream LayerWithRequestedResolution(Resolution res) { + VideoStream s; + s.requested_resolution = res; + return s; +} + +} // namespace + +TEST(EncoderStreamFactory, SinglecastRequestedResolution) { + VideoEncoder::EncoderInfo encoder_info; + auto factory = rtc::make_ref_counted( + "VP8", kMaxQp, + /* is_screenshare= */ false, + /* conference_mode= */ false, encoder_info); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.push_back( + LayerWithRequestedResolution({.width = 640, .height = 360})); + auto streams = factory->CreateEncoderStreams(1280, 720, encoder_config); + EXPECT_EQ(GetStreamResolutions(streams), (std::vector{ + {.width = 640, .height = 360}, + })); +} + +TEST(EncoderStreamFactory, SinglecastRequestedResolutionWithAdaptation) { + VideoSourceRestrictions restrictions( + /* max_pixels_per_frame= */ (320 * 320), + /* target_pixels_per_frame= */ absl::nullopt, + /* max_frame_rate= */ absl::nullopt); + VideoEncoder::EncoderInfo encoder_info; + auto factory = rtc::make_ref_counted( + "VP8", kMaxQp, + /* is_screenshare= */ false, + /* conference_mode= */ false, encoder_info, restrictions); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.push_back( + LayerWithRequestedResolution({.width = 640, .height = 360})); + auto streams = factory->CreateEncoderStreams(1280, 720, encoder_config); + EXPECT_EQ(GetStreamResolutions(streams), (std::vector{ + {.width = 320, .height = 180}, + })); +} + +} // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index deebc87fbc..3178d9cd57 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -33,6 +33,7 @@ #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_encoder.h" #include "call/adaptation/resource_adaptation_processor.h" +#include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_adapter.h" #include "modules/video_coding/include/video_codec_initializer.h" #include "modules/video_coding/svc/svc_rate_allocator.h" @@ -508,7 +509,7 @@ void ApplyEncoderBitrateLimitsIfSingleActiveStream( } absl::optional ParseVp9LowTierCoreCountThreshold( - const webrtc::FieldTrialsView& trials) { + const FieldTrialsView& trials) { FieldTrialFlag disable_low_tier("Disabled"); FieldTrialParameter max_core_count("max_core_count", 2); ParseFieldTrial({&disable_low_tier, &max_core_count}, @@ -519,6 +520,22 @@ absl::optional ParseVp9LowTierCoreCountThreshold( return max_core_count.Get(); } +absl::optional MergeRestrictions( + const std::vector>& list) { + absl::optional return_value; + for (const auto& res : list) { + if (!res) { + continue; + } + if (!return_value) { + return_value = *res; + continue; + } + return_value->UpdateMin(*res); + } + return return_value; +} + } // namespace VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings() @@ -949,7 +966,9 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_config_.video_format.name, encoder_config_.max_qp, encoder_config_.content_type == webrtc::VideoEncoderConfig::ContentType::kScreen, - encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo()); + encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo(), + MergeRestrictions({latest_restrictions_, animate_restrictions_}), + &field_trials_); streams = factory->CreateEncoderStreams( last_frame_info_->width, last_frame_info_->height, encoder_config_); @@ -2313,6 +2332,11 @@ void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( RTC_LOG(LS_INFO) << "Updating sink restrictions from " << (reason ? reason->Name() : std::string("")) << " to " << restrictions.ToString(); + + // TODO(webrtc:14451) Split video_source_sink_controller_ + // so that ownership on restrictions/wants is kept on &encoder_queue_ + latest_restrictions_ = restrictions; + worker_queue_->PostTask(SafeTask( task_safety_.flag(), [this, restrictions = std::move(restrictions)]() { RTC_DCHECK_RUN_ON(worker_queue_); @@ -2454,6 +2478,17 @@ void VideoStreamEncoder::CheckForAnimatedContent( RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent " "animation detection."; } + // TODO(webrtc:14451) Split video_source_sink_controller_ + // so that ownership on restrictions/wants is kept on &encoder_queue_ + if (should_cap_resolution) { + animate_restrictions_ = + VideoSourceRestrictions(kMaxAnimationPixels, + /* target_pixels_per_frame= */ absl::nullopt, + /* max_frame_rate= */ absl::nullopt); + } else { + animate_restrictions_.reset(); + } + worker_queue_->PostTask( SafeTask(task_safety_.flag(), [this, should_cap_resolution]() { RTC_DCHECK_RUN_ON(worker_queue_); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 7a4fc324b1..9af2e0bcff 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -457,6 +457,20 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, const absl::optional vp9_low_tier_core_threshold_; + // These are copies of restrictions (glorified max_pixel_count) set by + // a) OnVideoSourceRestrictionsUpdated + // b) CheckForAnimatedContent + // They are used to scale down encoding resolution if needed when using + // requested_resolution. + // + // TODO(webrtc:14451) Split video_source_sink_controller_ + // so that ownership on restrictions/wants is kept on &encoder_queue_, that + // these extra copies would not be needed. + absl::optional latest_restrictions_ + RTC_GUARDED_BY(&encoder_queue_); + absl::optional animate_restrictions_ + RTC_GUARDED_BY(&encoder_queue_); + // Used to cancel any potentially pending tasks to the worker thread. // Refrenced by tasks running on `encoder_queue_` so need to be destroyed // after stopping that queue. Must be created and destroyed on