diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index a51f93015d..80b2c0584a 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -20,6 +20,8 @@ rtc_library("resource_adaptation") { "resource_consumer.h", "resource_consumer_configuration.cc", "resource_consumer_configuration.h", + "video_source_restrictions.cc", + "video_source_restrictions.h", ] deps = [ "../../rtc_base:checks", diff --git a/call/adaptation/resource_adaptation_module_interface.cc b/call/adaptation/resource_adaptation_module_interface.cc index 941278bdac..e89d1eff2c 100644 --- a/call/adaptation/resource_adaptation_module_interface.cc +++ b/call/adaptation/resource_adaptation_module_interface.cc @@ -10,39 +10,8 @@ #include "call/adaptation/resource_adaptation_module_interface.h" -#include "rtc_base/checks.h" - namespace webrtc { -VideoSourceRestrictions::VideoSourceRestrictions( - absl::optional max_pixels_per_frame, - absl::optional target_pixels_per_frame, - absl::optional max_frame_rate) - : max_pixels_per_frame_(std::move(max_pixels_per_frame)), - target_pixels_per_frame_(std::move(target_pixels_per_frame)), - max_frame_rate_(std::move(max_frame_rate)) { - RTC_DCHECK(!max_pixels_per_frame_.has_value() || - max_pixels_per_frame_.value() < - static_cast(std::numeric_limits::max())); - RTC_DCHECK(!max_frame_rate_.has_value() || - max_frame_rate_.value() < std::numeric_limits::max()); - RTC_DCHECK(!max_frame_rate_.has_value() || max_frame_rate_.value() > 0.0); -} - -const absl::optional& VideoSourceRestrictions::max_pixels_per_frame() - const { - return max_pixels_per_frame_; -} - -const absl::optional& VideoSourceRestrictions::target_pixels_per_frame() - const { - return target_pixels_per_frame_; -} - -const absl::optional& VideoSourceRestrictions::max_frame_rate() const { - return max_frame_rate_; -} - ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {} ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {} diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h index 929011ac56..825b914d6e 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -11,37 +11,10 @@ #ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ #define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ -#include -#include - -#include "absl/types/optional.h" +#include "call/adaptation/video_source_restrictions.h" namespace webrtc { -// Describes optional restrictions to the resolution and frame rate of a video -// source. -class VideoSourceRestrictions { - public: - // All values must be positive or nullopt. - // TODO(hbos): Support expressing "disable this stream"? - VideoSourceRestrictions(absl::optional max_pixels_per_frame, - absl::optional target_pixels_per_frame, - absl::optional max_frame_rate); - - const absl::optional& max_pixels_per_frame() const; - const absl::optional& target_pixels_per_frame() const; - const absl::optional& max_frame_rate() const; - - private: - // These map to rtc::VideoSinkWants's |max_pixel_count| and - // |target_pixel_count|. - // TODO(hbos): It's not clear what "target" means; either make it well-defined - // or remove it in favor of only using |max_pixels_per_frame_|. - absl::optional max_pixels_per_frame_; - absl::optional target_pixels_per_frame_; - absl::optional max_frame_rate_; -}; - // The listener is responsible for carrying out the reconfiguration of the video // source such that the VideoSourceRestrictions are fulfilled. class ResourceAdaptationModuleListener { diff --git a/call/adaptation/video_source_restrictions.cc b/call/adaptation/video_source_restrictions.cc new file mode 100644 index 0000000000..b848bf80bc --- /dev/null +++ b/call/adaptation/video_source_restrictions.cc @@ -0,0 +1,68 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_source_restrictions.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +VideoSourceRestrictions::VideoSourceRestrictions() + : max_pixels_per_frame_(absl::nullopt), + target_pixels_per_frame_(absl::nullopt), + max_frame_rate_(absl::nullopt) {} + +VideoSourceRestrictions::VideoSourceRestrictions( + absl::optional max_pixels_per_frame, + absl::optional target_pixels_per_frame, + absl::optional max_frame_rate) + : max_pixels_per_frame_(std::move(max_pixels_per_frame)), + target_pixels_per_frame_(std::move(target_pixels_per_frame)), + max_frame_rate_(std::move(max_frame_rate)) { + RTC_DCHECK(!max_pixels_per_frame_.has_value() || + max_pixels_per_frame_.value() < + static_cast(std::numeric_limits::max())); + RTC_DCHECK(!max_frame_rate_.has_value() || + max_frame_rate_.value() < std::numeric_limits::max()); + RTC_DCHECK(!max_frame_rate_.has_value() || max_frame_rate_.value() > 0.0); +} + +const absl::optional& VideoSourceRestrictions::max_pixels_per_frame() + const { + return max_pixels_per_frame_; +} + +const absl::optional& VideoSourceRestrictions::target_pixels_per_frame() + const { + return target_pixels_per_frame_; +} + +const absl::optional& VideoSourceRestrictions::max_frame_rate() const { + return max_frame_rate_; +} + +void VideoSourceRestrictions::set_max_pixels_per_frame( + absl::optional max_pixels_per_frame) { + max_pixels_per_frame_ = std::move(max_pixels_per_frame); +} + +void VideoSourceRestrictions::set_target_pixels_per_frame( + absl::optional target_pixels_per_frame) { + target_pixels_per_frame_ = std::move(target_pixels_per_frame); +} + +void VideoSourceRestrictions::set_max_frame_rate( + absl::optional max_frame_rate) { + max_frame_rate_ = std::move(max_frame_rate); +} + +} // namespace webrtc diff --git a/call/adaptation/video_source_restrictions.h b/call/adaptation/video_source_restrictions.h new file mode 100644 index 0000000000..a992084d06 --- /dev/null +++ b/call/adaptation/video_source_restrictions.h @@ -0,0 +1,62 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ +#define CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ + +#include + +#include "absl/types/optional.h" + +namespace webrtc { + +// Describes optional restrictions to the resolution and frame rate of a video +// source. +class VideoSourceRestrictions { + public: + // Constructs without any restrictions. + VideoSourceRestrictions(); + // All values must be positive or nullopt. + // TODO(hbos): Support expressing "disable this stream"? + VideoSourceRestrictions(absl::optional max_pixels_per_frame, + absl::optional target_pixels_per_frame, + absl::optional max_frame_rate); + + bool operator==(const VideoSourceRestrictions& rhs) const { + return max_pixels_per_frame_ == rhs.max_pixels_per_frame_ && + target_pixels_per_frame_ == rhs.target_pixels_per_frame_ && + max_frame_rate_ == rhs.max_frame_rate_; + } + bool operator!=(const VideoSourceRestrictions& rhs) const { + return !(*this == rhs); + } + + const absl::optional& max_pixels_per_frame() const; + const absl::optional& target_pixels_per_frame() const; + const absl::optional& max_frame_rate() const; + + void set_max_pixels_per_frame(absl::optional max_pixels_per_frame); + void set_target_pixels_per_frame( + absl::optional target_pixels_per_frame); + void set_max_frame_rate(absl::optional max_frame_rate); + + private: + // These map to rtc::VideoSinkWants's |max_pixel_count| and + // |target_pixel_count|. + // TODO(hbos): It's not clear what "target" means; either make it well-defined + // or remove it in favor of only using |max_pixels_per_frame_|. + absl::optional max_pixels_per_frame_; + absl::optional target_pixels_per_frame_; + absl::optional max_frame_rate_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ diff --git a/video/BUILD.gn b/video/BUILD.gn index daa3a57764..c77973ef44 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -186,6 +186,8 @@ rtc_library("video_stream_encoder_impl") { "overuse_frame_detector.h", "overuse_frame_detector_resource_adaptation_module.cc", "overuse_frame_detector_resource_adaptation_module.h", + "video_source_sink_controller.cc", + "video_source_sink_controller.h", "video_stream_encoder.cc", "video_stream_encoder.h", ] @@ -564,6 +566,7 @@ if (rtc_include_tests) { "video_receive_stream_unittest.cc", "video_send_stream_impl_unittest.cc", "video_send_stream_tests.cc", + "video_source_sink_controller_unittest.cc", "video_stream_decoder_impl_unittest.cc", "video_stream_encoder_unittest.cc", ] @@ -609,6 +612,7 @@ if (rtc_include_tests) { "../call:simulated_network", "../call:simulated_packet_receiver", "../call:video_stream_api", + "../call/adaptation:resource_adaptation", "../common_video", "../common_video/test:utilities", "../media:rtc_audio_video", diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index 9b3cbf6b36..673eebd09e 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -18,6 +18,7 @@ #include "absl/algorithm/container.h" #include "api/video/video_source_interface.h" +#include "call/adaptation/video_source_restrictions.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/strings/string_builder.h" @@ -40,161 +41,71 @@ bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { degradation_preference == DegradationPreference::BALANCED; } -// Constructs VideoSourceRestrictions from |target_pixel_count|, -// |max_pixel_count| and |max_framerate_fps|. Other rtc::VideoSinkWants -// information such as |rotation_applied| is lost in the conversion. -VideoSourceRestrictions VideoSinkWantsToVideoSourceRestrictions( - rtc::VideoSinkWants active_sink_wants) { - return VideoSourceRestrictions( - active_sink_wants.max_pixel_count != std::numeric_limits::max() - ? absl::optional(active_sink_wants.max_pixel_count) - : absl::nullopt, - active_sink_wants.target_pixel_count.has_value() - ? absl::optional(rtc::dchecked_cast( - active_sink_wants.target_pixel_count.value())) - : absl::nullopt, - active_sink_wants.max_framerate_fps != std::numeric_limits::max() - ? absl::optional(active_sink_wants.max_framerate_fps) - : absl::nullopt); -} - -// Constructs rtc::VideoSinkWants from max_pixels_per_frame(), -// target_pixels_per_frame() and max_frame_rate(). The rest of the members, such -// as |rotation_applied|, are obtained from the |baseline_sink_wants|. -rtc::VideoSinkWants VideoSourceRestrictionsToVideoSinkWants( - const rtc::VideoSinkWants& baseline_sink_wants, - VideoSourceRestrictions restrictions) { - rtc::VideoSinkWants sink_wants = baseline_sink_wants; - sink_wants.max_pixel_count = - restrictions.max_pixels_per_frame().has_value() - ? static_cast(restrictions.max_pixels_per_frame().value()) - : std::numeric_limits::max(); - sink_wants.target_pixel_count = - restrictions.target_pixels_per_frame().has_value() - ? absl::optional(rtc::dchecked_cast( - restrictions.target_pixels_per_frame().value())) - : absl::nullopt; - sink_wants.max_framerate_fps = - restrictions.max_frame_rate().has_value() - ? static_cast(restrictions.max_frame_rate().value()) - : std::numeric_limits::max(); - return sink_wants; -} - } // namespace -// VideoSourceProxy is responsible ensuring thread safety between calls to -// OveruseFrameDetectorResourceAdaptationModule::SetSource that will happen on -// libjingle's worker thread when a video capturer is connected to the encoder -// and the encoder task queue (encoder_queue_) where the encoder reports its -// VideoSinkWants. -class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { +// VideoSourceRestrictor is responsible for keeping track of current +// VideoSourceRestrictions and how to modify them in response to adapting up or +// down. It is not reponsible for determining when we should adapt up or down - +// for that, see OveruseFrameDetectorResourceAdaptationModule::AdaptUp() and +// AdaptDown() - only how to modify the source/sink restrictions when this +// happens. Note that it is also not responsible for reconfigruring the +// source/sink, it is only a keeper of desired restrictions. +// +// Thread safety is ensured between SetHasInputVideoAndDegradationPreference() +// calls on the worker thread and adaptation logic on the encoder task queue +// using a lock. +class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { public: - explicit VideoSourceProxy(rtc::VideoSinkInterface* sink) - : sink_(sink), - degradation_preference_(DegradationPreference::DISABLED), - source_(nullptr), - max_framerate_(std::numeric_limits::max()), - max_pixels_(std::numeric_limits::max()), - resolution_alignment_(1) {} + explicit VideoSourceRestrictor( + VideoSourceSinkController* video_source_sink_controller) + : video_source_sink_controller_(video_source_sink_controller), + has_input_video_(false), + degradation_preference_(DegradationPreference::DISABLED) {} - VideoSourceRestrictions ToVideoSourceRestrictions() { - return VideoSinkWantsToVideoSourceRestrictions(GetActiveSinkWants()); - } - - void ApplyVideoSourceRestrictions(VideoSourceRestrictions restrictions) { + VideoSourceRestrictions source_restrictions() { rtc::CritScope lock(&crit_); - rtc::VideoSinkWants wants = VideoSourceRestrictionsToVideoSinkWants( - GetActiveSinkWantsInternal(), std::move(restrictions)); - if (!source_) - return; - source_->AddOrUpdateSink(sink_, wants); + return source_restrictions_; } - // Informs the sink of the new source settings. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. - void SetSource(rtc::VideoSourceInterface* source, - const DegradationPreference& degradation_preference) { + // Inform the restrictor of new source status and degradation preference. + // TODO(hbos): Can this be moved to the encoder queue? If so, the |crit_| lock + // can be removed and we only need a sequence checker. + void SetHasInputVideoAndDegradationPreference( + bool has_input_video, + DegradationPreference degradation_preference) { // Called on libjingle's worker thread. RTC_DCHECK_RUN_ON(&main_checker_); - rtc::VideoSourceInterface* old_source = nullptr; - rtc::VideoSinkWants wants; - { - rtc::CritScope lock(&crit_); - degradation_preference_ = degradation_preference; - old_source = source_; - source_ = source; - wants = GetActiveSinkWantsInternal(); - } - - if (old_source != source && old_source != nullptr) { - old_source->RemoveSink(sink_); - } - - if (!source) { - return; - } - - source->AddOrUpdateSink(sink_, wants); + rtc::CritScope lock(&crit_); + has_input_video_ = has_input_video; + degradation_preference_ = degradation_preference; } // Informs the sink of the new source settings. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. - void SetMaxFramerateAndAlignment(int max_framerate, - int resolution_alignment) { - RTC_DCHECK_GT(max_framerate, 0); - rtc::CritScope lock(&crit_); - if (max_framerate == max_framerate_ && - resolution_alignment == resolution_alignment_) { - return; - } - - RTC_LOG(LS_INFO) << "Set max framerate: " << max_framerate - << " and resolution alignment: " << resolution_alignment; - max_framerate_ = max_framerate; - resolution_alignment_ = resolution_alignment; - if (source_) { - source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); - } - } - - // Informs the sink of the new source settings. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. - void SetWantsRotationApplied(bool rotation_applied) { - rtc::CritScope lock(&crit_); - sink_wants_.rotation_applied = rotation_applied; - if (source_) { - source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); - } - } - - rtc::VideoSinkWants GetActiveSinkWants() { - rtc::CritScope lock(&crit_); - return GetActiveSinkWantsInternal(); - } - - // Informs the sink of the new source settings. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. + // TODO(https://crbug.com/webrtc/11222): Handle all sink updates in + // video_stream_encoder.cc. This method is only used when setting the + // degradation preference such that it moves in or out of the "balanced" + // state, or when clearing all counters. When moving the remaining degradation + // preference logic inside the VideoSourceSinkController to here, stop + // explicitly setting the controller's restrictions and instead inform the + // VideoStreamEncoder of updated restrictions using + // OnVideoSourceRestrictionsUpdated(). void ResetPixelFpsCount() { rtc::CritScope lock(&crit_); - sink_wants_.max_pixel_count = std::numeric_limits::max(); - sink_wants_.target_pixel_count.reset(); - sink_wants_.max_framerate_fps = std::numeric_limits::max(); - if (source_) - source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + // Clear all restrictions. + source_restrictions_ = VideoSourceRestrictions(); + video_source_sink_controller_->SetRestrictions(source_restrictions_); + video_source_sink_controller_->PushSourceSinkSettings(); } - // Updates the sink settings, but DOES NOT inform the sink of the new - // settings. Reapplying video source restrictions is required, see - // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more - // information. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. + // Updates the source_restrictions(). The source/sink has to be informed of + // this separately. bool RequestResolutionLowerThan(int pixel_count, int min_pixels_per_frame, bool* min_pixels_reached) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); - if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { + if (!has_input_video_ || + !IsResolutionScalingEnabled(degradation_preference_)) { // This can happen since |degradation_preference_| is set on libjingle's // worker thread but the adaptation is done on the encoder task queue. return false; @@ -202,7 +113,10 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { // The input video frame size will have a resolution less than or equal to // |max_pixel_count| depending on how the source can scale the frame size. const int pixels_wanted = (pixel_count * 3) / 5; - if (pixels_wanted >= sink_wants_.max_pixel_count) { + if (pixels_wanted >= + rtc::dchecked_cast( + source_restrictions_.max_pixels_per_frame().value_or( + std::numeric_limits::max()))) { return false; } if (pixels_wanted < min_pixels_per_frame) { @@ -211,16 +125,16 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { } RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted; - sink_wants_.max_pixel_count = pixels_wanted; - sink_wants_.target_pixel_count = absl::nullopt; + source_restrictions_.set_max_pixels_per_frame( + pixels_wanted != std::numeric_limits::max() + ? absl::optional(pixels_wanted) + : absl::nullopt); + source_restrictions_.set_target_pixels_per_frame(absl::nullopt); return true; } - // Updates the sink settings, but DOES NOT inform the sink of the new - // settings. Reapplying video source restrictions is required, see - // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more - // information. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. + // Updates the source_restrictions(). The source/sink has to be informed of + // this separately. int RequestFramerateLowerThan(int fps) { // Called on the encoder task queue. // The input video frame rate will be scaled down to 2/3, rounding down. @@ -238,15 +152,13 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { return (pixel_count * 5) / 3; } - // Updates the sink settings, but DOES NOT inform the sink of the new - // settings. Reapplying video source restrictions is required, see - // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more - // information. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. + // Updates the source_restrictions(). The source/sink has to be informed of + // this separately. bool RequestHigherResolutionThan(int pixel_count) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); - if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { + if (!has_input_video_ || + !IsResolutionScalingEnabled(degradation_preference_)) { // This can happen since |degradation_preference_| is set on libjingle's // worker thread but the adaptation is done on the encoder task queue. return false; @@ -255,21 +167,28 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { if (max_pixels_wanted != std::numeric_limits::max()) max_pixels_wanted = pixel_count * 4; - if (max_pixels_wanted <= sink_wants_.max_pixel_count) + if (max_pixels_wanted <= + rtc::dchecked_cast( + source_restrictions_.max_pixels_per_frame().value_or( + std::numeric_limits::max()))) { return false; - - sink_wants_.max_pixel_count = max_pixels_wanted; - if (max_pixels_wanted == std::numeric_limits::max()) { - // Remove any constraints. - sink_wants_.target_pixel_count.reset(); - } else { - sink_wants_.target_pixel_count = GetHigherResolutionThan(pixel_count); } + RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; + source_restrictions_.set_max_pixels_per_frame( + max_pixels_wanted != std::numeric_limits::max() + ? absl::optional(max_pixels_wanted) + : absl::nullopt); + source_restrictions_.set_target_pixels_per_frame( + max_pixels_wanted != std::numeric_limits::max() + ? absl::optional(GetHigherResolutionThan(pixel_count)) + : absl::nullopt); return true; } + // Updates the source_restrictions(). The source/sink has to be informed of + // this separately. // Request upgrade in framerate. Returns the new requested frame, or -1 if // no change requested. Note that maxint may be returned if limits due to // adaptation requests are removed completely. In that case, consider @@ -284,104 +203,61 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; } - // Updates the sink settings, but DOES NOT inform the sink of the new - // settings. Reapplying video source restrictions is required, see - // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more - // information. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. + // Updates the source_restrictions(). The source/sink has to be informed of + // this separately. bool RestrictFramerate(int fps) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); - if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) + if (!has_input_video_ || + !IsFramerateScalingEnabled(degradation_preference_)) return false; const int fps_wanted = std::max(kMinFramerateFps, fps); - if (fps_wanted >= sink_wants_.max_framerate_fps) + if (fps_wanted >= + rtc::dchecked_cast(source_restrictions_.max_frame_rate().value_or( + std::numeric_limits::max()))) return false; RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; - sink_wants_.max_framerate_fps = fps_wanted; + source_restrictions_.set_max_frame_rate( + fps_wanted != std::numeric_limits::max() + ? absl::optional(fps_wanted) + : absl::nullopt); return true; } - // Updates the sink settings, but DOES NOT inform the sink of the new - // settings. Reapplying video source restrictions is required, see - // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more - // information. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. + // Updates the source_restrictions(). The source/sink has to be informed of + // this separately. bool IncreaseFramerate(int fps) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); - if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) + if (!has_input_video_ || + !IsFramerateScalingEnabled(degradation_preference_)) return false; const int fps_wanted = std::max(kMinFramerateFps, fps); - if (fps_wanted <= sink_wants_.max_framerate_fps) + if (fps_wanted <= + rtc::dchecked_cast(source_restrictions_.max_frame_rate().value_or( + std::numeric_limits::max()))) return false; RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; - sink_wants_.max_framerate_fps = fps_wanted; - return true; - } - - // Informs the sink of the new source settings. - // Used in automatic animation detection for screenshare. - // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. - bool RestrictPixels(int max_pixels) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { - // This can happen since |degradation_preference_| is set on libjingle's - // worker thread but the adaptation is done on the encoder task queue. - return false; - } - max_pixels_ = max_pixels; - RTC_LOG(LS_INFO) << "Applying max pixel restriction: " << max_pixels; - source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + source_restrictions_.set_max_frame_rate( + fps_wanted != std::numeric_limits::max() + ? absl::optional(fps_wanted) + : absl::nullopt); return true; } private: - rtc::VideoSinkWants GetActiveSinkWantsInternal() - RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) { - rtc::VideoSinkWants wants = sink_wants_; - // Clear any constraints from the current sink wants that don't apply to - // the used degradation_preference. - switch (degradation_preference_) { - case DegradationPreference::BALANCED: - break; - case DegradationPreference::MAINTAIN_FRAMERATE: - wants.max_framerate_fps = std::numeric_limits::max(); - break; - case DegradationPreference::MAINTAIN_RESOLUTION: - wants.max_pixel_count = std::numeric_limits::max(); - wants.target_pixel_count.reset(); - break; - case DegradationPreference::DISABLED: - wants.max_pixel_count = std::numeric_limits::max(); - wants.target_pixel_count.reset(); - wants.max_framerate_fps = std::numeric_limits::max(); - } - // Limit to configured max framerate. - wants.max_framerate_fps = std::min(max_framerate_, wants.max_framerate_fps); - // Limit resolution due to automatic animation detection for screenshare. - wants.max_pixel_count = std::min(max_pixels_, wants.max_pixel_count); - wants.resolution_alignment = resolution_alignment_; - - return wants; - } - rtc::CriticalSection crit_; SequenceChecker main_checker_; - rtc::VideoSinkInterface* const sink_; - rtc::VideoSinkWants sink_wants_ RTC_GUARDED_BY(&crit_); + VideoSourceSinkController* const video_source_sink_controller_; + VideoSourceRestrictions source_restrictions_ RTC_GUARDED_BY(&crit_); + bool has_input_video_ RTC_GUARDED_BY(&crit_); DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_); - rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(&crit_); - int max_framerate_ RTC_GUARDED_BY(&crit_); - int max_pixels_ RTC_GUARDED_BY(&crit_); - int resolution_alignment_ RTC_GUARDED_BY(&crit_); - RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy); + RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor); }; // Class holding adaptation information. @@ -510,19 +386,21 @@ OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ToString( OveruseFrameDetectorResourceAdaptationModule:: OveruseFrameDetectorResourceAdaptationModule( VideoStreamEncoder* video_stream_encoder, - rtc::VideoSinkInterface* sink, + VideoSourceSinkController* video_source_sink_controller, std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer, ResourceAdaptationModuleListener* adaptation_listener) : encoder_queue_(nullptr), adaptation_listener_(adaptation_listener), video_stream_encoder_(video_stream_encoder), + video_source_sink_controller_(video_source_sink_controller), degradation_preference_(DegradationPreference::DISABLED), adapt_counters_(), balanced_settings_(), last_adaptation_request_(absl::nullopt), last_frame_pixel_count_(absl::nullopt), - source_proxy_(std::make_unique(sink)), + source_restrictor_(std::make_unique( + video_source_sink_controller)), overuse_detector_(std::move(overuse_detector)), codec_max_framerate_(-1), encoder_start_bitrate_bps_(0), @@ -575,13 +453,6 @@ void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() { overuse_detector_->StopCheckForOveruse(); } -void OveruseFrameDetectorResourceAdaptationModule::ApplyVideoSourceRestrictions( - VideoSourceRestrictions restrictions) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); - source_proxy_->ApplyVideoSourceRestrictions(std::move(restrictions)); -} - void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured( const VideoFrame& frame, int64_t time_when_first_seen_us) { @@ -636,10 +507,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetIsQualityScalerEnabled( is_quality_scaler_enabled_ = is_quality_scaler_enabled; } -void OveruseFrameDetectorResourceAdaptationModule::SetSource( - rtc::VideoSourceInterface* source, - const DegradationPreference& degradation_preference) { - source_proxy_->SetSource(source, degradation_preference); +void OveruseFrameDetectorResourceAdaptationModule:: + SetHasInputVideoAndDegradationPreference( + bool has_input_video, + DegradationPreference degradation_preference) { + source_restrictor_->SetHasInputVideoAndDegradationPreference( + has_input_video, degradation_preference); encoder_queue_->PostTask([this, degradation_preference] { RTC_DCHECK_RUN_ON(encoder_queue_); if (degradation_preference_ != degradation_preference) { @@ -650,7 +523,7 @@ void OveruseFrameDetectorResourceAdaptationModule::SetSource( degradation_preference_ == DegradationPreference::BALANCED) { // TODO(asapersson): Consider removing |adapt_counters_| map and use one // AdaptCounter for all modes. - source_proxy_->ResetPixelFpsCount(); + source_restrictor_->ResetPixelFpsCount(); adapt_counters_.clear(); } } @@ -658,38 +531,24 @@ void OveruseFrameDetectorResourceAdaptationModule::SetSource( }); } -void OveruseFrameDetectorResourceAdaptationModule:: - SetSourceWantsRotationApplied(bool rotation_applied) { - source_proxy_->SetWantsRotationApplied(rotation_applied); -} - -void OveruseFrameDetectorResourceAdaptationModule:: - SetSourceMaxFramerateAndAlignment(int max_framerate, - int resolution_alignment) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); - source_proxy_->SetMaxFramerateAndAlignment(max_framerate, - resolution_alignment); -} - -void OveruseFrameDetectorResourceAdaptationModule::SetSourceMaxPixels( - int max_pixels) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); - source_proxy_->RestrictPixels(max_pixels); -} - void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() { RTC_DCHECK(encoder_queue_); RTC_DCHECK_RUN_ON(encoder_queue_); + // We need the "sink wants" from the |video_source_sink_controller_| because + // the controller filters its current settings as "sink wants" differently + // depending degradation preferences. + // TODO(https://crbug.com/webrtc/11222): When degradation preference-related + // changes to settings are handled by this class instead, we can remove the + // dependency on the controller; the VideoSourceRestrictions outputted by this + // module will then be the "final" settings, including the max frame rate. + auto sink_wants = video_source_sink_controller_->CurrentSettingsToSinkWants(); // Get the current target framerate, ie the maximum framerate as specified by // the current codec configuration, or any limit imposed by cpu adaption in // maintain-resolution or balanced mode. This is used to make sure overuse // detection doesn't needlessly trigger in low and/or variable framerate // scenarios. int target_framerate = - std::min(codec_max_framerate_, - source_proxy_->GetActiveSinkWants().max_framerate_fps); + std::min(codec_max_framerate_, sink_wants.max_framerate_fps); overuse_detector_->OnTargetFramerateUpdated(target_framerate); } @@ -697,7 +556,7 @@ void OveruseFrameDetectorResourceAdaptationModule::ResetAdaptationCounters() { RTC_DCHECK(encoder_queue_); RTC_DCHECK_RUN_ON(encoder_queue_); last_adaptation_request_.reset(); - source_proxy_->ResetPixelFpsCount(); + source_restrictor_->ResetPixelFpsCount(); adapt_counters_.clear(); } @@ -741,13 +600,14 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { // Try scale up framerate, if higher. int fps = balanced_settings_.MaxFps(encoder_config_.codec_type, *last_frame_pixel_count_); - if (source_proxy_->IncreaseFramerate(fps)) { + if (source_restrictor_->IncreaseFramerate(fps)) { GetAdaptCounter().DecrementFramerate(reason, fps); // Reset framerate in case of fewer fps steps down than up. if (adapt_counter.FramerateCount() == 0 && fps != std::numeric_limits::max()) { RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - source_proxy_->IncreaseFramerate(std::numeric_limits::max()); + source_restrictor_->IncreaseFramerate( + std::numeric_limits::max()); } break; } @@ -776,7 +636,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; pixel_count = std::numeric_limits::max(); } - if (!source_proxy_->RequestHigherResolutionThan(pixel_count)) + if (!source_restrictor_->RequestHigherResolutionThan(pixel_count)) return; GetAdaptCounter().DecrementResolution(reason); break; @@ -790,7 +650,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { } const int requested_framerate = - source_proxy_->RequestHigherFramerateThan(fps); + source_restrictor_->RequestHigherFramerateThan(fps); if (requested_framerate == -1) { overuse_detector_->OnTargetFramerateUpdated(codec_max_framerate_); return; @@ -807,7 +667,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { // Tell the adaptation listener to reconfigure the source for us according to // the latest adaptation. adaptation_listener_->OnVideoSourceRestrictionsUpdated( - source_proxy_->ToVideoSourceRestrictions()); + source_restrictor_->source_restrictions()); last_adaptation_request_.emplace(adaptation_request); @@ -864,7 +724,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( // Try scale down framerate, if lower. int fps = balanced_settings_.MinFps(encoder_config_.codec_type, *last_frame_pixel_count_); - if (source_proxy_->RestrictFramerate(fps)) { + if (source_restrictor_->RestrictFramerate(fps)) { GetAdaptCounter().IncrementFramerate(reason); // Check if requested fps is higher (or close to) input fps. absl::optional min_diff = @@ -883,7 +743,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( case DegradationPreference::MAINTAIN_FRAMERATE: { // Scale down resolution. bool min_pixels_reached = false; - if (!source_proxy_->RequestResolutionLowerThan( + if (!source_restrictor_->RequestResolutionLowerThan( adaptation_request.input_pixel_count_, encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame, &min_pixels_reached)) { @@ -896,8 +756,9 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( } case DegradationPreference::MAINTAIN_RESOLUTION: { // Scale down framerate. - const int requested_framerate = source_proxy_->RequestFramerateLowerThan( - adaptation_request.framerate_fps_); + const int requested_framerate = + source_restrictor_->RequestFramerateLowerThan( + adaptation_request.framerate_fps_); if (requested_framerate == -1) return true; RTC_DCHECK_NE(codec_max_framerate_, -1); @@ -913,7 +774,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( // Tell the adaptation listener to reconfigure the source for us according to // the latest adaptation. adaptation_listener_->OnVideoSourceRestrictionsUpdated( - source_proxy_->ToVideoSourceRestrictions()); + source_restrictor_->source_restrictions()); last_adaptation_request_.emplace(adaptation_request); @@ -1006,8 +867,9 @@ bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( int pixels, uint32_t bitrate_bps) const { absl::optional bitrate_limits = - GetEncoderBitrateLimits(encoder_->GetEncoderInfo(), - source_proxy_->GetHigherResolutionThan(pixels)); + GetEncoderBitrateLimits( + encoder_->GetEncoderInfo(), + source_restrictor_->GetHigherResolutionThan(pixels)); if (!bitrate_limits.has_value() || bitrate_bps == 0) { return true; // No limit configured or bitrate provided. } diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 21e44a2d59..c5485c19cd 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -20,7 +20,6 @@ #include "absl/types/optional.h" #include "api/rtp_parameters.h" #include "api/video/video_frame.h" -#include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" #include "api/video/video_stream_encoder_observer.h" #include "api/video_codecs/video_encoder.h" @@ -28,6 +27,7 @@ #include "call/adaptation/resource_adaptation_module_interface.h" #include "rtc_base/experiments/balanced_degradation_settings.h" #include "video/overuse_frame_detector.h" +#include "video/video_source_sink_controller.h" namespace webrtc { @@ -51,7 +51,7 @@ class OveruseFrameDetectorResourceAdaptationModule public: OveruseFrameDetectorResourceAdaptationModule( VideoStreamEncoder* video_stream_encoder, - rtc::VideoSinkInterface* sink, + VideoSourceSinkController* video_source_controller, std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer, ResourceAdaptationModuleListener* adaptation_listener); @@ -75,10 +75,6 @@ class OveruseFrameDetectorResourceAdaptationModule ResourceAdaptationModuleListener* adaptation_listener) override; void StopCheckForOveruse() override; - // TODO(hbos): When VideoSourceProxy is refactored and reconfiguration logic - // is entirely moved to video_stream_encoder.cc, remove this method. - void ApplyVideoSourceRestrictions(VideoSourceRestrictions restrictions); - // Input to the OveruseFrameDetector, which are required for this module to // function. These map to OveruseFrameDetector methods. // TODO(hbos): Define virtual methods in ResourceAdaptationModuleInterface @@ -105,12 +101,9 @@ class OveruseFrameDetectorResourceAdaptationModule // method is called incorrectly. void SetIsQualityScalerEnabled(bool is_quality_scaler_enabled); - void SetSource(rtc::VideoSourceInterface* source, - const DegradationPreference& degradation_preference); - void SetSourceWantsRotationApplied(bool rotation_applied); - void SetSourceMaxFramerateAndAlignment(int max_framerate, - int resolution_alignment); - void SetSourceMaxPixels(int max_pixels); + void SetHasInputVideoAndDegradationPreference( + bool has_input_video, + DegradationPreference degradation_preference); // TODO(hbos): Can we get rid of this? Seems we should know whether the frame // rate has updated. @@ -182,7 +175,7 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional GetQpThresholds() const; private: - class VideoSourceProxy; + class VideoSourceRestrictor; struct AdaptationRequest { // The pixel count produced by the source at the time of the adaptation. @@ -200,13 +193,18 @@ class OveruseFrameDetectorResourceAdaptationModule bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const RTC_RUN_ON(encoder_queue_); - // TODO(hbos): Can we move the |source_proxy_| to the |encoder_queue_| and - // replace |encoder_queue_| with a sequence checker instead? + // TODO(hbos): Can we move the |source_restrictor_| to the |encoder_queue_| + // and replace |encoder_queue_| with a sequence checker instead? rtc::TaskQueue* encoder_queue_; ResourceAdaptationModuleListener* const adaptation_listener_ RTC_GUARDED_BY(encoder_queue_); // Used to query CpuOveruseOptions at StartCheckForOveruse(). VideoStreamEncoder* video_stream_encoder_ RTC_GUARDED_BY(encoder_queue_); + // TODO(https://crbug.com/webrtc/11222): When the VideoSourceSinkController is + // no longer aware of DegradationPreference, and the degradation + // preference-related logic resides within this class, we can remove this + // dependency on the VideoSourceSinkController. + VideoSourceSinkController* const video_source_sink_controller_; DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); // Counters used for deciding if the video resolution or framerate is // currently restricted, and if so, why, on a per degradation preference @@ -222,8 +220,8 @@ class OveruseFrameDetectorResourceAdaptationModule absl::optional last_adaptation_request_ RTC_GUARDED_BY(encoder_queue_); absl::optional last_frame_pixel_count_ RTC_GUARDED_BY(encoder_queue_); - // The source proxy may modify its source or sink off the |encoder_queue_|. - const std::unique_ptr source_proxy_; + // Keeps track of source restrictions that this adaptation module outputs. + const std::unique_ptr source_restrictor_; const std::unique_ptr overuse_detector_ RTC_PT_GUARDED_BY(encoder_queue_); int codec_max_framerate_ RTC_GUARDED_BY(encoder_queue_); diff --git a/video/video_source_sink_controller.cc b/video/video_source_sink_controller.cc new file mode 100644 index 0000000000..f3585766a0 --- /dev/null +++ b/video/video_source_sink_controller.cc @@ -0,0 +1,168 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/video_source_sink_controller.h" + +#include +#include +#include + +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +VideoSourceSinkController::VideoSourceSinkController( + rtc::VideoSinkInterface* sink, + rtc::VideoSourceInterface* source) + : sink_(sink), + source_(source), + degradation_preference_(DegradationPreference::DISABLED) { + RTC_DCHECK(sink_); +} + +void VideoSourceSinkController::SetSource( + rtc::VideoSourceInterface* source, + DegradationPreference degradation_preference) { + rtc::VideoSourceInterface* old_source; + rtc::VideoSinkWants wants; + { + rtc::CritScope lock(&crit_); + old_source = source_; + source_ = source; + degradation_preference_ = degradation_preference; + wants = CurrentSettingsToSinkWantsInternal(); + } + if (old_source != source && old_source) + old_source->RemoveSink(sink_); + if (!source) + return; + source->AddOrUpdateSink(sink_, wants); +} + +void VideoSourceSinkController::PushSourceSinkSettings() { + rtc::CritScope lock(&crit_); + if (!source_) + return; + source_->AddOrUpdateSink(sink_, CurrentSettingsToSinkWantsInternal()); +} + +VideoSourceRestrictions VideoSourceSinkController::restrictions() const { + rtc::CritScope lock(&crit_); + return restrictions_; +} + +absl::optional VideoSourceSinkController::pixels_per_frame_upper_limit() + const { + rtc::CritScope lock(&crit_); + return pixels_per_frame_upper_limit_; +} + +absl::optional VideoSourceSinkController::frame_rate_upper_limit() + const { + rtc::CritScope lock(&crit_); + return frame_rate_upper_limit_; +} + +bool VideoSourceSinkController::rotation_applied() const { + rtc::CritScope lock(&crit_); + return rotation_applied_; +} + +int VideoSourceSinkController::resolution_alignment() const { + rtc::CritScope lock(&crit_); + return resolution_alignment_; +} + +void VideoSourceSinkController::SetRestrictions( + VideoSourceRestrictions restrictions) { + rtc::CritScope lock(&crit_); + restrictions_ = std::move(restrictions); +} + +void VideoSourceSinkController::SetPixelsPerFrameUpperLimit( + absl::optional pixels_per_frame_upper_limit) { + rtc::CritScope lock(&crit_); + pixels_per_frame_upper_limit_ = std::move(pixels_per_frame_upper_limit); +} + +void VideoSourceSinkController::SetFrameRateUpperLimit( + absl::optional frame_rate_upper_limit) { + rtc::CritScope lock(&crit_); + frame_rate_upper_limit_ = std::move(frame_rate_upper_limit); +} + +void VideoSourceSinkController::SetRotationApplied(bool rotation_applied) { + rtc::CritScope lock(&crit_); + rotation_applied_ = rotation_applied; +} + +void VideoSourceSinkController::SetResolutionAlignment( + int resolution_alignment) { + rtc::CritScope lock(&crit_); + resolution_alignment_ = resolution_alignment; +} + +rtc::VideoSinkWants VideoSourceSinkController::CurrentSettingsToSinkWants() + const { + rtc::CritScope lock(&crit_); + return CurrentSettingsToSinkWantsInternal(); +} + +// RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_) +rtc::VideoSinkWants +VideoSourceSinkController::CurrentSettingsToSinkWantsInternal() const { + rtc::VideoSinkWants wants; + wants.rotation_applied = rotation_applied_; + // |wants.black_frames| is not used, it always has its default value false. + wants.max_pixel_count = + rtc::dchecked_cast(restrictions_.max_pixels_per_frame().value_or( + std::numeric_limits::max())); + wants.target_pixel_count = + restrictions_.target_pixels_per_frame().has_value() + ? absl::optional(rtc::dchecked_cast( + restrictions_.target_pixels_per_frame().value())) + : absl::nullopt; + wants.max_framerate_fps = + restrictions_.max_frame_rate().has_value() + ? static_cast(restrictions_.max_frame_rate().value()) + : std::numeric_limits::max(); + wants.resolution_alignment = resolution_alignment_; + { + // Clear any constraints from the current sink wants that don't apply to + // the used degradation_preference. + switch (degradation_preference_) { + case DegradationPreference::BALANCED: + break; + case DegradationPreference::MAINTAIN_FRAMERATE: + wants.max_framerate_fps = std::numeric_limits::max(); + break; + case DegradationPreference::MAINTAIN_RESOLUTION: + wants.max_pixel_count = std::numeric_limits::max(); + wants.target_pixel_count.reset(); + break; + case DegradationPreference::DISABLED: + wants.max_pixel_count = std::numeric_limits::max(); + wants.target_pixel_count.reset(); + wants.max_framerate_fps = std::numeric_limits::max(); + } + } + wants.max_pixel_count = + std::min(wants.max_pixel_count, + rtc::dchecked_cast(pixels_per_frame_upper_limit_.value_or( + std::numeric_limits::max()))); + wants.max_framerate_fps = + std::min(wants.max_framerate_fps, + frame_rate_upper_limit_.has_value() + ? static_cast(frame_rate_upper_limit_.value()) + : std::numeric_limits::max()); + return wants; +} + +} // namespace webrtc diff --git a/video/video_source_sink_controller.h b/video/video_source_sink_controller.h new file mode 100644 index 0000000000..79260363ea --- /dev/null +++ b/video/video_source_sink_controller.h @@ -0,0 +1,89 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_VIDEO_SOURCE_SINK_CONTROLLER_H_ +#define VIDEO_VIDEO_SOURCE_SINK_CONTROLLER_H_ + +#include "absl/types/optional.h" +#include "api/rtp_parameters.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "call/adaptation/resource_adaptation_module_interface.h" +#include "rtc_base/critical_section.h" + +namespace webrtc { + +// Responsible for configuring source/sink settings, i.e. performing +// rtc::VideoSourceInterface::AddOrUpdateSink(). It does this by +// storing settings internally which are converted to rtc::VideoSinkWants when +// PushSourceSinkSettings() is performed. +class VideoSourceSinkController { + public: + VideoSourceSinkController(rtc::VideoSinkInterface* sink, + rtc::VideoSourceInterface* source); + + // TODO(https://crbug.com/webrtc/11222): Remove dependency on + // DegradationPreference! How degradation preference affects + // VideoSourceRestrictions should not be a responsibility of the controller, + // but of the resource adaptation module. + void SetSource(rtc::VideoSourceInterface* source, + DegradationPreference degradation_preference); + // Must be called in order for changes to settings to have an effect. This + // allows you to modify multiple properties in a single push to the sink. + void PushSourceSinkSettings(); + + VideoSourceRestrictions restrictions() const; + absl::optional pixels_per_frame_upper_limit() const; + absl::optional frame_rate_upper_limit() const; + bool rotation_applied() const; + int resolution_alignment() const; + + // Updates the settings stored internally. In order for these settings to be + // applied to the sink, PushSourceSinkSettings() must subsequently be called. + void SetRestrictions(VideoSourceRestrictions restrictions); + void SetPixelsPerFrameUpperLimit( + absl::optional pixels_per_frame_upper_limit); + void SetFrameRateUpperLimit(absl::optional frame_rate_upper_limit); + void SetRotationApplied(bool rotation_applied); + void SetResolutionAlignment(int resolution_alignment); + + // TODO(https://crbug.com/webrtc/11222): Outside of testing, this is only used + // by OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate(). + // When the DegradationPreference logic has moved outside of this class, there + // will be no public need for this method other than testing reasons and this + // can be renamed "ForTesting". + rtc::VideoSinkWants CurrentSettingsToSinkWants() const; + + private: + rtc::VideoSinkWants CurrentSettingsToSinkWantsInternal() const + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + + // TODO(hbos): If everything is handled on the same sequence (i.e. + // VideoStreamEncoder's encoder queue) then |crit_| can be replaced by + // sequence checker. Investigate if we want to do this. + mutable rtc::CriticalSection crit_; + rtc::VideoSinkInterface* const sink_; + rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(&crit_); + DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_); + // Pixel and frame rate restrictions. + VideoSourceRestrictions restrictions_ RTC_GUARDED_BY(&crit_); + // Ensures that even if we are not restricted, the sink is never configured + // above this limit. Example: We are not CPU limited (no |restrictions_|) but + // our encoder is capped at 30 fps (= |frame_rate_upper_limit_|). + absl::optional pixels_per_frame_upper_limit_ RTC_GUARDED_BY(&crit_); + absl::optional frame_rate_upper_limit_ RTC_GUARDED_BY(&crit_); + bool rotation_applied_ RTC_GUARDED_BY(&crit_) = false; + int resolution_alignment_ RTC_GUARDED_BY(&crit_) = 1; +}; + +} // namespace webrtc + +#endif // VIDEO_VIDEO_SOURCE_SINK_CONTROLLER_H_ diff --git a/video/video_source_sink_controller_unittest.cc b/video/video_source_sink_controller_unittest.cc new file mode 100644 index 0000000000..61cfafd45c --- /dev/null +++ b/video/video_source_sink_controller_unittest.cc @@ -0,0 +1,164 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/video_source_sink_controller.h" + +#include + +#include "api/video/video_frame.h" +#include "api/video/video_source_interface.h" +#include "call/adaptation/video_source_restrictions.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using testing::_; + +namespace webrtc { + +namespace { + +constexpr int kIntUnconstrained = std::numeric_limits::max(); + +class MockVideoSinkWithVideoFrame : public rtc::VideoSinkInterface { + public: + ~MockVideoSinkWithVideoFrame() override {} + + MOCK_METHOD1(OnFrame, void(const VideoFrame& frame)); + MOCK_METHOD0(OnDiscardedFrame, void()); +}; + +class MockVideoSourceWithVideoFrame + : public rtc::VideoSourceInterface { + public: + ~MockVideoSourceWithVideoFrame() override {} + + MOCK_METHOD2(AddOrUpdateSink, + void(rtc::VideoSinkInterface*, + const rtc::VideoSinkWants&)); + MOCK_METHOD1(RemoveSink, void(rtc::VideoSinkInterface*)); +}; + +} // namespace + +TEST(VideoSourceSinkControllerTest, UnconstrainedByDefault) { + MockVideoSinkWithVideoFrame sink; + MockVideoSourceWithVideoFrame source; + VideoSourceSinkController controller(&sink, &source); + EXPECT_EQ(controller.restrictions(), VideoSourceRestrictions()); + EXPECT_FALSE(controller.pixels_per_frame_upper_limit().has_value()); + EXPECT_FALSE(controller.frame_rate_upper_limit().has_value()); + EXPECT_FALSE(controller.rotation_applied()); + EXPECT_EQ(controller.resolution_alignment(), 1); + + EXPECT_CALL(source, AddOrUpdateSink(_, _)) + .WillOnce([](rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) { + EXPECT_FALSE(wants.rotation_applied); + EXPECT_EQ(wants.max_pixel_count, kIntUnconstrained); + EXPECT_EQ(wants.target_pixel_count, absl::nullopt); + EXPECT_EQ(wants.max_framerate_fps, kIntUnconstrained); + EXPECT_EQ(wants.resolution_alignment, 1); + }); + controller.PushSourceSinkSettings(); +} + +TEST(VideoSourceSinkControllerTest, VideoRestrictionsToSinkWants) { + MockVideoSinkWithVideoFrame sink; + MockVideoSourceWithVideoFrame source; + VideoSourceSinkController controller(&sink, &source); + + // Balanced degradation preference gives us what we ask for. + EXPECT_CALL(source, AddOrUpdateSink(_, _)).Times(1); + controller.SetSource(&source, DegradationPreference::BALANCED); + + VideoSourceRestrictions restrictions = controller.restrictions(); + // max_pixels_per_frame() maps to |max_pixel_count|. + restrictions.set_max_pixels_per_frame(42u); + // target_pixels_per_frame() maps to |target_pixel_count|. + restrictions.set_target_pixels_per_frame(200u); + // max_frame_rate() maps to |max_framerate_fps|. + restrictions.set_max_frame_rate(30.0); + controller.SetRestrictions(restrictions); + EXPECT_CALL(source, AddOrUpdateSink(_, _)) + .WillOnce([](rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) { + EXPECT_EQ(wants.max_pixel_count, 42); + EXPECT_EQ(wants.target_pixel_count, 200); + EXPECT_EQ(wants.max_framerate_fps, 30); + }); + controller.PushSourceSinkSettings(); + + // Disabled degradation preference makes the "wants" unconstrained despite our + // restrictions. + EXPECT_CALL(source, AddOrUpdateSink(_, _)).Times(1); + controller.SetSource(&source, DegradationPreference::DISABLED); + EXPECT_CALL(source, AddOrUpdateSink(_, _)) + .WillOnce([](rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) { + EXPECT_EQ(wants.max_pixel_count, kIntUnconstrained); + EXPECT_FALSE(wants.target_pixel_count.has_value()); + EXPECT_EQ(wants.max_framerate_fps, kIntUnconstrained); + }); + controller.PushSourceSinkSettings(); + + // pixels_per_frame_upper_limit() caps |max_pixel_count| regardless of + // degradation preferences. + controller.SetPixelsPerFrameUpperLimit(24); + // frame_rate_upper_limit() caps |max_framerate_fps| regardless of degradation + // preferences. + controller.SetFrameRateUpperLimit(10.0); + + EXPECT_CALL(source, AddOrUpdateSink(_, _)) + .WillOnce([](rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) { + EXPECT_EQ(wants.max_pixel_count, 24); + EXPECT_EQ(wants.max_framerate_fps, 10); + }); + controller.PushSourceSinkSettings(); +} + +TEST(VideoSourceSinkControllerTest, RotationApplied) { + MockVideoSinkWithVideoFrame sink; + MockVideoSourceWithVideoFrame source; + VideoSourceSinkController controller(&sink, &source); + controller.SetRotationApplied(true); + EXPECT_TRUE(controller.rotation_applied()); + + EXPECT_CALL(source, AddOrUpdateSink(_, _)) + .WillOnce([](rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) { + EXPECT_TRUE(wants.rotation_applied); + }); + controller.PushSourceSinkSettings(); +} + +TEST(VideoSourceSinkControllerTest, ResolutionAlignment) { + MockVideoSinkWithVideoFrame sink; + MockVideoSourceWithVideoFrame source; + VideoSourceSinkController controller(&sink, &source); + controller.SetResolutionAlignment(13); + EXPECT_EQ(controller.resolution_alignment(), 13); + + EXPECT_CALL(source, AddOrUpdateSink(_, _)) + .WillOnce([](rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) { + EXPECT_EQ(wants.resolution_alignment, 13); + }); + controller.PushSourceSinkSettings(); +} + +TEST(VideoSourceSinkControllerTest, + PushSourceSinkSettingsWithoutSourceDoesNotCrash) { + MockVideoSinkWithVideoFrame sink; + VideoSourceSinkController controller(&sink, nullptr); + controller.PushSourceSinkSettings(); +} + +} // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 27aaa93278..bdc0324cca 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -318,10 +318,13 @@ VideoStreamEncoder::VideoStreamEncoder( automatic_animation_detection_experiment_( ParseAutomatincAnimationDetectionFieldTrial()), encoder_switch_requested_(false), + video_source_sink_controller_(std::make_unique( + /*sink=*/this, + /*source=*/nullptr)), resource_adaptation_module_( std::make_unique( /*video_stream_encoder=*/this, - /*sink=*/this, + video_source_sink_controller_.get(), std::move(overuse_detector), encoder_stats_observer, /*adaptation_listener=*/this)), @@ -344,7 +347,7 @@ VideoStreamEncoder::~VideoStreamEncoder() { void VideoStreamEncoder::Stop() { RTC_DCHECK_RUN_ON(&thread_checker_); - resource_adaptation_module_->SetSource(nullptr, DegradationPreference()); + video_source_sink_controller_->SetSource(nullptr, DegradationPreference()); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); resource_adaptation_module_->StopCheckForOveruse(); @@ -385,7 +388,9 @@ void VideoStreamEncoder::SetSource( rtc::VideoSourceInterface* source, const DegradationPreference& degradation_preference) { RTC_DCHECK_RUN_ON(&thread_checker_); - resource_adaptation_module_->SetSource(source, degradation_preference); + video_source_sink_controller_->SetSource(source, degradation_preference); + resource_adaptation_module_->SetHasInputVideoAndDegradationPreference( + source, degradation_preference); encoder_queue_.PostTask([this, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); if (encoder_) @@ -401,7 +406,8 @@ void VideoStreamEncoder::SetSource( } void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { - resource_adaptation_module_->SetSourceWantsRotationApplied(rotation_applied); + video_source_sink_controller_->SetRotationApplied(rotation_applied); + video_source_sink_controller_->PushSourceSinkSettings(); encoder_queue_.PostTask([this, sink] { RTC_DCHECK_RUN_ON(&encoder_queue_); sink_ = sink; @@ -602,8 +608,14 @@ void VideoStreamEncoder::ReconfigureEncoder() { for (const auto& stream : streams) { max_framerate = std::max(stream.max_framerate, max_framerate); } - resource_adaptation_module_->SetSourceMaxFramerateAndAlignment( - max_framerate, encoder_->GetEncoderInfo().requested_resolution_alignment); + int alignment = encoder_->GetEncoderInfo().requested_resolution_alignment; + if (max_framerate != + video_source_sink_controller_->frame_rate_upper_limit() || + alignment != video_source_sink_controller_->resolution_alignment()) { + video_source_sink_controller_->SetFrameRateUpperLimit(max_framerate); + video_source_sink_controller_->SetResolutionAlignment(alignment); + video_source_sink_controller_->PushSourceSinkSettings(); + } if (codec.maxBitrate == 0) { // max is one bit per pixel @@ -1731,10 +1743,8 @@ void VideoStreamEncoder::TriggerAdaptUp( void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions) { RTC_DCHECK_RUN_ON(&encoder_queue_); - // TODO(hbos): Move logic for reconfiguring the video source from the resource - // adaptation module to here. - resource_adaptation_module_->ApplyVideoSourceRestrictions( - std::move(restrictions)); + video_source_sink_controller_->SetRestrictions(std::move(restrictions)); + video_source_sink_controller_->PushSourceSinkSettings(); } void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image, @@ -1995,9 +2005,10 @@ void VideoStreamEncoder::CheckForAnimatedContent( RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent " "animation detection."; } - resource_adaptation_module_->SetSourceMaxPixels( - should_cap_resolution ? kMaxAnimationPixels - : std::numeric_limits::max()); + video_source_sink_controller_->SetPixelsPerFrameUpperLimit( + should_cap_resolution ? absl::optional(kMaxAnimationPixels) + : absl::nullopt); + video_source_sink_controller_->PushSourceSinkSettings(); } } diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 4d4b079715..6db3d53da0 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -27,6 +27,7 @@ #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "call/adaptation/resource_adaptation_module_interface.h" +#include "call/adaptation/video_source_restrictions.h" #include "modules/video_coding/utility/frame_dropper.h" #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/critical_section.h" @@ -43,6 +44,7 @@ #include "video/encoder_bitrate_adjuster.h" #include "video/frame_encode_metadata_writer.h" #include "video/overuse_frame_detector_resource_adaptation_module.h" +#include "video/video_source_sink_controller.h" namespace webrtc { @@ -406,6 +408,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // track of whether a request has been made or not. bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_); + std::unique_ptr video_source_sink_controller_; std::unique_ptr resource_adaptation_module_;