diff --git a/call/adaptation/resource_adaptation_module_interface.cc b/call/adaptation/resource_adaptation_module_interface.cc index 887fa24c5d..941278bdac 100644 --- a/call/adaptation/resource_adaptation_module_interface.cc +++ b/call/adaptation/resource_adaptation_module_interface.cc @@ -10,8 +10,41 @@ #include "call/adaptation/resource_adaptation_module_interface.h" +#include "rtc_base/checks.h" + namespace webrtc { +VideoSourceRestrictions::VideoSourceRestrictions( + absl::optional max_pixels_per_frame, + absl::optional target_pixels_per_frame, + absl::optional max_frame_rate) + : max_pixels_per_frame_(std::move(max_pixels_per_frame)), + target_pixels_per_frame_(std::move(target_pixels_per_frame)), + max_frame_rate_(std::move(max_frame_rate)) { + RTC_DCHECK(!max_pixels_per_frame_.has_value() || + max_pixels_per_frame_.value() < + static_cast(std::numeric_limits::max())); + RTC_DCHECK(!max_frame_rate_.has_value() || + max_frame_rate_.value() < std::numeric_limits::max()); + RTC_DCHECK(!max_frame_rate_.has_value() || max_frame_rate_.value() > 0.0); +} + +const absl::optional& VideoSourceRestrictions::max_pixels_per_frame() + const { + return max_pixels_per_frame_; +} + +const absl::optional& VideoSourceRestrictions::target_pixels_per_frame() + const { + return target_pixels_per_frame_; +} + +const absl::optional& VideoSourceRestrictions::max_frame_rate() const { + return max_frame_rate_; +} + +ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {} + ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {} } // namespace webrtc diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h index 0834d08321..929011ac56 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -11,8 +11,49 @@ #ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ #define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ +#include +#include + +#include "absl/types/optional.h" + namespace webrtc { +// Describes optional restrictions to the resolution and frame rate of a video +// source. +class VideoSourceRestrictions { + public: + // All values must be positive or nullopt. + // TODO(hbos): Support expressing "disable this stream"? + VideoSourceRestrictions(absl::optional max_pixels_per_frame, + absl::optional target_pixels_per_frame, + absl::optional max_frame_rate); + + const absl::optional& max_pixels_per_frame() const; + const absl::optional& target_pixels_per_frame() const; + const absl::optional& max_frame_rate() const; + + private: + // These map to rtc::VideoSinkWants's |max_pixel_count| and + // |target_pixel_count|. + // TODO(hbos): It's not clear what "target" means; either make it well-defined + // or remove it in favor of only using |max_pixels_per_frame_|. + absl::optional max_pixels_per_frame_; + absl::optional target_pixels_per_frame_; + absl::optional max_frame_rate_; +}; + +// The listener is responsible for carrying out the reconfiguration of the video +// source such that the VideoSourceRestrictions are fulfilled. +class ResourceAdaptationModuleListener { + public: + virtual ~ResourceAdaptationModuleListener(); + + // TODO(hbos): When we support the muli-stream use case, the arguments need to + // specify which video stream's source needs to be reconfigured. + virtual void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions) = 0; +}; + // Responsible for reconfiguring encoded streams based on resource consumption, // such as scaling down resolution or frame rate when CPU is overused. This // interface is meant to be injectable into VideoStreamEncoder. @@ -35,7 +76,8 @@ class ResourceAdaptationModuleInterface { // in a VideoStreamEncoder here directly then have a dependency on a different // build target). For the multi-stream use case we may consider making // ResourceAdaptationModuleInterface reference counted. - virtual void StartCheckForOveruse() = 0; + virtual void StartCheckForOveruse( + ResourceAdaptationModuleListener* adaptation_listener) = 0; virtual void StopCheckForOveruse() = 0; }; diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index afb4d6fb84..9b3cbf6b36 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -19,6 +19,7 @@ #include "absl/algorithm/container.h" #include "api/video/video_source_interface.h" #include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/strings/string_builder.h" #include "rtc_base/system/fallthrough.h" #include "video/video_stream_encoder.h" @@ -39,6 +40,47 @@ bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { degradation_preference == DegradationPreference::BALANCED; } +// Constructs VideoSourceRestrictions from |target_pixel_count|, +// |max_pixel_count| and |max_framerate_fps|. Other rtc::VideoSinkWants +// information such as |rotation_applied| is lost in the conversion. +VideoSourceRestrictions VideoSinkWantsToVideoSourceRestrictions( + rtc::VideoSinkWants active_sink_wants) { + return VideoSourceRestrictions( + active_sink_wants.max_pixel_count != std::numeric_limits::max() + ? absl::optional(active_sink_wants.max_pixel_count) + : absl::nullopt, + active_sink_wants.target_pixel_count.has_value() + ? absl::optional(rtc::dchecked_cast( + active_sink_wants.target_pixel_count.value())) + : absl::nullopt, + active_sink_wants.max_framerate_fps != std::numeric_limits::max() + ? absl::optional(active_sink_wants.max_framerate_fps) + : absl::nullopt); +} + +// Constructs rtc::VideoSinkWants from max_pixels_per_frame(), +// target_pixels_per_frame() and max_frame_rate(). The rest of the members, such +// as |rotation_applied|, are obtained from the |baseline_sink_wants|. +rtc::VideoSinkWants VideoSourceRestrictionsToVideoSinkWants( + const rtc::VideoSinkWants& baseline_sink_wants, + VideoSourceRestrictions restrictions) { + rtc::VideoSinkWants sink_wants = baseline_sink_wants; + sink_wants.max_pixel_count = + restrictions.max_pixels_per_frame().has_value() + ? static_cast(restrictions.max_pixels_per_frame().value()) + : std::numeric_limits::max(); + sink_wants.target_pixel_count = + restrictions.target_pixels_per_frame().has_value() + ? absl::optional(rtc::dchecked_cast( + restrictions.target_pixels_per_frame().value())) + : absl::nullopt; + sink_wants.max_framerate_fps = + restrictions.max_frame_rate().has_value() + ? static_cast(restrictions.max_frame_rate().value()) + : std::numeric_limits::max(); + return sink_wants; +} + } // namespace // VideoSourceProxy is responsible ensuring thread safety between calls to @@ -56,6 +98,21 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { max_pixels_(std::numeric_limits::max()), resolution_alignment_(1) {} + VideoSourceRestrictions ToVideoSourceRestrictions() { + return VideoSinkWantsToVideoSourceRestrictions(GetActiveSinkWants()); + } + + void ApplyVideoSourceRestrictions(VideoSourceRestrictions restrictions) { + rtc::CritScope lock(&crit_); + rtc::VideoSinkWants wants = VideoSourceRestrictionsToVideoSinkWants( + GetActiveSinkWantsInternal(), std::move(restrictions)); + if (!source_) + return; + source_->AddOrUpdateSink(sink_, wants); + } + + // Informs the sink of the new source settings. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. void SetSource(rtc::VideoSourceInterface* source, const DegradationPreference& degradation_preference) { // Called on libjingle's worker thread. @@ -81,6 +138,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { source->AddOrUpdateSink(sink_, wants); } + // Informs the sink of the new source settings. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. void SetMaxFramerateAndAlignment(int max_framerate, int resolution_alignment) { RTC_DCHECK_GT(max_framerate, 0); @@ -99,6 +158,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { } } + // Informs the sink of the new source settings. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. void SetWantsRotationApplied(bool rotation_applied) { rtc::CritScope lock(&crit_); sink_wants_.rotation_applied = rotation_applied; @@ -112,6 +173,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { return GetActiveSinkWantsInternal(); } + // Informs the sink of the new source settings. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. void ResetPixelFpsCount() { rtc::CritScope lock(&crit_); sink_wants_.max_pixel_count = std::numeric_limits::max(); @@ -121,6 +184,11 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); } + // Updates the sink settings, but DOES NOT inform the sink of the new + // settings. Reapplying video source restrictions is required, see + // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more + // information. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. bool RequestResolutionLowerThan(int pixel_count, int min_pixels_per_frame, bool* min_pixels_reached) { @@ -145,10 +213,14 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { << pixels_wanted; sink_wants_.max_pixel_count = pixels_wanted; sink_wants_.target_pixel_count = absl::nullopt; - source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); return true; } + // Updates the sink settings, but DOES NOT inform the sink of the new + // settings. Reapplying video source restrictions is required, see + // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more + // information. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. int RequestFramerateLowerThan(int fps) { // Called on the encoder task queue. // The input video frame rate will be scaled down to 2/3, rounding down. @@ -166,6 +238,11 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { return (pixel_count * 5) / 3; } + // Updates the sink settings, but DOES NOT inform the sink of the new + // settings. Reapplying video source restrictions is required, see + // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more + // information. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. bool RequestHigherResolutionThan(int pixel_count) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); @@ -190,7 +267,6 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { } RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; - source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); return true; } @@ -208,6 +284,11 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; } + // Updates the sink settings, but DOES NOT inform the sink of the new + // settings. Reapplying video source restrictions is required, see + // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more + // information. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. bool RestrictFramerate(int fps) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); @@ -220,10 +301,14 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; sink_wants_.max_framerate_fps = fps_wanted; - source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); return true; } + // Updates the sink settings, but DOES NOT inform the sink of the new + // settings. Reapplying video source restrictions is required, see + // ToVideoSourceRestrictions() and ApplyVideoSourceRestrictions() for more + // information. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. bool IncreaseFramerate(int fps) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); @@ -236,11 +321,12 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; sink_wants_.max_framerate_fps = fps_wanted; - source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); return true; } + // Informs the sink of the new source settings. // Used in automatic animation detection for screenshare. + // TODO(hbos): Handle all sink updates in video_stream_encoder.cc. bool RestrictPixels(int max_pixels) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); @@ -426,8 +512,10 @@ OveruseFrameDetectorResourceAdaptationModule:: VideoStreamEncoder* video_stream_encoder, rtc::VideoSinkInterface* sink, std::unique_ptr overuse_detector, - VideoStreamEncoderObserver* encoder_stats_observer) + VideoStreamEncoderObserver* encoder_stats_observer, + ResourceAdaptationModuleListener* adaptation_listener) : encoder_queue_(nullptr), + adaptation_listener_(adaptation_listener), video_stream_encoder_(video_stream_encoder), degradation_preference_(DegradationPreference::DISABLED), adapt_counters_(), @@ -442,6 +530,7 @@ OveruseFrameDetectorResourceAdaptationModule:: encoder_config_(), encoder_(nullptr), encoder_stats_observer_(encoder_stats_observer) { + RTC_DCHECK(adaptation_listener_); RTC_DCHECK(video_stream_encoder_); RTC_DCHECK(overuse_detector_); RTC_DCHECK(encoder_stats_observer_); @@ -464,10 +553,18 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoder( encoder_ = encoder; } -void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse() { +void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse( + ResourceAdaptationModuleListener* adaptation_listener) { RTC_DCHECK(encoder_queue_); RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(encoder_); + // TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside + // the interval between StartCheckForOveruse() and StopCheckForOveruse(), + // support configuring which |adaptation_listener_| to use on the fly. It is + // currently hardcoded for the entire lifetime of the module in order to + // support adaptation caused by VideoStreamEncoder or QualityScaler invoking + // AdaptUp() and AdaptDown() even when the OveruseDetector is inactive. + RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); overuse_detector_->StartCheckForOveruse( encoder_queue_, video_stream_encoder_->GetCpuOveruseOptions(), this); } @@ -478,6 +575,13 @@ void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() { overuse_detector_->StopCheckForOveruse(); } +void OveruseFrameDetectorResourceAdaptationModule::ApplyVideoSourceRestrictions( + VideoSourceRestrictions restrictions) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + source_proxy_->ApplyVideoSourceRestrictions(std::move(restrictions)); +} + void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured( const VideoFrame& frame, int64_t time_when_first_seen_us) { @@ -700,6 +804,11 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { return; } + // Tell the adaptation listener to reconfigure the source for us according to + // the latest adaptation. + adaptation_listener_->OnVideoSourceRestrictionsUpdated( + source_proxy_->ToVideoSourceRestrictions()); + last_adaptation_request_.emplace(adaptation_request); UpdateAdaptationStats(reason); @@ -801,6 +910,11 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( RTC_NOTREACHED(); } + // Tell the adaptation listener to reconfigure the source for us according to + // the latest adaptation. + adaptation_listener_->OnVideoSourceRestrictionsUpdated( + source_proxy_->ToVideoSourceRestrictions()); + last_adaptation_request_.emplace(adaptation_request); UpdateAdaptationStats(reason); diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 322677b6f3..21e44a2d59 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -53,7 +53,8 @@ class OveruseFrameDetectorResourceAdaptationModule VideoStreamEncoder* video_stream_encoder, rtc::VideoSinkInterface* sink, std::unique_ptr overuse_detector, - VideoStreamEncoderObserver* encoder_stats_observer); + VideoStreamEncoderObserver* encoder_stats_observer, + ResourceAdaptationModuleListener* adaptation_listener); ~OveruseFrameDetectorResourceAdaptationModule() override; void Initialize(rtc::TaskQueue* encoder_queue); @@ -70,9 +71,14 @@ class OveruseFrameDetectorResourceAdaptationModule } // ResourceAdaptationModuleInterface implementation. - void StartCheckForOveruse() override; + void StartCheckForOveruse( + ResourceAdaptationModuleListener* adaptation_listener) override; void StopCheckForOveruse() override; + // TODO(hbos): When VideoSourceProxy is refactored and reconfiguration logic + // is entirely moved to video_stream_encoder.cc, remove this method. + void ApplyVideoSourceRestrictions(VideoSourceRestrictions restrictions); + // Input to the OveruseFrameDetector, which are required for this module to // function. These map to OveruseFrameDetector methods. // TODO(hbos): Define virtual methods in ResourceAdaptationModuleInterface @@ -197,6 +203,8 @@ class OveruseFrameDetectorResourceAdaptationModule // TODO(hbos): Can we move the |source_proxy_| to the |encoder_queue_| and // replace |encoder_queue_| with a sequence checker instead? rtc::TaskQueue* encoder_queue_; + ResourceAdaptationModuleListener* const adaptation_listener_ + RTC_GUARDED_BY(encoder_queue_); // Used to query CpuOveruseOptions at StartCheckForOveruse(). VideoStreamEncoder* video_stream_encoder_ RTC_GUARDED_BY(encoder_queue_); DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index d2382b063d..27aaa93278 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -323,7 +323,8 @@ VideoStreamEncoder::VideoStreamEncoder( /*video_stream_encoder=*/this, /*sink=*/this, std::move(overuse_detector), - encoder_stats_observer)), + encoder_stats_observer, + /*adaptation_listener=*/this)), encoder_queue_(task_queue_factory->CreateTaskQueue( "EncoderQueue", TaskQueueFactory::Priority::NORMAL)) { @@ -684,7 +685,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { if (pending_encoder_creation_) { resource_adaptation_module_->StopCheckForOveruse(); - resource_adaptation_module_->StartCheckForOveruse(); + resource_adaptation_module_->StartCheckForOveruse(this); pending_encoder_creation_ = false; } @@ -1727,6 +1728,15 @@ void VideoStreamEncoder::TriggerAdaptUp( resource_adaptation_module_->AdaptUp(reason); } +void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions) { + RTC_DCHECK_RUN_ON(&encoder_queue_); + // TODO(hbos): Move logic for reconfiguring the video source from the resource + // adaptation module to here. + resource_adaptation_module_->ApplyVideoSourceRestrictions( + std::move(restrictions)); +} + void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image, int64_t time_sent_us, int temporal_index, diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index e3a063ff2f..4d4b079715 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -26,6 +26,7 @@ #include "api/video/video_stream_encoder_settings.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "call/adaptation/resource_adaptation_module_interface.h" #include "modules/video_coding/utility/frame_dropper.h" #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/critical_section.h" @@ -58,7 +59,8 @@ absl::optional GetEncoderBitrateLimits( // Call ConfigureEncoder with the codec settings. // Call Stop() when done. class VideoStreamEncoder : public VideoStreamEncoderInterface, - private EncodedImageCallback { + private EncodedImageCallback, + public ResourceAdaptationModuleListener { public: VideoStreamEncoder(Clock* clock, uint32_t number_of_cores, @@ -116,6 +118,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void TriggerAdaptUp(AdaptationObserverInterface::AdaptReason reason); bool TriggerAdaptDown(AdaptationObserverInterface::AdaptReason reason); + void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions) override; + private: class VideoFrameInfo { public: