From 382cc6d8a6c3e1887de28e948d4dd57922fcfe01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Tue, 7 Jan 2020 10:15:04 +0100 Subject: [PATCH] Add incomplete ResourceAdaptationModuleInterface. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This interface will be improved upon iteratively to aid reviewability. The initial version only handles starting and stopping the module; input and output of the module is still implementation-specific. TBR=sprang@webrtc.org Bug: webrtc:11222 Change-Id: Ie307cfe3d3211c84346c035f2c0e9a632f58221b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/162580 Commit-Queue: Henrik Boström Reviewed-by: Evan Shrubsole Cr-Commit-Position: refs/heads/master@{#30167} --- call/adaptation/BUILD.gn | 2 + .../resource_adaptation_module_interface.cc | 17 +++++++ .../resource_adaptation_module_interface.h | 44 ++++++++++++++++++ video/BUILD.gn | 1 + ...ame_detector_resource_adaptation_module.cc | 9 ++-- ...rame_detector_resource_adaptation_module.h | 17 +++++-- video/video_stream_encoder.cc | 45 +++++++++---------- video/video_stream_encoder.h | 4 ++ 8 files changed, 110 insertions(+), 29 deletions(-) create mode 100644 call/adaptation/resource_adaptation_module_interface.cc create mode 100644 call/adaptation/resource_adaptation_module_interface.h diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index 12d04a18b3..a51f93015d 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -12,6 +12,8 @@ rtc_library("resource_adaptation") { sources = [ "resource.cc", "resource.h", + "resource_adaptation_module_interface.cc", + "resource_adaptation_module_interface.h", "resource_adaptation_processor.cc", "resource_adaptation_processor.h", "resource_consumer.cc", diff --git a/call/adaptation/resource_adaptation_module_interface.cc b/call/adaptation/resource_adaptation_module_interface.cc new file mode 100644 index 0000000000..887fa24c5d --- /dev/null +++ b/call/adaptation/resource_adaptation_module_interface.cc @@ -0,0 +1,17 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/resource_adaptation_module_interface.h" + +namespace webrtc { + +ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {} + +} // namespace webrtc diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h new file mode 100644 index 0000000000..0834d08321 --- /dev/null +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -0,0 +1,44 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ +#define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ + +namespace webrtc { + +// Responsible for reconfiguring encoded streams based on resource consumption, +// such as scaling down resolution or frame rate when CPU is overused. This +// interface is meant to be injectable into VideoStreamEncoder. +// +// [UNDER CONSTRUCTION] This interface is work-in-progress. In the future it +// needs to be able to handle all the necessary input and output for resource +// adaptation decision making. +// +// TODO(https://crbug.com/webrtc/11222): Make this interface feature-complete so +// that a module (such as OveruseFrameDetectorResourceAdaptationModule) is fully +// operational through this abstract interface. +class ResourceAdaptationModuleInterface { + public: + virtual ~ResourceAdaptationModuleInterface(); + + // TODO(hbos): When input/output of the module is adequetly handled by this + // interface, these methods need to say which stream to start/stop, enabling + // multi-stream aware implementations of ResourceAdaptationModuleInterface. We + // don't want to do this before we have the right interfaces (e.g. if we pass + // in a VideoStreamEncoder here directly then have a dependency on a different + // build target). For the multi-stream use case we may consider making + // ResourceAdaptationModuleInterface reference counted. + virtual void StartCheckForOveruse() = 0; + virtual void StopCheckForOveruse() = 0; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ diff --git a/video/BUILD.gn b/video/BUILD.gn index 51cd40bdf7..8cfbc94efb 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -203,6 +203,7 @@ rtc_library("video_stream_encoder_impl") { "../api/video:video_rtp_headers", "../api/video:video_stream_encoder", "../api/video_codecs:video_codecs_api", + "../call/adaptation:resource_adaptation", "../common_video", "../modules:module_api_public", "../modules/video_coding", diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index b5c61b2818..2e840380c6 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -423,10 +423,12 @@ OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ToString( OveruseFrameDetectorResourceAdaptationModule:: OveruseFrameDetectorResourceAdaptationModule( + VideoStreamEncoder* video_stream_encoder, rtc::VideoSinkInterface* sink, std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer) : encoder_queue_(nullptr), + video_stream_encoder_(video_stream_encoder), degradation_preference_(DegradationPreference::DISABLED), adapt_counters_(), balanced_settings_(), @@ -440,6 +442,7 @@ OveruseFrameDetectorResourceAdaptationModule:: encoder_config_(), encoder_(nullptr), encoder_stats_observer_(encoder_stats_observer) { + RTC_DCHECK(video_stream_encoder_); RTC_DCHECK(overuse_detector_); RTC_DCHECK(encoder_stats_observer_); } @@ -461,12 +464,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoder( encoder_ = encoder; } -void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse( - const CpuOveruseOptions& options) { +void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse() { RTC_DCHECK(encoder_queue_); RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(encoder_); - overuse_detector_->StartCheckForOveruse(encoder_queue_, options, this); + overuse_detector_->StartCheckForOveruse( + encoder_queue_, video_stream_encoder_->GetCpuOveruseOptions(), this); } void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() { diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index d13a6a710e..322677b6f3 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -25,11 +25,14 @@ #include "api/video/video_stream_encoder_observer.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_config.h" +#include "call/adaptation/resource_adaptation_module_interface.h" #include "rtc_base/experiments/balanced_degradation_settings.h" #include "video/overuse_frame_detector.h" namespace webrtc { +class VideoStreamEncoder; + // This class is used by the VideoStreamEncoder and is responsible for adapting // resolution up or down based on encode usage percent. It keeps track of video // source settings, adaptation counters and may get influenced by @@ -43,9 +46,11 @@ namespace webrtc { // generic interface in VideoStreamEncoder, unblocking other modules from being // implemented and used. class OveruseFrameDetectorResourceAdaptationModule - : public AdaptationObserverInterface { + : public ResourceAdaptationModuleInterface, + public AdaptationObserverInterface { public: OveruseFrameDetectorResourceAdaptationModule( + VideoStreamEncoder* video_stream_encoder, rtc::VideoSinkInterface* sink, std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer); @@ -64,13 +69,15 @@ class OveruseFrameDetectorResourceAdaptationModule return degradation_preference_; } + // ResourceAdaptationModuleInterface implementation. + void StartCheckForOveruse() override; + void StopCheckForOveruse() override; + // Input to the OveruseFrameDetector, which are required for this module to // function. These map to OveruseFrameDetector methods. // TODO(hbos): Define virtual methods in ResourceAdaptationModuleInterface // for input that are more generic so that this class can be used without // assumptions about underlying implementation. - void StartCheckForOveruse(const CpuOveruseOptions& options); - void StopCheckForOveruse(); void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); void FrameSent(uint32_t timestamp, int64_t time_sent_in_us, @@ -187,7 +194,11 @@ class OveruseFrameDetectorResourceAdaptationModule bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const RTC_RUN_ON(encoder_queue_); + // TODO(hbos): Can we move the |source_proxy_| to the |encoder_queue_| and + // replace |encoder_queue_| with a sequence checker instead? rtc::TaskQueue* encoder_queue_; + // Used to query CpuOveruseOptions at StartCheckForOveruse(). + VideoStreamEncoder* video_stream_encoder_ RTC_GUARDED_BY(encoder_queue_); DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); // Counters used for deciding if the video resolution or framerate is // currently restricted, and if so, why, on a per degradation preference diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index ccf00deb4e..d2382b063d 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -84,26 +84,6 @@ bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { degradation_preference == DegradationPreference::BALANCED; } -// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle -// pipelining encoders better (multiple input frames before something comes -// out). This should effectively turn off CPU adaptations for systems that -// remotely cope with the load right now. -CpuOveruseOptions GetCpuOveruseOptions( - const VideoStreamEncoderSettings& settings, - bool full_overuse_time) { - CpuOveruseOptions options; - - if (full_overuse_time) { - options.low_encode_usage_threshold_percent = 150; - options.high_encode_usage_threshold_percent = 200; - } - if (settings.experiment_cpu_load_estimator) { - options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec; - } - - return options; -} - bool RequiresEncoderReset(const VideoCodec& prev_send_codec, const VideoCodec& new_send_codec, bool was_encode_called_since_last_initialization) { @@ -340,7 +320,8 @@ VideoStreamEncoder::VideoStreamEncoder( encoder_switch_requested_(false), resource_adaptation_module_( std::make_unique( - this, + /*video_stream_encoder=*/this, + /*sink=*/this, std::move(overuse_detector), encoder_stats_observer)), encoder_queue_(task_queue_factory->CreateTaskQueue( @@ -703,8 +684,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { if (pending_encoder_creation_) { resource_adaptation_module_->StopCheckForOveruse(); - resource_adaptation_module_->StartCheckForOveruse(GetCpuOveruseOptions( - settings_, encoder_->GetEncoderInfo().is_hardware_accelerated)); + resource_adaptation_module_->StartCheckForOveruse(); pending_encoder_creation_ = false; } @@ -1718,6 +1698,25 @@ bool VideoStreamEncoder::TryQualityRampup(int64_t now_ms) { return false; } +// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle +// pipelining encoders better (multiple input frames before something comes +// out). This should effectively turn off CPU adaptations for systems that +// remotely cope with the load right now. +CpuOveruseOptions VideoStreamEncoder::GetCpuOveruseOptions() const { + RTC_DCHECK_RUN_ON(&encoder_queue_); + CpuOveruseOptions options; + // Hardware accelerated encoders are assumed to be pipelined; give them + // additional overuse time. + if (encoder_->GetEncoderInfo().is_hardware_accelerated) { + options.low_encode_usage_threshold_percent = 150; + options.high_encode_usage_threshold_percent = 200; + } + if (settings_.experiment_cpu_load_estimator) { + options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec; + } + return options; +} + bool VideoStreamEncoder::TriggerAdaptDown( AdaptationObserverInterface::AdaptReason reason) { return resource_adaptation_module_->AdaptDown(reason); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index f180688a67..e3a063ff2f 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -100,6 +100,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, uint8_t fraction_lost, int64_t round_trip_time_ms) override; + // If an OveruseFrameDetectorResourceAdaptationModule is used, this method is + // used by the module to configure its OveruseFrameDetector. + CpuOveruseOptions GetCpuOveruseOptions() const; + protected: // Used for testing. For example the |ScalingObserverInterface| methods must // be called on |encoder_queue_|.