diff --git a/video/BUILD.gn b/video/BUILD.gn index 0f9829e9e2..51cd40bdf7 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -184,11 +184,14 @@ rtc_library("video_stream_encoder_impl") { "frame_encode_metadata_writer.h", "overuse_frame_detector.cc", "overuse_frame_detector.h", + "overuse_frame_detector_resource_adaptation_module.cc", + "overuse_frame_detector_resource_adaptation_module.h", "video_stream_encoder.cc", "video_stream_encoder.h", ] deps = [ + "../api:rtp_parameters", "../api/units:data_rate", "../api/video:encoded_image", "../api/video:video_bitrate_allocation", diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc new file mode 100644 index 0000000000..b5c61b2818 --- /dev/null +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -0,0 +1,900 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/overuse_frame_detector_resource_adaptation_module.h" + +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "api/video/video_source_interface.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/system/fallthrough.h" +#include "video/video_stream_encoder.h" + +namespace webrtc { + +namespace { + +const int kMinFramerateFps = 2; + +bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { + return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE || + degradation_preference == DegradationPreference::BALANCED; +} + +bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { + return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION || + degradation_preference == DegradationPreference::BALANCED; +} + +} // namespace + +// VideoSourceProxy is responsible ensuring thread safety between calls to +// OveruseFrameDetectorResourceAdaptationModule::SetSource that will happen on +// libjingle's worker thread when a video capturer is connected to the encoder +// and the encoder task queue (encoder_queue_) where the encoder reports its +// VideoSinkWants. +class OveruseFrameDetectorResourceAdaptationModule::VideoSourceProxy { + public: + explicit VideoSourceProxy(rtc::VideoSinkInterface* sink) + : sink_(sink), + degradation_preference_(DegradationPreference::DISABLED), + source_(nullptr), + max_framerate_(std::numeric_limits::max()), + max_pixels_(std::numeric_limits::max()), + resolution_alignment_(1) {} + + void SetSource(rtc::VideoSourceInterface* source, + const DegradationPreference& degradation_preference) { + // Called on libjingle's worker thread. + RTC_DCHECK_RUN_ON(&main_checker_); + rtc::VideoSourceInterface* old_source = nullptr; + rtc::VideoSinkWants wants; + { + rtc::CritScope lock(&crit_); + degradation_preference_ = degradation_preference; + old_source = source_; + source_ = source; + wants = GetActiveSinkWantsInternal(); + } + + if (old_source != source && old_source != nullptr) { + old_source->RemoveSink(sink_); + } + + if (!source) { + return; + } + + source->AddOrUpdateSink(sink_, wants); + } + + void SetMaxFramerateAndAlignment(int max_framerate, + int resolution_alignment) { + RTC_DCHECK_GT(max_framerate, 0); + rtc::CritScope lock(&crit_); + if (max_framerate == max_framerate_ && + resolution_alignment == resolution_alignment_) { + return; + } + + RTC_LOG(LS_INFO) << "Set max framerate: " << max_framerate + << " and resolution alignment: " << resolution_alignment; + max_framerate_ = max_framerate; + resolution_alignment_ = resolution_alignment; + if (source_) { + source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + } + } + + void SetWantsRotationApplied(bool rotation_applied) { + rtc::CritScope lock(&crit_); + sink_wants_.rotation_applied = rotation_applied; + if (source_) { + source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + } + } + + rtc::VideoSinkWants GetActiveSinkWants() { + rtc::CritScope lock(&crit_); + return GetActiveSinkWantsInternal(); + } + + void ResetPixelFpsCount() { + rtc::CritScope lock(&crit_); + sink_wants_.max_pixel_count = std::numeric_limits::max(); + sink_wants_.target_pixel_count.reset(); + sink_wants_.max_framerate_fps = std::numeric_limits::max(); + if (source_) + source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + } + + bool RequestResolutionLowerThan(int pixel_count, + int min_pixels_per_frame, + bool* min_pixels_reached) { + // Called on the encoder task queue. + rtc::CritScope lock(&crit_); + if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { + // This can happen since |degradation_preference_| is set on libjingle's + // worker thread but the adaptation is done on the encoder task queue. + return false; + } + // The input video frame size will have a resolution less than or equal to + // |max_pixel_count| depending on how the source can scale the frame size. + const int pixels_wanted = (pixel_count * 3) / 5; + if (pixels_wanted >= sink_wants_.max_pixel_count) { + return false; + } + if (pixels_wanted < min_pixels_per_frame) { + *min_pixels_reached = true; + return false; + } + RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " + << pixels_wanted; + sink_wants_.max_pixel_count = pixels_wanted; + sink_wants_.target_pixel_count = absl::nullopt; + source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + return true; + } + + int RequestFramerateLowerThan(int fps) { + // Called on the encoder task queue. + // The input video frame rate will be scaled down to 2/3, rounding down. + int framerate_wanted = (fps * 2) / 3; + return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; + } + + int GetHigherResolutionThan(int pixel_count) const { + // On step down we request at most 3/5 the pixel count of the previous + // resolution, so in order to take "one step up" we request a resolution + // as close as possible to 5/3 of the current resolution. The actual pixel + // count selected depends on the capabilities of the source. In order to + // not take a too large step up, we cap the requested pixel count to be at + // most four time the current number of pixels. + return (pixel_count * 5) / 3; + } + + bool RequestHigherResolutionThan(int pixel_count) { + // Called on the encoder task queue. + rtc::CritScope lock(&crit_); + if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { + // This can happen since |degradation_preference_| is set on libjingle's + // worker thread but the adaptation is done on the encoder task queue. + return false; + } + int max_pixels_wanted = pixel_count; + if (max_pixels_wanted != std::numeric_limits::max()) + max_pixels_wanted = pixel_count * 4; + + if (max_pixels_wanted <= sink_wants_.max_pixel_count) + return false; + + sink_wants_.max_pixel_count = max_pixels_wanted; + if (max_pixels_wanted == std::numeric_limits::max()) { + // Remove any constraints. + sink_wants_.target_pixel_count.reset(); + } else { + sink_wants_.target_pixel_count = GetHigherResolutionThan(pixel_count); + } + RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " + << max_pixels_wanted; + source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + return true; + } + + // Request upgrade in framerate. Returns the new requested frame, or -1 if + // no change requested. Note that maxint may be returned if limits due to + // adaptation requests are removed completely. In that case, consider + // |max_framerate_| to be the current limit (assuming the capturer complies). + int RequestHigherFramerateThan(int fps) { + // Called on the encoder task queue. + // The input frame rate will be scaled up to the last step, with rounding. + int framerate_wanted = fps; + if (fps != std::numeric_limits::max()) + framerate_wanted = (fps * 3) / 2; + + return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; + } + + bool RestrictFramerate(int fps) { + // Called on the encoder task queue. + rtc::CritScope lock(&crit_); + if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) + return false; + + const int fps_wanted = std::max(kMinFramerateFps, fps); + if (fps_wanted >= sink_wants_.max_framerate_fps) + return false; + + RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; + sink_wants_.max_framerate_fps = fps_wanted; + source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + return true; + } + + bool IncreaseFramerate(int fps) { + // Called on the encoder task queue. + rtc::CritScope lock(&crit_); + if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) + return false; + + const int fps_wanted = std::max(kMinFramerateFps, fps); + if (fps_wanted <= sink_wants_.max_framerate_fps) + return false; + + RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; + sink_wants_.max_framerate_fps = fps_wanted; + source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + return true; + } + + // Used in automatic animation detection for screenshare. + bool RestrictPixels(int max_pixels) { + // Called on the encoder task queue. + rtc::CritScope lock(&crit_); + if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { + // This can happen since |degradation_preference_| is set on libjingle's + // worker thread but the adaptation is done on the encoder task queue. + return false; + } + max_pixels_ = max_pixels; + RTC_LOG(LS_INFO) << "Applying max pixel restriction: " << max_pixels; + source_->AddOrUpdateSink(sink_, GetActiveSinkWantsInternal()); + return true; + } + + private: + rtc::VideoSinkWants GetActiveSinkWantsInternal() + RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) { + rtc::VideoSinkWants wants = sink_wants_; + // Clear any constraints from the current sink wants that don't apply to + // the used degradation_preference. + switch (degradation_preference_) { + case DegradationPreference::BALANCED: + break; + case DegradationPreference::MAINTAIN_FRAMERATE: + wants.max_framerate_fps = std::numeric_limits::max(); + break; + case DegradationPreference::MAINTAIN_RESOLUTION: + wants.max_pixel_count = std::numeric_limits::max(); + wants.target_pixel_count.reset(); + break; + case DegradationPreference::DISABLED: + wants.max_pixel_count = std::numeric_limits::max(); + wants.target_pixel_count.reset(); + wants.max_framerate_fps = std::numeric_limits::max(); + } + // Limit to configured max framerate. + wants.max_framerate_fps = std::min(max_framerate_, wants.max_framerate_fps); + // Limit resolution due to automatic animation detection for screenshare. + wants.max_pixel_count = std::min(max_pixels_, wants.max_pixel_count); + wants.resolution_alignment = resolution_alignment_; + + return wants; + } + + rtc::CriticalSection crit_; + SequenceChecker main_checker_; + rtc::VideoSinkInterface* const sink_; + rtc::VideoSinkWants sink_wants_ RTC_GUARDED_BY(&crit_); + DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_); + rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(&crit_); + int max_framerate_ RTC_GUARDED_BY(&crit_); + int max_pixels_ RTC_GUARDED_BY(&crit_); + int resolution_alignment_ RTC_GUARDED_BY(&crit_); + + RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy); +}; + +// Class holding adaptation information. +OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::AdaptCounter() { + fps_counters_.resize(kScaleReasonSize); + resolution_counters_.resize(kScaleReasonSize); + static_assert(kScaleReasonSize == 2, "Update MoveCount."); +} + +OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::~AdaptCounter() {} + +std::string +OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ToString() const { + rtc::StringBuilder ss; + ss << "Downgrade counts: fps: {" << ToString(fps_counters_); + ss << "}, resolution: {" << ToString(resolution_counters_) << "}"; + return ss.Release(); +} + +VideoStreamEncoderObserver::AdaptationSteps +OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::Counts( + int reason) const { + VideoStreamEncoderObserver::AdaptationSteps counts; + counts.num_framerate_reductions = fps_counters_[reason]; + counts.num_resolution_reductions = resolution_counters_[reason]; + return counts; +} + +void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter:: + IncrementFramerate(int reason) { + ++(fps_counters_[reason]); +} + +void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter:: + IncrementResolution(int reason) { + ++(resolution_counters_[reason]); +} + +void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter:: + DecrementFramerate(int reason) { + if (fps_counters_[reason] == 0) { + // Balanced mode: Adapt up is in a different order, switch reason. + // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3). + // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0} + // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0} + // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0} + // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0} + RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; + RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; + MoveCount(&resolution_counters_, reason); + MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize); + } + --(fps_counters_[reason]); + RTC_DCHECK_GE(fps_counters_[reason], 0); +} + +void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter:: + DecrementResolution(int reason) { + if (resolution_counters_[reason] == 0) { + // Balanced mode: Adapt up is in a different order, switch reason. + RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; + RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; + MoveCount(&fps_counters_, reason); + MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize); + } + --(resolution_counters_[reason]); + RTC_DCHECK_GE(resolution_counters_[reason], 0); +} + +void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter:: + DecrementFramerate(int reason, int cur_fps) { + DecrementFramerate(reason); + // Reset if at max fps (i.e. in case of fewer steps up than down). + if (cur_fps == std::numeric_limits::max()) + absl::c_fill(fps_counters_, 0); +} + +int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::FramerateCount() + const { + return Count(fps_counters_); +} + +int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter:: + ResolutionCount() const { + return Count(resolution_counters_); +} + +int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::FramerateCount( + int reason) const { + return fps_counters_[reason]; +} + +int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ResolutionCount( + int reason) const { + return resolution_counters_[reason]; +} + +int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::TotalCount( + int reason) const { + return FramerateCount(reason) + ResolutionCount(reason); +} + +int OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::Count( + const std::vector& counters) const { + return absl::c_accumulate(counters, 0); +} + +void OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::MoveCount( + std::vector* counters, + int from_reason) { + int to_reason = (from_reason + 1) % kScaleReasonSize; + ++((*counters)[to_reason]); + --((*counters)[from_reason]); +} + +std::string +OveruseFrameDetectorResourceAdaptationModule::AdaptCounter::ToString( + const std::vector& counters) const { + rtc::StringBuilder ss; + for (size_t reason = 0; reason < kScaleReasonSize; ++reason) { + ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; + } + return ss.Release(); +} + +OveruseFrameDetectorResourceAdaptationModule:: + OveruseFrameDetectorResourceAdaptationModule( + rtc::VideoSinkInterface* sink, + std::unique_ptr overuse_detector, + VideoStreamEncoderObserver* encoder_stats_observer) + : encoder_queue_(nullptr), + degradation_preference_(DegradationPreference::DISABLED), + adapt_counters_(), + balanced_settings_(), + last_adaptation_request_(absl::nullopt), + last_frame_pixel_count_(absl::nullopt), + source_proxy_(std::make_unique(sink)), + overuse_detector_(std::move(overuse_detector)), + codec_max_framerate_(-1), + encoder_start_bitrate_bps_(0), + is_quality_scaler_enabled_(false), + encoder_config_(), + encoder_(nullptr), + encoder_stats_observer_(encoder_stats_observer) { + RTC_DCHECK(overuse_detector_); + RTC_DCHECK(encoder_stats_observer_); +} + +OveruseFrameDetectorResourceAdaptationModule:: + ~OveruseFrameDetectorResourceAdaptationModule() {} + +void OveruseFrameDetectorResourceAdaptationModule::Initialize( + rtc::TaskQueue* encoder_queue) { + RTC_DCHECK(!encoder_queue_); + encoder_queue_ = encoder_queue; + RTC_DCHECK(encoder_queue_); +} + +void OveruseFrameDetectorResourceAdaptationModule::SetEncoder( + VideoEncoder* encoder) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + encoder_ = encoder; +} + +void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse( + const CpuOveruseOptions& options) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK(encoder_); + overuse_detector_->StartCheckForOveruse(encoder_queue_, options, this); +} + +void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + overuse_detector_->StopCheckForOveruse(); +} + +void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured( + const VideoFrame& frame, + int64_t time_when_first_seen_us) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + overuse_detector_->FrameCaptured(frame, time_when_first_seen_us); +} + +void OveruseFrameDetectorResourceAdaptationModule::FrameSent( + uint32_t timestamp, + int64_t time_sent_in_us, + int64_t capture_time_us, + absl::optional encode_duration_us) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, + encode_duration_us); +} + +void OveruseFrameDetectorResourceAdaptationModule::SetLastFramePixelCount( + absl::optional last_frame_pixel_count) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + last_frame_pixel_count_ = last_frame_pixel_count; +} + +void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig( + VideoEncoderConfig encoder_config) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + encoder_config_ = std::move(encoder_config); +} + +void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFramerate( + int codec_max_framerate) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + codec_max_framerate_ = codec_max_framerate; +} + +void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps( + uint32_t encoder_start_bitrate_bps) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + encoder_start_bitrate_bps_ = encoder_start_bitrate_bps; +} + +void OveruseFrameDetectorResourceAdaptationModule::SetIsQualityScalerEnabled( + bool is_quality_scaler_enabled) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + is_quality_scaler_enabled_ = is_quality_scaler_enabled; +} + +void OveruseFrameDetectorResourceAdaptationModule::SetSource( + rtc::VideoSourceInterface* source, + const DegradationPreference& degradation_preference) { + source_proxy_->SetSource(source, degradation_preference); + encoder_queue_->PostTask([this, degradation_preference] { + RTC_DCHECK_RUN_ON(encoder_queue_); + if (degradation_preference_ != degradation_preference) { + // Reset adaptation state, so that we're not tricked into thinking there's + // an already pending request of the same type. + last_adaptation_request_.reset(); + if (degradation_preference == DegradationPreference::BALANCED || + degradation_preference_ == DegradationPreference::BALANCED) { + // TODO(asapersson): Consider removing |adapt_counters_| map and use one + // AdaptCounter for all modes. + source_proxy_->ResetPixelFpsCount(); + adapt_counters_.clear(); + } + } + degradation_preference_ = degradation_preference; + }); +} + +void OveruseFrameDetectorResourceAdaptationModule:: + SetSourceWantsRotationApplied(bool rotation_applied) { + source_proxy_->SetWantsRotationApplied(rotation_applied); +} + +void OveruseFrameDetectorResourceAdaptationModule:: + SetSourceMaxFramerateAndAlignment(int max_framerate, + int resolution_alignment) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + source_proxy_->SetMaxFramerateAndAlignment(max_framerate, + resolution_alignment); +} + +void OveruseFrameDetectorResourceAdaptationModule::SetSourceMaxPixels( + int max_pixels) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + source_proxy_->RestrictPixels(max_pixels); +} + +void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + // Get the current target framerate, ie the maximum framerate as specified by + // the current codec configuration, or any limit imposed by cpu adaption in + // maintain-resolution or balanced mode. This is used to make sure overuse + // detection doesn't needlessly trigger in low and/or variable framerate + // scenarios. + int target_framerate = + std::min(codec_max_framerate_, + source_proxy_->GetActiveSinkWants().max_framerate_fps); + overuse_detector_->OnTargetFramerateUpdated(target_framerate); +} + +void OveruseFrameDetectorResourceAdaptationModule::ResetAdaptationCounters() { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + last_adaptation_request_.reset(); + source_proxy_->ResetPixelFpsCount(); + adapt_counters_.clear(); +} + +void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + const AdaptCounter& adapt_counter = GetConstAdaptCounter(); + int num_downgrades = adapt_counter.TotalCount(reason); + if (num_downgrades == 0) + return; + RTC_DCHECK_GT(num_downgrades, 0); + + AdaptationRequest adaptation_request = { + *last_frame_pixel_count_, encoder_stats_observer_->GetInputFrameRate(), + AdaptationRequest::Mode::kAdaptUp}; + + bool adapt_up_requested = + last_adaptation_request_ && + last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; + + if (EffectiveDegradataionPreference() == + DegradationPreference::MAINTAIN_FRAMERATE) { + if (adapt_up_requested && + adaptation_request.input_pixel_count_ <= + last_adaptation_request_->input_pixel_count_) { + // Don't request higher resolution if the current resolution is not + // higher than the last time we asked for the resolution to be higher. + return; + } + } + + switch (EffectiveDegradataionPreference()) { + case DegradationPreference::BALANCED: { + // Check if quality should be increased based on bitrate. + if (reason == kQuality && + !balanced_settings_.CanAdaptUp(*last_frame_pixel_count_, + encoder_start_bitrate_bps_)) { + return; + } + // Try scale up framerate, if higher. + int fps = balanced_settings_.MaxFps(encoder_config_.codec_type, + *last_frame_pixel_count_); + if (source_proxy_->IncreaseFramerate(fps)) { + GetAdaptCounter().DecrementFramerate(reason, fps); + // Reset framerate in case of fewer fps steps down than up. + if (adapt_counter.FramerateCount() == 0 && + fps != std::numeric_limits::max()) { + RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; + source_proxy_->IncreaseFramerate(std::numeric_limits::max()); + } + break; + } + // Check if resolution should be increased based on bitrate. + if (reason == kQuality && + !balanced_settings_.CanAdaptUpResolution( + *last_frame_pixel_count_, encoder_start_bitrate_bps_)) { + return; + } + // Scale up resolution. + RTC_FALLTHROUGH(); + } + case DegradationPreference::MAINTAIN_FRAMERATE: { + // Check if resolution should be increased based on bitrate and + // limits specified by encoder capabilities. + if (reason == kQuality && + !CanAdaptUpResolution(*last_frame_pixel_count_, + encoder_start_bitrate_bps_)) { + return; + } + + // Scale up resolution. + int pixel_count = adaptation_request.input_pixel_count_; + if (adapt_counter.ResolutionCount() == 1) { + RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; + pixel_count = std::numeric_limits::max(); + } + if (!source_proxy_->RequestHigherResolutionThan(pixel_count)) + return; + GetAdaptCounter().DecrementResolution(reason); + break; + } + case DegradationPreference::MAINTAIN_RESOLUTION: { + // Scale up framerate. + int fps = adaptation_request.framerate_fps_; + if (adapt_counter.FramerateCount() == 1) { + RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; + fps = std::numeric_limits::max(); + } + + const int requested_framerate = + source_proxy_->RequestHigherFramerateThan(fps); + if (requested_framerate == -1) { + overuse_detector_->OnTargetFramerateUpdated(codec_max_framerate_); + return; + } + overuse_detector_->OnTargetFramerateUpdated( + std::min(codec_max_framerate_, requested_framerate)); + GetAdaptCounter().DecrementFramerate(reason); + break; + } + case DegradationPreference::DISABLED: + return; + } + + last_adaptation_request_.emplace(adaptation_request); + + UpdateAdaptationStats(reason); + + RTC_LOG(LS_INFO) << adapt_counter.ToString(); +} + +bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( + AdaptReason reason) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + AdaptationRequest adaptation_request = { + *last_frame_pixel_count_, encoder_stats_observer_->GetInputFrameRate(), + AdaptationRequest::Mode::kAdaptDown}; + + bool downgrade_requested = + last_adaptation_request_ && + last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; + + bool did_adapt = true; + + switch (EffectiveDegradataionPreference()) { + case DegradationPreference::BALANCED: + break; + case DegradationPreference::MAINTAIN_FRAMERATE: + if (downgrade_requested && + adaptation_request.input_pixel_count_ >= + last_adaptation_request_->input_pixel_count_) { + // Don't request lower resolution if the current resolution is not + // lower than the last time we asked for the resolution to be lowered. + return true; + } + break; + case DegradationPreference::MAINTAIN_RESOLUTION: + if (adaptation_request.framerate_fps_ <= 0 || + (downgrade_requested && + adaptation_request.framerate_fps_ < kMinFramerateFps)) { + // If no input fps estimate available, can't determine how to scale down + // framerate. Otherwise, don't request lower framerate if we don't have + // a valid frame rate. Since framerate, unlike resolution, is a measure + // we have to estimate, and can fluctuate naturally over time, don't + // make the same kind of limitations as for resolution, but trust the + // overuse detector to not trigger too often. + return true; + } + break; + case DegradationPreference::DISABLED: + return true; + } + + switch (EffectiveDegradataionPreference()) { + case DegradationPreference::BALANCED: { + // Try scale down framerate, if lower. + int fps = balanced_settings_.MinFps(encoder_config_.codec_type, + *last_frame_pixel_count_); + if (source_proxy_->RestrictFramerate(fps)) { + GetAdaptCounter().IncrementFramerate(reason); + // Check if requested fps is higher (or close to) input fps. + absl::optional min_diff = + balanced_settings_.MinFpsDiff(*last_frame_pixel_count_); + if (min_diff && adaptation_request.framerate_fps_ > 0) { + int fps_diff = adaptation_request.framerate_fps_ - fps; + if (fps_diff < min_diff.value()) { + did_adapt = false; + } + } + break; + } + // Scale down resolution. + RTC_FALLTHROUGH(); + } + case DegradationPreference::MAINTAIN_FRAMERATE: { + // Scale down resolution. + bool min_pixels_reached = false; + if (!source_proxy_->RequestResolutionLowerThan( + adaptation_request.input_pixel_count_, + encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame, + &min_pixels_reached)) { + if (min_pixels_reached) + encoder_stats_observer_->OnMinPixelLimitReached(); + return true; + } + GetAdaptCounter().IncrementResolution(reason); + break; + } + case DegradationPreference::MAINTAIN_RESOLUTION: { + // Scale down framerate. + const int requested_framerate = source_proxy_->RequestFramerateLowerThan( + adaptation_request.framerate_fps_); + if (requested_framerate == -1) + return true; + RTC_DCHECK_NE(codec_max_framerate_, -1); + overuse_detector_->OnTargetFramerateUpdated( + std::min(codec_max_framerate_, requested_framerate)); + GetAdaptCounter().IncrementFramerate(reason); + break; + } + case DegradationPreference::DISABLED: + RTC_NOTREACHED(); + } + + last_adaptation_request_.emplace(adaptation_request); + + UpdateAdaptationStats(reason); + + RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString(); + return did_adapt; +} + +// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. +void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( + AdaptReason reason) { + switch (reason) { + case kCpu: + encoder_stats_observer_->OnAdaptationChanged( + VideoStreamEncoderObserver::AdaptationReason::kCpu, + GetActiveCounts(kCpu), GetActiveCounts(kQuality)); + break; + case kQuality: + encoder_stats_observer_->OnAdaptationChanged( + VideoStreamEncoderObserver::AdaptationReason::kQuality, + GetActiveCounts(kCpu), GetActiveCounts(kQuality)); + break; + } +} + +VideoStreamEncoderObserver::AdaptationSteps +OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( + AdaptReason reason) { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + VideoStreamEncoderObserver::AdaptationSteps counts = + GetConstAdaptCounter().Counts(reason); + switch (reason) { + case kCpu: + if (!IsFramerateScalingEnabled(degradation_preference_)) + counts.num_framerate_reductions = absl::nullopt; + if (!IsResolutionScalingEnabled(degradation_preference_)) + counts.num_resolution_reductions = absl::nullopt; + break; + case kQuality: + if (!IsFramerateScalingEnabled(degradation_preference_) || + !is_quality_scaler_enabled_) { + counts.num_framerate_reductions = absl::nullopt; + } + if (!IsResolutionScalingEnabled(degradation_preference_) || + !is_quality_scaler_enabled_) { + counts.num_resolution_reductions = absl::nullopt; + } + break; + } + return counts; +} + +DegradationPreference OveruseFrameDetectorResourceAdaptationModule:: + EffectiveDegradataionPreference() { + // Balanced mode for screenshare works via automatic animation detection: + // Resolution is capped for fullscreen animated content. + // Adapatation is done only via framerate downgrade. + // Thus effective degradation preference is MAINTAIN_RESOLUTION. + return (encoder_config_.content_type == + VideoEncoderConfig::ContentType::kScreen && + degradation_preference_ == DegradationPreference::BALANCED) + ? DegradationPreference::MAINTAIN_RESOLUTION + : degradation_preference_; +} + +OveruseFrameDetectorResourceAdaptationModule::AdaptCounter& +OveruseFrameDetectorResourceAdaptationModule::GetAdaptCounter() { + return adapt_counters_[degradation_preference_]; +} + +const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter& +OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + return adapt_counters_[degradation_preference_]; +} + +absl::optional +OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK(last_frame_pixel_count_.has_value()); + return balanced_settings_.GetQpThresholds(encoder_config_.codec_type, + last_frame_pixel_count_.value()); +} + +bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( + int pixels, + uint32_t bitrate_bps) const { + absl::optional bitrate_limits = + GetEncoderBitrateLimits(encoder_->GetEncoderInfo(), + source_proxy_->GetHigherResolutionThan(pixels)); + if (!bitrate_limits.has_value() || bitrate_bps == 0) { + return true; // No limit configured or bitrate provided. + } + RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels); + return bitrate_bps >= + static_cast(bitrate_limits->min_start_bitrate_bps); +} + +} // namespace webrtc diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h new file mode 100644 index 0000000000..d13a6a710e --- /dev/null +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -0,0 +1,221 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ +#define VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ + +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/rtp_parameters.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "api/video/video_stream_encoder_observer.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_config.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" +#include "video/overuse_frame_detector.h" + +namespace webrtc { + +// This class is used by the VideoStreamEncoder and is responsible for adapting +// resolution up or down based on encode usage percent. It keeps track of video +// source settings, adaptation counters and may get influenced by +// VideoStreamEncoder's quality scaler through AdaptUp() and AdaptDown() calls. +// TODO(hbos): Reduce the coupling with VideoStreamEncoder. +// TODO(hbos): Add unittests specific to this class, it is currently only tested +// indirectly in video_stream_encoder_unittest.cc and other tests exercising +// VideoStreamEncoder. +// TODO(hbos): Create and implement an abstract interface +// ResourceAdaptationModuleInterface and make this class inherit it. Use the +// generic interface in VideoStreamEncoder, unblocking other modules from being +// implemented and used. +class OveruseFrameDetectorResourceAdaptationModule + : public AdaptationObserverInterface { + public: + OveruseFrameDetectorResourceAdaptationModule( + rtc::VideoSinkInterface* sink, + std::unique_ptr overuse_detector, + VideoStreamEncoderObserver* encoder_stats_observer); + ~OveruseFrameDetectorResourceAdaptationModule() override; + + void Initialize(rtc::TaskQueue* encoder_queue); + // Sets the encoder to reconfigure based on overuse. + // TODO(hbos): Don't reconfigure the encoder directly. Instead, define the + // output of a resource adaptation module as a struct and let the + // VideoStreamEncoder handle the interaction with the actual encoder. + void SetEncoder(VideoEncoder* encoder); + + DegradationPreference degradation_preference() const { + RTC_DCHECK(encoder_queue_); + RTC_DCHECK_RUN_ON(encoder_queue_); + return degradation_preference_; + } + + // Input to the OveruseFrameDetector, which are required for this module to + // function. These map to OveruseFrameDetector methods. + // TODO(hbos): Define virtual methods in ResourceAdaptationModuleInterface + // for input that are more generic so that this class can be used without + // assumptions about underlying implementation. + void StartCheckForOveruse(const CpuOveruseOptions& options); + void StopCheckForOveruse(); + void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); + void FrameSent(uint32_t timestamp, + int64_t time_sent_in_us, + int64_t capture_time_us, + absl::optional encode_duration_us); + + // Various other settings and feedback mechanisms. + // TODO(hbos): Find a common interface that would make sense for a generic + // resource adaptation module. Unify code paths where possible. Do we really + // need this many public methods? + void SetLastFramePixelCount(absl::optional last_frame_pixel_count); + void SetEncoderConfig(VideoEncoderConfig encoder_config); + void SetCodecMaxFramerate(int codec_max_framerate); + void SetEncoderStartBitrateBps(uint32_t encoder_start_bitrate_bps); + // Inform the detector whether or not the quality scaler is enabled. This + // helps GetActiveCounts() return absl::nullopt when appropriate. + // TODO(hbos): This feels really hacky, can we report the right values without + // this boolean? It would be really easy to report the wrong thing if this + // method is called incorrectly. + void SetIsQualityScalerEnabled(bool is_quality_scaler_enabled); + + void SetSource(rtc::VideoSourceInterface* source, + const DegradationPreference& degradation_preference); + void SetSourceWantsRotationApplied(bool rotation_applied); + void SetSourceMaxFramerateAndAlignment(int max_framerate, + int resolution_alignment); + void SetSourceMaxPixels(int max_pixels); + + // TODO(hbos): Can we get rid of this? Seems we should know whether the frame + // rate has updated. + void RefreshTargetFramerate(); + void ResetAdaptationCounters(); + + class AdaptCounter final { + public: + AdaptCounter(); + ~AdaptCounter(); + + // Get number of adaptation downscales for |reason|. + VideoStreamEncoderObserver::AdaptationSteps Counts(int reason) const; + + std::string ToString() const; + + void IncrementFramerate(int reason); + void IncrementResolution(int reason); + void DecrementFramerate(int reason); + void DecrementResolution(int reason); + void DecrementFramerate(int reason, int cur_fps); + + // Gets the total number of downgrades (for all adapt reasons). + int FramerateCount() const; + int ResolutionCount() const; + + // Gets the total number of downgrades for |reason|. + int FramerateCount(int reason) const; + int ResolutionCount(int reason) const; + int TotalCount(int reason) const; + + private: + std::string ToString(const std::vector& counters) const; + int Count(const std::vector& counters) const; + void MoveCount(std::vector* counters, int from_reason); + + // Degradation counters holding number of framerate/resolution reductions + // per adapt reason. + std::vector fps_counters_; + std::vector resolution_counters_; + }; + + // AdaptationObserverInterface implementation. Used both "internally" as + // feedback from |overuse_detector_|, and externally from VideoStreamEncoder: + // - It is wired to the VideoStreamEncoder::quality_scaler_. + // - It is invoked by VideoStreamEncoder::MaybeEncodeVideoFrame(). + // TODO(hbos): Decouple quality scaling and resource adaptation, or find an + // interface for reconfiguring externally. + // TODO(hbos): VideoStreamEncoder should not be responsible for any part of + // the adaptation. + void AdaptUp(AdaptReason reason) override; + bool AdaptDown(AdaptReason reason) override; + + // Used by VideoStreamEncoder when ConfigureQualityScaler() occurs and the + // |encoder_stats_observer_| is called outside of this class. + // TODO(hbos): Decouple quality scaling and resource adaptation logic and make + // this method private. + VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( + AdaptReason reason); + + // Used by VideoStreamEncoder::MaybeEncodeVideoFrame(). + // TODO(hbos): VideoStreamEncoder should not be responsible for any part of + // the adaptation. Move this logic to this module? + const AdaptCounter& GetConstAdaptCounter(); + + // Used by VideoStreamEncoder::ConfigureQualityScaler(). + // TODO(hbos): Decouple quality scaling and resource adaptation logic and + // delete this method. + absl::optional GetQpThresholds() const; + + private: + class VideoSourceProxy; + + struct AdaptationRequest { + // The pixel count produced by the source at the time of the adaptation. + int input_pixel_count_; + // Framerate received from the source at the time of the adaptation. + int framerate_fps_; + // Indicates if request was to adapt up or down. + enum class Mode { kAdaptUp, kAdaptDown } mode_; + }; + + void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(encoder_queue_); + DegradationPreference EffectiveDegradataionPreference() + RTC_RUN_ON(encoder_queue_); + AdaptCounter& GetAdaptCounter() RTC_RUN_ON(encoder_queue_); + bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const + RTC_RUN_ON(encoder_queue_); + + rtc::TaskQueue* encoder_queue_; + DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); + // Counters used for deciding if the video resolution or framerate is + // currently restricted, and if so, why, on a per degradation preference + // basis. + // TODO(sprang): Replace this with a state holding a relative overuse measure + // instead, that can be translated into suitable down-scale or fps limit. + std::map adapt_counters_ + RTC_GUARDED_BY(encoder_queue_); + const BalancedDegradationSettings balanced_settings_ + RTC_GUARDED_BY(encoder_queue_); + // Stores a snapshot of the last adaptation request triggered by an AdaptUp + // or AdaptDown signal. + absl::optional last_adaptation_request_ + RTC_GUARDED_BY(encoder_queue_); + absl::optional last_frame_pixel_count_ RTC_GUARDED_BY(encoder_queue_); + // The source proxy may modify its source or sink off the |encoder_queue_|. + const std::unique_ptr source_proxy_; + const std::unique_ptr overuse_detector_ + RTC_PT_GUARDED_BY(encoder_queue_); + int codec_max_framerate_ RTC_GUARDED_BY(encoder_queue_); + uint32_t encoder_start_bitrate_bps_ RTC_GUARDED_BY(encoder_queue_); + bool is_quality_scaler_enabled_ RTC_GUARDED_BY(encoder_queue_); + VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(encoder_queue_); + VideoEncoder* encoder_ RTC_GUARDED_BY(encoder_queue_); + VideoStreamEncoderObserver* const encoder_stats_observer_ + RTC_GUARDED_BY(encoder_queue_); +}; + +} // namespace webrtc + +#endif // VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 446b1b62e9..ccf00deb4e 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -45,7 +45,6 @@ namespace { // Time interval for logging frame counts. const int64_t kFrameLogIntervalMs = 60000; -const int kMinFramerateFps = 2; // Time to keep a single cached pending frame in paused state. const int64_t kPendingFrameTimeoutMs = 1000; @@ -214,268 +213,42 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo( } } // namespace -// VideoSourceProxy is responsible ensuring thread safety between calls to -// VideoStreamEncoder::SetSource that will happen on libjingle's worker thread -// when a video capturer is connected to the encoder and the encoder task queue -// (encoder_queue_) where the encoder reports its VideoSinkWants. -class VideoStreamEncoder::VideoSourceProxy { - public: - explicit VideoSourceProxy(VideoStreamEncoder* video_stream_encoder) - : video_stream_encoder_(video_stream_encoder), - degradation_preference_(DegradationPreference::DISABLED), - source_(nullptr), - max_framerate_(std::numeric_limits::max()), - max_pixels_(std::numeric_limits::max()), - resolution_alignment_(1) {} +absl::optional GetEncoderBitrateLimits( + const VideoEncoder::EncoderInfo& encoder_info, + int frame_size_pixels) { + std::vector bitrate_limits = + encoder_info.resolution_bitrate_limits; - void SetSource(rtc::VideoSourceInterface* source, - const DegradationPreference& degradation_preference) { - // Called on libjingle's worker thread. - RTC_DCHECK_RUN_ON(&main_checker_); - rtc::VideoSourceInterface* old_source = nullptr; - rtc::VideoSinkWants wants; - { - rtc::CritScope lock(&crit_); - degradation_preference_ = degradation_preference; - old_source = source_; - source_ = source; - wants = GetActiveSinkWantsInternal(); + // Sort the list of bitrate limits by resolution. + sort(bitrate_limits.begin(), bitrate_limits.end(), + [](const VideoEncoder::ResolutionBitrateLimits& lhs, + const VideoEncoder::ResolutionBitrateLimits& rhs) { + return lhs.frame_size_pixels < rhs.frame_size_pixels; + }); + + for (size_t i = 0; i < bitrate_limits.size(); ++i) { + RTC_DCHECK_GT(bitrate_limits[i].min_bitrate_bps, 0); + RTC_DCHECK_GT(bitrate_limits[i].min_start_bitrate_bps, 0); + RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, + bitrate_limits[i].min_bitrate_bps); + if (i > 0) { + // The bitrate limits aren't expected to decrease with resolution. + RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, + bitrate_limits[i - 1].min_bitrate_bps); + RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, + bitrate_limits[i - 1].min_start_bitrate_bps); + RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, + bitrate_limits[i - 1].max_bitrate_bps); } - if (old_source != source && old_source != nullptr) { - old_source->RemoveSink(video_stream_encoder_); - } - - if (!source) { - return; - } - - source->AddOrUpdateSink(video_stream_encoder_, wants); - } - - void SetMaxFramerateAndAlignment(int max_framerate, - int resolution_alignment) { - RTC_DCHECK_GT(max_framerate, 0); - rtc::CritScope lock(&crit_); - if (max_framerate == max_framerate_ && - resolution_alignment == resolution_alignment_) { - return; - } - - RTC_LOG(LS_INFO) << "Set max framerate: " << max_framerate - << " and resolution alignment: " << resolution_alignment; - max_framerate_ = max_framerate; - resolution_alignment_ = resolution_alignment; - if (source_) { - source_->AddOrUpdateSink(video_stream_encoder_, - GetActiveSinkWantsInternal()); + if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) { + return absl::optional( + bitrate_limits[i]); } } - void SetWantsRotationApplied(bool rotation_applied) { - rtc::CritScope lock(&crit_); - sink_wants_.rotation_applied = rotation_applied; - if (source_) { - source_->AddOrUpdateSink(video_stream_encoder_, - GetActiveSinkWantsInternal()); - } - } - - rtc::VideoSinkWants GetActiveSinkWants() { - rtc::CritScope lock(&crit_); - return GetActiveSinkWantsInternal(); - } - - void ResetPixelFpsCount() { - rtc::CritScope lock(&crit_); - sink_wants_.max_pixel_count = std::numeric_limits::max(); - sink_wants_.target_pixel_count.reset(); - sink_wants_.max_framerate_fps = std::numeric_limits::max(); - if (source_) - source_->AddOrUpdateSink(video_stream_encoder_, - GetActiveSinkWantsInternal()); - } - - bool RequestResolutionLowerThan(int pixel_count, - int min_pixels_per_frame, - bool* min_pixels_reached) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { - // This can happen since |degradation_preference_| is set on libjingle's - // worker thread but the adaptation is done on the encoder task queue. - return false; - } - // The input video frame size will have a resolution less than or equal to - // |max_pixel_count| depending on how the source can scale the frame size. - const int pixels_wanted = (pixel_count * 3) / 5; - if (pixels_wanted >= sink_wants_.max_pixel_count) { - return false; - } - if (pixels_wanted < min_pixels_per_frame) { - *min_pixels_reached = true; - return false; - } - RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " - << pixels_wanted; - sink_wants_.max_pixel_count = pixels_wanted; - sink_wants_.target_pixel_count = absl::nullopt; - source_->AddOrUpdateSink(video_stream_encoder_, - GetActiveSinkWantsInternal()); - return true; - } - - int RequestFramerateLowerThan(int fps) { - // Called on the encoder task queue. - // The input video frame rate will be scaled down to 2/3, rounding down. - int framerate_wanted = (fps * 2) / 3; - return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; - } - - int GetHigherResolutionThan(int pixel_count) const { - // On step down we request at most 3/5 the pixel count of the previous - // resolution, so in order to take "one step up" we request a resolution - // as close as possible to 5/3 of the current resolution. The actual pixel - // count selected depends on the capabilities of the source. In order to - // not take a too large step up, we cap the requested pixel count to be at - // most four time the current number of pixels. - return (pixel_count * 5) / 3; - } - - bool RequestHigherResolutionThan(int pixel_count) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { - // This can happen since |degradation_preference_| is set on libjingle's - // worker thread but the adaptation is done on the encoder task queue. - return false; - } - int max_pixels_wanted = pixel_count; - if (max_pixels_wanted != std::numeric_limits::max()) - max_pixels_wanted = pixel_count * 4; - - if (max_pixels_wanted <= sink_wants_.max_pixel_count) - return false; - - sink_wants_.max_pixel_count = max_pixels_wanted; - if (max_pixels_wanted == std::numeric_limits::max()) { - // Remove any constraints. - sink_wants_.target_pixel_count.reset(); - } else { - sink_wants_.target_pixel_count = GetHigherResolutionThan(pixel_count); - } - RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " - << max_pixels_wanted; - source_->AddOrUpdateSink(video_stream_encoder_, - GetActiveSinkWantsInternal()); - return true; - } - - // Request upgrade in framerate. Returns the new requested frame, or -1 if - // no change requested. Note that maxint may be returned if limits due to - // adaptation requests are removed completely. In that case, consider - // |max_framerate_| to be the current limit (assuming the capturer complies). - int RequestHigherFramerateThan(int fps) { - // Called on the encoder task queue. - // The input frame rate will be scaled up to the last step, with rounding. - int framerate_wanted = fps; - if (fps != std::numeric_limits::max()) - framerate_wanted = (fps * 3) / 2; - - return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; - } - - bool RestrictFramerate(int fps) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) - return false; - - const int fps_wanted = std::max(kMinFramerateFps, fps); - if (fps_wanted >= sink_wants_.max_framerate_fps) - return false; - - RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; - sink_wants_.max_framerate_fps = fps_wanted; - source_->AddOrUpdateSink(video_stream_encoder_, - GetActiveSinkWantsInternal()); - return true; - } - - bool IncreaseFramerate(int fps) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) - return false; - - const int fps_wanted = std::max(kMinFramerateFps, fps); - if (fps_wanted <= sink_wants_.max_framerate_fps) - return false; - - RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; - sink_wants_.max_framerate_fps = fps_wanted; - source_->AddOrUpdateSink(video_stream_encoder_, - GetActiveSinkWantsInternal()); - return true; - } - - // Used in automatic animation detection for screenshare. - bool RestrictPixels(int max_pixels) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { - // This can happen since |degradation_preference_| is set on libjingle's - // worker thread but the adaptation is done on the encoder task queue. - return false; - } - max_pixels_ = max_pixels; - RTC_LOG(LS_INFO) << "Applying max pixel restriction: " << max_pixels; - source_->AddOrUpdateSink(video_stream_encoder_, - GetActiveSinkWantsInternal()); - return true; - } - - private: - rtc::VideoSinkWants GetActiveSinkWantsInternal() - RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) { - rtc::VideoSinkWants wants = sink_wants_; - // Clear any constraints from the current sink wants that don't apply to - // the used degradation_preference. - switch (degradation_preference_) { - case DegradationPreference::BALANCED: - break; - case DegradationPreference::MAINTAIN_FRAMERATE: - wants.max_framerate_fps = std::numeric_limits::max(); - break; - case DegradationPreference::MAINTAIN_RESOLUTION: - wants.max_pixel_count = std::numeric_limits::max(); - wants.target_pixel_count.reset(); - break; - case DegradationPreference::DISABLED: - wants.max_pixel_count = std::numeric_limits::max(); - wants.target_pixel_count.reset(); - wants.max_framerate_fps = std::numeric_limits::max(); - } - // Limit to configured max framerate. - wants.max_framerate_fps = std::min(max_framerate_, wants.max_framerate_fps); - // Limit resolution due to automatic animation detection for screenshare. - wants.max_pixel_count = std::min(max_pixels_, wants.max_pixel_count); - wants.resolution_alignment = resolution_alignment_; - return wants; - } - - rtc::CriticalSection crit_; - SequenceChecker main_checker_; - VideoStreamEncoder* const video_stream_encoder_; - rtc::VideoSinkWants sink_wants_ RTC_GUARDED_BY(&crit_); - DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_); - rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(&crit_); - int max_framerate_ RTC_GUARDED_BY(&crit_); - int max_pixels_ RTC_GUARDED_BY(&crit_); - int resolution_alignment_ RTC_GUARDED_BY(&crit_); - - RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy); -}; + return absl::nullopt; +} VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings() : rate_control(), @@ -519,12 +292,10 @@ VideoStreamEncoder::VideoStreamEncoder( quality_rampup_done_(false), quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), - source_proxy_(new VideoSourceProxy(this)), sink_(nullptr), settings_(settings), rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), - overuse_detector_(std::move(overuse_detector)), encoder_stats_observer_(encoder_stats_observer), encoder_initialized_(false), max_framerate_(-1), @@ -541,7 +312,6 @@ VideoStreamEncoder::VideoStreamEncoder( was_encode_called_since_last_initialization_(false), encoder_failed_(false), clock_(clock), - degradation_preference_(DegradationPreference::DISABLED), posted_frames_waiting_for_encode_(0), last_captured_timestamp_(0), delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - @@ -568,12 +338,17 @@ VideoStreamEncoder::VideoStreamEncoder( automatic_animation_detection_experiment_( ParseAutomatincAnimationDetectionFieldTrial()), encoder_switch_requested_(false), + resource_adaptation_module_( + std::make_unique( + this, + std::move(overuse_detector), + encoder_stats_observer)), encoder_queue_(task_queue_factory->CreateTaskQueue( "EncoderQueue", TaskQueueFactory::Priority::NORMAL)) { RTC_DCHECK(encoder_stats_observer); - RTC_DCHECK(overuse_detector_); RTC_DCHECK_GE(number_of_cores, 1); + resource_adaptation_module_->Initialize(encoder_queue()); for (auto& state : encoder_buffer_state_) state.fill(std::numeric_limits::max()); @@ -587,14 +362,15 @@ VideoStreamEncoder::~VideoStreamEncoder() { void VideoStreamEncoder::Stop() { RTC_DCHECK_RUN_ON(&thread_checker_); - source_proxy_->SetSource(nullptr, DegradationPreference()); + resource_adaptation_module_->SetSource(nullptr, DegradationPreference()); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); - overuse_detector_->StopCheckForOveruse(); + resource_adaptation_module_->StopCheckForOveruse(); rate_allocator_ = nullptr; bitrate_observer_ = nullptr; ReleaseEncoder(); quality_scaler_ = nullptr; + resource_adaptation_module_->SetIsQualityScalerEnabled(false); shutdown_event_.Set(); }); @@ -627,23 +403,9 @@ void VideoStreamEncoder::SetSource( rtc::VideoSourceInterface* source, const DegradationPreference& degradation_preference) { RTC_DCHECK_RUN_ON(&thread_checker_); - source_proxy_->SetSource(source, degradation_preference); + resource_adaptation_module_->SetSource(source, degradation_preference); encoder_queue_.PostTask([this, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); - if (degradation_preference_ != degradation_preference) { - // Reset adaptation state, so that we're not tricked into thinking there's - // an already pending request of the same type. - last_adaptation_request_.reset(); - if (degradation_preference == DegradationPreference::BALANCED || - degradation_preference_ == DegradationPreference::BALANCED) { - // TODO(asapersson): Consider removing |adapt_counters_| map and use one - // AdaptCounter for all modes. - source_proxy_->ResetPixelFpsCount(); - adapt_counters_.clear(); - } - } - degradation_preference_ = degradation_preference; - if (encoder_) ConfigureQualityScaler(encoder_->GetEncoderInfo()); @@ -651,13 +413,13 @@ void VideoStreamEncoder::SetSource( max_framerate_ != -1) { // If frame rate scaling is no longer allowed, remove any potential // allowance for longer frame intervals. - overuse_detector_->OnTargetFramerateUpdated(max_framerate_); + resource_adaptation_module_->RefreshTargetFramerate(); } }); } void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { - source_proxy_->SetWantsRotationApplied(rotation_applied); + resource_adaptation_module_->SetSourceWantsRotationApplied(rotation_applied); encoder_queue_.PostTask([this, sink] { RTC_DCHECK_RUN_ON(&encoder_queue_); sink_ = sink; @@ -668,6 +430,8 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { encoder_queue_.PostTask([this, start_bitrate_bps] { RTC_DCHECK_RUN_ON(&encoder_queue_); encoder_start_bitrate_bps_ = start_bitrate_bps; + resource_adaptation_module_->SetEncoderStartBitrateBps( + encoder_start_bitrate_bps_); set_start_bitrate_bps_ = start_bitrate_bps; set_start_bitrate_time_ms_ = clock_->TimeInMilliseconds(); }); @@ -685,6 +449,7 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, (!encoder_ || encoder_config_.video_format != config.video_format || max_data_payload_length_ != max_data_payload_length); encoder_config_ = std::move(config); + resource_adaptation_module_->SetEncoderConfig(encoder_config_.Copy()); max_data_payload_length_ = max_data_payload_length; pending_encoder_reconfiguration_ = true; @@ -700,49 +465,14 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, encoder_config_.video_format); if (HasInternalSource()) { last_frame_info_ = VideoFrameInfo(176, 144, false); + resource_adaptation_module_->SetLastFramePixelCount( + last_frame_info_->pixel_count()); ReconfigureEncoder(); } } }); } -static absl::optional -GetEncoderBitrateLimits(const VideoEncoder::EncoderInfo& encoder_info, - int frame_size_pixels) { - std::vector bitrate_limits = - encoder_info.resolution_bitrate_limits; - - // Sort the list of bitrate limits by resolution. - sort(bitrate_limits.begin(), bitrate_limits.end(), - [](const VideoEncoder::ResolutionBitrateLimits& lhs, - const VideoEncoder::ResolutionBitrateLimits& rhs) { - return lhs.frame_size_pixels < rhs.frame_size_pixels; - }); - - for (size_t i = 0; i < bitrate_limits.size(); ++i) { - RTC_DCHECK_GT(bitrate_limits[i].min_bitrate_bps, 0); - RTC_DCHECK_GT(bitrate_limits[i].min_start_bitrate_bps, 0); - RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, - bitrate_limits[i].min_bitrate_bps); - if (i > 0) { - // The bitrate limits aren't expected to decrease with resolution. - RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, - bitrate_limits[i - 1].min_bitrate_bps); - RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, - bitrate_limits[i - 1].min_start_bitrate_bps); - RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, - bitrate_limits[i - 1].max_bitrate_bps); - } - - if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) { - return absl::optional( - bitrate_limits[i]); - } - } - - return absl::nullopt; -} - // TODO(bugs.webrtc.org/8807): Currently this always does a hard // reconfiguration, but this isn't always necessary. Add in logic to only update // the VideoBitrateAllocator and call OnEncoderConfigurationChanged with a @@ -793,6 +523,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_ = settings_.encoder_factory->CreateVideoEncoder( encoder_config_.video_format); + resource_adaptation_module_->SetEncoder(encoder_.get()); // TODO(nisse): What to do if creating the encoder fails? Crash, // or just discard incoming frames? RTC_CHECK(encoder_); @@ -882,13 +613,14 @@ void VideoStreamEncoder::ReconfigureEncoder() { // Make sure the start bit rate is sane... RTC_DCHECK_LE(codec.startBitrate, 1000000); max_framerate_ = codec.maxFramerate; + resource_adaptation_module_->SetCodecMaxFramerate(max_framerate_); // Inform source about max configured framerate. int max_framerate = 0; for (const auto& stream : streams) { max_framerate = std::max(stream.max_framerate, max_framerate); } - source_proxy_->SetMaxFramerateAndAlignment( + resource_adaptation_module_->SetSourceMaxFramerateAndAlignment( max_framerate, encoder_->GetEncoderInfo().requested_resolution_alignment); if (codec.maxBitrate == 0) { @@ -970,12 +702,9 @@ void VideoStreamEncoder::ReconfigureEncoder() { } if (pending_encoder_creation_) { - overuse_detector_->StopCheckForOveruse(); - overuse_detector_->StartCheckForOveruse( - &encoder_queue_, - GetCpuOveruseOptions( - settings_, encoder_->GetEncoderInfo().is_hardware_accelerated), - this); + resource_adaptation_module_->StopCheckForOveruse(); + resource_adaptation_module_->StartCheckForOveruse(GetCpuOveruseOptions( + settings_, encoder_->GetEncoderInfo().is_hardware_accelerated)); pending_encoder_creation_ = false; } @@ -1030,14 +759,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { std::move(streams), encoder_config_.content_type, encoder_config_.min_transmit_bitrate_bps); - // Get the current target framerate, ie the maximum framerate as specified by - // the current codec configuration, or any limit imposed by cpu adaption in - // maintain-resolution or balanced mode. This is used to make sure overuse - // detection doesn't needlessly trigger in low and/or variable framerate - // scenarios. - int target_framerate = std::min( - max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); - overuse_detector_->OnTargetFramerateUpdated(target_framerate); + resource_adaptation_module_->RefreshTargetFramerate(); ConfigureQualityScaler(info); } @@ -1047,7 +769,8 @@ void VideoStreamEncoder::ConfigureQualityScaler( RTC_DCHECK_RUN_ON(&encoder_queue_); const auto scaling_settings = encoder_info.scaling_settings; const bool quality_scaling_allowed = - IsResolutionScalingEnabled(degradation_preference_) && + IsResolutionScalingEnabled( + resource_adaptation_module_->degradation_preference()) && scaling_settings.thresholds; if (quality_scaling_allowed) { @@ -1062,24 +785,26 @@ void VideoStreamEncoder::ConfigureQualityScaler( } // Since the interface is non-public, std::make_unique can't do this // upcast. - AdaptationObserverInterface* observer = this; + AdaptationObserverInterface* observer = resource_adaptation_module_.get(); quality_scaler_ = std::make_unique( &encoder_queue_, observer, experimental_thresholds ? *experimental_thresholds : *(scaling_settings.thresholds)); + resource_adaptation_module_->SetIsQualityScalerEnabled(true); has_seen_first_significant_bwe_change_ = false; initial_framedrop_ = 0; } } else { quality_scaler_.reset(nullptr); + resource_adaptation_module_->SetIsQualityScalerEnabled(false); initial_framedrop_ = kMaxInitialFramedrop; } - if (degradation_preference_ == DegradationPreference::BALANCED && + if (resource_adaptation_module_->degradation_preference() == + DegradationPreference::BALANCED && quality_scaler_ && last_frame_info_) { absl::optional thresholds = - balanced_settings_.GetQpThresholds(encoder_config_.codec_type, - last_frame_info_->pixel_count()); + resource_adaptation_module_->GetQpThresholds(); if (thresholds) { quality_scaler_->SetQpThresholds(*thresholds); } @@ -1087,7 +812,10 @@ void VideoStreamEncoder::ConfigureQualityScaler( encoder_stats_observer_->OnAdaptationChanged( VideoStreamEncoderObserver::AdaptationReason::kNone, - GetActiveCounts(kCpu), GetActiveCounts(kQuality)); + resource_adaptation_module_->GetActiveCounts( + AdaptationObserverInterface::AdaptReason::kCpu), + resource_adaptation_module_->GetActiveCounts( + AdaptationObserverInterface::AdaptReason::kQuality)); } void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { @@ -1322,6 +1050,8 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, pending_encoder_reconfiguration_ = true; last_frame_info_ = VideoFrameInfo(video_frame.width(), video_frame.height(), video_frame.is_texture()); + resource_adaptation_module_->SetLastFramePixelCount( + last_frame_info_->pixel_count()); RTC_LOG(LS_INFO) << "Video frame parameters changed: dimensions=" << last_frame_info_->width << "x" << last_frame_info_->height @@ -1376,15 +1106,22 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, if (DropDueToSize(video_frame.size())) { RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; - int fps_count = GetConstAdaptCounter().FramerateCount(kQuality); - int res_count = GetConstAdaptCounter().ResolutionCount(kQuality); - AdaptDown(kQuality); - if (degradation_preference_ == DegradationPreference::BALANCED && - GetConstAdaptCounter().FramerateCount(kQuality) > fps_count) { + int fps_count = + resource_adaptation_module_->GetConstAdaptCounter().FramerateCount( + AdaptationObserverInterface::AdaptReason::kQuality); + int res_count = + resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount( + AdaptationObserverInterface::AdaptReason::kQuality); + TriggerAdaptDown(AdaptationObserverInterface::AdaptReason::kQuality); + if (resource_adaptation_module_->degradation_preference() == + DegradationPreference::BALANCED && + resource_adaptation_module_->GetConstAdaptCounter().FramerateCount( + AdaptationObserverInterface::AdaptReason::kQuality) > fps_count) { // Adapt framerate in same step as resolution. - AdaptDown(kQuality); + TriggerAdaptDown(AdaptationObserverInterface::AdaptReason::kQuality); } - if (GetConstAdaptCounter().ResolutionCount(kQuality) > res_count) { + if (resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount( + AdaptationObserverInterface::AdaptReason::kQuality) > res_count) { encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); } ++initial_framedrop_; @@ -1404,12 +1141,12 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, initial_framedrop_ = kMaxInitialFramedrop; if (!quality_rampup_done_ && TryQualityRampup(now_ms) && - GetConstAdaptCounter().ResolutionCount(kQuality) > 0 && - GetConstAdaptCounter().TotalCount(kCpu) == 0) { + resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount( + AdaptationObserverInterface::AdaptReason::kQuality) > 0 && + resource_adaptation_module_->GetConstAdaptCounter().TotalCount( + AdaptationObserverInterface::AdaptReason::kCpu) == 0) { RTC_LOG(LS_INFO) << "Reset quality limitations."; - last_adaptation_request_.reset(); - source_proxy_->ResetPixelFpsCount(); - adapt_counters_.clear(); + resource_adaptation_module_->ResetAdaptationCounters(); quality_rampup_done_ = true; } @@ -1593,7 +1330,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); - overuse_detector_->FrameCaptured(out_frame, time_when_posted_us); + resource_adaptation_module_->FrameCaptured(out_frame, time_when_posted_us); RTC_DCHECK_LE(send_codec_.width, out_frame.width()); RTC_DCHECK_LE(send_codec_.height, out_frame.height()); @@ -1922,6 +1659,8 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, encoder_start_bitrate_bps_ = target_bitrate.bps() != 0 ? target_bitrate.bps() : encoder_start_bitrate_bps_; + resource_adaptation_module_->SetEncoderStartBitrateBps( + encoder_start_bitrate_bps_); if (video_suspension_changed) { RTC_LOG(LS_INFO) << "Video suspend state changed to: " @@ -1979,277 +1718,14 @@ bool VideoStreamEncoder::TryQualityRampup(int64_t now_ms) { return false; } -bool VideoStreamEncoder::AdaptDown(AdaptReason reason) { - RTC_DCHECK_RUN_ON(&encoder_queue_); - AdaptationRequest adaptation_request = { - last_frame_info_->pixel_count(), - encoder_stats_observer_->GetInputFrameRate(), - AdaptationRequest::Mode::kAdaptDown}; - - bool downgrade_requested = - last_adaptation_request_ && - last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; - - bool did_adapt = true; - - switch (EffectiveDegradataionPreference()) { - case DegradationPreference::BALANCED: - break; - case DegradationPreference::MAINTAIN_FRAMERATE: - if (downgrade_requested && - adaptation_request.input_pixel_count_ >= - last_adaptation_request_->input_pixel_count_) { - // Don't request lower resolution if the current resolution is not - // lower than the last time we asked for the resolution to be lowered. - return true; - } - break; - case DegradationPreference::MAINTAIN_RESOLUTION: - if (adaptation_request.framerate_fps_ <= 0 || - (downgrade_requested && - adaptation_request.framerate_fps_ < kMinFramerateFps)) { - // If no input fps estimate available, can't determine how to scale down - // framerate. Otherwise, don't request lower framerate if we don't have - // a valid frame rate. Since framerate, unlike resolution, is a measure - // we have to estimate, and can fluctuate naturally over time, don't - // make the same kind of limitations as for resolution, but trust the - // overuse detector to not trigger too often. - return true; - } - break; - case DegradationPreference::DISABLED: - return true; - } - - switch (EffectiveDegradataionPreference()) { - case DegradationPreference::BALANCED: { - // Try scale down framerate, if lower. - int fps = balanced_settings_.MinFps(encoder_config_.codec_type, - last_frame_info_->pixel_count()); - if (source_proxy_->RestrictFramerate(fps)) { - GetAdaptCounter().IncrementFramerate(reason); - // Check if requested fps is higher (or close to) input fps. - absl::optional min_diff = - balanced_settings_.MinFpsDiff(last_frame_info_->pixel_count()); - if (min_diff && adaptation_request.framerate_fps_ > 0) { - int fps_diff = adaptation_request.framerate_fps_ - fps; - if (fps_diff < min_diff.value()) { - did_adapt = false; - } - } - break; - } - // Scale down resolution. - RTC_FALLTHROUGH(); - } - case DegradationPreference::MAINTAIN_FRAMERATE: { - // Scale down resolution. - bool min_pixels_reached = false; - if (!source_proxy_->RequestResolutionLowerThan( - adaptation_request.input_pixel_count_, - encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame, - &min_pixels_reached)) { - if (min_pixels_reached) - encoder_stats_observer_->OnMinPixelLimitReached(); - return true; - } - GetAdaptCounter().IncrementResolution(reason); - break; - } - case DegradationPreference::MAINTAIN_RESOLUTION: { - // Scale down framerate. - const int requested_framerate = source_proxy_->RequestFramerateLowerThan( - adaptation_request.framerate_fps_); - if (requested_framerate == -1) - return true; - RTC_DCHECK_NE(max_framerate_, -1); - overuse_detector_->OnTargetFramerateUpdated( - std::min(max_framerate_, requested_framerate)); - GetAdaptCounter().IncrementFramerate(reason); - break; - } - case DegradationPreference::DISABLED: - RTC_NOTREACHED(); - } - - last_adaptation_request_.emplace(adaptation_request); - - UpdateAdaptationStats(reason); - - RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString(); - return did_adapt; +bool VideoStreamEncoder::TriggerAdaptDown( + AdaptationObserverInterface::AdaptReason reason) { + return resource_adaptation_module_->AdaptDown(reason); } -void VideoStreamEncoder::AdaptUp(AdaptReason reason) { - RTC_DCHECK_RUN_ON(&encoder_queue_); - - const AdaptCounter& adapt_counter = GetConstAdaptCounter(); - int num_downgrades = adapt_counter.TotalCount(reason); - if (num_downgrades == 0) - return; - RTC_DCHECK_GT(num_downgrades, 0); - - AdaptationRequest adaptation_request = { - last_frame_info_->pixel_count(), - encoder_stats_observer_->GetInputFrameRate(), - AdaptationRequest::Mode::kAdaptUp}; - - bool adapt_up_requested = - last_adaptation_request_ && - last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; - - if (EffectiveDegradataionPreference() == - DegradationPreference::MAINTAIN_FRAMERATE) { - if (adapt_up_requested && - adaptation_request.input_pixel_count_ <= - last_adaptation_request_->input_pixel_count_) { - // Don't request higher resolution if the current resolution is not - // higher than the last time we asked for the resolution to be higher. - return; - } - } - - switch (EffectiveDegradataionPreference()) { - case DegradationPreference::BALANCED: { - // Check if quality should be increased based on bitrate. - if (reason == kQuality && - !balanced_settings_.CanAdaptUp(last_frame_info_->pixel_count(), - encoder_start_bitrate_bps_)) { - return; - } - // Try scale up framerate, if higher. - int fps = balanced_settings_.MaxFps(encoder_config_.codec_type, - last_frame_info_->pixel_count()); - if (source_proxy_->IncreaseFramerate(fps)) { - GetAdaptCounter().DecrementFramerate(reason, fps); - // Reset framerate in case of fewer fps steps down than up. - if (adapt_counter.FramerateCount() == 0 && - fps != std::numeric_limits::max()) { - RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - source_proxy_->IncreaseFramerate(std::numeric_limits::max()); - } - break; - } - // Check if resolution should be increased based on bitrate. - if (reason == kQuality && - !balanced_settings_.CanAdaptUpResolution( - last_frame_info_->pixel_count(), encoder_start_bitrate_bps_)) { - return; - } - // Scale up resolution. - RTC_FALLTHROUGH(); - } - case DegradationPreference::MAINTAIN_FRAMERATE: { - // Check if resolution should be increased based on bitrate and - // limits specified by encoder capabilities. - if (reason == kQuality && - !CanAdaptUpResolution(last_frame_info_->pixel_count(), - encoder_start_bitrate_bps_)) { - return; - } - - // Scale up resolution. - int pixel_count = adaptation_request.input_pixel_count_; - if (adapt_counter.ResolutionCount() == 1) { - RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; - pixel_count = std::numeric_limits::max(); - } - if (!source_proxy_->RequestHigherResolutionThan(pixel_count)) - return; - GetAdaptCounter().DecrementResolution(reason); - break; - } - case DegradationPreference::MAINTAIN_RESOLUTION: { - // Scale up framerate. - int fps = adaptation_request.framerate_fps_; - if (adapt_counter.FramerateCount() == 1) { - RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; - fps = std::numeric_limits::max(); - } - - const int requested_framerate = - source_proxy_->RequestHigherFramerateThan(fps); - if (requested_framerate == -1) { - overuse_detector_->OnTargetFramerateUpdated(max_framerate_); - return; - } - overuse_detector_->OnTargetFramerateUpdated( - std::min(max_framerate_, requested_framerate)); - GetAdaptCounter().DecrementFramerate(reason); - break; - } - case DegradationPreference::DISABLED: - return; - } - - last_adaptation_request_.emplace(adaptation_request); - - UpdateAdaptationStats(reason); - - RTC_LOG(LS_INFO) << adapt_counter.ToString(); -} - -bool VideoStreamEncoder::CanAdaptUpResolution(int pixels, - uint32_t bitrate_bps) const { - absl::optional bitrate_limits = - GetEncoderBitrateLimits(encoder_info_, - source_proxy_->GetHigherResolutionThan(pixels)); - if (!bitrate_limits.has_value() || bitrate_bps == 0) { - return true; // No limit configured or bitrate provided. - } - RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels); - return bitrate_bps >= - static_cast(bitrate_limits->min_start_bitrate_bps); -} - -// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. -void VideoStreamEncoder::UpdateAdaptationStats(AdaptReason reason) { - switch (reason) { - case kCpu: - encoder_stats_observer_->OnAdaptationChanged( - VideoStreamEncoderObserver::AdaptationReason::kCpu, - GetActiveCounts(kCpu), GetActiveCounts(kQuality)); - break; - case kQuality: - encoder_stats_observer_->OnAdaptationChanged( - VideoStreamEncoderObserver::AdaptationReason::kQuality, - GetActiveCounts(kCpu), GetActiveCounts(kQuality)); - break; - } -} - -VideoStreamEncoderObserver::AdaptationSteps VideoStreamEncoder::GetActiveCounts( - AdaptReason reason) { - VideoStreamEncoderObserver::AdaptationSteps counts = - GetConstAdaptCounter().Counts(reason); - switch (reason) { - case kCpu: - if (!IsFramerateScalingEnabled(degradation_preference_)) - counts.num_framerate_reductions = absl::nullopt; - if (!IsResolutionScalingEnabled(degradation_preference_)) - counts.num_resolution_reductions = absl::nullopt; - break; - case kQuality: - if (!IsFramerateScalingEnabled(degradation_preference_) || - !quality_scaler_) { - counts.num_framerate_reductions = absl::nullopt; - } - if (!IsResolutionScalingEnabled(degradation_preference_) || - !quality_scaler_) { - counts.num_resolution_reductions = absl::nullopt; - } - break; - } - return counts; -} - -VideoStreamEncoder::AdaptCounter& VideoStreamEncoder::GetAdaptCounter() { - return adapt_counters_[degradation_preference_]; -} - -const VideoStreamEncoder::AdaptCounter& -VideoStreamEncoder::GetConstAdaptCounter() { - return adapt_counters_[degradation_preference_]; +void VideoStreamEncoder::TriggerAdaptUp( + AdaptationObserverInterface::AdaptReason reason) { + resource_adaptation_module_->AdaptUp(reason); } void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image, @@ -2294,7 +1770,7 @@ void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image, } } - overuse_detector_->FrameSent( + resource_adaptation_module_->FrameSent( encoded_image.Timestamp(), time_sent_us, encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec, encode_duration_us); @@ -2320,116 +1796,6 @@ void VideoStreamEncoder::ReleaseEncoder() { TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release"); } -// Class holding adaptation information. -VideoStreamEncoder::AdaptCounter::AdaptCounter() { - fps_counters_.resize(kScaleReasonSize); - resolution_counters_.resize(kScaleReasonSize); - static_assert(kScaleReasonSize == 2, "Update MoveCount."); -} - -VideoStreamEncoder::AdaptCounter::~AdaptCounter() {} - -std::string VideoStreamEncoder::AdaptCounter::ToString() const { - rtc::StringBuilder ss; - ss << "Downgrade counts: fps: {" << ToString(fps_counters_); - ss << "}, resolution: {" << ToString(resolution_counters_) << "}"; - return ss.Release(); -} - -VideoStreamEncoderObserver::AdaptationSteps -VideoStreamEncoder::AdaptCounter::Counts(int reason) const { - VideoStreamEncoderObserver::AdaptationSteps counts; - counts.num_framerate_reductions = fps_counters_[reason]; - counts.num_resolution_reductions = resolution_counters_[reason]; - return counts; -} - -void VideoStreamEncoder::AdaptCounter::IncrementFramerate(int reason) { - ++(fps_counters_[reason]); -} - -void VideoStreamEncoder::AdaptCounter::IncrementResolution(int reason) { - ++(resolution_counters_[reason]); -} - -void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason) { - if (fps_counters_[reason] == 0) { - // Balanced mode: Adapt up is in a different order, switch reason. - // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3). - // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0} - // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0} - // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0} - // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0} - RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; - RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; - MoveCount(&resolution_counters_, reason); - MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize); - } - --(fps_counters_[reason]); - RTC_DCHECK_GE(fps_counters_[reason], 0); -} - -void VideoStreamEncoder::AdaptCounter::DecrementResolution(int reason) { - if (resolution_counters_[reason] == 0) { - // Balanced mode: Adapt up is in a different order, switch reason. - RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; - RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; - MoveCount(&fps_counters_, reason); - MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize); - } - --(resolution_counters_[reason]); - RTC_DCHECK_GE(resolution_counters_[reason], 0); -} - -void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason, - int cur_fps) { - DecrementFramerate(reason); - // Reset if at max fps (i.e. in case of fewer steps up than down). - if (cur_fps == std::numeric_limits::max()) - absl::c_fill(fps_counters_, 0); -} - -int VideoStreamEncoder::AdaptCounter::FramerateCount() const { - return Count(fps_counters_); -} - -int VideoStreamEncoder::AdaptCounter::ResolutionCount() const { - return Count(resolution_counters_); -} - -int VideoStreamEncoder::AdaptCounter::FramerateCount(int reason) const { - return fps_counters_[reason]; -} - -int VideoStreamEncoder::AdaptCounter::ResolutionCount(int reason) const { - return resolution_counters_[reason]; -} - -int VideoStreamEncoder::AdaptCounter::TotalCount(int reason) const { - return FramerateCount(reason) + ResolutionCount(reason); -} - -int VideoStreamEncoder::AdaptCounter::Count( - const std::vector& counters) const { - return absl::c_accumulate(counters, 0); -} - -void VideoStreamEncoder::AdaptCounter::MoveCount(std::vector* counters, - int from_reason) { - int to_reason = (from_reason + 1) % kScaleReasonSize; - ++((*counters)[to_reason]); - --((*counters)[from_reason]); -} - -std::string VideoStreamEncoder::AdaptCounter::ToString( - const std::vector& counters) const { - rtc::StringBuilder ss; - for (size_t reason = 0; reason < kScaleReasonSize; ++reason) { - ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; - } - return ss.Release(); -} - bool VideoStreamEncoder::EncoderSwitchExperiment::IsBitrateBelowThreshold( const DataRate& target_bitrate) { DataRate rate = @@ -2562,7 +1928,8 @@ void VideoStreamEncoder::CheckForAnimatedContent( if (!automatic_animation_detection_experiment_.enabled || encoder_config_.content_type != VideoEncoderConfig::ContentType::kScreen || - degradation_preference_ != DegradationPreference::BALANCED) { + resource_adaptation_module_->degradation_preference() != + DegradationPreference::BALANCED) { return; } @@ -2619,23 +1986,10 @@ void VideoStreamEncoder::CheckForAnimatedContent( RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent " "animation detection."; } - source_proxy_->RestrictPixels(should_cap_resolution - ? kMaxAnimationPixels - : std::numeric_limits::max()); + resource_adaptation_module_->SetSourceMaxPixels( + should_cap_resolution ? kMaxAnimationPixels + : std::numeric_limits::max()); } } -DegradationPreference VideoStreamEncoder::EffectiveDegradataionPreference() - const { - // Balanced mode for screenshare works via automatic animation detection: - // Resolution is capped for fullscreen animated content. - // Adapatation is done only via framerate downgrade. - // Thus effective degradation preference is MAINTAIN_RESOLUTION. - return (encoder_config_.content_type == - VideoEncoderConfig::ContentType::kScreen && - degradation_preference_ == DegradationPreference::BALANCED) - ? DegradationPreference::MAINTAIN_RESOLUTION - : degradation_preference_; -} - } // namespace webrtc diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 9517944049..f180688a67 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -30,7 +30,6 @@ #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/critical_section.h" #include "rtc_base/event.h" -#include "rtc_base/experiments/balanced_degradation_settings.h" #include "rtc_base/experiments/quality_rampup_experiment.h" #include "rtc_base/experiments/quality_scaler_settings.h" #include "rtc_base/experiments/rate_control_settings.h" @@ -42,10 +41,14 @@ #include "system_wrappers/include/clock.h" #include "video/encoder_bitrate_adjuster.h" #include "video/frame_encode_metadata_writer.h" -#include "video/overuse_frame_detector.h" +#include "video/overuse_frame_detector_resource_adaptation_module.h" namespace webrtc { +absl::optional GetEncoderBitrateLimits( + const VideoEncoder::EncoderInfo& encoder_info, + int frame_size_pixels); + // VideoStreamEncoder represent a video encoder that accepts raw video frames as // input and produces an encoded bit stream. // Usage: @@ -55,9 +58,7 @@ namespace webrtc { // Call ConfigureEncoder with the codec settings. // Call Stop() when done. class VideoStreamEncoder : public VideoStreamEncoderInterface, - private EncodedImageCallback, - // Protected only to provide access to tests. - protected AdaptationObserverInterface { + private EncodedImageCallback { public: VideoStreamEncoder(Clock* clock, uint32_t number_of_cores, @@ -104,14 +105,14 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // be called on |encoder_queue_|. rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } - // AdaptationObserverInterface implementation. // These methods are protected for easier testing. - void AdaptUp(AdaptReason reason) override; - bool AdaptDown(AdaptReason reason) override; + // TODO(hbos): When "DropDueToSize" no longer causes TriggerAdaptDown(), these + // methods are only used for testing and can be removed in favor of the test + // invoking AdaptUp() or AdaptDown() on a test-injected adaptation module. + void TriggerAdaptUp(AdaptationObserverInterface::AdaptReason reason); + bool TriggerAdaptDown(AdaptationObserverInterface::AdaptReason reason); private: - class VideoSourceProxy; - class VideoFrameInfo { public: VideoFrameInfo(int width, int height, bool is_texture) @@ -183,50 +184,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void SetEncoderRates(const EncoderRateSettings& rate_settings) RTC_RUN_ON(&encoder_queue_); - // Class holding adaptation information. - class AdaptCounter final { - public: - AdaptCounter(); - ~AdaptCounter(); - - // Get number of adaptation downscales for |reason|. - VideoStreamEncoderObserver::AdaptationSteps Counts(int reason) const; - - std::string ToString() const; - - void IncrementFramerate(int reason); - void IncrementResolution(int reason); - void DecrementFramerate(int reason); - void DecrementResolution(int reason); - void DecrementFramerate(int reason, int cur_fps); - - // Gets the total number of downgrades (for all adapt reasons). - int FramerateCount() const; - int ResolutionCount() const; - - // Gets the total number of downgrades for |reason|. - int FramerateCount(int reason) const; - int ResolutionCount(int reason) const; - int TotalCount(int reason) const; - - private: - std::string ToString(const std::vector& counters) const; - int Count(const std::vector& counters) const; - void MoveCount(std::vector* counters, int from_reason); - - // Degradation counters holding number of framerate/resolution reductions - // per adapt reason. - std::vector fps_counters_; - std::vector resolution_counters_; - }; - - AdaptCounter& GetAdaptCounter() RTC_RUN_ON(&encoder_queue_); - const AdaptCounter& GetConstAdaptCounter() RTC_RUN_ON(&encoder_queue_); - void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(&encoder_queue_); - VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts( - AdaptReason reason) RTC_RUN_ON(&encoder_queue_); - bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const - RTC_RUN_ON(&encoder_queue_); void RunPostEncode(EncodedImage encoded_image, int64_t time_sent_us, int temporal_index, @@ -238,10 +195,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, int64_t time_when_posted_in_ms) RTC_RUN_ON(&encoder_queue_); - // Calculates degradation preference used in adaptation down or up. - DegradationPreference EffectiveDegradataionPreference() const - RTC_RUN_ON(&encoder_queue_); - rtc::Event shutdown_event_; const uint32_t number_of_cores_; @@ -255,14 +208,11 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, const bool quality_scaling_experiment_enabled_; - const std::unique_ptr source_proxy_; EncoderSink* sink_; const VideoStreamEncoderSettings settings_; const RateControlSettings rate_control_settings_; const QualityScalerSettings quality_scaler_settings_; - const std::unique_ptr overuse_detector_ - RTC_PT_GUARDED_BY(&encoder_queue_); std::unique_ptr quality_scaler_ RTC_GUARDED_BY(&encoder_queue_) RTC_PT_GUARDED_BY(&encoder_queue_); @@ -277,8 +227,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, bool encoder_initialized_; std::unique_ptr rate_allocator_ RTC_GUARDED_BY(&encoder_queue_) RTC_PT_GUARDED_BY(&encoder_queue_); - // The maximum frame rate of the current codec configuration, as determined - // at the last ReconfigureEncoder() call. int max_framerate_ RTC_GUARDED_BY(&encoder_queue_); // Set when ConfigureEncoder has been called in order to lazy reconfigure the @@ -308,30 +256,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, bool encoder_failed_ RTC_GUARDED_BY(&encoder_queue_); Clock* const clock_; - // Counters used for deciding if the video resolution or framerate is - // currently restricted, and if so, why, on a per degradation preference - // basis. - // TODO(sprang): Replace this with a state holding a relative overuse measure - // instead, that can be translated into suitable down-scale or fps limit. - std::map adapt_counters_ - RTC_GUARDED_BY(&encoder_queue_); - // Set depending on degradation preferences. - DegradationPreference degradation_preference_ RTC_GUARDED_BY(&encoder_queue_); - - const BalancedDegradationSettings balanced_settings_; - - struct AdaptationRequest { - // The pixel count produced by the source at the time of the adaptation. - int input_pixel_count_; - // Framerate received from the source at the time of the adaptation. - int framerate_fps_; - // Indicates if request was to adapt up or down. - enum class Mode { kAdaptUp, kAdaptDown } mode_; - }; - // Stores a snapshot of the last adaptation request triggered by an AdaptUp - // or AdaptDown signal. - absl::optional last_adaptation_request_ - RTC_GUARDED_BY(&encoder_queue_); rtc::RaceChecker incoming_frame_race_checker_ RTC_GUARDED_BY(incoming_frame_race_checker_); @@ -473,6 +397,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // track of whether a request has been made or not. bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_); + std::unique_ptr + resource_adaptation_module_; + // All public methods are proxied to |encoder_queue_|. It must must be // destroyed first to make sure no tasks are run that use other members. rtc::TaskQueue encoder_queue_; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 57b2b35e7c..4b4cd2d67f 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -155,17 +155,20 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { new CpuOveruseDetectorProxy(stats_proxy)), task_queue_factory) {} - void PostTaskAndWait(bool down, AdaptReason reason) { + void PostTaskAndWait(bool down, + AdaptationObserverInterface::AdaptReason reason) { PostTaskAndWait(down, reason, /*expected_results=*/true); } - void PostTaskAndWait(bool down, AdaptReason reason, bool expected_results) { + void PostTaskAndWait(bool down, + AdaptationObserverInterface::AdaptReason reason, + bool expected_results) { rtc::Event event; encoder_queue()->PostTask([this, &event, reason, down, expected_results] { if (down) - EXPECT_EQ(expected_results, AdaptDown(reason)); + EXPECT_EQ(expected_results, TriggerAdaptDown(reason)); else - AdaptUp(reason); + TriggerAdaptUp(reason); event.Set(); }); ASSERT_TRUE(event.Wait(5000)); @@ -180,24 +183,29 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { } void TriggerCpuOveruse() { - PostTaskAndWait(/*down=*/true, AdaptReason::kCpu); + PostTaskAndWait(/*down=*/true, + AdaptationObserverInterface::AdaptReason::kCpu); } void TriggerCpuNormalUsage() { - PostTaskAndWait(/*down=*/false, AdaptReason::kCpu); + PostTaskAndWait(/*down=*/false, + AdaptationObserverInterface::AdaptReason::kCpu); } void TriggerQualityLow() { - PostTaskAndWait(/*down=*/true, AdaptReason::kQuality); + PostTaskAndWait(/*down=*/true, + AdaptationObserverInterface::AdaptReason::kQuality); } void TriggerQualityLowExpectFalse() { - PostTaskAndWait(/*down=*/true, AdaptReason::kQuality, + PostTaskAndWait(/*down=*/true, + AdaptationObserverInterface::AdaptReason::kQuality, /*expected_results=*/false); } void TriggerQualityHigh() { - PostTaskAndWait(/*down=*/false, AdaptReason::kQuality); + PostTaskAndWait(/*down=*/false, + AdaptationObserverInterface::AdaptReason::kQuality); } CpuOveruseDetectorProxy* overuse_detector_proxy_;