/* * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "call/adaptation/resource_adaptation_processor.h" #include #include #include #include "absl/algorithm/container.h" #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" #include "rtc_base/strings/string_builder.h" #include "rtc_base/task_utils/to_queued_task.h" namespace webrtc { ResourceAdaptationProcessor::ResourceListenerDelegate::ResourceListenerDelegate( ResourceAdaptationProcessor* processor) : resource_adaptation_queue_(nullptr), processor_(processor) {} void ResourceAdaptationProcessor::ResourceListenerDelegate:: SetResourceAdaptationQueue(TaskQueueBase* resource_adaptation_queue) { RTC_DCHECK(!resource_adaptation_queue_); RTC_DCHECK(resource_adaptation_queue); resource_adaptation_queue_ = resource_adaptation_queue; RTC_DCHECK_RUN_ON(resource_adaptation_queue_); } void ResourceAdaptationProcessor::ResourceListenerDelegate:: OnProcessorDestroyed() { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); processor_ = nullptr; } void ResourceAdaptationProcessor::ResourceListenerDelegate:: OnResourceUsageStateMeasured(rtc::scoped_refptr resource, ResourceUsageState usage_state) { if (!resource_adaptation_queue_->IsCurrent()) { resource_adaptation_queue_->PostTask(ToQueuedTask( [this_ref = rtc::scoped_refptr(this), resource, usage_state] { this_ref->OnResourceUsageStateMeasured(resource, usage_state); })); return; } RTC_DCHECK_RUN_ON(resource_adaptation_queue_); if (processor_) { processor_->OnResourceUsageStateMeasured(resource, usage_state); } } ResourceAdaptationProcessor::MitigationResultAndLogMessage:: MitigationResultAndLogMessage() : result(MitigationResult::kAdaptationApplied), message() {} ResourceAdaptationProcessor::MitigationResultAndLogMessage:: MitigationResultAndLogMessage(MitigationResult result, std::string message) : result(result), message(std::move(message)) {} ResourceAdaptationProcessor::ResourceAdaptationProcessor( VideoStreamInputStateProvider* input_state_provider, VideoStreamEncoderObserver* encoder_stats_observer) : resource_adaptation_queue_(nullptr), resource_listener_delegate_( new rtc::RefCountedObject(this)), is_resource_adaptation_enabled_(false), input_state_provider_(input_state_provider), encoder_stats_observer_(encoder_stats_observer), resources_(), degradation_preference_(DegradationPreference::DISABLED), effective_degradation_preference_(DegradationPreference::DISABLED), is_screenshare_(false), stream_adapter_(std::make_unique()), last_reported_source_restrictions_(), previous_mitigation_results_(), processing_in_progress_(false) {} ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(!is_resource_adaptation_enabled_); RTC_DCHECK(restrictions_listeners_.empty()) << "There are restrictions listener(s) depending on a " << "ResourceAdaptationProcessor being destroyed."; RTC_DCHECK(resources_.empty()) << "There are resource(s) attached to a ResourceAdaptationProcessor " << "being destroyed."; RTC_DCHECK(adaptation_constraints_.empty()) << "There are constaint(s) attached to a ResourceAdaptationProcessor " << "being destroyed."; RTC_DCHECK(adaptation_listeners_.empty()) << "There are listener(s) attached to a ResourceAdaptationProcessor " << "being destroyed."; resource_listener_delegate_->OnProcessorDestroyed(); } void ResourceAdaptationProcessor::SetResourceAdaptationQueue( TaskQueueBase* resource_adaptation_queue) { RTC_DCHECK(!resource_adaptation_queue_); RTC_DCHECK(resource_adaptation_queue); resource_adaptation_queue_ = resource_adaptation_queue; resource_listener_delegate_->SetResourceAdaptationQueue( resource_adaptation_queue); RTC_DCHECK_RUN_ON(resource_adaptation_queue_); } DegradationPreference ResourceAdaptationProcessor::degradation_preference() const { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); return degradation_preference_; } DegradationPreference ResourceAdaptationProcessor::effective_degradation_preference() const { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); return effective_degradation_preference_; } void ResourceAdaptationProcessor::StartResourceAdaptation() { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); if (is_resource_adaptation_enabled_) return; for (const auto& resource : resources_) { resource->SetResourceListener(resource_listener_delegate_); } is_resource_adaptation_enabled_ = true; } void ResourceAdaptationProcessor::StopResourceAdaptation() { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); if (!is_resource_adaptation_enabled_) return; for (const auto& resource : resources_) { resource->SetResourceListener(nullptr); } is_resource_adaptation_enabled_ = false; } void ResourceAdaptationProcessor::AddRestrictionsListener( VideoSourceRestrictionsListener* restrictions_listener) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(std::find(restrictions_listeners_.begin(), restrictions_listeners_.end(), restrictions_listener) == restrictions_listeners_.end()); restrictions_listeners_.push_back(restrictions_listener); } void ResourceAdaptationProcessor::RemoveRestrictionsListener( VideoSourceRestrictionsListener* restrictions_listener) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); auto it = std::find(restrictions_listeners_.begin(), restrictions_listeners_.end(), restrictions_listener); RTC_DCHECK(it != restrictions_listeners_.end()); restrictions_listeners_.erase(it); } void ResourceAdaptationProcessor::AddResource( rtc::scoped_refptr resource) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); // TODO(hbos): Allow adding resources while |is_resource_adaptation_enabled_| // by registering as a listener of the resource on adding it. RTC_DCHECK(!is_resource_adaptation_enabled_); RTC_DCHECK(std::find(resources_.begin(), resources_.end(), resource) == resources_.end()); resources_.push_back(resource); } std::vector> ResourceAdaptationProcessor::GetResources() const { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); return resources_; } void ResourceAdaptationProcessor::RemoveResource( rtc::scoped_refptr resource) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); // TODO(hbos): Allow removing resources while // |is_resource_adaptation_enabled_| by unregistering as a listener of the // resource on removing it. RTC_DCHECK(!is_resource_adaptation_enabled_); auto it = std::find(resources_.begin(), resources_.end(), resource); RTC_DCHECK(it != resources_.end()); resources_.erase(it); } void ResourceAdaptationProcessor::AddAdaptationConstraint( AdaptationConstraint* adaptation_constraint) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(std::find(adaptation_constraints_.begin(), adaptation_constraints_.end(), adaptation_constraint) == adaptation_constraints_.end()); adaptation_constraints_.push_back(adaptation_constraint); } void ResourceAdaptationProcessor::RemoveAdaptationConstraint( AdaptationConstraint* adaptation_constraint) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); auto it = std::find(adaptation_constraints_.begin(), adaptation_constraints_.end(), adaptation_constraint); RTC_DCHECK(it != adaptation_constraints_.end()); adaptation_constraints_.erase(it); } void ResourceAdaptationProcessor::AddAdaptationListener( AdaptationListener* adaptation_listener) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(std::find(adaptation_listeners_.begin(), adaptation_listeners_.end(), adaptation_listener) == adaptation_listeners_.end()); adaptation_listeners_.push_back(adaptation_listener); } void ResourceAdaptationProcessor::RemoveAdaptationListener( AdaptationListener* adaptation_listener) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); auto it = std::find(adaptation_listeners_.begin(), adaptation_listeners_.end(), adaptation_listener); RTC_DCHECK(it != adaptation_listeners_.end()); adaptation_listeners_.erase(it); } void ResourceAdaptationProcessor::SetDegradationPreference( DegradationPreference degradation_preference) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); degradation_preference_ = degradation_preference; MaybeUpdateEffectiveDegradationPreference(); } void ResourceAdaptationProcessor::SetIsScreenshare(bool is_screenshare) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); is_screenshare_ = is_screenshare; MaybeUpdateEffectiveDegradationPreference(); } void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); effective_degradation_preference_ = (is_screenshare_ && degradation_preference_ == DegradationPreference::BALANCED) ? DegradationPreference::MAINTAIN_RESOLUTION : degradation_preference_; stream_adapter_->SetDegradationPreference(effective_degradation_preference_); MaybeUpdateVideoSourceRestrictions(nullptr); } void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_LOG(INFO) << "Resetting restrictions"; stream_adapter_->ClearRestrictions(); adaptation_limits_by_resources_.clear(); for (auto restrictions_listener : restrictions_listeners_) { restrictions_listener->OnResourceLimitationChanged( nullptr, adaptation_limits_by_resources_); } MaybeUpdateVideoSourceRestrictions(nullptr); } void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions( rtc::scoped_refptr reason) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); VideoSourceRestrictions new_source_restrictions = FilterRestrictionsByDegradationPreference( stream_adapter_->source_restrictions(), effective_degradation_preference_); if (last_reported_source_restrictions_ != new_source_restrictions) { RTC_LOG(INFO) << "Reporting new restrictions (in " << DegradationPreferenceToString( effective_degradation_preference_) << "): " << new_source_restrictions.ToString(); last_reported_source_restrictions_ = std::move(new_source_restrictions); for (auto* restrictions_listener : restrictions_listeners_) { restrictions_listener->OnVideoSourceRestrictionsUpdated( last_reported_source_restrictions_, stream_adapter_->adaptation_counters(), reason); } } } void ResourceAdaptationProcessor::OnResourceUsageStateMeasured( rtc::scoped_refptr resource, ResourceUsageState usage_state) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); MitigationResultAndLogMessage result_and_message; switch (usage_state) { case ResourceUsageState::kOveruse: result_and_message = OnResourceOveruse(resource); break; case ResourceUsageState::kUnderuse: result_and_message = OnResourceUnderuse(resource); break; } // Maybe log the result of the operation. auto it = previous_mitigation_results_.find(resource.get()); if (it != previous_mitigation_results_.end() && it->second == result_and_message.result) { // This resource has previously reported the same result and we haven't // successfully adapted since - don't log to avoid spam. return; } RTC_LOG(INFO) << "Resource \"" << resource->Name() << "\" signalled " << ResourceUsageStateToString(usage_state) << ". " << result_and_message.message; if (result_and_message.result == MitigationResult::kAdaptationApplied) { previous_mitigation_results_.clear(); } else { previous_mitigation_results_.insert( std::make_pair(resource.get(), result_and_message.result)); } } bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation( const VideoStreamInputState& input_state) const { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); return input_state.HasInputFrameSizeAndFramesPerSecond() && (effective_degradation_preference_ != DegradationPreference::MAINTAIN_RESOLUTION || input_state.frames_per_second() >= kMinFrameRateFps); } ResourceAdaptationProcessor::MitigationResultAndLogMessage ResourceAdaptationProcessor::OnResourceUnderuse( rtc::scoped_refptr reason_resource) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(!processing_in_progress_); processing_in_progress_ = true; if (effective_degradation_preference_ == DegradationPreference::DISABLED) { processing_in_progress_ = false; return MitigationResultAndLogMessage( MitigationResult::kDisabled, "Not adapting up because DegradationPreference is disabled"); } VideoStreamInputState input_state = input_state_provider_->InputState(); if (!HasSufficientInputForAdaptation(input_state)) { processing_in_progress_ = false; return MitigationResultAndLogMessage( MitigationResult::kInsufficientInput, "Not adapting up because input is insufficient"); } // Update video input states and encoder settings for accurate adaptation. stream_adapter_->SetInput(input_state); // How can this stream be adapted up? Adaptation adaptation = stream_adapter_->GetAdaptationUp(); if (adaptation.status() != Adaptation::Status::kValid) { processing_in_progress_ = false; rtc::StringBuilder message; message << "Not adapting up because VideoStreamAdapter returned " << Adaptation::StatusToString(adaptation.status()); return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter, message.Release()); } VideoSourceRestrictions restrictions_before = stream_adapter_->source_restrictions(); VideoStreamAdapter::RestrictionsWithCounters peek_restrictions = stream_adapter_->PeekNextRestrictions(adaptation); VideoSourceRestrictions restrictions_after = peek_restrictions.restrictions; // Check that resource is most limited... std::vector> most_limited_resources; VideoAdaptationCounters most_limited_restrictions; std::tie(most_limited_resources, most_limited_restrictions) = FindMostLimitedResources(); RTC_DCHECK(!most_limited_resources.empty()) << "Can not have no limited resources when adaptation status is valid. " "Should be kLimitReached."; for (const auto* constraint : adaptation_constraints_) { if (!constraint->IsAdaptationUpAllowed(input_state, restrictions_before, restrictions_after, reason_resource)) { processing_in_progress_ = false; rtc::StringBuilder message; message << "Not adapting up because constraint \"" << constraint->Name() << "\" disallowed it"; return MitigationResultAndLogMessage( MitigationResult::kRejectedByConstraint, message.Release()); } } // If the most restricted resource is less limited than current restrictions // then proceed with adapting up. if (most_limited_restrictions.Total() >= stream_adapter_->adaptation_counters().Total()) { // If |reason_resource| is not one of the most limiting resources then abort // adaptation. if (absl::c_find(most_limited_resources, reason_resource) == most_limited_resources.end()) { processing_in_progress_ = false; rtc::StringBuilder message; message << "Resource \"" << reason_resource->Name() << "\" was not the most limited resource."; return MitigationResultAndLogMessage( MitigationResult::kNotMostLimitedResource, message.Release()); } UpdateResourceLimitations(reason_resource, peek_restrictions); if (most_limited_resources.size() > 1) { // If there are multiple most limited resources, all must signal underuse // before the adaptation is applied. processing_in_progress_ = false; rtc::StringBuilder message; message << "Resource \"" << reason_resource->Name() << "\" was not the only most limited resource."; return MitigationResultAndLogMessage( MitigationResult::kSharedMostLimitedResource, message.Release()); } } // Apply adaptation. stream_adapter_->ApplyAdaptation(adaptation); for (auto* adaptation_listener : adaptation_listeners_) { adaptation_listener->OnAdaptationApplied( input_state, restrictions_before, restrictions_after, reason_resource); } // Update VideoSourceRestrictions based on adaptation. This also informs the // |restrictions_listeners_|. MaybeUpdateVideoSourceRestrictions(reason_resource); processing_in_progress_ = false; rtc::StringBuilder message; message << "Adapted up successfully. Unfiltered adaptations: " << stream_adapter_->adaptation_counters().ToString(); return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied, message.Release()); } ResourceAdaptationProcessor::MitigationResultAndLogMessage ResourceAdaptationProcessor::OnResourceOveruse( rtc::scoped_refptr reason_resource) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(!processing_in_progress_); processing_in_progress_ = true; if (effective_degradation_preference_ == DegradationPreference::DISABLED) { processing_in_progress_ = false; return MitigationResultAndLogMessage( MitigationResult::kDisabled, "Not adapting down because DegradationPreference is disabled"); } VideoStreamInputState input_state = input_state_provider_->InputState(); if (!HasSufficientInputForAdaptation(input_state)) { processing_in_progress_ = false; return MitigationResultAndLogMessage( MitigationResult::kInsufficientInput, "Not adapting down because input is insufficient"); } // Update video input states and encoder settings for accurate adaptation. stream_adapter_->SetInput(input_state); // How can this stream be adapted up? Adaptation adaptation = stream_adapter_->GetAdaptationDown(); if (adaptation.min_pixel_limit_reached()) { encoder_stats_observer_->OnMinPixelLimitReached(); } if (adaptation.status() != Adaptation::Status::kValid) { processing_in_progress_ = false; rtc::StringBuilder message; message << "Not adapting down because VideoStreamAdapter returned " << Adaptation::StatusToString(adaptation.status()); return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter, message.Release()); } // Apply adaptation. VideoSourceRestrictions restrictions_before = stream_adapter_->source_restrictions(); VideoStreamAdapter::RestrictionsWithCounters peek_next_restrictions = stream_adapter_->PeekNextRestrictions(adaptation); VideoSourceRestrictions restrictions_after = peek_next_restrictions.restrictions; UpdateResourceLimitations(reason_resource, peek_next_restrictions); stream_adapter_->ApplyAdaptation(adaptation); for (auto* adaptation_listener : adaptation_listeners_) { adaptation_listener->OnAdaptationApplied( input_state, restrictions_before, restrictions_after, reason_resource); } // Update VideoSourceRestrictions based on adaptation. This also informs the // |restrictions_listeners_|. MaybeUpdateVideoSourceRestrictions(reason_resource); processing_in_progress_ = false; rtc::StringBuilder message; message << "Adapted down successfully. Unfiltered adaptations: " << stream_adapter_->adaptation_counters().ToString(); return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied, message.Release()); } void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize( rtc::scoped_refptr reason_resource) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_LOG(INFO) << "TriggerAdaptationDueToFrameDroppedDueToSize called"; VideoAdaptationCounters counters_before = stream_adapter_->adaptation_counters(); OnResourceOveruse(reason_resource); if (degradation_preference_ == DegradationPreference::BALANCED && stream_adapter_->adaptation_counters().fps_adaptations > counters_before.fps_adaptations) { // Oops, we adapted frame rate. Adapt again, maybe it will adapt resolution! // Though this is not guaranteed... OnResourceOveruse(reason_resource); } if (stream_adapter_->adaptation_counters().resolution_adaptations > counters_before.resolution_adaptations) { encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); } } std::pair>, VideoAdaptationCounters> ResourceAdaptationProcessor::FindMostLimitedResources() const { std::vector> most_limited_resources; VideoAdaptationCounters most_limited_restrictions; for (const auto& resource_and_adaptation_limit_ : adaptation_limits_by_resources_) { const VideoAdaptationCounters& counters = resource_and_adaptation_limit_.second; if (counters.Total() > most_limited_restrictions.Total()) { most_limited_restrictions = counters; most_limited_resources.clear(); most_limited_resources.push_back(resource_and_adaptation_limit_.first); } else if (most_limited_restrictions == counters) { most_limited_resources.push_back(resource_and_adaptation_limit_.first); } } return std::make_pair(std::move(most_limited_resources), most_limited_restrictions); } void ResourceAdaptationProcessor::UpdateResourceLimitations( rtc::scoped_refptr reason_resource, const VideoStreamAdapter::RestrictionsWithCounters& peek_next_restrictions) { adaptation_limits_by_resources_[reason_resource] = peek_next_restrictions.adaptation_counters; for (auto restrictions_listener : restrictions_listeners_) { restrictions_listener->OnResourceLimitationChanged( reason_resource, adaptation_limits_by_resources_); } } } // namespace webrtc