b93ad75c94
Some restrictions previously were preventing adaptation up caused by the quality resource. However, it makes sense to use the same restrictions in the case of other resources. This CL removes now unneeded wire-up of reason/resource causing adaptation. Bug: webrtc:11771 Change-Id: Iec301a59d2a41d32d23b6be340f3b5637d697e52 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/181580 Commit-Queue: Evan Shrubsole <eshr@google.com> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Cr-Commit-Position: refs/heads/master@{#31926}
392 lines
16 KiB
C++
392 lines
16 KiB
C++
/*
|
|
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "call/adaptation/resource_adaptation_processor.h"
|
|
|
|
#include <algorithm>
|
|
#include <string>
|
|
#include <utility>
|
|
|
|
#include "absl/algorithm/container.h"
|
|
#include "api/video/video_adaptation_counters.h"
|
|
#include "call/adaptation/video_stream_adapter.h"
|
|
#include "rtc_base/logging.h"
|
|
#include "rtc_base/ref_counted_object.h"
|
|
#include "rtc_base/strings/string_builder.h"
|
|
#include "rtc_base/synchronization/sequence_checker.h"
|
|
#include "rtc_base/task_utils/to_queued_task.h"
|
|
|
|
namespace webrtc {
|
|
|
|
ResourceAdaptationProcessor::ResourceListenerDelegate::ResourceListenerDelegate(
|
|
ResourceAdaptationProcessor* processor)
|
|
: resource_adaptation_queue_(nullptr), processor_(processor) {}
|
|
|
|
void ResourceAdaptationProcessor::ResourceListenerDelegate::
|
|
SetResourceAdaptationQueue(TaskQueueBase* resource_adaptation_queue) {
|
|
RTC_DCHECK(!resource_adaptation_queue_);
|
|
RTC_DCHECK(resource_adaptation_queue);
|
|
resource_adaptation_queue_ = resource_adaptation_queue;
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::ResourceListenerDelegate::
|
|
OnProcessorDestroyed() {
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
processor_ = nullptr;
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::ResourceListenerDelegate::
|
|
OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource,
|
|
ResourceUsageState usage_state) {
|
|
if (!resource_adaptation_queue_->IsCurrent()) {
|
|
resource_adaptation_queue_->PostTask(ToQueuedTask(
|
|
[this_ref = rtc::scoped_refptr<ResourceListenerDelegate>(this),
|
|
resource, usage_state] {
|
|
this_ref->OnResourceUsageStateMeasured(resource, usage_state);
|
|
}));
|
|
return;
|
|
}
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
if (processor_) {
|
|
processor_->OnResourceUsageStateMeasured(resource, usage_state);
|
|
}
|
|
}
|
|
|
|
ResourceAdaptationProcessor::MitigationResultAndLogMessage::
|
|
MitigationResultAndLogMessage()
|
|
: result(MitigationResult::kAdaptationApplied), message() {}
|
|
|
|
ResourceAdaptationProcessor::MitigationResultAndLogMessage::
|
|
MitigationResultAndLogMessage(MitigationResult result, std::string message)
|
|
: result(result), message(std::move(message)) {}
|
|
|
|
ResourceAdaptationProcessor::ResourceAdaptationProcessor(
|
|
VideoStreamEncoderObserver* encoder_stats_observer,
|
|
VideoStreamAdapter* stream_adapter)
|
|
: resource_adaptation_queue_(nullptr),
|
|
resource_listener_delegate_(
|
|
new rtc::RefCountedObject<ResourceListenerDelegate>(this)),
|
|
encoder_stats_observer_(encoder_stats_observer),
|
|
resources_(),
|
|
stream_adapter_(stream_adapter),
|
|
last_reported_source_restrictions_(),
|
|
previous_mitigation_results_() {
|
|
RTC_DCHECK(stream_adapter_);
|
|
}
|
|
|
|
ResourceAdaptationProcessor::~ResourceAdaptationProcessor() {
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
RTC_DCHECK(resources_.empty())
|
|
<< "There are resource(s) attached to a ResourceAdaptationProcessor "
|
|
<< "being destroyed.";
|
|
stream_adapter_->RemoveRestrictionsListener(this);
|
|
resource_listener_delegate_->OnProcessorDestroyed();
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::SetResourceAdaptationQueue(
|
|
TaskQueueBase* resource_adaptation_queue) {
|
|
RTC_DCHECK(!resource_adaptation_queue_);
|
|
RTC_DCHECK(resource_adaptation_queue);
|
|
resource_adaptation_queue_ = resource_adaptation_queue;
|
|
resource_listener_delegate_->SetResourceAdaptationQueue(
|
|
resource_adaptation_queue);
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
// Now that we have the adaptation queue we can attach as adaptation listener.
|
|
stream_adapter_->AddRestrictionsListener(this);
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::AddResourceLimitationsListener(
|
|
ResourceLimitationsListener* limitations_listener) {
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
RTC_DCHECK(std::find(resource_limitations_listeners_.begin(),
|
|
resource_limitations_listeners_.end(),
|
|
limitations_listener) ==
|
|
resource_limitations_listeners_.end());
|
|
resource_limitations_listeners_.push_back(limitations_listener);
|
|
}
|
|
void ResourceAdaptationProcessor::RemoveResourceLimitationsListener(
|
|
ResourceLimitationsListener* limitations_listener) {
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
auto it =
|
|
std::find(resource_limitations_listeners_.begin(),
|
|
resource_limitations_listeners_.end(), limitations_listener);
|
|
RTC_DCHECK(it != resource_limitations_listeners_.end());
|
|
resource_limitations_listeners_.erase(it);
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::AddResource(
|
|
rtc::scoped_refptr<Resource> resource) {
|
|
RTC_DCHECK(resource);
|
|
{
|
|
MutexLock crit(&resources_lock_);
|
|
RTC_DCHECK(absl::c_find(resources_, resource) == resources_.end())
|
|
<< "Resource \"" << resource->Name() << "\" was already registered.";
|
|
resources_.push_back(resource);
|
|
}
|
|
resource->SetResourceListener(resource_listener_delegate_);
|
|
RTC_LOG(INFO) << "Registered resource \"" << resource->Name() << "\".";
|
|
}
|
|
|
|
std::vector<rtc::scoped_refptr<Resource>>
|
|
ResourceAdaptationProcessor::GetResources() const {
|
|
MutexLock crit(&resources_lock_);
|
|
return resources_;
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::RemoveResource(
|
|
rtc::scoped_refptr<Resource> resource) {
|
|
RTC_DCHECK(resource);
|
|
RTC_LOG(INFO) << "Removing resource \"" << resource->Name() << "\".";
|
|
resource->SetResourceListener(nullptr);
|
|
{
|
|
MutexLock crit(&resources_lock_);
|
|
auto it = absl::c_find(resources_, resource);
|
|
RTC_DCHECK(it != resources_.end()) << "Resource \"" << resource->Name()
|
|
<< "\" was not a registered resource.";
|
|
resources_.erase(it);
|
|
}
|
|
RemoveLimitationsImposedByResource(std::move(resource));
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::RemoveLimitationsImposedByResource(
|
|
rtc::scoped_refptr<Resource> resource) {
|
|
if (!resource_adaptation_queue_->IsCurrent()) {
|
|
resource_adaptation_queue_->PostTask(ToQueuedTask(
|
|
[this, resource]() { RemoveLimitationsImposedByResource(resource); }));
|
|
return;
|
|
}
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
auto resource_adaptation_limits =
|
|
adaptation_limits_by_resources_.find(resource);
|
|
if (resource_adaptation_limits != adaptation_limits_by_resources_.end()) {
|
|
VideoStreamAdapter::RestrictionsWithCounters adaptation_limits =
|
|
resource_adaptation_limits->second;
|
|
adaptation_limits_by_resources_.erase(resource_adaptation_limits);
|
|
if (adaptation_limits_by_resources_.empty()) {
|
|
// Only the resource being removed was adapted so clear restrictions.
|
|
stream_adapter_->ClearRestrictions();
|
|
return;
|
|
}
|
|
|
|
VideoStreamAdapter::RestrictionsWithCounters most_limited =
|
|
FindMostLimitedResources().second;
|
|
|
|
if (adaptation_limits.counters.Total() <= most_limited.counters.Total()) {
|
|
// The removed limitations were less limited than the most limited
|
|
// resource. Don't change the current restrictions.
|
|
return;
|
|
}
|
|
|
|
// Apply the new most limited resource as the next restrictions.
|
|
Adaptation adapt_to = stream_adapter_->GetAdaptationTo(
|
|
most_limited.counters, most_limited.restrictions);
|
|
RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid);
|
|
stream_adapter_->ApplyAdaptation(adapt_to, nullptr);
|
|
|
|
RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to "
|
|
"next most limited restrictions: "
|
|
<< most_limited.restrictions.ToString() << " with counters "
|
|
<< most_limited.counters.ToString();
|
|
}
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::OnResourceUsageStateMeasured(
|
|
rtc::scoped_refptr<Resource> resource,
|
|
ResourceUsageState usage_state) {
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
RTC_DCHECK(resource);
|
|
// |resource| could have been removed after signalling.
|
|
{
|
|
MutexLock crit(&resources_lock_);
|
|
if (absl::c_find(resources_, resource) == resources_.end()) {
|
|
RTC_LOG(INFO) << "Ignoring signal from removed resource \""
|
|
<< resource->Name() << "\".";
|
|
return;
|
|
}
|
|
}
|
|
MitigationResultAndLogMessage result_and_message;
|
|
switch (usage_state) {
|
|
case ResourceUsageState::kOveruse:
|
|
result_and_message = OnResourceOveruse(resource);
|
|
break;
|
|
case ResourceUsageState::kUnderuse:
|
|
result_and_message = OnResourceUnderuse(resource);
|
|
break;
|
|
}
|
|
// Maybe log the result of the operation.
|
|
auto it = previous_mitigation_results_.find(resource.get());
|
|
if (it != previous_mitigation_results_.end() &&
|
|
it->second == result_and_message.result) {
|
|
// This resource has previously reported the same result and we haven't
|
|
// successfully adapted since - don't log to avoid spam.
|
|
return;
|
|
}
|
|
RTC_LOG(INFO) << "Resource \"" << resource->Name() << "\" signalled "
|
|
<< ResourceUsageStateToString(usage_state) << ". "
|
|
<< result_and_message.message;
|
|
if (result_and_message.result == MitigationResult::kAdaptationApplied) {
|
|
previous_mitigation_results_.clear();
|
|
} else {
|
|
previous_mitigation_results_.insert(
|
|
std::make_pair(resource.get(), result_and_message.result));
|
|
}
|
|
}
|
|
|
|
ResourceAdaptationProcessor::MitigationResultAndLogMessage
|
|
ResourceAdaptationProcessor::OnResourceUnderuse(
|
|
rtc::scoped_refptr<Resource> reason_resource) {
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
// How can this stream be adapted up?
|
|
Adaptation adaptation = stream_adapter_->GetAdaptationUp();
|
|
if (adaptation.status() != Adaptation::Status::kValid) {
|
|
rtc::StringBuilder message;
|
|
message << "Not adapting up because VideoStreamAdapter returned "
|
|
<< Adaptation::StatusToString(adaptation.status());
|
|
return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter,
|
|
message.Release());
|
|
}
|
|
// Check that resource is most limited.
|
|
std::vector<rtc::scoped_refptr<Resource>> most_limited_resources;
|
|
VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions;
|
|
std::tie(most_limited_resources, most_limited_restrictions) =
|
|
FindMostLimitedResources();
|
|
|
|
// If the most restricted resource is less limited than current restrictions
|
|
// then proceed with adapting up.
|
|
if (!most_limited_resources.empty() &&
|
|
most_limited_restrictions.counters.Total() >=
|
|
stream_adapter_->adaptation_counters().Total()) {
|
|
// If |reason_resource| is not one of the most limiting resources then abort
|
|
// adaptation.
|
|
if (absl::c_find(most_limited_resources, reason_resource) ==
|
|
most_limited_resources.end()) {
|
|
rtc::StringBuilder message;
|
|
message << "Resource \"" << reason_resource->Name()
|
|
<< "\" was not the most limited resource.";
|
|
return MitigationResultAndLogMessage(
|
|
MitigationResult::kNotMostLimitedResource, message.Release());
|
|
}
|
|
|
|
if (most_limited_resources.size() > 1) {
|
|
// If there are multiple most limited resources, all must signal underuse
|
|
// before the adaptation is applied.
|
|
UpdateResourceLimitations(reason_resource, adaptation.restrictions(),
|
|
adaptation.counters());
|
|
rtc::StringBuilder message;
|
|
message << "Resource \"" << reason_resource->Name()
|
|
<< "\" was not the only most limited resource.";
|
|
return MitigationResultAndLogMessage(
|
|
MitigationResult::kSharedMostLimitedResource, message.Release());
|
|
}
|
|
}
|
|
// Apply adaptation.
|
|
stream_adapter_->ApplyAdaptation(adaptation, reason_resource);
|
|
rtc::StringBuilder message;
|
|
message << "Adapted up successfully. Unfiltered adaptations: "
|
|
<< stream_adapter_->adaptation_counters().ToString();
|
|
return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied,
|
|
message.Release());
|
|
}
|
|
|
|
ResourceAdaptationProcessor::MitigationResultAndLogMessage
|
|
ResourceAdaptationProcessor::OnResourceOveruse(
|
|
rtc::scoped_refptr<Resource> reason_resource) {
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
// How can this stream be adapted up?
|
|
Adaptation adaptation = stream_adapter_->GetAdaptationDown();
|
|
if (adaptation.min_pixel_limit_reached()) {
|
|
encoder_stats_observer_->OnMinPixelLimitReached();
|
|
}
|
|
if (adaptation.status() != Adaptation::Status::kValid) {
|
|
rtc::StringBuilder message;
|
|
message << "Not adapting down because VideoStreamAdapter returned "
|
|
<< Adaptation::StatusToString(adaptation.status());
|
|
return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter,
|
|
message.Release());
|
|
}
|
|
// Apply adaptation.
|
|
UpdateResourceLimitations(reason_resource, adaptation.restrictions(),
|
|
adaptation.counters());
|
|
stream_adapter_->ApplyAdaptation(adaptation, reason_resource);
|
|
rtc::StringBuilder message;
|
|
message << "Adapted down successfully. Unfiltered adaptations: "
|
|
<< stream_adapter_->adaptation_counters().ToString();
|
|
return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied,
|
|
message.Release());
|
|
}
|
|
|
|
std::pair<std::vector<rtc::scoped_refptr<Resource>>,
|
|
VideoStreamAdapter::RestrictionsWithCounters>
|
|
ResourceAdaptationProcessor::FindMostLimitedResources() const {
|
|
std::vector<rtc::scoped_refptr<Resource>> most_limited_resources;
|
|
VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions{
|
|
VideoSourceRestrictions(), VideoAdaptationCounters()};
|
|
|
|
for (const auto& resource_and_adaptation_limit_ :
|
|
adaptation_limits_by_resources_) {
|
|
const auto& restrictions_with_counters =
|
|
resource_and_adaptation_limit_.second;
|
|
if (restrictions_with_counters.counters.Total() >
|
|
most_limited_restrictions.counters.Total()) {
|
|
most_limited_restrictions = restrictions_with_counters;
|
|
most_limited_resources.clear();
|
|
most_limited_resources.push_back(resource_and_adaptation_limit_.first);
|
|
} else if (most_limited_restrictions.counters ==
|
|
restrictions_with_counters.counters) {
|
|
most_limited_resources.push_back(resource_and_adaptation_limit_.first);
|
|
}
|
|
}
|
|
return std::make_pair(std::move(most_limited_resources),
|
|
most_limited_restrictions);
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::UpdateResourceLimitations(
|
|
rtc::scoped_refptr<Resource> reason_resource,
|
|
const VideoSourceRestrictions& restrictions,
|
|
const VideoAdaptationCounters& counters) {
|
|
auto& adaptation_limits = adaptation_limits_by_resources_[reason_resource];
|
|
if (adaptation_limits.restrictions == restrictions &&
|
|
adaptation_limits.counters == counters) {
|
|
return;
|
|
}
|
|
adaptation_limits = {restrictions, counters};
|
|
|
|
std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters> limitations;
|
|
for (const auto& p : adaptation_limits_by_resources_) {
|
|
limitations.insert(std::make_pair(p.first, p.second.counters));
|
|
}
|
|
for (auto limitations_listener : resource_limitations_listeners_) {
|
|
limitations_listener->OnResourceLimitationChanged(reason_resource,
|
|
limitations);
|
|
}
|
|
}
|
|
|
|
void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated(
|
|
VideoSourceRestrictions restrictions,
|
|
const VideoAdaptationCounters& adaptation_counters,
|
|
rtc::scoped_refptr<Resource> reason,
|
|
const VideoSourceRestrictions& unfiltered_restrictions) {
|
|
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
|
if (reason) {
|
|
UpdateResourceLimitations(reason, unfiltered_restrictions,
|
|
adaptation_counters);
|
|
} else if (adaptation_counters.Total() == 0) {
|
|
// Adaptations are cleared.
|
|
adaptation_limits_by_resources_.clear();
|
|
previous_mitigation_results_.clear();
|
|
for (auto limitations_listener : resource_limitations_listeners_) {
|
|
limitations_listener->OnResourceLimitationChanged(nullptr, {});
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace webrtc
|