admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Only allow most limited resource to trigger adapt up

Browse Source 
A more detailed explaination is in the bug, but this changes
the way that adaptation happens when multiple resources are
limited. Only the one that is most limited can trigger an
adaptation up. If multiple resources are most limited both
need to underuse to adapt up.

Some of the changes in this patch to make it all work:

* VideoStreamEncoder unittests that did not reflect this
new behaviour have been changed.

* PeekNextRestrictions returns the adaptation counters as
well as the restrictions.

* Adaptation statstics have changed so that when adapting
up all resources are tagged as triggering the adaptation.
Additionally the statistics for the current adaptation is
now the total number of adaptations per reason, rather then
the number of adaptations due to that reason.

* PreventAdaptUpDueToActiveCounts is removed as most limited
resource is a strong implementation of that.

Bug: webrtc:11553
Change-Id: If1545a201c8e019598edf82657a1befde8b05268
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/176128
Commit-Queue: Evan Shrubsole <eshr@google.com>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31497}
This commit is contained in:
Evan Shrubsole 2020-06-11 10:45:29 +02:00 committed by Commit Bot
parent 8a89b5bc0d
commit 64469037b7
15 changed files with 400 additions and 442 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
call/adaptation/resource_adaptation_processor.cc
View File

@ -248,7 +248,11 @@ void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
RTC_LOG(INFO) << "Resetting restrictions";
stream_adapter_->ClearRestrictions();
adaptations_counts_by_resource_.clear();
adaptation_limits_by_resources_.clear();
for (auto restrictions_listener : restrictions_listeners_) {
restrictions_listener->OnResourceLimitationChanged(
nullptr, adaptation_limits_by_resources_);
}
MaybeUpdateVideoSourceRestrictions(nullptr);
}
@ -270,9 +274,6 @@ void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions(
last_reported_source_restrictions_,
stream_adapter_->adaptation_counters(), reason);
}
if (reason) {
UpdateResourceDegradationCounts(reason);
}
}
}
@ -336,13 +337,6 @@ ResourceAdaptationProcessor::OnResourceUnderuse(
MitigationResult::kInsufficientInput,
"Not adapting up because input is insufficient");
}
if (!IsResourceAllowedToAdaptUp(reason_resource)) {
processing_in_progress_ = false;
return MitigationResultAndLogMessage(
MitigationResult::kRejectedByAdaptationCounts,
"Not adapting up because this resource has not previously adapted down "
"(according to adaptation counters)");
}
// Update video input states and encoder settings for accurate adaptation.
stream_adapter_->SetInput(input_state);
// How can this stream be adapted up?
@ -355,11 +349,20 @@ ResourceAdaptationProcessor::OnResourceUnderuse(
return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter,
message.Release());
}
// Are all resources OK with this adaptation being applied?
VideoSourceRestrictions restrictions_before =
stream_adapter_->source_restrictions();
VideoSourceRestrictions restrictions_after =
VideoStreamAdapter::RestrictionsWithCounters peek_restrictions =
stream_adapter_->PeekNextRestrictions(adaptation);
VideoSourceRestrictions restrictions_after = peek_restrictions.restrictions;
// Check that resource is most limited...
std::vector<rtc::scoped_refptr<Resource>> most_limited_resources;
VideoAdaptationCounters most_limited_restrictions;
std::tie(most_limited_resources, most_limited_restrictions) =
FindMostLimitedResources();
RTC_DCHECK(!most_limited_resources.empty())
<< "Can not have no limited resources when adaptation status is valid. "
"Should be kLimitReached.";
for (const auto* constraint : adaptation_constraints_) {
if (!constraint->IsAdaptationUpAllowed(input_state, restrictions_before,
restrictions_after,
@ -372,6 +375,34 @@ ResourceAdaptationProcessor::OnResourceUnderuse(
MitigationResult::kRejectedByConstraint, message.Release());
}
}
// If the most restricted resource is less limited than current restrictions
// then proceed with adapting up.
if (most_limited_restrictions.Total() >=
stream_adapter_->adaptation_counters().Total()) {
// If |reason_resource| is not one of the most limiting resources then abort
// adaptation.
if (absl::c_find(most_limited_resources, reason_resource) ==
most_limited_resources.end()) {
processing_in_progress_ = false;
rtc::StringBuilder message;
message << "Resource \"" << reason_resource->Name()
<< "\" was not the most limited resource.";
return MitigationResultAndLogMessage(
MitigationResult::kNotMostLimitedResource, message.Release());
}
UpdateResourceLimitations(reason_resource, peek_restrictions);
if (most_limited_resources.size() > 1) {
// If there are multiple most limited resources, all must signal underuse
// before the adaptation is applied.
processing_in_progress_ = false;
rtc::StringBuilder message;
message << "Resource \"" << reason_resource->Name()
<< "\" was not the only most limited resource.";
return MitigationResultAndLogMessage(
MitigationResult::kSharedMostLimitedResource, message.Release());
}
}
// Apply adaptation.
stream_adapter_->ApplyAdaptation(adaptation);
for (auto* adaptation_listener : adaptation_listeners_) {
@ -426,8 +457,11 @@ ResourceAdaptationProcessor::OnResourceOveruse(
// Apply adaptation.
VideoSourceRestrictions restrictions_before =
stream_adapter_->source_restrictions();
VideoSourceRestrictions restrictions_after =
VideoStreamAdapter::RestrictionsWithCounters peek_next_restrictions =
stream_adapter_->PeekNextRestrictions(adaptation);
VideoSourceRestrictions restrictions_after =
peek_next_restrictions.restrictions;
UpdateResourceLimitations(reason_resource, peek_next_restrictions);
stream_adapter_->ApplyAdaptation(adaptation);
for (auto* adaptation_listener : adaptation_listeners_) {
adaptation_listener->OnAdaptationApplied(
@ -464,27 +498,38 @@ void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize(
}
}
void ResourceAdaptationProcessor::UpdateResourceDegradationCounts(
rtc::scoped_refptr<Resource> resource) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
RTC_DCHECK(resource);
int delta = stream_adapter_->adaptation_counters().Total();
for (const auto& adaptations : adaptations_counts_by_resource_) {
delta -= adaptations.second;
}
std::pair<std::vector<rtc::scoped_refptr<Resource>>, VideoAdaptationCounters>
ResourceAdaptationProcessor::FindMostLimitedResources() const {
std::vector<rtc::scoped_refptr<Resource>> most_limited_resources;
VideoAdaptationCounters most_limited_restrictions;
// Default value is 0, inserts the value if missing.
adaptations_counts_by_resource_[resource] += delta;
RTC_DCHECK_GE(adaptations_counts_by_resource_[resource], 0);
for (const auto& resource_and_adaptation_limit_ :
adaptation_limits_by_resources_) {
const VideoAdaptationCounters& counters =
resource_and_adaptation_limit_.second;
if (counters.Total() > most_limited_restrictions.Total()) {
most_limited_restrictions = counters;
most_limited_resources.clear();
most_limited_resources.push_back(resource_and_adaptation_limit_.first);
} else if (most_limited_restrictions == counters) {
most_limited_resources.push_back(resource_and_adaptation_limit_.first);
}
}
return std::make_pair(std::move(most_limited_resources),
most_limited_restrictions);
}
bool ResourceAdaptationProcessor::IsResourceAllowedToAdaptUp(
rtc::scoped_refptr<Resource> resource) const {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
RTC_DCHECK(resource);
const auto& adaptations = adaptations_counts_by_resource_.find(resource);
return adaptations != adaptations_counts_by_resource_.end() &&
adaptations->second > 0;
void ResourceAdaptationProcessor::UpdateResourceLimitations(
rtc::scoped_refptr<Resource> reason_resource,
const VideoStreamAdapter::RestrictionsWithCounters&
peek_next_restrictions) {
adaptation_limits_by_resources_[reason_resource] =
peek_next_restrictions.adaptation_counters;
for (auto restrictions_listener : restrictions_listeners_) {
restrictions_listener->OnResourceLimitationChanged(
reason_resource, adaptation_limits_by_resources_);
}
}
} // namespace webrtc

33
call/adaptation/resource_adaptation_processor.h
View File

@ -14,6 +14,7 @@
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
@ -124,7 +125,8 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
enum class MitigationResult {
kDisabled,
kInsufficientInput,
kRejectedByAdaptationCounts,
kNotMostLimitedResource,
kSharedMostLimitedResource,
kRejectedByAdapter,
kRejectedByConstraint,
kAdaptationApplied,
@ -151,17 +153,19 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
// If the filtered source restrictions are different than
// |last_reported_source_restrictions_|, inform the listeners.
void MaybeUpdateVideoSourceRestrictions(rtc::scoped_refptr<Resource> reason);
// Updates the number of times the resource has degraded based on the latest
// degradation applied.
void UpdateResourceDegradationCounts(rtc::scoped_refptr<Resource> resource);
// Returns true if a Resource has been overused in the pass and is responsible
// for creating a VideoSourceRestriction. The current algorithm counts the
// number of times the resource caused an adaptation and allows adapting up
// if that number is non-zero. This is consistent with how adaptation has
// traditionally been handled.
// TODO(crbug.com/webrtc/11553) Change this algorithm to look at the resources
// restrictions rather than just the counters.
bool IsResourceAllowedToAdaptUp(rtc::scoped_refptr<Resource> resource) const;
void UpdateResourceLimitations(
rtc::scoped_refptr<Resource> reason_resource,
const VideoStreamAdapter::RestrictionsWithCounters&
peek_next_restrictions) RTC_RUN_ON(resource_adaptation_queue_);
// Searches |adaptation_limits_by_resources_| for each resource with the
// highest total adaptation counts. Adaptation up may only occur if the
// resource performing the adaptation is the only most limited resource. This
// function returns the list of all most limited resources as well as the
// corresponding adaptation of that resource.
std::pair<std::vector<rtc::scoped_refptr<Resource>>, VideoAdaptationCounters>
FindMostLimitedResources() const RTC_RUN_ON(resource_adaptation_queue_);
TaskQueueBase* resource_adaptation_queue_;
rtc::scoped_refptr<ResourceListenerDelegate> resource_listener_delegate_;
@ -181,8 +185,9 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
std::vector<AdaptationListener*> adaptation_listeners_
RTC_GUARDED_BY(resource_adaptation_queue_);
// Purely used for statistics, does not ensure mapped resources stay alive.
std::map<const Resource*, int> adaptations_counts_by_resource_
RTC_GUARDED_BY(resource_adaptation_queue_);
std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>
adaptation_limits_by_resources_
RTC_GUARDED_BY(resource_adaptation_queue_);
// Adaptation strategy settings.
DegradationPreference degradation_preference_
RTC_GUARDED_BY(resource_adaptation_queue_);

5
call/adaptation/resource_adaptation_processor_interface.cc
View File

@ -12,8 +12,9 @@
namespace webrtc {
VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() {}
VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default;
ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() {}
ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() =
default;
} // namespace webrtc

9
call/adaptation/resource_adaptation_processor_interface.h
View File

@ -11,6 +11,8 @@
#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_
#define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_
#include <map>
#include "absl/types/optional.h"
#include "api/adaptation/resource.h"
#include "api/rtp_parameters.h"
@ -38,6 +40,13 @@ class VideoSourceRestrictionsListener {
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
rtc::scoped_refptr<Resource> reason) = 0;
// The limitations on a resource were changed. This does not mean the current
// video restrictions have changed.
virtual void OnResourceLimitationChanged(
rtc::scoped_refptr<Resource> resource,
const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
resource_limitations) {}
};
// The Resource Adaptation Processor is responsible for reacting to resource

93
call/adaptation/resource_adaptation_processor_unittest.cc
View File

@ -305,6 +305,36 @@ TEST_F(ResourceAdaptationProcessorTest,
EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total());
}
TEST_F(ResourceAdaptationProcessorTest, OnlyMostLimitedResourceMayAdaptUp) {
processor_->SetDegradationPreference(
DegradationPreference::MAINTAIN_FRAMERATE);
processor_->StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
other_resource_->SetUsageState(ResourceUsageState::kOveruse);
EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// |other_resource_| is most limited, resource_ can't adapt up.
resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
other_resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// |resource_| and |other_resource_| are now most limited, so both must
// signal underuse to adapt up.
other_resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
}
TEST_F(ResourceAdaptationProcessorTest,
MultipleResourcesCanTriggerMultipleAdaptations) {
processor_->SetDegradationPreference(
@ -321,20 +351,81 @@ TEST_F(ResourceAdaptationProcessorTest,
EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// resource_ is not most limited so can't adapt from underuse.
resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
other_resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// Does not trigger adaptation since resource has no adaptations left.
// resource_ is still not most limited so can't adapt from underuse.
resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// However it will be after overuse
resource_->SetUsageState(ResourceUsageState::kOveruse);
EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// Now other_resource_ can't adapt up as it is not most restricted.
other_resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// resource_ is limited at 3 adaptations and other_resource_ 2.
// With the most limited resource signalling underuse in the following
// order we get back to unrestricted video.
resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// Both resource_ and other_resource_ are most limited.
other_resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// Again both are most limited.
resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
other_resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total());
}
TEST_F(ResourceAdaptationProcessorTest,
MostLimitedResourceAdaptationWorksAfterChangingDegradataionPreference) {
processor_->SetDegradationPreference(
DegradationPreference::MAINTAIN_FRAMERATE);
processor_->StartResourceAdaptation();
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
// Adapt down until we can't anymore.
resource_->SetUsageState(ResourceUsageState::kOveruse);
RestrictSource(restrictions_listener_.restrictions());
resource_->SetUsageState(ResourceUsageState::kOveruse);
RestrictSource(restrictions_listener_.restrictions());
resource_->SetUsageState(ResourceUsageState::kOveruse);
RestrictSource(restrictions_listener_.restrictions());
resource_->SetUsageState(ResourceUsageState::kOveruse);
RestrictSource(restrictions_listener_.restrictions());
resource_->SetUsageState(ResourceUsageState::kOveruse);
RestrictSource(restrictions_listener_.restrictions());
int last_total = restrictions_listener_.adaptation_counters().Total();
processor_->SetDegradationPreference(
DegradationPreference::MAINTAIN_RESOLUTION);
// resource_ can not adapt up since we have never reduced FPS.
resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(last_total, restrictions_listener_.adaptation_counters().Total());
other_resource_->SetUsageState(ResourceUsageState::kOveruse);
EXPECT_EQ(last_total + 1,
restrictions_listener_.adaptation_counters().Total());
RestrictSource(restrictions_listener_.restrictions());
// other_resource_ is most limited so should be able to adapt up.
other_resource_->SetUsageState(ResourceUsageState::kUnderuse);
EXPECT_EQ(last_total, restrictions_listener_.adaptation_counters().Total());
}
TEST_F(ResourceAdaptationProcessorTest, AdaptingTriggersOnAdaptationApplied) {

10
call/adaptation/video_stream_adapter.cc
View File

@ -513,16 +513,18 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() const {
}
}
VideoSourceRestrictions VideoStreamAdapter::PeekNextRestrictions(
const Adaptation& adaptation) const {
VideoStreamAdapter::RestrictionsWithCounters
VideoStreamAdapter::PeekNextRestrictions(const Adaptation& adaptation) const {
RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_);
RTC_LOG(LS_INFO) << "PeekNextRestrictions called";
if (adaptation.status() != Adaptation::Status::kValid)
return source_restrictor_->source_restrictions();
return {source_restrictor_->source_restrictions(),
source_restrictor_->adaptation_counters()};
VideoSourceRestrictor restrictor_copy = *source_restrictor_;
restrictor_copy.ApplyAdaptationStep(adaptation.step(),
degradation_preference_);
return restrictor_copy.source_restrictions();
return {restrictor_copy.source_restrictions(),
restrictor_copy.adaptation_counters()};
}
void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) {

8
call/adaptation/video_stream_adapter.h
View File

@ -129,10 +129,16 @@ class VideoStreamAdapter {
// status code indicating the reason why we cannot adapt.
Adaptation GetAdaptationUp() const;
Adaptation GetAdaptationDown() const;
struct RestrictionsWithCounters {
VideoSourceRestrictions restrictions;
VideoAdaptationCounters adaptation_counters;
};
// Returns the restrictions that result from applying the adaptation, without
// actually applying it. If the adaptation is not valid, current restrictions
// are returned.
VideoSourceRestrictions PeekNextRestrictions(
RestrictionsWithCounters PeekNextRestrictions(
const Adaptation& adaptation) const;
// Updates source_restrictions() based according to the Adaptation.
void ApplyAdaptation(const Adaptation& adaptation);

19
call/adaptation/video_stream_adapter_unittest.cc
View File

@ -686,26 +686,35 @@ TEST(VideoStreamAdapterTest, PeekNextRestrictions) {
{
Adaptation adaptation = adapter.GetAdaptationUp();
EXPECT_EQ(Adaptation::Status::kLimitReached, adaptation.status());
EXPECT_EQ(adapter.PeekNextRestrictions(adaptation),
VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters =
adapter.PeekNextRestrictions(adaptation);
EXPECT_EQ(restrictions_with_counters.restrictions,
adapter.source_restrictions());
EXPECT_EQ(0, restrictions_with_counters.adaptation_counters.Total());
}
// When we adapt down.
{
Adaptation adaptation = adapter.GetAdaptationDown();
EXPECT_EQ(Adaptation::Status::kValid, adaptation.status());
VideoSourceRestrictions next_restrictions =
VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters =
adapter.PeekNextRestrictions(adaptation);
fake_stream.ApplyAdaptation(adaptation);
EXPECT_EQ(next_restrictions, adapter.source_restrictions());
EXPECT_EQ(restrictions_with_counters.restrictions,
adapter.source_restrictions());
EXPECT_EQ(restrictions_with_counters.adaptation_counters,
adapter.adaptation_counters());
}
// When we adapt up.
{
Adaptation adaptation = adapter.GetAdaptationUp();
EXPECT_EQ(Adaptation::Status::kValid, adaptation.status());
VideoSourceRestrictions next_restrictions =
VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters =
adapter.PeekNextRestrictions(adaptation);
fake_stream.ApplyAdaptation(adaptation);
EXPECT_EQ(next_restrictions, adapter.source_restrictions());
EXPECT_EQ(restrictions_with_counters.restrictions,
adapter.source_restrictions());
EXPECT_EQ(restrictions_with_counters.adaptation_counters,
adapter.adaptation_counters());
}
}

1
video/adaptation/BUILD.gn
View File

@ -66,7 +66,6 @@ if (rtc_include_tests) {
sources = [
"overuse_frame_detector_unittest.cc",
"quality_scaler_resource_unittest.cc",
"video_stream_encoder_resource_manager_unittest.cc",
]
deps = [
":video_adaptation",

202
video/adaptation/video_stream_encoder_resource_manager.cc
View File

@ -10,11 +10,9 @@
#include "video/adaptation/video_stream_encoder_resource_manager.h"
#include <algorithm>
#include <cmath>
#include <limits>
#include <memory>
#include <string>
#include <utility>
#include "absl/algorithm/container.h"
@ -56,15 +54,6 @@ std::string ToString(VideoAdaptationReason reason) {
}
}
VideoAdaptationReason OtherReason(VideoAdaptationReason reason) {
switch (reason) {
case VideoAdaptationReason::kQuality:
return VideoAdaptationReason::kCpu;
case VideoAdaptationReason::kCpu:
return VideoAdaptationReason::kQuality;
}
}
} // namespace
class VideoStreamEncoderResourceManager::InitialFrameDropper {
@ -139,56 +128,6 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
int initial_framedrop_;
};
VideoStreamEncoderResourceManager::ActiveCountsConstraint::
ActiveCountsConstraint(VideoStreamEncoderResourceManager* manager)
: manager_(manager),
resource_adaptation_queue_(nullptr),
adaptation_processor_(nullptr) {}
void VideoStreamEncoderResourceManager::ActiveCountsConstraint::
SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue) {
resource_adaptation_queue_ = resource_adaptation_queue;
}
void VideoStreamEncoderResourceManager::ActiveCountsConstraint::
SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
adaptation_processor_ = adaptation_processor;
}
bool VideoStreamEncoderResourceManager::ActiveCountsConstraint::
IsAdaptationUpAllowed(const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
rtc::scoped_refptr<Resource> reason_resource) const {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
RTC_DCHECK(adaptation_processor_);
VideoAdaptationReason reason =
manager_->GetReasonFromResource(reason_resource);
{
// This is the same as |resource_adaptation_queue_|, but need to
// RTC_DCHECK_RUN_ON() both to avoid compiler error when accessing
// |manager_->active_counts_|.
RTC_DCHECK_RUN_ON(manager_->resource_adaptation_queue_);
// We can't adapt up if we're already at the highest setting.
// Note that this only includes counts relevant to the current degradation
// preference. e.g. we previously adapted resolution, now prefer adpating
// fps, only count the fps adaptations and not the previous resolution
// adaptations.
// TODO(hbos): Why would the reason matter? If a particular resource doesn't
// want us to go up it should prevent us from doing so itself rather than to
// have this catch-all reason- and stats-based approach.
int num_downgrades =
FilterVideoAdaptationCountersByDegradationPreference(
manager_->active_counts_[reason],
adaptation_processor_->effective_degradation_preference())
.Total();
RTC_DCHECK_GE(num_downgrades, 0);
return num_downgrades > 0;
}
}
VideoStreamEncoderResourceManager::BitrateConstraint::BitrateConstraint(
VideoStreamEncoderResourceManager* manager)
: manager_(manager),
@ -328,9 +267,7 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
Clock* clock,
bool experiment_cpu_load_estimator,
std::unique_ptr<OveruseFrameDetector> overuse_detector)
: active_counts_constraint_(
new rtc::RefCountedObject<ActiveCountsConstraint>(this)),
bitrate_constraint_(new rtc::RefCountedObject<BitrateConstraint>(this)),
: bitrate_constraint_(new rtc::RefCountedObject<BitrateConstraint>(this)),
balanced_constraint_(new rtc::RefCountedObject<BalancedConstraint>(this)),
encode_usage_resource_(
EncodeUsageResource::Create(std::move(overuse_detector))),
@ -369,8 +306,6 @@ void VideoStreamEncoderResourceManager::Initialize(
RTC_DCHECK(resource_adaptation_queue);
encoder_queue_ = encoder_queue;
resource_adaptation_queue_ = resource_adaptation_queue;
active_counts_constraint_->SetAdaptationQueue(
resource_adaptation_queue_->Get());
bitrate_constraint_->SetAdaptationQueue(resource_adaptation_queue_->Get());
balanced_constraint_->SetAdaptationQueue(resource_adaptation_queue_->Get());
encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
@ -385,7 +320,6 @@ void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
adaptation_processor_ = adaptation_processor;
active_counts_constraint_->SetAdaptationProcessor(adaptation_processor);
balanced_constraint_->SetAdaptationProcessor(adaptation_processor);
quality_scaler_resource_->SetAdaptationProcessor(adaptation_processor);
}
@ -441,7 +375,7 @@ VideoStreamEncoderResourceManager::MappedResources() const {
std::vector<AdaptationConstraint*>
VideoStreamEncoderResourceManager::AdaptationConstraints() const {
return {active_counts_constraint_, bitrate_constraint_, balanced_constraint_};
return {bitrate_constraint_, balanced_constraint_};
}
std::vector<AdaptationListener*>
@ -660,29 +594,11 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
const VideoAdaptationCounters& adaptation_counters,
rtc::scoped_refptr<Resource> reason) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
VideoAdaptationCounters previous_adaptation_counters =
active_counts_[VideoAdaptationReason::kQuality] +
active_counts_[VideoAdaptationReason::kCpu];
int adaptation_counters_total_abs_diff = std::abs(
adaptation_counters.Total() - previous_adaptation_counters.Total());
if (reason) {
// A resource signal triggered this adaptation. The adaptation counters have
// to be updated every time the adaptation counter is incremented or
// decremented due to a resource.
RTC_DCHECK_EQ(adaptation_counters_total_abs_diff, 1);
VideoAdaptationReason reason_type = GetReasonFromResource(reason);
UpdateAdaptationStats(adaptation_counters, reason_type);
} else if (adaptation_counters.Total() == 0) {
// TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback.
if (!reason && adaptation_counters.Total() == 0) {
// Adaptation was manually reset - clear the per-reason counters too.
ResetActiveCounts();
encoder_stats_observer_->ClearAdaptationStats();
} else {
// If a reason did not increase or decrease the Total() by 1 and the
// restrictions were not just reset, the adaptation counters MUST not have
// been modified and there is nothing to do stats-wise.
RTC_DCHECK_EQ(adaptation_counters_total_abs_diff, 0);
}
RTC_LOG(LS_INFO) << ActiveCountsToString();
// The VideoStreamEncoder makes the manager outlive the encoder queue. This
// means that if the task gets executed, |this| has not been freed yet.
@ -693,6 +609,37 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
});
}
void VideoStreamEncoderResourceManager::OnResourceLimitationChanged(
rtc::scoped_refptr<Resource> resource,
const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
resource_limitations) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
if (!resource) {
ResetActiveCounts();
return;
}
std::map<VideoAdaptationReason, VideoAdaptationCounters> limitations;
for (auto& resource_counter : resource_limitations) {
std::map<VideoAdaptationReason, VideoAdaptationCounters>::iterator it;
bool inserted;
std::tie(it, inserted) = limitations.emplace(
GetReasonFromResource(resource_counter.first), resource_counter.second);
if (!inserted && it->second.Total() < resource_counter.second.Total()) {
it->second = resource_counter.second;
}
}
VideoAdaptationReason adaptation_reason = GetReasonFromResource(resource);
if (active_counts_[adaptation_reason] != limitations[adaptation_reason]) {
active_counts_[adaptation_reason] = limitations[adaptation_reason];
encoder_stats_observer_->OnAdaptationChanged(
adaptation_reason, active_counts_[VideoAdaptationReason::kCpu],
active_counts_[VideoAdaptationReason::kQuality]);
}
RTC_LOG(LS_INFO) << ActiveCountsToString();
}
void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() {
RTC_DCHECK_RUN_ON(encoder_queue_);
absl::optional<double> codec_max_frame_rate =
@ -714,84 +661,6 @@ void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() {
encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
}
void VideoStreamEncoderResourceManager::OnAdaptationCountChanged(
const VideoAdaptationCounters& adaptation_count,
VideoAdaptationCounters* active_count,
VideoAdaptationCounters* other_active) {
RTC_DCHECK(active_count);
RTC_DCHECK(other_active);
const int active_total = active_count->Total();
const int other_total = other_active->Total();
const VideoAdaptationCounters prev_total = *active_count + *other_active;
const int delta_resolution_adaptations =
adaptation_count.resolution_adaptations -
prev_total.resolution_adaptations;
const int delta_fps_adaptations =
adaptation_count.fps_adaptations - prev_total.fps_adaptations;
RTC_DCHECK_EQ(
std::abs(delta_resolution_adaptations) + std::abs(delta_fps_adaptations),
1)
<< "Adaptation took more than one step!";
if (delta_resolution_adaptations > 0) {
++active_count->resolution_adaptations;
} else if (delta_resolution_adaptations < 0) {
if (active_count->resolution_adaptations == 0) {
RTC_DCHECK_GT(active_count->fps_adaptations, 0) << "No downgrades left";
RTC_DCHECK_GT(other_active->resolution_adaptations, 0)
<< "No resolution adaptation to borrow from";
// Lend an fps adaptation to other and take one resolution adaptation.
--active_count->fps_adaptations;
++other_active->fps_adaptations;
--other_active->resolution_adaptations;
} else {
--active_count->resolution_adaptations;
}
}
if (delta_fps_adaptations > 0) {
++active_count->fps_adaptations;
} else if (delta_fps_adaptations < 0) {
if (active_count->fps_adaptations == 0) {
RTC_DCHECK_GT(active_count->resolution_adaptations, 0)
<< "No downgrades left";
RTC_DCHECK_GT(other_active->fps_adaptations, 0)
<< "No fps adaptation to borrow from";
// Lend a resolution adaptation to other and take one fps adaptation.
--active_count->resolution_adaptations;
++other_active->resolution_adaptations;
--other_active->fps_adaptations;
} else {
--active_count->fps_adaptations;
}
}
RTC_DCHECK(*active_count + *other_active == adaptation_count);
RTC_DCHECK_EQ(other_active->Total(), other_total);
RTC_DCHECK_EQ(
active_count->Total(),
active_total + delta_resolution_adaptations + delta_fps_adaptations);
RTC_DCHECK_GE(active_count->resolution_adaptations, 0);
RTC_DCHECK_GE(active_count->fps_adaptations, 0);
RTC_DCHECK_GE(other_active->resolution_adaptations, 0);
RTC_DCHECK_GE(other_active->fps_adaptations, 0);
}
void VideoStreamEncoderResourceManager::UpdateAdaptationStats(
const VideoAdaptationCounters& total_counts,
VideoAdaptationReason reason) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
// Update active counts
VideoAdaptationCounters& active_count = active_counts_[reason];
VideoAdaptationCounters& other_active = active_counts_[OtherReason(reason)];
OnAdaptationCountChanged(total_counts, &active_count, &other_active);
encoder_stats_observer_->OnAdaptationChanged(
reason, active_counts_[VideoAdaptationReason::kCpu],
active_counts_[VideoAdaptationReason::kQuality]);
}
void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
VideoStreamEncoderObserver::AdaptationSettings cpu_settings(
@ -862,6 +731,7 @@ void VideoStreamEncoderResourceManager::ResetActiveCounts() {
active_counts_.clear();
active_counts_[VideoAdaptationReason::kCpu] = VideoAdaptationCounters();
active_counts_[VideoAdaptationReason::kQuality] = VideoAdaptationCounters();
encoder_stats_observer_->ClearAdaptationStats();
}
std::string VideoStreamEncoderResourceManager::ActiveCountsToString() const {

55
video/adaptation/video_stream_encoder_resource_manager.h
View File

@ -133,18 +133,10 @@ class VideoStreamEncoderResourceManager
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
rtc::scoped_refptr<Resource> reason) override;
// For reasons of adaptation and statistics, we not only count the total
// number of adaptations, but we also count the number of adaptations per
// reason.
// This method takes the new total number of adaptations and allocates that to
// the "active" count - number of adaptations for the current reason.
// The "other" count is the number of adaptations for the other reason.
// This must be called for each adaptation step made.
static void OnAdaptationCountChanged(
const VideoAdaptationCounters& adaptation_count,
VideoAdaptationCounters* active_count,
VideoAdaptationCounters* other_active);
void OnResourceLimitationChanged(
rtc::scoped_refptr<Resource> resource,
const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
resource_limitations) override;
private:
class InitialFrameDropper;
@ -163,8 +155,6 @@ class VideoStreamEncoderResourceManager
void UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
void UpdateAdaptationStats(const VideoAdaptationCounters& total_counts,
VideoAdaptationReason reason);
void UpdateStatsAdaptationSettings() const;
// Checks to see if we should execute the quality rampup experiment. The
@ -178,36 +168,6 @@ class VideoStreamEncoderResourceManager
std::string ActiveCountsToString() const;
// TODO(hbos): Add tests for manager's constraints.
// Does not trigger adaptations, only prevents adapting up based on
// |active_counts_|.
class ActiveCountsConstraint : public rtc::RefCountInterface,
public AdaptationConstraint {
public:
explicit ActiveCountsConstraint(VideoStreamEncoderResourceManager* manager);
~ActiveCountsConstraint() override = default;
void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue);
void SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor);
// AdaptationConstraint implementation.
std::string Name() const override { return "ActiveCountsConstraint"; }
bool IsAdaptationUpAllowed(
const VideoStreamInputState& input_state,
const VideoSourceRestrictions& restrictions_before,
const VideoSourceRestrictions& restrictions_after,
rtc::scoped_refptr<Resource> reason_resource) const override;
private:
// The |manager_| must be alive as long as this resource is added to the
// ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called.
VideoStreamEncoderResourceManager* const manager_;
TaskQueueBase* resource_adaptation_queue_;
ResourceAdaptationProcessorInterface* adaptation_processor_
RTC_GUARDED_BY(resource_adaptation_queue_);
};
// Does not trigger adaptations, only prevents adapting up resolution.
class BitrateConstraint : public rtc::RefCountInterface,
public AdaptationConstraint {
@ -272,7 +232,6 @@ class VideoStreamEncoderResourceManager
RTC_GUARDED_BY(resource_adaptation_queue_);
};
const rtc::scoped_refptr<ActiveCountsConstraint> active_counts_constraint_;
const rtc::scoped_refptr<BitrateConstraint> bitrate_constraint_;
const rtc::scoped_refptr<BalancedConstraint> balanced_constraint_;
const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;
@ -324,10 +283,8 @@ class VideoStreamEncoderResourceManager
// One AdaptationCounter for each reason, tracking the number of times we have
// adapted for each reason. The sum of active_counts_ MUST always equal the
// total adaptation provided by the VideoSourceRestrictions.
// TODO(https://crbug.com/webrtc/11542): When we have an adaptation queue,
// guard the activec counts by it instead. The |encoder_stats_observer_| is
// thread-safe anyway, and active counts are used by
// ActiveCountsConstraint to make decisions.
// TODO(bugs.webrtc.org/11553) Remove active counts by computing them on the
// fly. This require changes to MaybePerformQualityRampupExperiment.
std::unordered_map<VideoAdaptationReason, VideoAdaptationCounters>
active_counts_ RTC_GUARDED_BY(resource_adaptation_queue_);
};

98
video/adaptation/video_stream_encoder_resource_manager_unittest.cc
View File

@ -1,98 +0,0 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/adaptation/video_stream_encoder_resource_manager.h"
#include "api/video/video_adaptation_counters.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
TEST(VideoStreamEncoderResourceManagerTest, FirstAdaptationDown_Fps) {
VideoAdaptationCounters cpu;
VideoAdaptationCounters qp;
VideoAdaptationCounters total(0, 1);
VideoStreamEncoderResourceManager::OnAdaptationCountChanged(total, &cpu, &qp);
VideoAdaptationCounters expected_cpu(0, 1);
VideoAdaptationCounters expected_qp;
EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp);
}
TEST(VideoStreamEncoderResourceManagerTest, FirstAdaptationDown_Resolution) {
VideoAdaptationCounters cpu;
VideoAdaptationCounters qp;
VideoAdaptationCounters total(1, 0);
VideoStreamEncoderResourceManager::OnAdaptationCountChanged(total, &cpu, &qp);
VideoAdaptationCounters expected_cpu(1, 0);
VideoAdaptationCounters expected_qp;
EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp);
}
TEST(VideoStreamEncoderResourceManagerTest, LastAdaptUp_Fps) {
VideoAdaptationCounters cpu(0, 1);
VideoAdaptationCounters qp;
VideoAdaptationCounters total;
VideoStreamEncoderResourceManager::OnAdaptationCountChanged(total, &cpu, &qp);
VideoAdaptationCounters expected_cpu;
VideoAdaptationCounters expected_qp;
EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp);
}
TEST(VideoStreamEncoderResourceManagerTest, LastAdaptUp_Resolution) {
VideoAdaptationCounters cpu(1, 0);
VideoAdaptationCounters qp;
VideoAdaptationCounters total;
VideoStreamEncoderResourceManager::OnAdaptationCountChanged(total, &cpu, &qp);
VideoAdaptationCounters expected_cpu;
VideoAdaptationCounters expected_qp;
EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp);
}
TEST(VideoStreamEncoderResourceManagerTest, AdaptUpWithBorrow_Resolution) {
VideoAdaptationCounters cpu(0, 1);
VideoAdaptationCounters qp(1, 0);
VideoAdaptationCounters total(0, 1);
// CPU adaptation for resolution, but no resolution adaptation left from CPU.
// We then borrow the resolution adaptation from qp, and give qp the fps
// adaptation from CPU.
VideoStreamEncoderResourceManager::OnAdaptationCountChanged(total, &cpu, &qp);
VideoAdaptationCounters expected_cpu(0, 0);
VideoAdaptationCounters expected_qp(0, 1);
EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp);
}
TEST(VideoStreamEncoderResourceManagerTest, AdaptUpWithBorrow_Fps) {
VideoAdaptationCounters cpu(1, 0);
VideoAdaptationCounters qp(0, 1);
VideoAdaptationCounters total(1, 0);
// CPU adaptation for fps, but no fps adaptation left from CPU. We then borrow
// the fps adaptation from qp, and give qp the resolution adaptation from CPU.
VideoStreamEncoderResourceManager::OnAdaptationCountChanged(total, &cpu, &qp);
VideoAdaptationCounters expected_cpu(0, 0);
VideoAdaptationCounters expected_qp(1, 0);
EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp);
}
} // namespace webrtc

44
video/send_statistics_proxy.cc
View File

@ -717,9 +717,11 @@ void SendStatisticsProxy::OnSuspendChange(bool is_suspended) {
uma_container_->quality_adapt_timer_.Stop(now_ms);
} else {
// Start adaptation stats if scaling is enabled.
if (adaptations_.MaskedCpuCounts().resolution_adaptations.has_value())
if (adaptation_limitations_.MaskedCpuCounts()
.resolution_adaptations.has_value())
uma_container_->cpu_adapt_timer_.Start(now_ms);
if (adaptations_.MaskedQualityCounts().resolution_adaptations.has_value())
if (adaptation_limitations_.MaskedQualityCounts()
.resolution_adaptations.has_value())
uma_container_->quality_adapt_timer_.Start(now_ms);
// Stop pause explicitly for stats that may be zero/not updated for some
// time.
@ -1021,7 +1023,7 @@ void SendStatisticsProxy::OnSendEncodedImage(
}
absl::optional<int> downscales =
adaptations_.MaskedQualityCounts().resolution_adaptations;
adaptation_limitations_.MaskedQualityCounts().resolution_adaptations;
stats_.bw_limited_resolution |=
(downscales.has_value() && downscales.value() > 0);
@ -1056,7 +1058,8 @@ void SendStatisticsProxy::OnIncomingFrame(int width, int height) {
uma_container_->input_fps_counter_.Add(1);
uma_container_->input_width_counter_.Add(width);
uma_container_->input_height_counter_.Add(height);
if (adaptations_.MaskedCpuCounts().resolution_adaptations.has_value()) {
if (adaptation_limitations_.MaskedCpuCounts()
.resolution_adaptations.has_value()) {
uma_container_->cpu_limited_frame_counter_.Add(
stats_.cpu_limited_resolution);
}
@ -1090,8 +1093,8 @@ void SendStatisticsProxy::OnFrameDropped(DropReason reason) {
void SendStatisticsProxy::ClearAdaptationStats() {
rtc::CritScope lock(&crit_);
adaptations_.set_cpu_counts(VideoAdaptationCounters());
adaptations_.set_quality_counts(VideoAdaptationCounters());
adaptation_limitations_.set_cpu_counts(VideoAdaptationCounters());
adaptation_limitations_.set_quality_counts(VideoAdaptationCounters());
UpdateAdaptationStats();
}
@ -1099,10 +1102,10 @@ void SendStatisticsProxy::UpdateAdaptationSettings(
VideoStreamEncoderObserver::AdaptationSettings cpu_settings,
VideoStreamEncoderObserver::AdaptationSettings quality_settings) {
rtc::CritScope lock(&crit_);
adaptations_.UpdateMaskingSettings(cpu_settings, quality_settings);
SetAdaptTimer(adaptations_.MaskedCpuCounts(),
adaptation_limitations_.UpdateMaskingSettings(cpu_settings, quality_settings);
SetAdaptTimer(adaptation_limitations_.MaskedCpuCounts(),
&uma_container_->cpu_adapt_timer_);
SetAdaptTimer(adaptations_.MaskedQualityCounts(),
SetAdaptTimer(adaptation_limitations_.MaskedQualityCounts(),
&uma_container_->quality_adapt_timer_);
UpdateAdaptationStats();
}
@ -1113,9 +1116,10 @@ void SendStatisticsProxy::OnAdaptationChanged(
const VideoAdaptationCounters& quality_counters) {
rtc::CritScope lock(&crit_);
MaskedAdaptationCounts receiver = adaptations_.MaskedQualityCounts();
adaptations_.set_cpu_counts(cpu_counters);
adaptations_.set_quality_counts(quality_counters);
MaskedAdaptationCounts receiver =
adaptation_limitations_.MaskedQualityCounts();
adaptation_limitations_.set_cpu_counts(cpu_counters);
adaptation_limitations_.set_quality_counts(quality_counters);
switch (reason) {
case VideoAdaptationReason::kCpu:
++stats_.number_of_cpu_adapt_changes;
@ -1123,7 +1127,7 @@ void SendStatisticsProxy::OnAdaptationChanged(
case VideoAdaptationReason::kQuality:
TryUpdateInitialQualityResolutionAdaptUp(
receiver.resolution_adaptations,
adaptations_.MaskedQualityCounts().resolution_adaptations);
adaptation_limitations_.MaskedQualityCounts().resolution_adaptations);
++stats_.number_of_quality_adapt_changes;
break;
}
@ -1131,8 +1135,8 @@ void SendStatisticsProxy::OnAdaptationChanged(
}
void SendStatisticsProxy::UpdateAdaptationStats() {
auto cpu_counts = adaptations_.MaskedCpuCounts();
auto quality_counts = adaptations_.MaskedQualityCounts();
auto cpu_counts = adaptation_limitations_.MaskedCpuCounts();
auto quality_counts = adaptation_limitations_.MaskedQualityCounts();
bool is_cpu_limited = cpu_counts.resolution_adaptations > 0 ||
cpu_counts.num_framerate_reductions > 0;
@ -1459,6 +1463,16 @@ void SendStatisticsProxy::Adaptations::set_quality_counts(
const VideoAdaptationCounters& quality_counts) {
quality_counts_ = quality_counts;
}
VideoAdaptationCounters SendStatisticsProxy::Adaptations::cpu_counts() const {
return cpu_counts_;
}
VideoAdaptationCounters SendStatisticsProxy::Adaptations::quality_counts()
const {
return quality_counts_;
}
void SendStatisticsProxy::Adaptations::UpdateMaskingSettings(
VideoStreamEncoderObserver::AdaptationSettings cpu_settings,
VideoStreamEncoderObserver::AdaptationSettings quality_settings) {

5
video/send_statistics_proxy.h
View File

@ -240,6 +240,9 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver,
void set_cpu_counts(const VideoAdaptationCounters& cpu_counts);
void set_quality_counts(const VideoAdaptationCounters& quality_counts);
VideoAdaptationCounters cpu_counts() const;
VideoAdaptationCounters quality_counts() const;
void UpdateMaskingSettings(AdaptationSettings cpu_settings,
AdaptationSettings quality_settings);
@ -299,7 +302,7 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver,
bool bw_limited_layers_ RTC_GUARDED_BY(crit_);
// Indicastes if the encoder internally downscales input image.
bool internal_encoder_scaler_ RTC_GUARDED_BY(crit_);
Adaptations adaptations_ RTC_GUARDED_BY(crit_);
Adaptations adaptation_limitations_ RTC_GUARDED_BY(crit_);
struct EncoderChangeEvent {
std::string previous_encoder_implementation;

151
video/video_stream_encoder_unittest.cc
View File

@ -2032,7 +2032,8 @@ TEST_F(VideoStreamEncoderTest,
EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
// Change the degradation preference back. CPU underuse should now adapt.
// Change the degradation preference back. CPU underuse should not adapt since
// QP is most limited.
video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
&video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
video_source_.IncomingCapturedFrame(
@ -2052,7 +2053,15 @@ TEST_F(VideoStreamEncoderTest,
CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
sink_.WaitForEncodedFrame(ntp_time);
ntp_time += kFrameIntervalMs;
EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
// Trigger QP underuse, fps should return to normal.
video_stream_encoder_->TriggerQualityHigh();
video_source_.IncomingCapturedFrame(
CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
sink_.WaitForEncodedFrame(ntp_time);
ntp_time += kFrameIntervalMs;
EXPECT_THAT(video_source_.sink_wants(), FpsMax());
video_stream_encoder_->Stop();
}
@ -3610,8 +3619,21 @@ TEST_F(VideoStreamEncoderTest,
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger cpu adapt up, expect upscaled resolution (480x270).
// Trigger quality adapt up, expect upscaled resolution (480x270).
video_stream_encoder_->TriggerQualityHigh();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger quality and cpu adapt up since both are most limited, expect
// upscaled resolution (640x360).
video_stream_encoder_->TriggerCpuUnderuse();
video_stream_encoder_->TriggerQualityHigh();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
@ -3619,32 +3641,24 @@ TEST_F(VideoStreamEncoderTest,
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger cpu adapt up, expect upscaled resolution (640x360).
video_stream_encoder_->TriggerCpuUnderuse();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger cpu adapt up, expect upscaled resolution (960x540).
// Trigger quality and cpu adapt up since both are most limited, expect
// upscaled resolution (960x540).
video_stream_encoder_->TriggerCpuUnderuse();
video_stream_encoder_->TriggerQualityHigh();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
last_wants = source.sink_wants();
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
// Trigger cpu adapt up, expect no change since not most limited (960x540).
// However the stats will change since the CPU resource is no longer limited.
video_stream_encoder_->TriggerCpuUnderuse();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
@ -3653,7 +3667,7 @@ TEST_F(VideoStreamEncoderTest,
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger quality adapt up, expect no restriction (1280x720).
video_stream_encoder_->TriggerQualityHigh();
@ -3665,7 +3679,7 @@ TEST_F(VideoStreamEncoderTest,
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
video_stream_encoder_->Stop();
}
@ -4602,7 +4616,7 @@ TEST_F(VideoStreamEncoderTest,
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger adapt down, expect expect increased fps (320x180@10fps).
// Trigger adapt up, expect expect increased fps (320x180@10fps).
video_stream_encoder_->TriggerQualityHigh();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
@ -4742,42 +4756,58 @@ TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger cpu adapt up, expect increased fps (640x360@30fps).
// Trigger cpu adapt up, expect no change since QP is most limited.
{
// Store current sink wants since we expect no change and if there is no
// change then last_wants() is not updated.
auto previous_sink_wants = source.sink_wants();
video_stream_encoder_->TriggerCpuUnderuse();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
}
// Trigger quality adapt up, expect increased fps (640x360@30fps).
video_stream_encoder_->TriggerQualityHigh();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger quality adapt up and Cpu adapt up since both are most limited,
// expect increased resolution (960x540@30fps).
video_stream_encoder_->TriggerQualityHigh();
video_stream_encoder_->TriggerCpuUnderuse();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsMax());
EXPECT_EQ(source.sink_wants().max_pixel_count,
source.last_wants().max_pixel_count);
EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
// Trigger quality adapt up and Cpu adapt up since both are most limited,
// expect no restriction (1280x720fps@30fps).
video_stream_encoder_->TriggerQualityHigh();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
video_stream_encoder_->TriggerCpuUnderuse();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
@ -4789,13 +4819,13 @@ TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger adapt up, expect no change.
video_stream_encoder_->TriggerQualityHigh();
EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
video_stream_encoder_->Stop();
}
@ -4848,14 +4878,28 @@ TEST_F(VideoStreamEncoderTest,
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
video_stream_encoder_->TriggerCpuUnderuse();
// Trigger cpu adapt up, expect no change because quality is most limited.
{
auto previous_sink_wants = source.sink_wants();
// Store current sink wants since we expect no change ind if there is no
// change then last__wants() is not updated.
video_stream_encoder_->TriggerCpuUnderuse();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
}
// Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
video_stream_encoder_->TriggerQualityHigh();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
@ -4863,12 +4907,13 @@ TEST_F(VideoStreamEncoderTest,
EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger quality adapt up, expect increased fps (640x360@30fps).
// Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
video_stream_encoder_->TriggerQualityHigh();
video_stream_encoder_->TriggerCpuUnderuse();
timestamp_ms += kFrameIntervalMs;
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(timestamp_ms);
@ -4878,13 +4923,13 @@ TEST_F(VideoStreamEncoderTest,
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
// Trigger adapt up, expect no change.
video_stream_encoder_->TriggerQualityHigh();
EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
video_stream_encoder_->Stop();
}