admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Adaptation] Rename and move resource adaptation module/processor stuff.

Browse Source 
Rename:
- call/adaptation/resource_adaptation_module_interface.[h/cc] -->
  call/adaptation/resource_adaptation_processor_interface.[h/cc]
- call/adaptation/resource_adaptation_processor.[h/cc] -->
  call/adaptation/new_resource_adaptation_processor_poc.[h/cc]

Move + Rename:
- video/overuse_frame_detector_resource_adaptation_module.[h/cc] -->
  video/adaptation/resource_adaptation_processor.[h/cc]

Move:
- video/encode_usage_resource.[h/cc] --> video/adaptation/...
- video/overuse_frame_detector.[h/cc] --> video/adaptation/...
- video/quality_scaler_resource.[h/cc] --> video/adaptation/...

Unittests are also moved. In order to avoid a circular dependency,
VideoStreamEncoder::kDefaultLastFrameInfo[Width/Height] is moved and
renamed to kDefaultInputPixels[Width/Height] in
video/adaptation/resource_adaptation_processor.[h/cc].

Bug: webrtc:11222
Change-Id: Icf920e8a7362002b1c63c42b2d9e2e63c990b532
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/170117
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30754}
This commit is contained in:
Henrik Boström 2020-03-10 19:08:05 +01:00 committed by Commit Bot
parent 0759249370
commit 62057627ef
23 changed files with 236 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/video/BUILD.gn
View File

@ -265,6 +265,7 @@ rtc_library("video_stream_encoder_create") {
":video_stream_encoder", ":video_stream_encoder",
"../../api:scoped_refptr", "../../api:scoped_refptr",
"../../video:video_stream_encoder_impl", "../../video:video_stream_encoder_impl",
"../../video/adaptation:video_adaptation",
"../task_queue", "../task_queue",
"../video_codecs:video_codecs_api", "../video_codecs:video_codecs_api",
] ]

2
api/video/video_stream_encoder_create.cc
View File

@ -12,7 +12,7 @@
#include <memory> #include <memory>
#include "video/overuse_frame_detector.h" #include "video/adaptation/overuse_frame_detector.h"
#include "video/video_stream_encoder.h" #include "video/video_stream_encoder.h"
namespace webrtc { namespace webrtc {

10
call/adaptation/BUILD.gn
View File

@ -12,12 +12,12 @@ rtc_library("resource_adaptation") {
sources = [ sources = [
"encoder_settings.cc", "encoder_settings.cc",
"encoder_settings.h", "encoder_settings.h",
"new_resource_adaptation_processor_poc.cc",
"new_resource_adaptation_processor_poc.h",
"resource.cc", "resource.cc",
"resource.h", "resource.h",
"resource_adaptation_module_interface.cc", "resource_adaptation_processor_interface.cc",
"resource_adaptation_module_interface.h", "resource_adaptation_processor_interface.h",
"resource_adaptation_processor.cc",
"resource_adaptation_processor.h",
"resource_consumer.cc", "resource_consumer.cc",
"resource_consumer.h", "resource_consumer.h",
"resource_consumer_configuration.cc", "resource_consumer_configuration.cc",
@ -41,7 +41,7 @@ if (rtc_include_tests) {
testonly = true testonly = true
sources = [ sources = [
"resource_adaptation_processor_unittest.cc", "new_resource_adaptation_processor_poc_unittest.cc",
"resource_unittest.cc", "resource_unittest.cc",
] ]
deps = [ deps = [

14
call/adaptation/resource_adaptation_processor.cc → call/adaptation/new_resource_adaptation_processor_poc.cc
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2019 The WebRTC Project Authors. All rights reserved. * Copyright 2020 The WebRTC Project Authors. All rights reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "call/adaptation/resource_adaptation_processor.h" #include "call/adaptation/new_resource_adaptation_processor_poc.h"
#include <limits> #include <limits>
#include <utility> #include <utility>
@ -48,7 +48,7 @@ ConsumerConfigurationPair::ConsumerConfigurationPair(
: consumer(consumer), configuration(configuration) {} : consumer(consumer), configuration(configuration) {}
absl::optional<ConsumerConfigurationPair> absl::optional<ConsumerConfigurationPair>
ResourceAdaptationProcessor::FindNextConfiguration() { NewResourceAdaptationProcessorPoc::FindNextConfiguration() {
ResourceUsageState overall_usage = ResourceUsageState::kUnderuse; ResourceUsageState overall_usage = ResourceUsageState::kUnderuse;
for (auto& resource : resources_) { for (auto& resource : resources_) {
if (resource->usage_state() == ResourceUsageState::kStable) { if (resource->usage_state() == ResourceUsageState::kStable) {
@ -92,8 +92,8 @@ ResourceAdaptationProcessor::FindNextConfiguration() {
} }
} }
ResourceConsumer* ResourceConsumer* NewResourceAdaptationProcessorPoc::
ResourceAdaptationProcessor::FindMostExpensiveConsumerThatCanBeAdaptedDown() { FindMostExpensiveConsumerThatCanBeAdaptedDown() {
ResourceConsumer* max_cost_consumer = nullptr; ResourceConsumer* max_cost_consumer = nullptr;
double max_cost = -1.0; double max_cost = -1.0;
for (auto& consumer : consumers_) { for (auto& consumer : consumers_) {
@ -108,8 +108,8 @@ ResourceAdaptationProcessor::FindMostExpensiveConsumerThatCanBeAdaptedDown() {
return max_cost_consumer; return max_cost_consumer;
} }
ResourceConsumer* ResourceConsumer* NewResourceAdaptationProcessorPoc::
ResourceAdaptationProcessor::FindLeastExpensiveConsumerThatCanBeAdaptedUp() { FindLeastExpensiveConsumerThatCanBeAdaptedUp() {
ResourceConsumer* min_cost_consumer = nullptr; ResourceConsumer* min_cost_consumer = nullptr;
double min_cost = std::numeric_limits<double>::infinity(); double min_cost = std::numeric_limits<double>::infinity();
for (auto& consumer : consumers_) { for (auto& consumer : consumers_) {

14
call/adaptation/resource_adaptation_processor.h → call/adaptation/new_resource_adaptation_processor_poc.h
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2019 The WebRTC Project Authors. All rights reserved. * Copyright 2020 The WebRTC Project Authors. All rights reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ #ifndef CALL_ADAPTATION_NEW_RESOURCE_ADAPTATION_PROCESSOR_POC_H_
#define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ #define CALL_ADAPTATION_NEW_RESOURCE_ADAPTATION_PROCESSOR_POC_H_
#include <memory> #include <memory>
#include <utility> #include <utility>
@ -55,7 +55,7 @@ struct ConsumerConfigurationPair {
// //
// This class owns all resources, consumers and configurations. As long as it is // This class owns all resources, consumers and configurations. As long as it is
// alive, raw pointers to these are safe to use. // alive, raw pointers to these are safe to use.
class ResourceAdaptationProcessor { class NewResourceAdaptationProcessorPoc {
public: public:
const std::vector<std::unique_ptr<Resource>>& resources() const { const std::vector<std::unique_ptr<Resource>>& resources() const {
return resources_; return resources_;
@ -70,7 +70,7 @@ class ResourceAdaptationProcessor {
// Takes on ownership of the argument. A raw pointer is returned to the object // Takes on ownership of the argument. A raw pointer is returned to the object
// for convenience; it is valid for the lifetime of the // for convenience; it is valid for the lifetime of the
// ResourceAdaptationProcessor. // NewResourceAdaptationProcessorPoc.
// T = any subclass of Resource // T = any subclass of Resource
template <typename T> template <typename T>
T* AddResource(std::unique_ptr<T> resource) { T* AddResource(std::unique_ptr<T> resource) {
@ -96,7 +96,7 @@ class ResourceAdaptationProcessor {
// Based on the current state of the resources and consumers, finds the // Based on the current state of the resources and consumers, finds the
// consumer that should be reconfigured up or down in order to maximies // consumer that should be reconfigured up or down in order to maximies
// quality without overusing any resources, as described in // quality without overusing any resources, as described in
// ResourceAdaptationProcessor's class description. // NewResourceAdaptationProcessorPoc's class description.
// //
// When this is used in a real system, care needs to be taken for how often // When this is used in a real system, care needs to be taken for how often
// FindNextConfiguration() is called. There may be a delay between // FindNextConfiguration() is called. There may be a delay between
@ -115,4 +115,4 @@ class ResourceAdaptationProcessor {
} // namespace webrtc } // namespace webrtc
#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ #endif // CALL_ADAPTATION_NEW_RESOURCE_ADAPTATION_PROCESSOR_POC_H_

51
call/adaptation/resource_adaptation_processor_unittest.cc → call/adaptation/new_resource_adaptation_processor_poc_unittest.cc
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2019 The WebRTC Project Authors. All rights reserved. * Copyright 2020 The WebRTC Project Authors. All rights reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "call/adaptation/resource_adaptation_processor.h" #include "call/adaptation/new_resource_adaptation_processor_poc.h"
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include "call/adaptation/resource.h" #include "call/adaptation/resource.h"
@ -33,7 +33,8 @@ void ConnectNeighbors(ResourceConsumerConfiguration* upper,
} }
std::vector<FakeResourceConsumerConfiguration*> std::vector<FakeResourceConsumerConfiguration*>
AddStandardResolutionConfigurations(ResourceAdaptationProcessor* processor) { AddStandardResolutionConfigurations(
NewResourceAdaptationProcessorPoc* processor) {
std::vector<FakeResourceConsumerConfiguration*> configs; std::vector<FakeResourceConsumerConfiguration*> configs;
configs.push_back(processor->AddConfiguration( configs.push_back(processor->AddConfiguration(
std::make_unique<FakeResourceConsumerConfiguration>(1920, 1080, 30.0, std::make_unique<FakeResourceConsumerConfiguration>(1920, 1080, 30.0,
@ -53,9 +54,9 @@ AddStandardResolutionConfigurations(ResourceAdaptationProcessor* processor) {
return configs; return configs;
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
SingleStreamAndResourceDontAdaptDownWhenStable) { SingleStreamAndResourceDontAdaptDownWhenStable) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kStable)); std::make_unique<FakeResource>(ResourceUsageState::kStable));
auto resolution_configs = AddStandardResolutionConfigurations(&processor); auto resolution_configs = AddStandardResolutionConfigurations(&processor);
@ -64,9 +65,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration()); EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration());
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
SingleStreamAndResourceAdaptDownOnOveruse) { SingleStreamAndResourceAdaptDownOnOveruse) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kOveruse)); std::make_unique<FakeResource>(ResourceUsageState::kOveruse));
auto resolution_configs = AddStandardResolutionConfigurations(&processor); auto resolution_configs = AddStandardResolutionConfigurations(&processor);
@ -78,9 +79,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_EQ(resolution_configs[k720pIndex], next_config->configuration); EXPECT_EQ(resolution_configs[k720pIndex], next_config->configuration);
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
SingleStreamAndResourceDontAdaptOnOveruseIfMinResolution) { SingleStreamAndResourceDontAdaptOnOveruseIfMinResolution) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kOveruse)); std::make_unique<FakeResource>(ResourceUsageState::kOveruse));
auto resolution_configs = AddStandardResolutionConfigurations(&processor); auto resolution_configs = AddStandardResolutionConfigurations(&processor);
@ -89,9 +90,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration()); EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration());
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
SingleStreamAndResourceAdaptUpOnUnderuse) { SingleStreamAndResourceAdaptUpOnUnderuse) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kUnderuse)); std::make_unique<FakeResource>(ResourceUsageState::kUnderuse));
auto resolution_configs = AddStandardResolutionConfigurations(&processor); auto resolution_configs = AddStandardResolutionConfigurations(&processor);
@ -103,9 +104,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_EQ(resolution_configs[k1080pIndex], next_config->configuration); EXPECT_EQ(resolution_configs[k1080pIndex], next_config->configuration);
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
SingleStreamAndResourceDontAdaptOnUnderuseIfMaxResolution) { SingleStreamAndResourceDontAdaptOnUnderuseIfMaxResolution) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kUnderuse)); std::make_unique<FakeResource>(ResourceUsageState::kUnderuse));
auto resolution_configs = AddStandardResolutionConfigurations(&processor); auto resolution_configs = AddStandardResolutionConfigurations(&processor);
@ -114,9 +115,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration()); EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration());
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
MultipleStreamsLargestStreamGetsAdaptedDownOnOveruse) { MultipleStreamsLargestStreamGetsAdaptedDownOnOveruse) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kOveruse)); std::make_unique<FakeResource>(ResourceUsageState::kOveruse));
auto resolution_configs = AddStandardResolutionConfigurations(&processor); auto resolution_configs = AddStandardResolutionConfigurations(&processor);
@ -137,9 +138,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_EQ(second_stream, next_config->consumer); EXPECT_EQ(second_stream, next_config->consumer);
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
MultipleStreamsSmallestStreamGetsAdaptedUpOnUnderuse) { MultipleStreamsSmallestStreamGetsAdaptedUpOnUnderuse) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kUnderuse)); std::make_unique<FakeResource>(ResourceUsageState::kUnderuse));
auto resolution_configs = AddStandardResolutionConfigurations(&processor); auto resolution_configs = AddStandardResolutionConfigurations(&processor);
@ -161,9 +162,9 @@ TEST(ResourceAdaptationProcessorTest,
} }
// If both streams are equally valid to adapt down, the first one is preferred. // If both streams are equally valid to adapt down, the first one is preferred.
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
MultipleStreamsAdaptFirstStreamWhenBothStreamsHaveSameCost) { MultipleStreamsAdaptFirstStreamWhenBothStreamsHaveSameCost) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kOveruse)); std::make_unique<FakeResource>(ResourceUsageState::kOveruse));
auto resolution_configs = AddStandardResolutionConfigurations(&processor); auto resolution_configs = AddStandardResolutionConfigurations(&processor);
@ -176,9 +177,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_EQ(first_stream, next_config->consumer); EXPECT_EQ(first_stream, next_config->consumer);
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
MultipleResourcesAdaptDownIfAnyIsOverused) { MultipleResourcesAdaptDownIfAnyIsOverused) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
auto* first_resource = processor.AddResource( auto* first_resource = processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kOveruse)); std::make_unique<FakeResource>(ResourceUsageState::kOveruse));
auto* second_resource = processor.AddResource( auto* second_resource = processor.AddResource(
@ -194,9 +195,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_TRUE(processor.FindNextConfiguration().has_value()); EXPECT_TRUE(processor.FindNextConfiguration().has_value());
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
MultipleResourcesAdaptUpIfAllAreUnderused) { MultipleResourcesAdaptUpIfAllAreUnderused) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
processor.AddResource( processor.AddResource(
std::make_unique<FakeResource>(ResourceUsageState::kUnderuse)); std::make_unique<FakeResource>(ResourceUsageState::kUnderuse));
auto* second_resource = processor.AddResource( auto* second_resource = processor.AddResource(
@ -211,9 +212,9 @@ TEST(ResourceAdaptationProcessorTest,
EXPECT_TRUE(processor.FindNextConfiguration().has_value()); EXPECT_TRUE(processor.FindNextConfiguration().has_value());
} }
TEST(ResourceAdaptationProcessorTest, TEST(NewResourceAdaptationProcessorPocTest,
HighestPreferredNeighborIsPickedWhenAdapting) { HighestPreferredNeighborIsPickedWhenAdapting) {
ResourceAdaptationProcessor processor; NewResourceAdaptationProcessorPoc processor;
// Set up the following graph, where (#) is the preference. // Set up the following graph, where (#) is the preference.
// //
// Downward arrows Upward arrows // Downward arrows Upward arrows

8
call/adaptation/resource_adaptation_module_interface.cc → call/adaptation/resource_adaptation_processor_interface.cc
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2019 The WebRTC Project Authors. All rights reserved. * Copyright 2020 The WebRTC Project Authors. All rights reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,12 +8,12 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "call/adaptation/resource_adaptation_module_interface.h" #include "call/adaptation/resource_adaptation_processor_interface.h"
namespace webrtc { namespace webrtc {
ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {} ResourceAdaptationProcessorListener::~ResourceAdaptationProcessorListener() {}
ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {} ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() {}
} // namespace webrtc } // namespace webrtc

35
call/adaptation/resource_adaptation_module_interface.h → call/adaptation/resource_adaptation_processor_interface.h
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2019 The WebRTC Project Authors. All rights reserved. * Copyright 2020 The WebRTC Project Authors. All rights reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ #ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_
#define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ #define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include "api/rtp_parameters.h" #include "api/rtp_parameters.h"
@ -22,12 +22,10 @@ namespace webrtc {
// The listener is responsible for carrying out the reconfiguration of the video // The listener is responsible for carrying out the reconfiguration of the video
// source such that the VideoSourceRestrictions are fulfilled. // source such that the VideoSourceRestrictions are fulfilled.
class ResourceAdaptationModuleListener { class ResourceAdaptationProcessorListener {
public: public:
virtual ~ResourceAdaptationModuleListener(); virtual ~ResourceAdaptationProcessorListener();
// TODO(hbos): When we support the muli-stream use case, the arguments need to
// specify which video stream's source needs to be reconfigured.
virtual void OnVideoSourceRestrictionsUpdated( virtual void OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions) = 0; VideoSourceRestrictions restrictions) = 0;
}; };
@ -35,27 +33,12 @@ class ResourceAdaptationModuleListener {
// Responsible for reconfiguring encoded streams based on resource consumption, // Responsible for reconfiguring encoded streams based on resource consumption,
// such as scaling down resolution or frame rate when CPU is overused. This // such as scaling down resolution or frame rate when CPU is overused. This
// interface is meant to be injectable into VideoStreamEncoder. // interface is meant to be injectable into VideoStreamEncoder.
// class ResourceAdaptationProcessorInterface {
// [UNDER CONSTRUCTION] This interface is work-in-progress. In the future it
// needs to be able to handle all the necessary input and output for resource
// adaptation decision making.
//
// TODO(https://crbug.com/webrtc/11222): Make this interface feature-complete so
// that a module (such as OveruseFrameDetectorResourceAdaptationModule) is fully
// operational through this abstract interface.
class ResourceAdaptationModuleInterface {
public: public:
virtual ~ResourceAdaptationModuleInterface(); virtual ~ResourceAdaptationProcessorInterface();
// TODO(hbos): When input/output of the module is adequetly handled by this
// interface, these methods need to say which stream to start/stop, enabling
// multi-stream aware implementations of ResourceAdaptationModuleInterface. We
// don't want to do this before we have the right interfaces (e.g. if we pass
// in a VideoStreamEncoder here directly then have a dependency on a different
// build target). For the multi-stream use case we may consider making
// ResourceAdaptationModuleInterface reference counted.
virtual void StartResourceAdaptation( virtual void StartResourceAdaptation(
ResourceAdaptationModuleListener* adaptation_listener) = 0; ResourceAdaptationProcessorListener* adaptation_listener) = 0;
virtual void StopResourceAdaptation() = 0; virtual void StopResourceAdaptation() = 0;
// The resource must out-live the module. // The resource must out-live the module.
@ -129,4 +112,4 @@ class ResourceAdaptationModuleInterface {
} // namespace webrtc } // namespace webrtc
#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ #endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_

10
video/BUILD.gn
View File

@ -177,20 +177,12 @@ rtc_library("video_stream_encoder_impl") {
# visibility = [ "../api/video:video_stream_encoder_create" ] # visibility = [ "../api/video:video_stream_encoder_create" ]
sources = [ sources = [
"encode_usage_resource.cc",
"encode_usage_resource.h",
"encoder_bitrate_adjuster.cc", "encoder_bitrate_adjuster.cc",
"encoder_bitrate_adjuster.h", "encoder_bitrate_adjuster.h",
"encoder_overshoot_detector.cc", "encoder_overshoot_detector.cc",
"encoder_overshoot_detector.h", "encoder_overshoot_detector.h",
"frame_encode_metadata_writer.cc", "frame_encode_metadata_writer.cc",
"frame_encode_metadata_writer.h", "frame_encode_metadata_writer.h",
"overuse_frame_detector.cc",
"overuse_frame_detector.h",
"overuse_frame_detector_resource_adaptation_module.cc",
"overuse_frame_detector_resource_adaptation_module.h",
"quality_scaler_resource.cc",
"quality_scaler_resource.h",
"video_source_sink_controller.cc", "video_source_sink_controller.cc",
"video_source_sink_controller.h", "video_source_sink_controller.h",
"video_stream_encoder.cc", "video_stream_encoder.cc",
@ -505,8 +497,6 @@ if (rtc_include_tests) {
"end_to_end_tests/stats_tests.cc", "end_to_end_tests/stats_tests.cc",
"end_to_end_tests/transport_feedback_tests.cc", "end_to_end_tests/transport_feedback_tests.cc",
"frame_encode_metadata_writer_unittest.cc", "frame_encode_metadata_writer_unittest.cc",
"overuse_frame_detector_resource_adaptation_unittest.cc",
"overuse_frame_detector_unittest.cc",
"picture_id_tests.cc", "picture_id_tests.cc",
"quality_limitation_reason_tracker_unittest.cc", "quality_limitation_reason_tracker_unittest.cc",
"quality_scaling_tests.cc", "quality_scaling_tests.cc",

32
video/adaptation/BUILD.gn
View File

@ -12,22 +12,43 @@ rtc_library("video_adaptation") {
sources = [ sources = [
"adaptation_counters.cc", "adaptation_counters.cc",
"adaptation_counters.h", "adaptation_counters.h",
"encode_usage_resource.cc",
"encode_usage_resource.h",
"overuse_frame_detector.cc",
"overuse_frame_detector.h",
"quality_scaler_resource.cc",
"quality_scaler_resource.h",
"resource_adaptation_processor.cc",
"resource_adaptation_processor.h",
"video_stream_adapter.cc", "video_stream_adapter.cc",
"video_stream_adapter.h", "video_stream_adapter.h",
] ]
deps = [ deps = [
"../../api:rtp_parameters", "../../api:rtp_parameters",
"../../api/task_queue:task_queue",
"../../api/video:video_frame",
"../../api/video:video_stream_encoder", "../../api/video:video_stream_encoder",
"../../api/video_codecs:video_codecs_api", "../../api/video_codecs:video_codecs_api",
"../../call/adaptation:resource_adaptation", "../../call/adaptation:resource_adaptation",
"../../modules/video_coding:video_coding_utility", "../../modules/video_coding:video_coding_utility",
"../../rtc_base:checks", "../../rtc_base:checks",
"../../rtc_base:logging", "../../rtc_base:logging",
"../../rtc_base:macromagic",
"../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_event", "../../rtc_base:rtc_event",
"../../rtc_base:rtc_numerics", "../../rtc_base:rtc_numerics",
"../../rtc_base:timeutils",
"../../rtc_base/experiments:balanced_degradation_settings", "../../rtc_base/experiments:balanced_degradation_settings",
"../../rtc_base/experiments:field_trial_parser",
"../../rtc_base/experiments:quality_rampup_experiment",
"../../rtc_base/experiments:quality_scaler_settings",
"../../rtc_base/synchronization:sequence_checker",
"../../rtc_base/task_utils:repeating_task",
"../../system_wrappers:field_trial",
"../../system_wrappers:system_wrappers",
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/base:core_headers",
"//third_party/abseil-cpp/absl/types:optional", "//third_party/abseil-cpp/absl/types:optional",
] ]
} }
@ -37,14 +58,23 @@ if (rtc_include_tests) {
testonly = true testonly = true
defines = [] defines = []
sources = [ "adaptation_counters_unittest.cc" ] sources = [
"adaptation_counters_unittest.cc",
"overuse_frame_detector_unittest.cc",
"resource_adaptation_processor_unittest.cc",
]
deps = [ deps = [
":video_adaptation", ":video_adaptation",
"../../api/video:encoded_image",
"../../api/video:video_frame_i420",
"../../modules/video_coding:video_coding_utility",
"../../rtc_base:checks", "../../rtc_base:checks",
"../../rtc_base:logging", "../../rtc_base:logging",
"../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_base_tests_utils",
"../../rtc_base:rtc_event", "../../rtc_base:rtc_event",
"../../rtc_base:rtc_numerics", "../../rtc_base:rtc_numerics",
"../../rtc_base:task_queue_for_test",
"//test:test_support", "//test:test_support",
"//testing/gtest", "//testing/gtest",
"//third_party/abseil-cpp/absl/types:optional", "//third_party/abseil-cpp/absl/types:optional",

2
video/encode_usage_resource.cc → video/adaptation/encode_usage_resource.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "video/encode_usage_resource.h" #include "video/adaptation/encode_usage_resource.h"
#include <limits> #include <limits>
#include <utility> #include <utility>

14
video/encode_usage_resource.h → video/adaptation/encode_usage_resource.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef VIDEO_ENCODE_USAGE_RESOURCE_H_ #ifndef VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_
#define VIDEO_ENCODE_USAGE_RESOURCE_H_ #define VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_
#include <memory> #include <memory>
#include <string> #include <string>
@ -17,15 +17,15 @@
#include "absl/types/optional.h" #include "absl/types/optional.h"
#include "call/adaptation/resource.h" #include "call/adaptation/resource.h"
#include "modules/video_coding/utility/quality_scaler.h" #include "modules/video_coding/utility/quality_scaler.h"
#include "video/overuse_frame_detector.h" #include "video/adaptation/overuse_frame_detector.h"
namespace webrtc { namespace webrtc {
// Handles interaction with the OveruseDetector. // Handles interaction with the OveruseDetector.
// TODO(hbos): Add unittests specific to this class, it is currently only tested // TODO(hbos): Add unittests specific to this class, it is currently only tested
// indirectly by usage in the OveruseFrameDetectorResourceAdaptationModule // indirectly by usage in the ResourceAdaptationProcessor (which is only tested
// (which is only tested because of its usage in VideoStreamEncoder); all tests // because of its usage in VideoStreamEncoder); all tests are currently in
// are currently in video_stream_encoder_unittest.cc. // video_stream_encoder_unittest.cc.
// TODO(https://crbug.com/webrtc/11222): Move this class to the // TODO(https://crbug.com/webrtc/11222): Move this class to the
// video/adaptation/ subdirectory. // video/adaptation/ subdirectory.
class EncodeUsageResource : public Resource, class EncodeUsageResource : public Resource,
@ -63,4 +63,4 @@ class EncodeUsageResource : public Resource,
} // namespace webrtc } // namespace webrtc
#endif // VIDEO_ENCODE_USAGE_RESOURCE_H_ #endif // VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_

4
video/overuse_frame_detector.cc → video/adaptation/overuse_frame_detector.cc
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "video/overuse_frame_detector.h" #include "video/adaptation/overuse_frame_detector.h"
#include <math.h> #include <math.h>
#include <stdio.h> #include <stdio.h>

8
video/overuse_frame_detector.h → video/adaptation/overuse_frame_detector.h
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef VIDEO_OVERUSE_FRAME_DETECTOR_H_ #ifndef VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_
#define VIDEO_OVERUSE_FRAME_DETECTOR_H_ #define VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_
#include <list> #include <list>
#include <memory> #include <memory>
@ -155,4 +155,4 @@ class OveruseFrameDetector {
} // namespace webrtc } // namespace webrtc
#endif // VIDEO_OVERUSE_FRAME_DETECTOR_H_ #endif // VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_

4
video/overuse_frame_detector_unittest.cc → video/adaptation/overuse_frame_detector_unittest.cc
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "video/overuse_frame_detector.h" #include "video/adaptation/overuse_frame_detector.h"
#include <memory> #include <memory>

2
video/quality_scaler_resource.cc → video/adaptation/quality_scaler_resource.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "video/quality_scaler_resource.h" #include "video/adaptation/quality_scaler_resource.h"
#include <utility> #include <utility>

12
video/quality_scaler_resource.h → video/adaptation/quality_scaler_resource.h
View File

@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef VIDEO_QUALITY_SCALER_RESOURCE_H_ #ifndef VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_
#define VIDEO_QUALITY_SCALER_RESOURCE_H_ #define VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_
#include <memory> #include <memory>
#include <string> #include <string>
@ -22,9 +22,9 @@ namespace webrtc {
// Handles interaction with the QualityScaler. // Handles interaction with the QualityScaler.
// TODO(hbos): Add unittests specific to this class, it is currently only tested // TODO(hbos): Add unittests specific to this class, it is currently only tested
// indirectly by usage in the OveruseFrameDetectorResourceAdaptationModule // indirectly by usage in the ResourceAdaptationProcessor (which is only tested
// (which is only tested because of its usage in VideoStreamEncoder); all tests // because of its usage in VideoStreamEncoder); all tests are currently in
// are currently in video_stream_encoder_unittest.cc. // video_stream_encoder_unittest.cc.
// TODO(https://crbug.com/webrtc/11222): Move this class to the // TODO(https://crbug.com/webrtc/11222): Move this class to the
// video/adaptation/ subdirectory. // video/adaptation/ subdirectory.
class QualityScalerResource : public Resource, class QualityScalerResource : public Resource,
@ -57,4 +57,4 @@ class QualityScalerResource : public Resource,
} // namespace webrtc } // namespace webrtc
#endif // VIDEO_QUALITY_SCALER_RESOURCE_H_ #endif // VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_

112
video/overuse_frame_detector_resource_adaptation_module.cc → video/adaptation/resource_adaptation_processor.cc
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2019 The WebRTC Project Authors. All rights reserved. * Copyright 2020 The WebRTC Project Authors. All rights reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "video/overuse_frame_detector_resource_adaptation_module.h" #include "video/adaptation/resource_adaptation_processor.h"
#include <algorithm> #include <algorithm>
#include <limits> #include <limits>
@ -26,10 +26,12 @@
#include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/strings/string_builder.h" #include "rtc_base/strings/string_builder.h"
#include "rtc_base/time_utils.h" #include "rtc_base/time_utils.h"
#include "video/video_stream_encoder.h"
namespace webrtc { namespace webrtc {
const int kDefaultInputPixelsWidth = 176;
const int kDefaultInputPixelsHeight = 144;
namespace { namespace {
bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
@ -93,7 +95,7 @@ AdaptationCounters ApplyDegradationPreference(
} // namespace } // namespace
class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper { class ResourceAdaptationProcessor::InitialFrameDropper {
public: public:
explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource) explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource)
: quality_scaler_resource_(quality_scaler_resource), : quality_scaler_resource_(quality_scaler_resource),
@ -164,13 +166,12 @@ class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper {
int initial_framedrop_; int initial_framedrop_;
}; };
OveruseFrameDetectorResourceAdaptationModule:: ResourceAdaptationProcessor::ResourceAdaptationProcessor(
OveruseFrameDetectorResourceAdaptationModule( Clock* clock,
Clock* clock, bool experiment_cpu_load_estimator,
bool experiment_cpu_load_estimator, std::unique_ptr<OveruseFrameDetector> overuse_detector,
std::unique_ptr<OveruseFrameDetector> overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer,
VideoStreamEncoderObserver* encoder_stats_observer, ResourceAdaptationProcessorListener* adaptation_listener)
ResourceAdaptationModuleListener* adaptation_listener)
: adaptation_listener_(adaptation_listener), : adaptation_listener_(adaptation_listener),
clock_(clock), clock_(clock),
state_(State::kStopped), state_(State::kStopped),
@ -200,13 +201,12 @@ OveruseFrameDetectorResourceAdaptationModule::
AdaptationObserverInterface::AdaptReason::kQuality); AdaptationObserverInterface::AdaptReason::kQuality);
} }
OveruseFrameDetectorResourceAdaptationModule:: ResourceAdaptationProcessor::~ResourceAdaptationProcessor() {
~OveruseFrameDetectorResourceAdaptationModule() {
RTC_DCHECK_EQ(state_, State::kStopped); RTC_DCHECK_EQ(state_, State::kStopped);
} }
void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( void ResourceAdaptationProcessor::StartResourceAdaptation(
ResourceAdaptationModuleListener* adaptation_listener) { ResourceAdaptationProcessorListener* adaptation_listener) {
RTC_DCHECK_EQ(state_, State::kStopped); RTC_DCHECK_EQ(state_, State::kStopped);
RTC_DCHECK(encoder_settings_.has_value()); RTC_DCHECK(encoder_settings_.has_value());
// TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener
@ -220,7 +220,7 @@ void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation(
state_ = State::kStarted; state_ = State::kStarted;
} }
void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { void ResourceAdaptationProcessor::StopResourceAdaptation() {
encode_usage_resource_->StopCheckForOveruse(); encode_usage_resource_->StopCheckForOveruse();
quality_scaler_resource_->StopCheckForOveruse(); quality_scaler_resource_->StopCheckForOveruse();
for (auto& resource_and_reason : resources_) { for (auto& resource_and_reason : resources_) {
@ -229,12 +229,11 @@ void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() {
state_ = State::kStopped; state_ = State::kStopped;
} }
void OveruseFrameDetectorResourceAdaptationModule::AddResource( void ResourceAdaptationProcessor::AddResource(Resource* resource) {
Resource* resource) {
return AddResource(resource, AdaptationObserverInterface::AdaptReason::kCpu); return AddResource(resource, AdaptationObserverInterface::AdaptReason::kCpu);
} }
void OveruseFrameDetectorResourceAdaptationModule::AddResource( void ResourceAdaptationProcessor::AddResource(
Resource* resource, Resource* resource,
AdaptationObserverInterface::AdaptReason reason) { AdaptationObserverInterface::AdaptReason reason) {
RTC_DCHECK(resource); RTC_DCHECK(resource);
@ -246,13 +245,12 @@ void OveruseFrameDetectorResourceAdaptationModule::AddResource(
resources_.emplace_back(resource, reason); resources_.emplace_back(resource, reason);
} }
void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( void ResourceAdaptationProcessor::SetHasInputVideo(bool has_input_video) {
bool has_input_video) {
// While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS. // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS.
has_input_video_ = has_input_video; has_input_video_ = has_input_video;
} }
void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( void ResourceAdaptationProcessor::SetDegradationPreference(
DegradationPreference degradation_preference) { DegradationPreference degradation_preference) {
degradation_preference_ = degradation_preference; degradation_preference_ = degradation_preference;
if (stream_adapter_->SetDegradationPreference(degradation_preference) == if (stream_adapter_->SetDegradationPreference(degradation_preference) ==
@ -263,7 +261,7 @@ void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference(
MaybeUpdateVideoSourceRestrictions(); MaybeUpdateVideoSourceRestrictions();
} }
void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings( void ResourceAdaptationProcessor::SetEncoderSettings(
EncoderSettings encoder_settings) { EncoderSettings encoder_settings) {
encoder_settings_ = std::move(encoder_settings); encoder_settings_ = std::move(encoder_settings);
@ -273,40 +271,36 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings(
MaybeUpdateTargetFrameRate(); MaybeUpdateTargetFrameRate();
} }
void OveruseFrameDetectorResourceAdaptationModule::SetStartBitrate( void ResourceAdaptationProcessor::SetStartBitrate(DataRate start_bitrate) {
DataRate start_bitrate) {
if (!start_bitrate.IsZero()) if (!start_bitrate.IsZero())
encoder_target_bitrate_bps_ = start_bitrate.bps(); encoder_target_bitrate_bps_ = start_bitrate.bps();
initial_frame_dropper_->SetStartBitrate(start_bitrate, initial_frame_dropper_->SetStartBitrate(start_bitrate,
clock_->TimeInMicroseconds()); clock_->TimeInMicroseconds());
} }
void OveruseFrameDetectorResourceAdaptationModule::SetTargetBitrate( void ResourceAdaptationProcessor::SetTargetBitrate(DataRate target_bitrate) {
DataRate target_bitrate) {
if (!target_bitrate.IsZero()) if (!target_bitrate.IsZero())
encoder_target_bitrate_bps_ = target_bitrate.bps(); encoder_target_bitrate_bps_ = target_bitrate.bps();
initial_frame_dropper_->SetTargetBitrate(target_bitrate, initial_frame_dropper_->SetTargetBitrate(target_bitrate,
clock_->TimeInMilliseconds()); clock_->TimeInMilliseconds());
} }
void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates( void ResourceAdaptationProcessor::SetEncoderRates(
const VideoEncoder::RateControlParameters& encoder_rates) { const VideoEncoder::RateControlParameters& encoder_rates) {
encoder_rates_ = encoder_rates; encoder_rates_ = encoder_rates;
} }
void OveruseFrameDetectorResourceAdaptationModule:: void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() {
ResetVideoSourceRestrictions() {
stream_adapter_->ClearRestrictions(); stream_adapter_->ClearRestrictions();
active_counts_.fill(AdaptationCounters()); active_counts_.fill(AdaptationCounters());
MaybeUpdateVideoSourceRestrictions(); MaybeUpdateVideoSourceRestrictions();
} }
void OveruseFrameDetectorResourceAdaptationModule::OnFrame( void ResourceAdaptationProcessor::OnFrame(const VideoFrame& frame) {
const VideoFrame& frame) {
last_input_frame_size_ = frame.size(); last_input_frame_size_ = frame.size();
} }
void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { void ResourceAdaptationProcessor::OnFrameDroppedDueToSize() {
AdaptationCounters counters_before = stream_adapter_->adaptation_counters(); AdaptationCounters counters_before = stream_adapter_->adaptation_counters();
OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
if (degradation_preference() == DegradationPreference::BALANCED && if (degradation_preference() == DegradationPreference::BALANCED &&
@ -322,14 +316,14 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() {
initial_frame_dropper_->OnFrameDroppedDueToSize(); initial_frame_dropper_->OnFrameDroppedDueToSize();
} }
void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted( void ResourceAdaptationProcessor::OnEncodeStarted(
const VideoFrame& cropped_frame, const VideoFrame& cropped_frame,
int64_t time_when_first_seen_us) { int64_t time_when_first_seen_us) {
encode_usage_resource_->OnEncodeStarted(cropped_frame, encode_usage_resource_->OnEncodeStarted(cropped_frame,
time_when_first_seen_us); time_when_first_seen_us);
} }
void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted( void ResourceAdaptationProcessor::OnEncodeCompleted(
const EncodedImage& encoded_image, const EncodedImage& encoded_image,
int64_t time_sent_in_us, int64_t time_sent_in_us,
absl::optional<int> encode_duration_us) { absl::optional<int> encode_duration_us) {
@ -343,21 +337,21 @@ void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted(
quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
} }
void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped( void ResourceAdaptationProcessor::OnFrameDropped(
EncodedImageCallback::DropReason reason) { EncodedImageCallback::DropReason reason) {
quality_scaler_resource_->OnFrameDropped(reason); quality_scaler_resource_->OnFrameDropped(reason);
} }
bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const { bool ResourceAdaptationProcessor::DropInitialFrames() const {
return initial_frame_dropper_->DropInitialFrames(); return initial_frame_dropper_->DropInitialFrames();
} }
void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() { void ResourceAdaptationProcessor::OnMaybeEncodeFrame() {
initial_frame_dropper_->OnMaybeEncodeFrame(); initial_frame_dropper_->OnMaybeEncodeFrame();
MaybePerformQualityRampupExperiment(); MaybePerformQualityRampupExperiment();
} }
void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( void ResourceAdaptationProcessor::UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds) { absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
if (qp_thresholds.has_value()) { if (qp_thresholds.has_value()) {
quality_scaler_resource_->StopCheckForOveruse(); quality_scaler_resource_->StopCheckForOveruse();
@ -368,7 +362,7 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings(
initial_frame_dropper_->OnQualityScalerSettingsUpdated(); initial_frame_dropper_->OnQualityScalerSettingsUpdated();
} }
void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( void ResourceAdaptationProcessor::ConfigureQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info) { const VideoEncoder::EncoderInfo& encoder_info) {
const auto scaling_settings = encoder_info.scaling_settings; const auto scaling_settings = encoder_info.scaling_settings;
const bool quality_scaling_allowed = const bool quality_scaling_allowed =
@ -414,7 +408,7 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler(
} }
ResourceListenerResponse ResourceListenerResponse
OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured( ResourceAdaptationProcessor::OnResourceUsageStateMeasured(
const Resource& resource) { const Resource& resource) {
const auto& registered_resource = const auto& registered_resource =
absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) { absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) {
@ -443,7 +437,7 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured(
} }
} }
void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( void ResourceAdaptationProcessor::OnResourceUnderuse(
AdaptationObserverInterface::AdaptReason reason) { AdaptationObserverInterface::AdaptReason reason) {
// We can't adapt up if we're already at the highest setting. // We can't adapt up if we're already at the highest setting.
// Note that this only includes counts relevant to the current degradation // Note that this only includes counts relevant to the current degradation
@ -484,8 +478,7 @@ void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse(
RTC_LOG(LS_INFO) << ActiveCountsToString(); RTC_LOG(LS_INFO) << ActiveCountsToString();
} }
ResourceListenerResponse ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse(
OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse(
AdaptationObserverInterface::AdaptReason reason) { AdaptationObserverInterface::AdaptReason reason) {
if (!has_input_video_) if (!has_input_video_)
return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency;
@ -516,8 +509,7 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse(
// pipelining encoders better (multiple input frames before something comes // pipelining encoders better (multiple input frames before something comes
// out). This should effectively turn off CPU adaptations for systems that // out). This should effectively turn off CPU adaptations for systems that
// remotely cope with the load right now. // remotely cope with the load right now.
CpuOveruseOptions CpuOveruseOptions ResourceAdaptationProcessor::GetCpuOveruseOptions() const {
OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const {
// This is already ensured by the only caller of this method: // This is already ensured by the only caller of this method:
// StartResourceAdaptation(). // StartResourceAdaptation().
RTC_DCHECK(encoder_settings_.has_value()); RTC_DCHECK(encoder_settings_.has_value());
@ -534,8 +526,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const {
return options; return options;
} }
int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault() int ResourceAdaptationProcessor::LastInputFrameSizeOrDefault() const {
const {
// The dependency on this hardcoded resolution is inherited from old code, // The dependency on this hardcoded resolution is inherited from old code,
// which used this resolution as a stand-in for not knowing the resolution // which used this resolution as a stand-in for not knowing the resolution
// yet. // yet.
@ -543,13 +534,11 @@ int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault()
// DCHECK passed all the tests but adding it does change the requirements of // DCHECK passed all the tests but adding it does change the requirements of
// this class (= not being allowed to call OnResourceUnderuse() or // this class (= not being allowed to call OnResourceUnderuse() or
// OnResourceOveruse() before OnFrame()) and deserves a standalone CL. // OnResourceOveruse() before OnFrame()) and deserves a standalone CL.
return last_input_frame_size_.value_or( return last_input_frame_size_.value_or(kDefaultInputPixelsWidth *
VideoStreamEncoder::kDefaultLastFrameInfoWidth * kDefaultInputPixelsHeight);
VideoStreamEncoder::kDefaultLastFrameInfoHeight);
} }
void OveruseFrameDetectorResourceAdaptationModule:: void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions() {
MaybeUpdateVideoSourceRestrictions() {
VideoSourceRestrictions new_restrictions = ApplyDegradationPreference( VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
stream_adapter_->source_restrictions(), degradation_preference_); stream_adapter_->source_restrictions(), degradation_preference_);
if (video_source_restrictions_ != new_restrictions) { if (video_source_restrictions_ != new_restrictions) {
@ -560,8 +549,7 @@ void OveruseFrameDetectorResourceAdaptationModule::
} }
} }
void OveruseFrameDetectorResourceAdaptationModule:: void ResourceAdaptationProcessor::MaybeUpdateTargetFrameRate() {
MaybeUpdateTargetFrameRate() {
absl::optional<double> codec_max_frame_rate = absl::optional<double> codec_max_frame_rate =
encoder_settings_.has_value() encoder_settings_.has_value()
? absl::optional<double>( ? absl::optional<double>(
@ -583,7 +571,7 @@ void OveruseFrameDetectorResourceAdaptationModule::
encode_usage_resource_->SetTargetFrameRate(target_frame_rate); encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
} }
void OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( void ResourceAdaptationProcessor::OnAdaptationCountChanged(
const AdaptationCounters& adaptation_count, const AdaptationCounters& adaptation_count,
AdaptationCounters* active_count, AdaptationCounters* active_count,
AdaptationCounters* other_active) { AdaptationCounters* other_active) {
@ -641,7 +629,7 @@ void OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
} }
// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( void ResourceAdaptationProcessor::UpdateAdaptationStats(
AdaptationObserverInterface::AdaptReason reason) { AdaptationObserverInterface::AdaptReason reason) {
// Update active counts // Update active counts
AdaptationCounters& active_count = active_counts_[reason]; AdaptationCounters& active_count = active_counts_[reason];
@ -668,7 +656,7 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats(
} }
VideoStreamEncoderObserver::AdaptationSteps VideoStreamEncoderObserver::AdaptationSteps
OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( ResourceAdaptationProcessor::GetActiveCounts(
AdaptationObserverInterface::AdaptReason reason) { AdaptationObserverInterface::AdaptReason reason) {
// TODO(https://crbug.com/webrtc/11392) Ideally this shuold be moved out of // TODO(https://crbug.com/webrtc/11392) Ideally this shuold be moved out of
// this class and into the encoder_stats_observer_. // this class and into the encoder_stats_observer_.
@ -700,7 +688,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
} }
VideoStreamAdapter::VideoInputMode VideoStreamAdapter::VideoInputMode
OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const { ResourceAdaptationProcessor::GetVideoInputMode() const {
if (!has_input_video_) if (!has_input_video_)
return VideoStreamAdapter::VideoInputMode::kNoVideo; return VideoStreamAdapter::VideoInputMode::kNoVideo;
return (encoder_settings_.has_value() && return (encoder_settings_.has_value() &&
@ -710,8 +698,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const {
: VideoStreamAdapter::VideoInputMode::kNormalVideo; : VideoStreamAdapter::VideoInputMode::kNormalVideo;
} }
void OveruseFrameDetectorResourceAdaptationModule:: void ResourceAdaptationProcessor::MaybePerformQualityRampupExperiment() {
MaybePerformQualityRampupExperiment() {
if (!quality_scaler_resource_->is_started()) if (!quality_scaler_resource_->is_started())
return; return;
@ -747,8 +734,7 @@ void OveruseFrameDetectorResourceAdaptationModule::
} }
} }
std::string OveruseFrameDetectorResourceAdaptationModule::ActiveCountsToString() std::string ResourceAdaptationProcessor::ActiveCountsToString() const {
const {
rtc::StringBuilder ss; rtc::StringBuilder ss;
ss << "Downgrade counts: fps: {"; ss << "Downgrade counts: fps: {";

47
video/overuse_frame_detector_resource_adaptation_module.h → video/adaptation/resource_adaptation_processor.h
View File

@ -1,5 +1,5 @@
/* /*
* Copyright 2019 The WebRTC Project Authors. All rights reserved. * Copyright 2020 The WebRTC Project Authors. All rights reserved.
* *
* Use of this source code is governed by a BSD-style license * Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source * that can be found in the LICENSE file in the root of the source
@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#ifndef VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ #ifndef VIDEO_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_
#define VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ #define VIDEO_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_
#include <map> #include <map>
#include <memory> #include <memory>
@ -26,20 +26,24 @@
#include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_config.h" #include "api/video_codecs/video_encoder_config.h"
#include "call/adaptation/resource.h" #include "call/adaptation/resource.h"
#include "call/adaptation/resource_adaptation_module_interface.h" #include "call/adaptation/resource_adaptation_processor_interface.h"
#include "rtc_base/experiments/quality_rampup_experiment.h" #include "rtc_base/experiments/quality_rampup_experiment.h"
#include "rtc_base/experiments/quality_scaler_settings.h" #include "rtc_base/experiments/quality_scaler_settings.h"
#include "rtc_base/strings/string_builder.h" #include "rtc_base/strings/string_builder.h"
#include "system_wrappers/include/clock.h" #include "system_wrappers/include/clock.h"
#include "video/adaptation/adaptation_counters.h" #include "video/adaptation/adaptation_counters.h"
#include "video/adaptation/encode_usage_resource.h"
#include "video/adaptation/overuse_frame_detector.h"
#include "video/adaptation/quality_scaler_resource.h"
#include "video/adaptation/video_stream_adapter.h" #include "video/adaptation/video_stream_adapter.h"
#include "video/encode_usage_resource.h"
#include "video/overuse_frame_detector.h"
#include "video/quality_scaler_resource.h"
namespace webrtc { namespace webrtc {
class VideoStreamEncoder; // The assumed input frame size if we have not yet received a frame.
// TODO(hbos): This is 144p - why are we assuming super low quality? Seems like
// a bad heuristic.
extern const int kDefaultInputPixelsWidth;
extern const int kDefaultInputPixelsHeight;
// This class is used by the VideoStreamEncoder and is responsible for adapting // This class is used by the VideoStreamEncoder and is responsible for adapting
// resolution up or down based on encode usage percent. It keeps track of video // resolution up or down based on encode usage percent. It keeps track of video
@ -51,29 +55,26 @@ class VideoStreamEncoder;
// TODO(hbos): Add unittests specific to this class, it is currently only tested // TODO(hbos): Add unittests specific to this class, it is currently only tested
// indirectly in video_stream_encoder_unittest.cc and other tests exercising // indirectly in video_stream_encoder_unittest.cc and other tests exercising
// VideoStreamEncoder. // VideoStreamEncoder.
// TODO(https://crbug.com/webrtc/11222): Rename this class to something more class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
// appropriate and move it to the video/adaptation/ subdirectory. public ResourceListener {
class OveruseFrameDetectorResourceAdaptationModule
: public ResourceAdaptationModuleInterface,
public ResourceListener {
public: public:
// The module can be constructed on any sequence, but must be initialized and // The processor can be constructed on any sequence, but must be initialized
// used on a single sequence, e.g. the encoder queue. // and used on a single sequence, e.g. the encoder queue.
OveruseFrameDetectorResourceAdaptationModule( ResourceAdaptationProcessor(
Clock* clock, Clock* clock,
bool experiment_cpu_load_estimator, bool experiment_cpu_load_estimator,
std::unique_ptr<OveruseFrameDetector> overuse_detector, std::unique_ptr<OveruseFrameDetector> overuse_detector,
VideoStreamEncoderObserver* encoder_stats_observer, VideoStreamEncoderObserver* encoder_stats_observer,
ResourceAdaptationModuleListener* adaptation_listener); ResourceAdaptationProcessorListener* adaptation_listener);
~OveruseFrameDetectorResourceAdaptationModule() override; ~ResourceAdaptationProcessor() override;
DegradationPreference degradation_preference() const { DegradationPreference degradation_preference() const {
return degradation_preference_; return degradation_preference_;
} }
// ResourceAdaptationModuleInterface implementation. // ResourceAdaptationProcessorInterface implementation.
void StartResourceAdaptation( void StartResourceAdaptation(
ResourceAdaptationModuleListener* adaptation_listener) override; ResourceAdaptationProcessorListener* adaptation_listener) override;
void StopResourceAdaptation() override; void StopResourceAdaptation() override;
// Uses a default AdaptReason of kCpu. // Uses a default AdaptReason of kCpu.
void AddResource(Resource* resource) override; void AddResource(Resource* resource) override;
@ -167,7 +168,7 @@ class OveruseFrameDetectorResourceAdaptationModule
std::string ActiveCountsToString() const; std::string ActiveCountsToString() const;
ResourceAdaptationModuleListener* const adaptation_listener_; ResourceAdaptationProcessorListener* const adaptation_listener_;
Clock* clock_; Clock* clock_;
State state_; State state_;
const bool experiment_cpu_load_estimator_; const bool experiment_cpu_load_estimator_;
@ -180,7 +181,7 @@ class OveruseFrameDetectorResourceAdaptationModule
// owned by the adapter, this class has no buisness relying on implementation // owned by the adapter, this class has no buisness relying on implementation
// details of the adapter. // details of the adapter.
DegradationPreference degradation_preference_; DegradationPreference degradation_preference_;
// Keeps track of source restrictions that this adaptation module outputs. // Keeps track of source restrictions that this adaptation processor outputs.
const std::unique_ptr<VideoStreamAdapter> stream_adapter_; const std::unique_ptr<VideoStreamAdapter> stream_adapter_;
const std::unique_ptr<EncodeUsageResource> encode_usage_resource_; const std::unique_ptr<EncodeUsageResource> encode_usage_resource_;
const std::unique_ptr<QualityScalerResource> quality_scaler_resource_; const std::unique_ptr<QualityScalerResource> quality_scaler_resource_;
@ -221,4 +222,4 @@ class OveruseFrameDetectorResourceAdaptationModule
} // namespace webrtc } // namespace webrtc
#endif // VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ #endif // VIDEO_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_

35
video/overuse_frame_detector_resource_adaptation_unittest.cc → video/adaptation/resource_adaptation_processor_unittest.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "video/overuse_frame_detector_resource_adaptation_module.h" #include "video/adaptation/resource_adaptation_processor.h"
#include "test/gmock.h" #include "test/gmock.h"
#include "test/gtest.h" #include "test/gtest.h"
@ -16,62 +16,55 @@
namespace webrtc { namespace webrtc {
TEST(OveruseFrameDetectorResourceAdaptationModuleTest, TEST(ResourceAdaptationProcessorTest, FirstAdaptationDown_Fps) {
FirstAdaptationDown_Fps) {
AdaptationCounters cpu; AdaptationCounters cpu;
AdaptationCounters qp; AdaptationCounters qp;
AdaptationCounters total(0, 1); AdaptationCounters total(0, 1);
OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp);
total, &cpu, &qp);
AdaptationCounters expected_cpu(0, 1); AdaptationCounters expected_cpu(0, 1);
AdaptationCounters expected_qp; AdaptationCounters expected_qp;
EXPECT_EQ(expected_cpu, cpu); EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp); EXPECT_EQ(expected_qp, qp);
} }
TEST(OveruseFrameDetectorResourceAdaptationModuleTest, TEST(ResourceAdaptationProcessorTest, FirstAdaptationDown_Resolution) {
FirstAdaptationDown_Resolution) {
AdaptationCounters cpu; AdaptationCounters cpu;
AdaptationCounters qp; AdaptationCounters qp;
AdaptationCounters total(1, 0); AdaptationCounters total(1, 0);
OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp);
total, &cpu, &qp);
AdaptationCounters expected_cpu(1, 0); AdaptationCounters expected_cpu(1, 0);
AdaptationCounters expected_qp; AdaptationCounters expected_qp;
EXPECT_EQ(expected_cpu, cpu); EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp); EXPECT_EQ(expected_qp, qp);
} }
TEST(OveruseFrameDetectorResourceAdaptationModuleTest, LastAdaptUp_Fps) { TEST(ResourceAdaptationProcessorTest, LastAdaptUp_Fps) {
AdaptationCounters cpu(0, 1); AdaptationCounters cpu(0, 1);
AdaptationCounters qp; AdaptationCounters qp;
AdaptationCounters total; AdaptationCounters total;
OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp);
total, &cpu, &qp);
AdaptationCounters expected_cpu; AdaptationCounters expected_cpu;
AdaptationCounters expected_qp; AdaptationCounters expected_qp;
EXPECT_EQ(expected_cpu, cpu); EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp); EXPECT_EQ(expected_qp, qp);
} }
TEST(OveruseFrameDetectorResourceAdaptationModuleTest, LastAdaptUp_Resolution) { TEST(ResourceAdaptationProcessorTest, LastAdaptUp_Resolution) {
AdaptationCounters cpu(1, 0); AdaptationCounters cpu(1, 0);
AdaptationCounters qp; AdaptationCounters qp;
AdaptationCounters total; AdaptationCounters total;
OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp);
total, &cpu, &qp);
AdaptationCounters expected_cpu; AdaptationCounters expected_cpu;
AdaptationCounters expected_qp; AdaptationCounters expected_qp;
EXPECT_EQ(expected_cpu, cpu); EXPECT_EQ(expected_cpu, cpu);
EXPECT_EQ(expected_qp, qp); EXPECT_EQ(expected_qp, qp);
} }
TEST(OveruseFrameDetectorResourceAdaptationModuleTest, TEST(ResourceAdaptationProcessorTest, AdaptUpWithBorrow_Resolution) {
AdaptUpWithBorrow_Resolution) {
AdaptationCounters cpu(0, 1); AdaptationCounters cpu(0, 1);
AdaptationCounters qp(1, 0); AdaptationCounters qp(1, 0);
AdaptationCounters total(0, 1); AdaptationCounters total(0, 1);
@ -79,8 +72,7 @@ TEST(OveruseFrameDetectorResourceAdaptationModuleTest,
// CPU adaptation for resolution, but no resolution adaptation left from CPU. // CPU adaptation for resolution, but no resolution adaptation left from CPU.
// We then borrow the resolution adaptation from qp, and give qp the fps // We then borrow the resolution adaptation from qp, and give qp the fps
// adaptation from CPU. // adaptation from CPU.
OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp);
total, &cpu, &qp);
AdaptationCounters expected_cpu(0, 0); AdaptationCounters expected_cpu(0, 0);
AdaptationCounters expected_qp(0, 1); AdaptationCounters expected_qp(0, 1);
@ -88,15 +80,14 @@ TEST(OveruseFrameDetectorResourceAdaptationModuleTest,
EXPECT_EQ(expected_qp, qp); EXPECT_EQ(expected_qp, qp);
} }
TEST(OveruseFrameDetectorResourceAdaptationModuleTest, AdaptUpWithBorrow_Fps) { TEST(ResourceAdaptationProcessorTest, AdaptUpWithBorrow_Fps) {
AdaptationCounters cpu(1, 0); AdaptationCounters cpu(1, 0);
AdaptationCounters qp(0, 1); AdaptationCounters qp(0, 1);
AdaptationCounters total(1, 0); AdaptationCounters total(1, 0);
// CPU adaptation for fps, but no fps adaptation left from CPU. We then borrow // CPU adaptation for fps, but no fps adaptation left from CPU. We then borrow
// the fps adaptation from qp, and give qp the resolution adaptation from CPU. // the fps adaptation from qp, and give qp the resolution adaptation from CPU.
OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp);
total, &cpu, &qp);
AdaptationCounters expected_cpu(0, 0); AdaptationCounters expected_cpu(0, 0);
AdaptationCounters expected_qp(1, 0); AdaptationCounters expected_qp(1, 0);

2
video/video_source_sink_controller.h
View File

@ -15,7 +15,7 @@
#include "api/video/video_frame.h" #include "api/video/video_frame.h"
#include "api/video/video_sink_interface.h" #include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h" #include "api/video/video_source_interface.h"
#include "call/adaptation/resource_adaptation_module_interface.h" #include "call/adaptation/video_source_restrictions.h"
#include "rtc_base/critical_section.h" #include "rtc_base/critical_section.h"
namespace webrtc { namespace webrtc {

58
video/video_stream_encoder.cc
View File

@ -24,7 +24,6 @@
#include "api/video/video_bitrate_allocator_factory.h" #include "api/video/video_bitrate_allocator_factory.h"
#include "api/video/video_codec_constants.h" #include "api/video/video_codec_constants.h"
#include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource_adaptation_module_interface.h"
#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
#include "modules/video_coding/include/video_codec_initializer.h" #include "modules/video_coding/include/video_codec_initializer.h"
#include "rtc_base/arraysize.h" #include "rtc_base/arraysize.h"
@ -37,6 +36,7 @@
#include "rtc_base/time_utils.h" #include "rtc_base/time_utils.h"
#include "rtc_base/trace_event.h" #include "rtc_base/trace_event.h"
#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/field_trial.h"
#include "video/adaptation/resource_adaptation_processor.h"
namespace webrtc { namespace webrtc {
@ -172,9 +172,6 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
} // namespace } // namespace
const int VideoStreamEncoder::kDefaultLastFrameInfoWidth = 176;
const int VideoStreamEncoder::kDefaultLastFrameInfoHeight = 144;
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings() VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
: rate_control(), : rate_control(),
encoder_target(DataRate::Zero()), encoder_target(DataRate::Zero()),
@ -260,8 +257,8 @@ VideoStreamEncoder::VideoStreamEncoder(
video_source_sink_controller_(std::make_unique<VideoSourceSinkController>( video_source_sink_controller_(std::make_unique<VideoSourceSinkController>(
/*sink=*/this, /*sink=*/this,
/*source=*/nullptr)), /*source=*/nullptr)),
resource_adaptation_module_( resource_adaptation_processor_(
std::make_unique<OveruseFrameDetectorResourceAdaptationModule>( std::make_unique<ResourceAdaptationProcessor>(
clock_, clock_,
settings_.experiment_cpu_load_estimator, settings_.experiment_cpu_load_estimator,
std::move(overuse_detector), std::move(overuse_detector),
@ -288,7 +285,7 @@ void VideoStreamEncoder::Stop() {
video_source_sink_controller_->SetSource(nullptr); video_source_sink_controller_->SetSource(nullptr);
encoder_queue_.PostTask([this] { encoder_queue_.PostTask([this] {
RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK_RUN_ON(&encoder_queue_);
resource_adaptation_module_->StopResourceAdaptation(); resource_adaptation_processor_->StopResourceAdaptation();
rate_allocator_ = nullptr; rate_allocator_ = nullptr;
bitrate_observer_ = nullptr; bitrate_observer_ = nullptr;
ReleaseEncoder(); ReleaseEncoder();
@ -327,11 +324,11 @@ void VideoStreamEncoder::SetSource(
video_source_sink_controller_->SetSource(source); video_source_sink_controller_->SetSource(source);
encoder_queue_.PostTask([this, source, degradation_preference] { encoder_queue_.PostTask([this, source, degradation_preference] {
RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK_RUN_ON(&encoder_queue_);
resource_adaptation_module_->SetHasInputVideo(source); resource_adaptation_processor_->SetHasInputVideo(source);
resource_adaptation_module_->SetDegradationPreference( resource_adaptation_processor_->SetDegradationPreference(
degradation_preference); degradation_preference);
if (encoder_) if (encoder_)
resource_adaptation_module_->ConfigureQualityScaler( resource_adaptation_processor_->ConfigureQualityScaler(
encoder_->GetEncoderInfo()); encoder_->GetEncoderInfo());
}); });
} }
@ -351,7 +348,7 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) {
encoder_target_bitrate_bps_ = encoder_target_bitrate_bps_ =
start_bitrate_bps != 0 ? absl::optional<uint32_t>(start_bitrate_bps) start_bitrate_bps != 0 ? absl::optional<uint32_t>(start_bitrate_bps)
: absl::nullopt; : absl::nullopt;
resource_adaptation_module_->SetStartBitrate( resource_adaptation_processor_->SetStartBitrate(
DataRate::BitsPerSec(start_bitrate_bps)); DataRate::BitsPerSec(start_bitrate_bps));
}); });
} }
@ -382,8 +379,8 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
codec_info_ = settings_.encoder_factory->QueryVideoEncoder( codec_info_ = settings_.encoder_factory->QueryVideoEncoder(
encoder_config_.video_format); encoder_config_.video_format);
if (HasInternalSource()) { if (HasInternalSource()) {
last_frame_info_ = VideoFrameInfo( last_frame_info_ = VideoFrameInfo(kDefaultInputPixelsWidth,
kDefaultLastFrameInfoWidth, kDefaultLastFrameInfoHeight, false); kDefaultInputPixelsHeight, false);
ReconfigureEncoder(); ReconfigureEncoder();
} }
} }
@ -650,7 +647,7 @@ void VideoStreamEncoder::ReconfigureEncoder() {
was_encode_called_since_last_initialization_ = false; was_encode_called_since_last_initialization_ = false;
} }
resource_adaptation_module_->SetEncoderSettings(EncoderSettings( resource_adaptation_processor_->SetEncoderSettings(EncoderSettings(
encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_)); encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_));
if (success) { if (success) {
@ -668,8 +665,8 @@ void VideoStreamEncoder::ReconfigureEncoder() {
} }
if (pending_encoder_creation_) { if (pending_encoder_creation_) {
resource_adaptation_module_->StopResourceAdaptation(); resource_adaptation_processor_->StopResourceAdaptation();
resource_adaptation_module_->StartResourceAdaptation(this); resource_adaptation_processor_->StartResourceAdaptation(this);
pending_encoder_creation_ = false; pending_encoder_creation_ = false;
} }
@ -724,7 +721,7 @@ void VideoStreamEncoder::ReconfigureEncoder() {
std::move(streams), encoder_config_.content_type, std::move(streams), encoder_config_.content_type,
encoder_config_.min_transmit_bitrate_bps); encoder_config_.min_transmit_bitrate_bps);
resource_adaptation_module_->ConfigureQualityScaler(info); resource_adaptation_processor_->ConfigureQualityScaler(info);
} }
void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) {
@ -961,14 +958,14 @@ void VideoStreamEncoder::SetEncoderRates(
frame_encode_metadata_writer_.OnSetRates( frame_encode_metadata_writer_.OnSetRates(
rate_settings.rate_control.bitrate, rate_settings.rate_control.bitrate,
static_cast<uint32_t>(rate_settings.rate_control.framerate_fps + 0.5)); static_cast<uint32_t>(rate_settings.rate_control.framerate_fps + 0.5));
resource_adaptation_module_->SetEncoderRates(rate_settings.rate_control); resource_adaptation_processor_->SetEncoderRates(rate_settings.rate_control);
} }
} }
void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
int64_t time_when_posted_us) { int64_t time_when_posted_us) {
RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK_RUN_ON(&encoder_queue_);
resource_adaptation_module_->OnFrame(video_frame); resource_adaptation_processor_->OnFrame(video_frame);
if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
video_frame.height() != last_frame_info_->height || video_frame.height() != last_frame_info_->height ||
@ -1030,7 +1027,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
if (DropDueToSize(video_frame.size())) { if (DropDueToSize(video_frame.size())) {
RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
resource_adaptation_module_->OnFrameDroppedDueToSize(); resource_adaptation_processor_->OnFrameDroppedDueToSize();
// Storing references to a native buffer risks blocking frame capture. // Storing references to a native buffer risks blocking frame capture.
if (video_frame.video_frame_buffer()->type() != if (video_frame.video_frame_buffer()->type() !=
VideoFrameBuffer::Type::kNative) { VideoFrameBuffer::Type::kNative) {
@ -1044,7 +1041,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
} }
return; return;
} }
resource_adaptation_module_->OnMaybeEncodeFrame(); resource_adaptation_processor_->OnMaybeEncodeFrame();
if (EncoderPaused()) { if (EncoderPaused()) {
// Storing references to a native buffer risks blocking frame capture. // Storing references to a native buffer risks blocking frame capture.
@ -1115,7 +1112,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
} }
if (encoder_info_ != info) { if (encoder_info_ != info) {
resource_adaptation_module_->SetEncoderSettings(EncoderSettings( resource_adaptation_processor_->SetEncoderSettings(EncoderSettings(
encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_)); encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_));
RTC_LOG(LS_INFO) << "Encoder settings changed from " RTC_LOG(LS_INFO) << "Encoder settings changed from "
<< encoder_info_.ToString() << " to " << info.ToString(); << encoder_info_.ToString() << " to " << info.ToString();
@ -1228,7 +1225,8 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
"Encode"); "Encode");
resource_adaptation_module_->OnEncodeStarted(out_frame, time_when_posted_us); resource_adaptation_processor_->OnEncodeStarted(out_frame,
time_when_posted_us);
RTC_DCHECK_LE(send_codec_.width, out_frame.width()); RTC_DCHECK_LE(send_codec_.width, out_frame.width());
RTC_DCHECK_LE(send_codec_.height, out_frame.height()); RTC_DCHECK_LE(send_codec_.height, out_frame.height());
@ -1503,7 +1501,7 @@ void VideoStreamEncoder::OnDroppedFrame(DropReason reason) {
sink_->OnDroppedFrame(reason); sink_->OnDroppedFrame(reason);
encoder_queue_.PostTask([this, reason] { encoder_queue_.PostTask([this, reason] {
RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK_RUN_ON(&encoder_queue_);
resource_adaptation_module_->OnFrameDropped(reason); resource_adaptation_processor_->OnFrameDropped(reason);
}); });
} }
@ -1602,7 +1600,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate,
if (target_bitrate.bps() != 0) if (target_bitrate.bps() != 0)
encoder_target_bitrate_bps_ = target_bitrate.bps(); encoder_target_bitrate_bps_ = target_bitrate.bps();
resource_adaptation_module_->SetTargetBitrate(target_bitrate); resource_adaptation_processor_->SetTargetBitrate(target_bitrate);
if (video_suspension_changed) { if (video_suspension_changed) {
RTC_LOG(LS_INFO) << "Video suspend state changed to: " RTC_LOG(LS_INFO) << "Video suspend state changed to: "
@ -1619,7 +1617,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate,
} }
bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
if (!resource_adaptation_module_->DropInitialFrames() || if (!resource_adaptation_processor_->DropInitialFrames() ||
!encoder_target_bitrate_bps_.has_value()) { !encoder_target_bitrate_bps_.has_value()) {
return false; return false;
} }
@ -1691,8 +1689,8 @@ void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image,
} }
} }
resource_adaptation_module_->OnEncodeCompleted(encoded_image, time_sent_us, resource_adaptation_processor_->OnEncodeCompleted(encoded_image, time_sent_us,
encode_duration_us); encode_duration_us);
if (bitrate_adjuster_) { if (bitrate_adjuster_) {
bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index); bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index);
} }
@ -1847,7 +1845,7 @@ void VideoStreamEncoder::CheckForAnimatedContent(
if (!automatic_animation_detection_experiment_.enabled || if (!automatic_animation_detection_experiment_.enabled ||
encoder_config_.content_type != encoder_config_.content_type !=
VideoEncoderConfig::ContentType::kScreen || VideoEncoderConfig::ContentType::kScreen ||
resource_adaptation_module_->degradation_preference() != resource_adaptation_processor_->degradation_preference() !=
DegradationPreference::BALANCED) { DegradationPreference::BALANCED) {
return; return;
} }
@ -1914,7 +1912,7 @@ void VideoStreamEncoder::CheckForAnimatedContent(
void VideoStreamEncoder::InjectAdaptationResource( void VideoStreamEncoder::InjectAdaptationResource(
Resource* resource, Resource* resource,
AdaptationObserverInterface::AdaptReason reason) { AdaptationObserverInterface::AdaptReason reason) {
resource_adaptation_module_->AddResource(resource, reason); resource_adaptation_processor_->AddResource(resource, reason);
} }
} // namespace webrtc } // namespace webrtc

21
video/video_stream_encoder.h
View File

@ -26,7 +26,7 @@
#include "api/video/video_stream_encoder_settings.h" #include "api/video/video_stream_encoder_settings.h"
#include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource_adaptation_module_interface.h" #include "call/adaptation/resource_adaptation_processor_interface.h"
#include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_source_restrictions.h"
#include "modules/video_coding/utility/frame_dropper.h" #include "modules/video_coding/utility/frame_dropper.h"
#include "rtc_base/critical_section.h" #include "rtc_base/critical_section.h"
@ -38,9 +38,9 @@
#include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/synchronization/sequence_checker.h"
#include "rtc_base/task_queue.h" #include "rtc_base/task_queue.h"
#include "system_wrappers/include/clock.h" #include "system_wrappers/include/clock.h"
#include "video/adaptation/resource_adaptation_processor.h"
#include "video/encoder_bitrate_adjuster.h" #include "video/encoder_bitrate_adjuster.h"
#include "video/frame_encode_metadata_writer.h" #include "video/frame_encode_metadata_writer.h"
#include "video/overuse_frame_detector_resource_adaptation_module.h"
#include "video/video_source_sink_controller.h" #include "video/video_source_sink_controller.h"
namespace webrtc { namespace webrtc {
@ -55,16 +55,8 @@ namespace webrtc {
// Call Stop() when done. // Call Stop() when done.
class VideoStreamEncoder : public VideoStreamEncoderInterface, class VideoStreamEncoder : public VideoStreamEncoderInterface,
private EncodedImageCallback, private EncodedImageCallback,
public ResourceAdaptationModuleListener { public ResourceAdaptationProcessorListener {
public: public:
// If the encoder is reconfigured with a source, but we've yet to receive any
// frames, this 144p resolution is picked as the default value of
// |last_frame_size_|.
// TODO(hbos): Can we avoid guesses and properly handle the case of
// |last_frame_info_| not having a value, deleting these constants?
static const int kDefaultLastFrameInfoWidth;
static const int kDefaultLastFrameInfoHeight;
VideoStreamEncoder(Clock* clock, VideoStreamEncoder(Clock* clock,
uint32_t number_of_cores, uint32_t number_of_cores,
VideoStreamEncoderObserver* encoder_stats_observer, VideoStreamEncoderObserver* encoder_stats_observer,
@ -402,7 +394,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_); bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_);
// The controller updates the sink wants based on restrictions that come from // The controller updates the sink wants based on restrictions that come from
// the resource adaptation module or adaptation due to bandwidth adaptation. // the resource adaptation processor or adaptation due to bandwidth
// adaptation.
// //
// This is used on the encoder queue, with a few exceptions: // This is used on the encoder queue, with a few exceptions:
// - VideoStreamEncoder::SetSource() invokes SetSource(). // - VideoStreamEncoder::SetSource() invokes SetSource().
@ -413,8 +406,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
// VideoSourceSinkController can be made single-threaded, and its lock can be // VideoSourceSinkController can be made single-threaded, and its lock can be
// replaced with a sequence checker. // replaced with a sequence checker.
std::unique_ptr<VideoSourceSinkController> video_source_sink_controller_; std::unique_ptr<VideoSourceSinkController> video_source_sink_controller_;
std::unique_ptr<OveruseFrameDetectorResourceAdaptationModule> std::unique_ptr<ResourceAdaptationProcessor> resource_adaptation_processor_
resource_adaptation_module_ RTC_GUARDED_BY(&encoder_queue_); RTC_GUARDED_BY(&encoder_queue_);
// All public methods are proxied to |encoder_queue_|. It must must be // All public methods are proxied to |encoder_queue_|. It must must be
// destroyed first to make sure no tasks are run that use other members. // destroyed first to make sure no tasks are run that use other members.