diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 3aae5fb654..c38e7bc947 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -265,6 +265,7 @@ rtc_library("video_stream_encoder_create") { ":video_stream_encoder", "../../api:scoped_refptr", "../../video:video_stream_encoder_impl", + "../../video/adaptation:video_adaptation", "../task_queue", "../video_codecs:video_codecs_api", ] diff --git a/api/video/video_stream_encoder_create.cc b/api/video/video_stream_encoder_create.cc index ac2f6b9819..3a2ebe79e1 100644 --- a/api/video/video_stream_encoder_create.cc +++ b/api/video/video_stream_encoder_create.cc @@ -12,7 +12,7 @@ #include -#include "video/overuse_frame_detector.h" +#include "video/adaptation/overuse_frame_detector.h" #include "video/video_stream_encoder.h" namespace webrtc { diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index 0288e247b4..32cf65990a 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -12,12 +12,12 @@ rtc_library("resource_adaptation") { sources = [ "encoder_settings.cc", "encoder_settings.h", + "new_resource_adaptation_processor_poc.cc", + "new_resource_adaptation_processor_poc.h", "resource.cc", "resource.h", - "resource_adaptation_module_interface.cc", - "resource_adaptation_module_interface.h", - "resource_adaptation_processor.cc", - "resource_adaptation_processor.h", + "resource_adaptation_processor_interface.cc", + "resource_adaptation_processor_interface.h", "resource_consumer.cc", "resource_consumer.h", "resource_consumer_configuration.cc", @@ -41,7 +41,7 @@ if (rtc_include_tests) { testonly = true sources = [ - "resource_adaptation_processor_unittest.cc", + "new_resource_adaptation_processor_poc_unittest.cc", "resource_unittest.cc", ] deps = [ diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/new_resource_adaptation_processor_poc.cc similarity index 91% rename from call/adaptation/resource_adaptation_processor.cc rename to call/adaptation/new_resource_adaptation_processor_poc.cc index 151480cb4d..dd14f768c6 100644 --- a/call/adaptation/resource_adaptation_processor.cc +++ b/call/adaptation/new_resource_adaptation_processor_poc.cc @@ -1,5 +1,5 @@ /* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "call/adaptation/resource_adaptation_processor.h" +#include "call/adaptation/new_resource_adaptation_processor_poc.h" #include #include @@ -48,7 +48,7 @@ ConsumerConfigurationPair::ConsumerConfigurationPair( : consumer(consumer), configuration(configuration) {} absl::optional -ResourceAdaptationProcessor::FindNextConfiguration() { +NewResourceAdaptationProcessorPoc::FindNextConfiguration() { ResourceUsageState overall_usage = ResourceUsageState::kUnderuse; for (auto& resource : resources_) { if (resource->usage_state() == ResourceUsageState::kStable) { @@ -92,8 +92,8 @@ ResourceAdaptationProcessor::FindNextConfiguration() { } } -ResourceConsumer* -ResourceAdaptationProcessor::FindMostExpensiveConsumerThatCanBeAdaptedDown() { +ResourceConsumer* NewResourceAdaptationProcessorPoc:: + FindMostExpensiveConsumerThatCanBeAdaptedDown() { ResourceConsumer* max_cost_consumer = nullptr; double max_cost = -1.0; for (auto& consumer : consumers_) { @@ -108,8 +108,8 @@ ResourceAdaptationProcessor::FindMostExpensiveConsumerThatCanBeAdaptedDown() { return max_cost_consumer; } -ResourceConsumer* -ResourceAdaptationProcessor::FindLeastExpensiveConsumerThatCanBeAdaptedUp() { +ResourceConsumer* NewResourceAdaptationProcessorPoc:: + FindLeastExpensiveConsumerThatCanBeAdaptedUp() { ResourceConsumer* min_cost_consumer = nullptr; double min_cost = std::numeric_limits::infinity(); for (auto& consumer : consumers_) { diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/new_resource_adaptation_processor_poc.h similarity index 91% rename from call/adaptation/resource_adaptation_processor.h rename to call/adaptation/new_resource_adaptation_processor_poc.h index 2855302beb..bf1a7e74e3 100644 --- a/call/adaptation/resource_adaptation_processor.h +++ b/call/adaptation/new_resource_adaptation_processor_poc.h @@ -1,5 +1,5 @@ /* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ -#define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ +#ifndef CALL_ADAPTATION_NEW_RESOURCE_ADAPTATION_PROCESSOR_POC_H_ +#define CALL_ADAPTATION_NEW_RESOURCE_ADAPTATION_PROCESSOR_POC_H_ #include #include @@ -55,7 +55,7 @@ struct ConsumerConfigurationPair { // // This class owns all resources, consumers and configurations. As long as it is // alive, raw pointers to these are safe to use. -class ResourceAdaptationProcessor { +class NewResourceAdaptationProcessorPoc { public: const std::vector>& resources() const { return resources_; @@ -70,7 +70,7 @@ class ResourceAdaptationProcessor { // Takes on ownership of the argument. A raw pointer is returned to the object // for convenience; it is valid for the lifetime of the - // ResourceAdaptationProcessor. + // NewResourceAdaptationProcessorPoc. // T = any subclass of Resource template T* AddResource(std::unique_ptr resource) { @@ -96,7 +96,7 @@ class ResourceAdaptationProcessor { // Based on the current state of the resources and consumers, finds the // consumer that should be reconfigured up or down in order to maximies // quality without overusing any resources, as described in - // ResourceAdaptationProcessor's class description. + // NewResourceAdaptationProcessorPoc's class description. // // When this is used in a real system, care needs to be taken for how often // FindNextConfiguration() is called. There may be a delay between @@ -115,4 +115,4 @@ class ResourceAdaptationProcessor { } // namespace webrtc -#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ +#endif // CALL_ADAPTATION_NEW_RESOURCE_ADAPTATION_PROCESSOR_POC_H_ diff --git a/call/adaptation/resource_adaptation_processor_unittest.cc b/call/adaptation/new_resource_adaptation_processor_poc_unittest.cc similarity index 89% rename from call/adaptation/resource_adaptation_processor_unittest.cc rename to call/adaptation/new_resource_adaptation_processor_poc_unittest.cc index df99aed48b..8f89985ed3 100644 --- a/call/adaptation/resource_adaptation_processor_unittest.cc +++ b/call/adaptation/new_resource_adaptation_processor_poc_unittest.cc @@ -1,5 +1,5 @@ /* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "call/adaptation/resource_adaptation_processor.h" +#include "call/adaptation/new_resource_adaptation_processor_poc.h" #include "absl/types/optional.h" #include "call/adaptation/resource.h" @@ -33,7 +33,8 @@ void ConnectNeighbors(ResourceConsumerConfiguration* upper, } std::vector -AddStandardResolutionConfigurations(ResourceAdaptationProcessor* processor) { +AddStandardResolutionConfigurations( + NewResourceAdaptationProcessorPoc* processor) { std::vector configs; configs.push_back(processor->AddConfiguration( std::make_unique(1920, 1080, 30.0, @@ -53,9 +54,9 @@ AddStandardResolutionConfigurations(ResourceAdaptationProcessor* processor) { return configs; } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, SingleStreamAndResourceDontAdaptDownWhenStable) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kStable)); auto resolution_configs = AddStandardResolutionConfigurations(&processor); @@ -64,9 +65,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration()); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, SingleStreamAndResourceAdaptDownOnOveruse) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kOveruse)); auto resolution_configs = AddStandardResolutionConfigurations(&processor); @@ -78,9 +79,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_EQ(resolution_configs[k720pIndex], next_config->configuration); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, SingleStreamAndResourceDontAdaptOnOveruseIfMinResolution) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kOveruse)); auto resolution_configs = AddStandardResolutionConfigurations(&processor); @@ -89,9 +90,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration()); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, SingleStreamAndResourceAdaptUpOnUnderuse) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kUnderuse)); auto resolution_configs = AddStandardResolutionConfigurations(&processor); @@ -103,9 +104,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_EQ(resolution_configs[k1080pIndex], next_config->configuration); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, SingleStreamAndResourceDontAdaptOnUnderuseIfMaxResolution) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kUnderuse)); auto resolution_configs = AddStandardResolutionConfigurations(&processor); @@ -114,9 +115,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_EQ(absl::nullopt, processor.FindNextConfiguration()); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, MultipleStreamsLargestStreamGetsAdaptedDownOnOveruse) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kOveruse)); auto resolution_configs = AddStandardResolutionConfigurations(&processor); @@ -137,9 +138,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_EQ(second_stream, next_config->consumer); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, MultipleStreamsSmallestStreamGetsAdaptedUpOnUnderuse) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kUnderuse)); auto resolution_configs = AddStandardResolutionConfigurations(&processor); @@ -161,9 +162,9 @@ TEST(ResourceAdaptationProcessorTest, } // If both streams are equally valid to adapt down, the first one is preferred. -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, MultipleStreamsAdaptFirstStreamWhenBothStreamsHaveSameCost) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kOveruse)); auto resolution_configs = AddStandardResolutionConfigurations(&processor); @@ -176,9 +177,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_EQ(first_stream, next_config->consumer); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, MultipleResourcesAdaptDownIfAnyIsOverused) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; auto* first_resource = processor.AddResource( std::make_unique(ResourceUsageState::kOveruse)); auto* second_resource = processor.AddResource( @@ -194,9 +195,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_TRUE(processor.FindNextConfiguration().has_value()); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, MultipleResourcesAdaptUpIfAllAreUnderused) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; processor.AddResource( std::make_unique(ResourceUsageState::kUnderuse)); auto* second_resource = processor.AddResource( @@ -211,9 +212,9 @@ TEST(ResourceAdaptationProcessorTest, EXPECT_TRUE(processor.FindNextConfiguration().has_value()); } -TEST(ResourceAdaptationProcessorTest, +TEST(NewResourceAdaptationProcessorPocTest, HighestPreferredNeighborIsPickedWhenAdapting) { - ResourceAdaptationProcessor processor; + NewResourceAdaptationProcessorPoc processor; // Set up the following graph, where (#) is the preference. // // Downward arrows Upward arrows diff --git a/call/adaptation/resource_adaptation_module_interface.cc b/call/adaptation/resource_adaptation_processor_interface.cc similarity index 57% rename from call/adaptation/resource_adaptation_module_interface.cc rename to call/adaptation/resource_adaptation_processor_interface.cc index e89d1eff2c..4e5251ce90 100644 --- a/call/adaptation/resource_adaptation_module_interface.cc +++ b/call/adaptation/resource_adaptation_processor_interface.cc @@ -1,5 +1,5 @@ /* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,12 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "call/adaptation/resource_adaptation_module_interface.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" namespace webrtc { -ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {} +ResourceAdaptationProcessorListener::~ResourceAdaptationProcessorListener() {} -ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {} +ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() {} } // namespace webrtc diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_processor_interface.h similarity index 76% rename from call/adaptation/resource_adaptation_module_interface.h rename to call/adaptation/resource_adaptation_processor_interface.h index 1248e17e02..04e4469069 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h @@ -1,5 +1,5 @@ /* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ -#define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ +#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_ +#define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_ #include "absl/types/optional.h" #include "api/rtp_parameters.h" @@ -22,12 +22,10 @@ namespace webrtc { // The listener is responsible for carrying out the reconfiguration of the video // source such that the VideoSourceRestrictions are fulfilled. -class ResourceAdaptationModuleListener { +class ResourceAdaptationProcessorListener { public: - virtual ~ResourceAdaptationModuleListener(); + virtual ~ResourceAdaptationProcessorListener(); - // TODO(hbos): When we support the muli-stream use case, the arguments need to - // specify which video stream's source needs to be reconfigured. virtual void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions) = 0; }; @@ -35,27 +33,12 @@ class ResourceAdaptationModuleListener { // Responsible for reconfiguring encoded streams based on resource consumption, // such as scaling down resolution or frame rate when CPU is overused. This // interface is meant to be injectable into VideoStreamEncoder. -// -// [UNDER CONSTRUCTION] This interface is work-in-progress. In the future it -// needs to be able to handle all the necessary input and output for resource -// adaptation decision making. -// -// TODO(https://crbug.com/webrtc/11222): Make this interface feature-complete so -// that a module (such as OveruseFrameDetectorResourceAdaptationModule) is fully -// operational through this abstract interface. -class ResourceAdaptationModuleInterface { +class ResourceAdaptationProcessorInterface { public: - virtual ~ResourceAdaptationModuleInterface(); + virtual ~ResourceAdaptationProcessorInterface(); - // TODO(hbos): When input/output of the module is adequetly handled by this - // interface, these methods need to say which stream to start/stop, enabling - // multi-stream aware implementations of ResourceAdaptationModuleInterface. We - // don't want to do this before we have the right interfaces (e.g. if we pass - // in a VideoStreamEncoder here directly then have a dependency on a different - // build target). For the multi-stream use case we may consider making - // ResourceAdaptationModuleInterface reference counted. virtual void StartResourceAdaptation( - ResourceAdaptationModuleListener* adaptation_listener) = 0; + ResourceAdaptationProcessorListener* adaptation_listener) = 0; virtual void StopResourceAdaptation() = 0; // The resource must out-live the module. @@ -129,4 +112,4 @@ class ResourceAdaptationModuleInterface { } // namespace webrtc -#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ +#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_ diff --git a/video/BUILD.gn b/video/BUILD.gn index 98a5ef8731..09cbca492a 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -177,20 +177,12 @@ rtc_library("video_stream_encoder_impl") { # visibility = [ "../api/video:video_stream_encoder_create" ] sources = [ - "encode_usage_resource.cc", - "encode_usage_resource.h", "encoder_bitrate_adjuster.cc", "encoder_bitrate_adjuster.h", "encoder_overshoot_detector.cc", "encoder_overshoot_detector.h", "frame_encode_metadata_writer.cc", "frame_encode_metadata_writer.h", - "overuse_frame_detector.cc", - "overuse_frame_detector.h", - "overuse_frame_detector_resource_adaptation_module.cc", - "overuse_frame_detector_resource_adaptation_module.h", - "quality_scaler_resource.cc", - "quality_scaler_resource.h", "video_source_sink_controller.cc", "video_source_sink_controller.h", "video_stream_encoder.cc", @@ -505,8 +497,6 @@ if (rtc_include_tests) { "end_to_end_tests/stats_tests.cc", "end_to_end_tests/transport_feedback_tests.cc", "frame_encode_metadata_writer_unittest.cc", - "overuse_frame_detector_resource_adaptation_unittest.cc", - "overuse_frame_detector_unittest.cc", "picture_id_tests.cc", "quality_limitation_reason_tracker_unittest.cc", "quality_scaling_tests.cc", diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index 95705cb649..f8bb6d8884 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -12,22 +12,43 @@ rtc_library("video_adaptation") { sources = [ "adaptation_counters.cc", "adaptation_counters.h", + "encode_usage_resource.cc", + "encode_usage_resource.h", + "overuse_frame_detector.cc", + "overuse_frame_detector.h", + "quality_scaler_resource.cc", + "quality_scaler_resource.h", + "resource_adaptation_processor.cc", + "resource_adaptation_processor.h", "video_stream_adapter.cc", "video_stream_adapter.h", ] deps = [ "../../api:rtp_parameters", + "../../api/task_queue:task_queue", + "../../api/video:video_frame", "../../api/video:video_stream_encoder", "../../api/video_codecs:video_codecs_api", "../../call/adaptation:resource_adaptation", "../../modules/video_coding:video_coding_utility", "../../rtc_base:checks", "../../rtc_base:logging", + "../../rtc_base:macromagic", "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_event", "../../rtc_base:rtc_numerics", + "../../rtc_base:timeutils", "../../rtc_base/experiments:balanced_degradation_settings", + "../../rtc_base/experiments:field_trial_parser", + "../../rtc_base/experiments:quality_rampup_experiment", + "../../rtc_base/experiments:quality_scaler_settings", + "../../rtc_base/synchronization:sequence_checker", + "../../rtc_base/task_utils:repeating_task", + "../../system_wrappers:field_trial", + "../../system_wrappers:system_wrappers", + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/base:core_headers", "//third_party/abseil-cpp/absl/types:optional", ] } @@ -37,14 +58,23 @@ if (rtc_include_tests) { testonly = true defines = [] - sources = [ "adaptation_counters_unittest.cc" ] + sources = [ + "adaptation_counters_unittest.cc", + "overuse_frame_detector_unittest.cc", + "resource_adaptation_processor_unittest.cc", + ] deps = [ ":video_adaptation", + "../../api/video:encoded_image", + "../../api/video:video_frame_i420", + "../../modules/video_coding:video_coding_utility", "../../rtc_base:checks", "../../rtc_base:logging", "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_base_tests_utils", "../../rtc_base:rtc_event", "../../rtc_base:rtc_numerics", + "../../rtc_base:task_queue_for_test", "//test:test_support", "//testing/gtest", "//third_party/abseil-cpp/absl/types:optional", diff --git a/video/encode_usage_resource.cc b/video/adaptation/encode_usage_resource.cc similarity index 98% rename from video/encode_usage_resource.cc rename to video/adaptation/encode_usage_resource.cc index 37bc23e158..385a8b9182 100644 --- a/video/encode_usage_resource.cc +++ b/video/adaptation/encode_usage_resource.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "video/encode_usage_resource.h" +#include "video/adaptation/encode_usage_resource.h" #include #include diff --git a/video/encode_usage_resource.h b/video/adaptation/encode_usage_resource.h similarity index 83% rename from video/encode_usage_resource.h rename to video/adaptation/encode_usage_resource.h index 119e9702d6..e626c2f50e 100644 --- a/video/encode_usage_resource.h +++ b/video/adaptation/encode_usage_resource.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef VIDEO_ENCODE_USAGE_RESOURCE_H_ -#define VIDEO_ENCODE_USAGE_RESOURCE_H_ +#ifndef VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_ +#define VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_ #include #include @@ -17,15 +17,15 @@ #include "absl/types/optional.h" #include "call/adaptation/resource.h" #include "modules/video_coding/utility/quality_scaler.h" -#include "video/overuse_frame_detector.h" +#include "video/adaptation/overuse_frame_detector.h" namespace webrtc { // Handles interaction with the OveruseDetector. // TODO(hbos): Add unittests specific to this class, it is currently only tested -// indirectly by usage in the OveruseFrameDetectorResourceAdaptationModule -// (which is only tested because of its usage in VideoStreamEncoder); all tests -// are currently in video_stream_encoder_unittest.cc. +// indirectly by usage in the ResourceAdaptationProcessor (which is only tested +// because of its usage in VideoStreamEncoder); all tests are currently in +// video_stream_encoder_unittest.cc. // TODO(https://crbug.com/webrtc/11222): Move this class to the // video/adaptation/ subdirectory. class EncodeUsageResource : public Resource, @@ -63,4 +63,4 @@ class EncodeUsageResource : public Resource, } // namespace webrtc -#endif // VIDEO_ENCODE_USAGE_RESOURCE_H_ +#endif // VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_ diff --git a/video/overuse_frame_detector.cc b/video/adaptation/overuse_frame_detector.cc similarity index 99% rename from video/overuse_frame_detector.cc rename to video/adaptation/overuse_frame_detector.cc index 9508470e54..64b67687e9 100644 --- a/video/overuse_frame_detector.cc +++ b/video/adaptation/overuse_frame_detector.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "video/overuse_frame_detector.h" +#include "video/adaptation/overuse_frame_detector.h" #include #include diff --git a/video/overuse_frame_detector.h b/video/adaptation/overuse_frame_detector.h similarity index 96% rename from video/overuse_frame_detector.h rename to video/adaptation/overuse_frame_detector.h index 4f64734944..e8c667dfdc 100644 --- a/video/overuse_frame_detector.h +++ b/video/adaptation/overuse_frame_detector.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef VIDEO_OVERUSE_FRAME_DETECTOR_H_ -#define VIDEO_OVERUSE_FRAME_DETECTOR_H_ +#ifndef VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ +#define VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ #include #include @@ -155,4 +155,4 @@ class OveruseFrameDetector { } // namespace webrtc -#endif // VIDEO_OVERUSE_FRAME_DETECTOR_H_ +#endif // VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ diff --git a/video/overuse_frame_detector_unittest.cc b/video/adaptation/overuse_frame_detector_unittest.cc similarity index 99% rename from video/overuse_frame_detector_unittest.cc rename to video/adaptation/overuse_frame_detector_unittest.cc index d77d4dcdfc..5ace2f2be8 100644 --- a/video/overuse_frame_detector_unittest.cc +++ b/video/adaptation/overuse_frame_detector_unittest.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "video/overuse_frame_detector.h" +#include "video/adaptation/overuse_frame_detector.h" #include diff --git a/video/quality_scaler_resource.cc b/video/adaptation/quality_scaler_resource.cc similarity index 97% rename from video/quality_scaler_resource.cc rename to video/adaptation/quality_scaler_resource.cc index 729cae3674..489c03589e 100644 --- a/video/quality_scaler_resource.cc +++ b/video/adaptation/quality_scaler_resource.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "video/quality_scaler_resource.h" +#include "video/adaptation/quality_scaler_resource.h" #include diff --git a/video/quality_scaler_resource.h b/video/adaptation/quality_scaler_resource.h similarity index 83% rename from video/quality_scaler_resource.h rename to video/adaptation/quality_scaler_resource.h index bd62af81a3..7708710dd5 100644 --- a/video/quality_scaler_resource.h +++ b/video/adaptation/quality_scaler_resource.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef VIDEO_QUALITY_SCALER_RESOURCE_H_ -#define VIDEO_QUALITY_SCALER_RESOURCE_H_ +#ifndef VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_ +#define VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_ #include #include @@ -22,9 +22,9 @@ namespace webrtc { // Handles interaction with the QualityScaler. // TODO(hbos): Add unittests specific to this class, it is currently only tested -// indirectly by usage in the OveruseFrameDetectorResourceAdaptationModule -// (which is only tested because of its usage in VideoStreamEncoder); all tests -// are currently in video_stream_encoder_unittest.cc. +// indirectly by usage in the ResourceAdaptationProcessor (which is only tested +// because of its usage in VideoStreamEncoder); all tests are currently in +// video_stream_encoder_unittest.cc. // TODO(https://crbug.com/webrtc/11222): Move this class to the // video/adaptation/ subdirectory. class QualityScalerResource : public Resource, @@ -57,4 +57,4 @@ class QualityScalerResource : public Resource, } // namespace webrtc -#endif // VIDEO_QUALITY_SCALER_RESOURCE_H_ +#endif // VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_ diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/adaptation/resource_adaptation_processor.cc similarity index 88% rename from video/overuse_frame_detector_resource_adaptation_module.cc rename to video/adaptation/resource_adaptation_processor.cc index 4dc2876d55..7958be4446 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/adaptation/resource_adaptation_processor.cc @@ -1,5 +1,5 @@ /* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "video/overuse_frame_detector_resource_adaptation_module.h" +#include "video/adaptation/resource_adaptation_processor.h" #include #include @@ -26,10 +26,12 @@ #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/strings/string_builder.h" #include "rtc_base/time_utils.h" -#include "video/video_stream_encoder.h" namespace webrtc { +const int kDefaultInputPixelsWidth = 176; +const int kDefaultInputPixelsHeight = 144; + namespace { bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { @@ -93,7 +95,7 @@ AdaptationCounters ApplyDegradationPreference( } // namespace -class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper { +class ResourceAdaptationProcessor::InitialFrameDropper { public: explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource) : quality_scaler_resource_(quality_scaler_resource), @@ -164,13 +166,12 @@ class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper { int initial_framedrop_; }; -OveruseFrameDetectorResourceAdaptationModule:: - OveruseFrameDetectorResourceAdaptationModule( - Clock* clock, - bool experiment_cpu_load_estimator, - std::unique_ptr overuse_detector, - VideoStreamEncoderObserver* encoder_stats_observer, - ResourceAdaptationModuleListener* adaptation_listener) +ResourceAdaptationProcessor::ResourceAdaptationProcessor( + Clock* clock, + bool experiment_cpu_load_estimator, + std::unique_ptr overuse_detector, + VideoStreamEncoderObserver* encoder_stats_observer, + ResourceAdaptationProcessorListener* adaptation_listener) : adaptation_listener_(adaptation_listener), clock_(clock), state_(State::kStopped), @@ -200,13 +201,12 @@ OveruseFrameDetectorResourceAdaptationModule:: AdaptationObserverInterface::AdaptReason::kQuality); } -OveruseFrameDetectorResourceAdaptationModule:: - ~OveruseFrameDetectorResourceAdaptationModule() { +ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { RTC_DCHECK_EQ(state_, State::kStopped); } -void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( - ResourceAdaptationModuleListener* adaptation_listener) { +void ResourceAdaptationProcessor::StartResourceAdaptation( + ResourceAdaptationProcessorListener* adaptation_listener) { RTC_DCHECK_EQ(state_, State::kStopped); RTC_DCHECK(encoder_settings_.has_value()); // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener @@ -220,7 +220,7 @@ void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( state_ = State::kStarted; } -void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { +void ResourceAdaptationProcessor::StopResourceAdaptation() { encode_usage_resource_->StopCheckForOveruse(); quality_scaler_resource_->StopCheckForOveruse(); for (auto& resource_and_reason : resources_) { @@ -229,12 +229,11 @@ void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { state_ = State::kStopped; } -void OveruseFrameDetectorResourceAdaptationModule::AddResource( - Resource* resource) { +void ResourceAdaptationProcessor::AddResource(Resource* resource) { return AddResource(resource, AdaptationObserverInterface::AdaptReason::kCpu); } -void OveruseFrameDetectorResourceAdaptationModule::AddResource( +void ResourceAdaptationProcessor::AddResource( Resource* resource, AdaptationObserverInterface::AdaptReason reason) { RTC_DCHECK(resource); @@ -246,13 +245,12 @@ void OveruseFrameDetectorResourceAdaptationModule::AddResource( resources_.emplace_back(resource, reason); } -void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( - bool has_input_video) { +void ResourceAdaptationProcessor::SetHasInputVideo(bool has_input_video) { // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS. has_input_video_ = has_input_video; } -void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( +void ResourceAdaptationProcessor::SetDegradationPreference( DegradationPreference degradation_preference) { degradation_preference_ = degradation_preference; if (stream_adapter_->SetDegradationPreference(degradation_preference) == @@ -263,7 +261,7 @@ void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( MaybeUpdateVideoSourceRestrictions(); } -void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings( +void ResourceAdaptationProcessor::SetEncoderSettings( EncoderSettings encoder_settings) { encoder_settings_ = std::move(encoder_settings); @@ -273,40 +271,36 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings( MaybeUpdateTargetFrameRate(); } -void OveruseFrameDetectorResourceAdaptationModule::SetStartBitrate( - DataRate start_bitrate) { +void ResourceAdaptationProcessor::SetStartBitrate(DataRate start_bitrate) { if (!start_bitrate.IsZero()) encoder_target_bitrate_bps_ = start_bitrate.bps(); initial_frame_dropper_->SetStartBitrate(start_bitrate, clock_->TimeInMicroseconds()); } -void OveruseFrameDetectorResourceAdaptationModule::SetTargetBitrate( - DataRate target_bitrate) { +void ResourceAdaptationProcessor::SetTargetBitrate(DataRate target_bitrate) { if (!target_bitrate.IsZero()) encoder_target_bitrate_bps_ = target_bitrate.bps(); initial_frame_dropper_->SetTargetBitrate(target_bitrate, clock_->TimeInMilliseconds()); } -void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates( +void ResourceAdaptationProcessor::SetEncoderRates( const VideoEncoder::RateControlParameters& encoder_rates) { encoder_rates_ = encoder_rates; } -void OveruseFrameDetectorResourceAdaptationModule:: - ResetVideoSourceRestrictions() { +void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { stream_adapter_->ClearRestrictions(); active_counts_.fill(AdaptationCounters()); MaybeUpdateVideoSourceRestrictions(); } -void OveruseFrameDetectorResourceAdaptationModule::OnFrame( - const VideoFrame& frame) { +void ResourceAdaptationProcessor::OnFrame(const VideoFrame& frame) { last_input_frame_size_ = frame.size(); } -void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { +void ResourceAdaptationProcessor::OnFrameDroppedDueToSize() { AdaptationCounters counters_before = stream_adapter_->adaptation_counters(); OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); if (degradation_preference() == DegradationPreference::BALANCED && @@ -322,14 +316,14 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { initial_frame_dropper_->OnFrameDroppedDueToSize(); } -void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted( +void ResourceAdaptationProcessor::OnEncodeStarted( const VideoFrame& cropped_frame, int64_t time_when_first_seen_us) { encode_usage_resource_->OnEncodeStarted(cropped_frame, time_when_first_seen_us); } -void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted( +void ResourceAdaptationProcessor::OnEncodeCompleted( const EncodedImage& encoded_image, int64_t time_sent_in_us, absl::optional encode_duration_us) { @@ -343,21 +337,21 @@ void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted( quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); } -void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped( +void ResourceAdaptationProcessor::OnFrameDropped( EncodedImageCallback::DropReason reason) { quality_scaler_resource_->OnFrameDropped(reason); } -bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const { +bool ResourceAdaptationProcessor::DropInitialFrames() const { return initial_frame_dropper_->DropInitialFrames(); } -void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() { +void ResourceAdaptationProcessor::OnMaybeEncodeFrame() { initial_frame_dropper_->OnMaybeEncodeFrame(); MaybePerformQualityRampupExperiment(); } -void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( +void ResourceAdaptationProcessor::UpdateQualityScalerSettings( absl::optional qp_thresholds) { if (qp_thresholds.has_value()) { quality_scaler_resource_->StopCheckForOveruse(); @@ -368,7 +362,7 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( initial_frame_dropper_->OnQualityScalerSettingsUpdated(); } -void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( +void ResourceAdaptationProcessor::ConfigureQualityScaler( const VideoEncoder::EncoderInfo& encoder_info) { const auto scaling_settings = encoder_info.scaling_settings; const bool quality_scaling_allowed = @@ -414,7 +408,7 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( } ResourceListenerResponse -OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured( +ResourceAdaptationProcessor::OnResourceUsageStateMeasured( const Resource& resource) { const auto& registered_resource = absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) { @@ -443,7 +437,7 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured( } } -void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( +void ResourceAdaptationProcessor::OnResourceUnderuse( AdaptationObserverInterface::AdaptReason reason) { // We can't adapt up if we're already at the highest setting. // Note that this only includes counts relevant to the current degradation @@ -484,8 +478,7 @@ void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( RTC_LOG(LS_INFO) << ActiveCountsToString(); } -ResourceListenerResponse -OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( +ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse( AdaptationObserverInterface::AdaptReason reason) { if (!has_input_video_) return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; @@ -516,8 +509,7 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( // pipelining encoders better (multiple input frames before something comes // out). This should effectively turn off CPU adaptations for systems that // remotely cope with the load right now. -CpuOveruseOptions -OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const { +CpuOveruseOptions ResourceAdaptationProcessor::GetCpuOveruseOptions() const { // This is already ensured by the only caller of this method: // StartResourceAdaptation(). RTC_DCHECK(encoder_settings_.has_value()); @@ -534,8 +526,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const { return options; } -int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault() - const { +int ResourceAdaptationProcessor::LastInputFrameSizeOrDefault() const { // The dependency on this hardcoded resolution is inherited from old code, // which used this resolution as a stand-in for not knowing the resolution // yet. @@ -543,13 +534,11 @@ int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault() // DCHECK passed all the tests but adding it does change the requirements of // this class (= not being allowed to call OnResourceUnderuse() or // OnResourceOveruse() before OnFrame()) and deserves a standalone CL. - return last_input_frame_size_.value_or( - VideoStreamEncoder::kDefaultLastFrameInfoWidth * - VideoStreamEncoder::kDefaultLastFrameInfoHeight); + return last_input_frame_size_.value_or(kDefaultInputPixelsWidth * + kDefaultInputPixelsHeight); } -void OveruseFrameDetectorResourceAdaptationModule:: - MaybeUpdateVideoSourceRestrictions() { +void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions() { VideoSourceRestrictions new_restrictions = ApplyDegradationPreference( stream_adapter_->source_restrictions(), degradation_preference_); if (video_source_restrictions_ != new_restrictions) { @@ -560,8 +549,7 @@ void OveruseFrameDetectorResourceAdaptationModule:: } } -void OveruseFrameDetectorResourceAdaptationModule:: - MaybeUpdateTargetFrameRate() { +void ResourceAdaptationProcessor::MaybeUpdateTargetFrameRate() { absl::optional codec_max_frame_rate = encoder_settings_.has_value() ? absl::optional( @@ -583,7 +571,7 @@ void OveruseFrameDetectorResourceAdaptationModule:: encode_usage_resource_->SetTargetFrameRate(target_frame_rate); } -void OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( +void ResourceAdaptationProcessor::OnAdaptationCountChanged( const AdaptationCounters& adaptation_count, AdaptationCounters* active_count, AdaptationCounters* other_active) { @@ -641,7 +629,7 @@ void OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( } // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. -void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( +void ResourceAdaptationProcessor::UpdateAdaptationStats( AdaptationObserverInterface::AdaptReason reason) { // Update active counts AdaptationCounters& active_count = active_counts_[reason]; @@ -668,7 +656,7 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( } VideoStreamEncoderObserver::AdaptationSteps -OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( +ResourceAdaptationProcessor::GetActiveCounts( AdaptationObserverInterface::AdaptReason reason) { // TODO(https://crbug.com/webrtc/11392) Ideally this shuold be moved out of // this class and into the encoder_stats_observer_. @@ -700,7 +688,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( } VideoStreamAdapter::VideoInputMode -OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const { +ResourceAdaptationProcessor::GetVideoInputMode() const { if (!has_input_video_) return VideoStreamAdapter::VideoInputMode::kNoVideo; return (encoder_settings_.has_value() && @@ -710,8 +698,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const { : VideoStreamAdapter::VideoInputMode::kNormalVideo; } -void OveruseFrameDetectorResourceAdaptationModule:: - MaybePerformQualityRampupExperiment() { +void ResourceAdaptationProcessor::MaybePerformQualityRampupExperiment() { if (!quality_scaler_resource_->is_started()) return; @@ -747,8 +734,7 @@ void OveruseFrameDetectorResourceAdaptationModule:: } } -std::string OveruseFrameDetectorResourceAdaptationModule::ActiveCountsToString() - const { +std::string ResourceAdaptationProcessor::ActiveCountsToString() const { rtc::StringBuilder ss; ss << "Downgrade counts: fps: {"; diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/adaptation/resource_adaptation_processor.h similarity index 86% rename from video/overuse_frame_detector_resource_adaptation_module.h rename to video/adaptation/resource_adaptation_processor.h index e959e2a8e0..589860db5f 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/adaptation/resource_adaptation_processor.h @@ -1,5 +1,5 @@ /* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ -#define VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ +#ifndef VIDEO_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ +#define VIDEO_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ #include #include @@ -26,20 +26,24 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_config.h" #include "call/adaptation/resource.h" -#include "call/adaptation/resource_adaptation_module_interface.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" #include "rtc_base/experiments/quality_rampup_experiment.h" #include "rtc_base/experiments/quality_scaler_settings.h" #include "rtc_base/strings/string_builder.h" #include "system_wrappers/include/clock.h" #include "video/adaptation/adaptation_counters.h" +#include "video/adaptation/encode_usage_resource.h" +#include "video/adaptation/overuse_frame_detector.h" +#include "video/adaptation/quality_scaler_resource.h" #include "video/adaptation/video_stream_adapter.h" -#include "video/encode_usage_resource.h" -#include "video/overuse_frame_detector.h" -#include "video/quality_scaler_resource.h" namespace webrtc { -class VideoStreamEncoder; +// The assumed input frame size if we have not yet received a frame. +// TODO(hbos): This is 144p - why are we assuming super low quality? Seems like +// a bad heuristic. +extern const int kDefaultInputPixelsWidth; +extern const int kDefaultInputPixelsHeight; // This class is used by the VideoStreamEncoder and is responsible for adapting // resolution up or down based on encode usage percent. It keeps track of video @@ -51,29 +55,26 @@ class VideoStreamEncoder; // TODO(hbos): Add unittests specific to this class, it is currently only tested // indirectly in video_stream_encoder_unittest.cc and other tests exercising // VideoStreamEncoder. -// TODO(https://crbug.com/webrtc/11222): Rename this class to something more -// appropriate and move it to the video/adaptation/ subdirectory. -class OveruseFrameDetectorResourceAdaptationModule - : public ResourceAdaptationModuleInterface, - public ResourceListener { +class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, + public ResourceListener { public: - // The module can be constructed on any sequence, but must be initialized and - // used on a single sequence, e.g. the encoder queue. - OveruseFrameDetectorResourceAdaptationModule( + // The processor can be constructed on any sequence, but must be initialized + // and used on a single sequence, e.g. the encoder queue. + ResourceAdaptationProcessor( Clock* clock, bool experiment_cpu_load_estimator, std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer, - ResourceAdaptationModuleListener* adaptation_listener); - ~OveruseFrameDetectorResourceAdaptationModule() override; + ResourceAdaptationProcessorListener* adaptation_listener); + ~ResourceAdaptationProcessor() override; DegradationPreference degradation_preference() const { return degradation_preference_; } - // ResourceAdaptationModuleInterface implementation. + // ResourceAdaptationProcessorInterface implementation. void StartResourceAdaptation( - ResourceAdaptationModuleListener* adaptation_listener) override; + ResourceAdaptationProcessorListener* adaptation_listener) override; void StopResourceAdaptation() override; // Uses a default AdaptReason of kCpu. void AddResource(Resource* resource) override; @@ -167,7 +168,7 @@ class OveruseFrameDetectorResourceAdaptationModule std::string ActiveCountsToString() const; - ResourceAdaptationModuleListener* const adaptation_listener_; + ResourceAdaptationProcessorListener* const adaptation_listener_; Clock* clock_; State state_; const bool experiment_cpu_load_estimator_; @@ -180,7 +181,7 @@ class OveruseFrameDetectorResourceAdaptationModule // owned by the adapter, this class has no buisness relying on implementation // details of the adapter. DegradationPreference degradation_preference_; - // Keeps track of source restrictions that this adaptation module outputs. + // Keeps track of source restrictions that this adaptation processor outputs. const std::unique_ptr stream_adapter_; const std::unique_ptr encode_usage_resource_; const std::unique_ptr quality_scaler_resource_; @@ -221,4 +222,4 @@ class OveruseFrameDetectorResourceAdaptationModule } // namespace webrtc -#endif // VIDEO_OVERUSE_FRAME_DETECTOR_RESOURCE_ADAPTATION_MODULE_H_ +#endif // VIDEO_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ diff --git a/video/overuse_frame_detector_resource_adaptation_unittest.cc b/video/adaptation/resource_adaptation_processor_unittest.cc similarity index 66% rename from video/overuse_frame_detector_resource_adaptation_unittest.cc rename to video/adaptation/resource_adaptation_processor_unittest.cc index c74920dbdf..40a44db061 100644 --- a/video/overuse_frame_detector_resource_adaptation_unittest.cc +++ b/video/adaptation/resource_adaptation_processor_unittest.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "video/overuse_frame_detector_resource_adaptation_module.h" +#include "video/adaptation/resource_adaptation_processor.h" #include "test/gmock.h" #include "test/gtest.h" @@ -16,62 +16,55 @@ namespace webrtc { -TEST(OveruseFrameDetectorResourceAdaptationModuleTest, - FirstAdaptationDown_Fps) { +TEST(ResourceAdaptationProcessorTest, FirstAdaptationDown_Fps) { AdaptationCounters cpu; AdaptationCounters qp; AdaptationCounters total(0, 1); - OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( - total, &cpu, &qp); + ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp); AdaptationCounters expected_cpu(0, 1); AdaptationCounters expected_qp; EXPECT_EQ(expected_cpu, cpu); EXPECT_EQ(expected_qp, qp); } -TEST(OveruseFrameDetectorResourceAdaptationModuleTest, - FirstAdaptationDown_Resolution) { +TEST(ResourceAdaptationProcessorTest, FirstAdaptationDown_Resolution) { AdaptationCounters cpu; AdaptationCounters qp; AdaptationCounters total(1, 0); - OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( - total, &cpu, &qp); + ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp); AdaptationCounters expected_cpu(1, 0); AdaptationCounters expected_qp; EXPECT_EQ(expected_cpu, cpu); EXPECT_EQ(expected_qp, qp); } -TEST(OveruseFrameDetectorResourceAdaptationModuleTest, LastAdaptUp_Fps) { +TEST(ResourceAdaptationProcessorTest, LastAdaptUp_Fps) { AdaptationCounters cpu(0, 1); AdaptationCounters qp; AdaptationCounters total; - OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( - total, &cpu, &qp); + ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp); AdaptationCounters expected_cpu; AdaptationCounters expected_qp; EXPECT_EQ(expected_cpu, cpu); EXPECT_EQ(expected_qp, qp); } -TEST(OveruseFrameDetectorResourceAdaptationModuleTest, LastAdaptUp_Resolution) { +TEST(ResourceAdaptationProcessorTest, LastAdaptUp_Resolution) { AdaptationCounters cpu(1, 0); AdaptationCounters qp; AdaptationCounters total; - OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( - total, &cpu, &qp); + ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp); AdaptationCounters expected_cpu; AdaptationCounters expected_qp; EXPECT_EQ(expected_cpu, cpu); EXPECT_EQ(expected_qp, qp); } -TEST(OveruseFrameDetectorResourceAdaptationModuleTest, - AdaptUpWithBorrow_Resolution) { +TEST(ResourceAdaptationProcessorTest, AdaptUpWithBorrow_Resolution) { AdaptationCounters cpu(0, 1); AdaptationCounters qp(1, 0); AdaptationCounters total(0, 1); @@ -79,8 +72,7 @@ TEST(OveruseFrameDetectorResourceAdaptationModuleTest, // CPU adaptation for resolution, but no resolution adaptation left from CPU. // We then borrow the resolution adaptation from qp, and give qp the fps // adaptation from CPU. - OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( - total, &cpu, &qp); + ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp); AdaptationCounters expected_cpu(0, 0); AdaptationCounters expected_qp(0, 1); @@ -88,15 +80,14 @@ TEST(OveruseFrameDetectorResourceAdaptationModuleTest, EXPECT_EQ(expected_qp, qp); } -TEST(OveruseFrameDetectorResourceAdaptationModuleTest, AdaptUpWithBorrow_Fps) { +TEST(ResourceAdaptationProcessorTest, AdaptUpWithBorrow_Fps) { AdaptationCounters cpu(1, 0); AdaptationCounters qp(0, 1); AdaptationCounters total(1, 0); // CPU adaptation for fps, but no fps adaptation left from CPU. We then borrow // the fps adaptation from qp, and give qp the resolution adaptation from CPU. - OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( - total, &cpu, &qp); + ResourceAdaptationProcessor::OnAdaptationCountChanged(total, &cpu, &qp); AdaptationCounters expected_cpu(0, 0); AdaptationCounters expected_qp(1, 0); diff --git a/video/video_source_sink_controller.h b/video/video_source_sink_controller.h index 379457cdf6..4811b2866e 100644 --- a/video/video_source_sink_controller.h +++ b/video/video_source_sink_controller.h @@ -15,7 +15,7 @@ #include "api/video/video_frame.h" #include "api/video/video_sink_interface.h" #include "api/video/video_source_interface.h" -#include "call/adaptation/resource_adaptation_module_interface.h" +#include "call/adaptation/video_source_restrictions.h" #include "rtc_base/critical_section.h" namespace webrtc { diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index d9eda8e302..8252cb29ef 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -24,7 +24,6 @@ #include "api/video/video_bitrate_allocator_factory.h" #include "api/video/video_codec_constants.h" #include "api/video_codecs/video_encoder.h" -#include "call/adaptation/resource_adaptation_module_interface.h" #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" #include "modules/video_coding/include/video_codec_initializer.h" #include "rtc_base/arraysize.h" @@ -37,6 +36,7 @@ #include "rtc_base/time_utils.h" #include "rtc_base/trace_event.h" #include "system_wrappers/include/field_trial.h" +#include "video/adaptation/resource_adaptation_processor.h" namespace webrtc { @@ -172,9 +172,6 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo( } // namespace -const int VideoStreamEncoder::kDefaultLastFrameInfoWidth = 176; -const int VideoStreamEncoder::kDefaultLastFrameInfoHeight = 144; - VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings() : rate_control(), encoder_target(DataRate::Zero()), @@ -260,8 +257,8 @@ VideoStreamEncoder::VideoStreamEncoder( video_source_sink_controller_(std::make_unique( /*sink=*/this, /*source=*/nullptr)), - resource_adaptation_module_( - std::make_unique( + resource_adaptation_processor_( + std::make_unique( clock_, settings_.experiment_cpu_load_estimator, std::move(overuse_detector), @@ -288,7 +285,7 @@ void VideoStreamEncoder::Stop() { video_source_sink_controller_->SetSource(nullptr); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); - resource_adaptation_module_->StopResourceAdaptation(); + resource_adaptation_processor_->StopResourceAdaptation(); rate_allocator_ = nullptr; bitrate_observer_ = nullptr; ReleaseEncoder(); @@ -327,11 +324,11 @@ void VideoStreamEncoder::SetSource( video_source_sink_controller_->SetSource(source); encoder_queue_.PostTask([this, source, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); - resource_adaptation_module_->SetHasInputVideo(source); - resource_adaptation_module_->SetDegradationPreference( + resource_adaptation_processor_->SetHasInputVideo(source); + resource_adaptation_processor_->SetDegradationPreference( degradation_preference); if (encoder_) - resource_adaptation_module_->ConfigureQualityScaler( + resource_adaptation_processor_->ConfigureQualityScaler( encoder_->GetEncoderInfo()); }); } @@ -351,7 +348,7 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { encoder_target_bitrate_bps_ = start_bitrate_bps != 0 ? absl::optional(start_bitrate_bps) : absl::nullopt; - resource_adaptation_module_->SetStartBitrate( + resource_adaptation_processor_->SetStartBitrate( DataRate::BitsPerSec(start_bitrate_bps)); }); } @@ -382,8 +379,8 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, codec_info_ = settings_.encoder_factory->QueryVideoEncoder( encoder_config_.video_format); if (HasInternalSource()) { - last_frame_info_ = VideoFrameInfo( - kDefaultLastFrameInfoWidth, kDefaultLastFrameInfoHeight, false); + last_frame_info_ = VideoFrameInfo(kDefaultInputPixelsWidth, + kDefaultInputPixelsHeight, false); ReconfigureEncoder(); } } @@ -650,7 +647,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { was_encode_called_since_last_initialization_ = false; } - resource_adaptation_module_->SetEncoderSettings(EncoderSettings( + resource_adaptation_processor_->SetEncoderSettings(EncoderSettings( encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_)); if (success) { @@ -668,8 +665,8 @@ void VideoStreamEncoder::ReconfigureEncoder() { } if (pending_encoder_creation_) { - resource_adaptation_module_->StopResourceAdaptation(); - resource_adaptation_module_->StartResourceAdaptation(this); + resource_adaptation_processor_->StopResourceAdaptation(); + resource_adaptation_processor_->StartResourceAdaptation(this); pending_encoder_creation_ = false; } @@ -724,7 +721,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { std::move(streams), encoder_config_.content_type, encoder_config_.min_transmit_bitrate_bps); - resource_adaptation_module_->ConfigureQualityScaler(info); + resource_adaptation_processor_->ConfigureQualityScaler(info); } void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { @@ -961,14 +958,14 @@ void VideoStreamEncoder::SetEncoderRates( frame_encode_metadata_writer_.OnSetRates( rate_settings.rate_control.bitrate, static_cast(rate_settings.rate_control.framerate_fps + 0.5)); - resource_adaptation_module_->SetEncoderRates(rate_settings.rate_control); + resource_adaptation_processor_->SetEncoderRates(rate_settings.rate_control); } } void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, int64_t time_when_posted_us) { RTC_DCHECK_RUN_ON(&encoder_queue_); - resource_adaptation_module_->OnFrame(video_frame); + resource_adaptation_processor_->OnFrame(video_frame); if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || video_frame.height() != last_frame_info_->height || @@ -1030,7 +1027,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, if (DropDueToSize(video_frame.size())) { RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; - resource_adaptation_module_->OnFrameDroppedDueToSize(); + resource_adaptation_processor_->OnFrameDroppedDueToSize(); // Storing references to a native buffer risks blocking frame capture. if (video_frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) { @@ -1044,7 +1041,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, } return; } - resource_adaptation_module_->OnMaybeEncodeFrame(); + resource_adaptation_processor_->OnMaybeEncodeFrame(); if (EncoderPaused()) { // Storing references to a native buffer risks blocking frame capture. @@ -1115,7 +1112,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, } if (encoder_info_ != info) { - resource_adaptation_module_->SetEncoderSettings(EncoderSettings( + resource_adaptation_processor_->SetEncoderSettings(EncoderSettings( encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_)); RTC_LOG(LS_INFO) << "Encoder settings changed from " << encoder_info_.ToString() << " to " << info.ToString(); @@ -1228,7 +1225,8 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); - resource_adaptation_module_->OnEncodeStarted(out_frame, time_when_posted_us); + resource_adaptation_processor_->OnEncodeStarted(out_frame, + time_when_posted_us); RTC_DCHECK_LE(send_codec_.width, out_frame.width()); RTC_DCHECK_LE(send_codec_.height, out_frame.height()); @@ -1503,7 +1501,7 @@ void VideoStreamEncoder::OnDroppedFrame(DropReason reason) { sink_->OnDroppedFrame(reason); encoder_queue_.PostTask([this, reason] { RTC_DCHECK_RUN_ON(&encoder_queue_); - resource_adaptation_module_->OnFrameDropped(reason); + resource_adaptation_processor_->OnFrameDropped(reason); }); } @@ -1602,7 +1600,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, if (target_bitrate.bps() != 0) encoder_target_bitrate_bps_ = target_bitrate.bps(); - resource_adaptation_module_->SetTargetBitrate(target_bitrate); + resource_adaptation_processor_->SetTargetBitrate(target_bitrate); if (video_suspension_changed) { RTC_LOG(LS_INFO) << "Video suspend state changed to: " @@ -1619,7 +1617,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, } bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { - if (!resource_adaptation_module_->DropInitialFrames() || + if (!resource_adaptation_processor_->DropInitialFrames() || !encoder_target_bitrate_bps_.has_value()) { return false; } @@ -1691,8 +1689,8 @@ void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image, } } - resource_adaptation_module_->OnEncodeCompleted(encoded_image, time_sent_us, - encode_duration_us); + resource_adaptation_processor_->OnEncodeCompleted(encoded_image, time_sent_us, + encode_duration_us); if (bitrate_adjuster_) { bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index); } @@ -1847,7 +1845,7 @@ void VideoStreamEncoder::CheckForAnimatedContent( if (!automatic_animation_detection_experiment_.enabled || encoder_config_.content_type != VideoEncoderConfig::ContentType::kScreen || - resource_adaptation_module_->degradation_preference() != + resource_adaptation_processor_->degradation_preference() != DegradationPreference::BALANCED) { return; } @@ -1914,7 +1912,7 @@ void VideoStreamEncoder::CheckForAnimatedContent( void VideoStreamEncoder::InjectAdaptationResource( Resource* resource, AdaptationObserverInterface::AdaptReason reason) { - resource_adaptation_module_->AddResource(resource, reason); + resource_adaptation_processor_->AddResource(resource, reason); } } // namespace webrtc diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 904a741f42..4963fb8141 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -26,7 +26,7 @@ #include "api/video/video_stream_encoder_settings.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" -#include "call/adaptation/resource_adaptation_module_interface.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" #include "call/adaptation/video_source_restrictions.h" #include "modules/video_coding/utility/frame_dropper.h" #include "rtc_base/critical_section.h" @@ -38,9 +38,9 @@ #include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/task_queue.h" #include "system_wrappers/include/clock.h" +#include "video/adaptation/resource_adaptation_processor.h" #include "video/encoder_bitrate_adjuster.h" #include "video/frame_encode_metadata_writer.h" -#include "video/overuse_frame_detector_resource_adaptation_module.h" #include "video/video_source_sink_controller.h" namespace webrtc { @@ -55,16 +55,8 @@ namespace webrtc { // Call Stop() when done. class VideoStreamEncoder : public VideoStreamEncoderInterface, private EncodedImageCallback, - public ResourceAdaptationModuleListener { + public ResourceAdaptationProcessorListener { public: - // If the encoder is reconfigured with a source, but we've yet to receive any - // frames, this 144p resolution is picked as the default value of - // |last_frame_size_|. - // TODO(hbos): Can we avoid guesses and properly handle the case of - // |last_frame_info_| not having a value, deleting these constants? - static const int kDefaultLastFrameInfoWidth; - static const int kDefaultLastFrameInfoHeight; - VideoStreamEncoder(Clock* clock, uint32_t number_of_cores, VideoStreamEncoderObserver* encoder_stats_observer, @@ -402,7 +394,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_); // The controller updates the sink wants based on restrictions that come from - // the resource adaptation module or adaptation due to bandwidth adaptation. + // the resource adaptation processor or adaptation due to bandwidth + // adaptation. // // This is used on the encoder queue, with a few exceptions: // - VideoStreamEncoder::SetSource() invokes SetSource(). @@ -413,8 +406,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // VideoSourceSinkController can be made single-threaded, and its lock can be // replaced with a sequence checker. std::unique_ptr video_source_sink_controller_; - std::unique_ptr - resource_adaptation_module_ RTC_GUARDED_BY(&encoder_queue_); + std::unique_ptr resource_adaptation_processor_ + RTC_GUARDED_BY(&encoder_queue_); // All public methods are proxied to |encoder_queue_|. It must must be // destroyed first to make sure no tasks are run that use other members.