From f46902c540c32f2c2507fde6998d5e1516158825 Mon Sep 17 00:00:00 2001 From: Jakob Ivarsson Date: Thu, 18 Jun 2020 15:48:39 +0200 Subject: [PATCH] Add a simple frame length controller. This will be used when adaptivePtime is enabled. Bug: chromium:1086942 Change-Id: I63c947c53a8c5b8e0825b78b847c3f7900197d6c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/177421 Reviewed-by: Minyue Li Commit-Queue: Jakob Ivarsson Cr-Commit-Position: refs/heads/master@{#31544} --- modules/audio_coding/BUILD.gn | 8 +- .../audio_network_adaptor/config.proto | 19 ++- .../controller_manager.cc | 15 +++ .../controller_manager_unittest.cc | 16 +++ .../frame_length_controller_v2.cc | 73 +++++++++++ .../frame_length_controller_v2.h | 44 +++++++ .../frame_length_controller_v2_unittest.cc | 121 ++++++++++++++++++ 7 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc create mode 100644 modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h create mode 100644 modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn index 99023613b2..eee17d53c4 100644 --- a/modules/audio_coding/BUILD.gn +++ b/modules/audio_coding/BUILD.gn @@ -882,6 +882,8 @@ rtc_library("audio_network_adaptor") { "audio_network_adaptor/fec_controller_plr_based.h", "audio_network_adaptor/frame_length_controller.cc", "audio_network_adaptor/frame_length_controller.h", + "audio_network_adaptor/frame_length_controller_v2.cc", + "audio_network_adaptor/frame_length_controller_v2.h", "audio_network_adaptor/include/audio_network_adaptor.h", "audio_network_adaptor/util/threshold_curve.h", ] @@ -902,7 +904,10 @@ rtc_library("audio_network_adaptor") { "../../system_wrappers", "../../system_wrappers:field_trial", ] - absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/types:optional", + ] if (rtc_enable_protobuf) { deps += [ @@ -1931,6 +1936,7 @@ if (rtc_include_tests) { "audio_network_adaptor/event_log_writer_unittest.cc", "audio_network_adaptor/fec_controller_plr_based_unittest.cc", "audio_network_adaptor/frame_length_controller_unittest.cc", + "audio_network_adaptor/frame_length_controller_v2_unittest.cc", "audio_network_adaptor/util/threshold_curve_unittest.cc", "codecs/builtin_audio_decoder_factory_unittest.cc", "codecs/builtin_audio_encoder_factory_unittest.cc", diff --git a/modules/audio_coding/audio_network_adaptor/config.proto b/modules/audio_coding/audio_network_adaptor/config.proto index 90c58e5c7d..347372e8d9 100644 --- a/modules/audio_coding/audio_network_adaptor/config.proto +++ b/modules/audio_coding/audio_network_adaptor/config.proto @@ -1,8 +1,10 @@ syntax = "proto2"; + +package webrtc.audio_network_adaptor.config; + option optimize_for = LITE_RUNTIME; option java_package = "org.webrtc.AudioNetworkAdaptor"; option java_outer_classname = "Config"; -package webrtc.audio_network_adaptor.config; message FecController { message Threshold { @@ -116,6 +118,19 @@ message FrameLengthController { optional int32 fl_60ms_to_40ms_bandwidth_bps = 12; } +message FrameLengthControllerV2 { + // FrameLengthControllerV2 chooses the frame length by taking the target + // bitrate and subtracting the overhead bitrate to obtain the remaining + // bitrate for the payload. The chosen frame length is the shortest possible + // where the payload bitrate is more than |min_payload_bitrate_bps|. + optional int32 min_payload_bitrate_bps = 1; + + // If true, uses the stable target bitrate to decide the frame length. This + // will result in less frame length toggling but spending more time at longer + // frame lengths compared to using the normal target bitrate. + optional bool use_slow_adaptation = 2; +} + message ChannelController { // Uplink bandwidth above which the number of encoded channels should switch // from 1 to 2. @@ -164,6 +179,7 @@ message Controller { DtxController dtx_controller = 24; BitrateController bitrate_controller = 25; FecControllerRplrBased fec_controller_rplr_based = 26; + FrameLengthControllerV2 frame_length_controller_v2 = 27; } } @@ -177,4 +193,3 @@ message ControllerManager { // made. optional float min_reordering_squared_distance = 3; } - diff --git a/modules/audio_coding/audio_network_adaptor/controller_manager.cc b/modules/audio_coding/audio_network_adaptor/controller_manager.cc index c7aad1da87..415b9fcf52 100644 --- a/modules/audio_coding/audio_network_adaptor/controller_manager.cc +++ b/modules/audio_coding/audio_network_adaptor/controller_manager.cc @@ -11,6 +11,7 @@ #include "modules/audio_coding/audio_network_adaptor/controller_manager.h" #include +#include #include #include @@ -20,6 +21,7 @@ #include "modules/audio_coding/audio_network_adaptor/dtx_controller.h" #include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h" #include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h" +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" #include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" #include "rtc_base/ignore_wundef.h" #include "rtc_base/logging.h" @@ -197,6 +199,14 @@ std::unique_ptr CreateBitrateController( initial_bitrate_bps, initial_frame_length_ms, fl_increase_overhead_offset, fl_decrease_overhead_offset))); } + +std::unique_ptr CreateFrameLengthControllerV2( + const audio_network_adaptor::config::FrameLengthControllerV2& config, + rtc::ArrayView encoder_frame_lengths_ms) { + return std::make_unique( + encoder_frame_lengths_ms, config.min_payload_bitrate_bps(), + config.use_slow_adaptation()); +} #endif // WEBRTC_ENABLE_PROTOBUF } // namespace @@ -277,6 +287,11 @@ std::unique_ptr ControllerManagerImpl::Create( controller_config.bitrate_controller(), initial_bitrate_bps, initial_frame_length_ms); break; + case audio_network_adaptor::config::Controller::kFrameLengthControllerV2: + controller = CreateFrameLengthControllerV2( + controller_config.frame_length_controller_v2(), + encoder_frame_lengths_ms); + break; default: RTC_NOTREACHED(); } diff --git a/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc b/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc index 4286434b5b..c71bbc9e2a 100644 --- a/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc +++ b/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc @@ -260,6 +260,14 @@ void AddFrameLengthControllerConfig( kChracteristicPacketLossFraction[1]); } +void AddFrameLengthControllerV2Config( + audio_network_adaptor::config::ControllerManager* config) { + auto controller = + config->add_controllers()->mutable_frame_length_controller_v2(); + controller->set_min_payload_bitrate_bps(16000); + controller->set_use_slow_adaptation(true); +} + constexpr int kInitialBitrateBps = 24000; constexpr size_t kIntialChannelsToEncode = 1; constexpr bool kInitialDtxEnabled = true; @@ -464,6 +472,14 @@ TEST(ControllerManagerTest, CreateFromConfigStringAndCheckReordering) { ControllerType::CHANNEL, ControllerType::DTX, ControllerType::BIT_RATE}); } + +TEST(ControllerManagerTest, CreateFrameLengthControllerV2) { + audio_network_adaptor::config::ControllerManager config; + AddFrameLengthControllerV2Config(&config); + auto states = CreateControllerManager(config.SerializeAsString()); + auto controllers = states.controller_manager->GetControllers(); + EXPECT_TRUE(controllers.size() == 1); +} #endif // WEBRTC_ENABLE_PROTOBUF } // namespace webrtc diff --git a/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc b/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc new file mode 100644 index 0000000000..36fc10ba82 --- /dev/null +++ b/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" + +#include + +#include "absl/algorithm/container.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +int OverheadBps(int overhead_bytes_per_packet, int frame_length_ms) { + return overhead_bytes_per_packet * 8 * 1000 / frame_length_ms; +} + +} // namespace + +FrameLengthControllerV2::FrameLengthControllerV2( + rtc::ArrayView encoder_frame_lengths_ms, + int min_payload_bitrate_bps, + bool use_slow_adaptation) + : encoder_frame_lengths_ms_(encoder_frame_lengths_ms.begin(), + encoder_frame_lengths_ms.end()), + min_payload_bitrate_bps_(min_payload_bitrate_bps), + use_slow_adaptation_(use_slow_adaptation) { + RTC_CHECK(!encoder_frame_lengths_ms_.empty()); + absl::c_sort(encoder_frame_lengths_ms_); +} + +void FrameLengthControllerV2::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.target_audio_bitrate_bps) { + target_bitrate_bps_ = network_metrics.target_audio_bitrate_bps; + } + if (network_metrics.overhead_bytes_per_packet) { + overhead_bytes_per_packet_ = network_metrics.overhead_bytes_per_packet; + } + if (network_metrics.uplink_bandwidth_bps) { + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; + } +} + +void FrameLengthControllerV2::MakeDecision(AudioEncoderRuntimeConfig* config) { + if (!target_bitrate_bps_ || !overhead_bytes_per_packet_ || + !uplink_bandwidth_bps_) { + return; + } + + auto it = + absl::c_find_if(encoder_frame_lengths_ms_, [&](int frame_length_ms) { + int target = use_slow_adaptation_ ? *uplink_bandwidth_bps_ + : *target_bitrate_bps_; + return target - + OverheadBps(*overhead_bytes_per_packet_, frame_length_ms) > + min_payload_bitrate_bps_; + }); + + // Longest frame length is chosen if none match our criteria. + config->frame_length_ms = it != encoder_frame_lengths_ms_.end() + ? *it + : encoder_frame_lengths_ms_.back(); +} + +} // namespace webrtc diff --git a/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h b/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h new file mode 100644 index 0000000000..d7102b0b44 --- /dev/null +++ b/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ + +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" + +namespace webrtc { + +class FrameLengthControllerV2 final : public Controller { + public: + FrameLengthControllerV2(rtc::ArrayView encoder_frame_lengths_ms, + int min_payload_bitrate_bps, + bool use_slow_adaptation); + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + std::vector encoder_frame_lengths_ms_; + const int min_payload_bitrate_bps_; + const bool use_slow_adaptation_; + + absl::optional uplink_bandwidth_bps_; + absl::optional target_bitrate_bps_; + absl::optional overhead_bytes_per_packet_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ diff --git a/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc b/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc new file mode 100644 index 0000000000..1c88f47c58 --- /dev/null +++ b/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" + +#include +#include + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr int kANASupportedFrameLengths[] = {20, 40, 60, 120}; +constexpr int kMinPayloadBitrateBps = 16000; + +} // namespace + +class FrameLengthControllerV2Test : public testing::Test { + protected: + AudioEncoderRuntimeConfig GetDecision() { + AudioEncoderRuntimeConfig config; + controller_->MakeDecision(&config); + return config; + } + + void SetOverhead(int overhead_bytes_per_packet) { + overhead_bytes_per_packet_ = overhead_bytes_per_packet; + Controller::NetworkMetrics metrics; + metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + controller_->UpdateNetworkMetrics(metrics); + } + + void SetTargetBitrate(int target_audio_bitrate_bps) { + target_audio_bitrate_bps_ = target_audio_bitrate_bps; + Controller::NetworkMetrics metrics; + metrics.target_audio_bitrate_bps = target_audio_bitrate_bps; + controller_->UpdateNetworkMetrics(metrics); + } + + void SetUplinkBandwidth(int uplink_bandwidth_bps) { + Controller::NetworkMetrics metrics; + metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller_->UpdateNetworkMetrics(metrics); + } + + void ExpectFrameLengthDecision(int expected_frame_length_ms) { + auto config = GetDecision(); + EXPECT_EQ(*config.frame_length_ms, expected_frame_length_ms); + } + + std::unique_ptr controller_ = + std::make_unique(kANASupportedFrameLengths, + kMinPayloadBitrateBps, + /*use_slow_adaptation=*/false); + absl::optional target_audio_bitrate_bps_; + absl::optional overhead_bytes_per_packet_; +}; + +// Don't return any decision if we haven't received all required network +// metrics. +TEST_F(FrameLengthControllerV2Test, RequireNetworkMetrics) { + auto config = GetDecision(); + EXPECT_FALSE(config.bitrate_bps); + EXPECT_FALSE(config.frame_length_ms); + + SetOverhead(30); + config = GetDecision(); + EXPECT_FALSE(config.frame_length_ms); + + SetTargetBitrate(32000); + config = GetDecision(); + EXPECT_FALSE(config.frame_length_ms); + + SetUplinkBandwidth(32000); + config = GetDecision(); + EXPECT_TRUE(config.frame_length_ms); +} + +TEST_F(FrameLengthControllerV2Test, UseFastAdaptation) { + SetOverhead(50); + SetTargetBitrate(50000); + SetUplinkBandwidth(50000); + ExpectFrameLengthDecision(20); + + SetTargetBitrate(20000); + ExpectFrameLengthDecision(120); + + SetTargetBitrate(30000); + ExpectFrameLengthDecision(40); + + SetTargetBitrate(25000); + ExpectFrameLengthDecision(60); +} + +TEST_F(FrameLengthControllerV2Test, UseSlowAdaptation) { + controller_ = std::make_unique( + kANASupportedFrameLengths, kMinPayloadBitrateBps, + /*use_slow_adaptation=*/true); + SetOverhead(50); + SetTargetBitrate(50000); + SetUplinkBandwidth(20000); + ExpectFrameLengthDecision(120); + + SetUplinkBandwidth(30000); + ExpectFrameLengthDecision(40); + + SetUplinkBandwidth(40000); + ExpectFrameLengthDecision(20); +} + +} // namespace webrtc