From 74158ff76177f1982d7f6930d73384e243997092 Mon Sep 17 00:00:00 2001 From: Jakob Ivarsson Date: Tue, 7 Sep 2021 14:24:56 +0200 Subject: [PATCH] Refactor delay manager. Split out `RelativeArrivalDelayTracker` and `DelayOptimizer` logic. This is in preparation for adding another `DelayOptimizer` specialized in handling reordered packets. Bug: webrtc:10178 Change-Id: Id3c1746d91980b171fa524f9b2b71cf11fc75f64 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/231224 Commit-Queue: Jakob Ivarsson Reviewed-by: Ivo Creusen Cr-Commit-Position: refs/heads/main@{#34938} --- modules/audio_coding/BUILD.gn | 6 + modules/audio_coding/neteq/decision_logic.cc | 17 +- .../neteq/decision_logic_unittest.cc | 5 +- modules/audio_coding/neteq/delay_manager.cc | 237 +++++--------- modules/audio_coding/neteq/delay_manager.h | 69 ++--- .../neteq/delay_manager_unittest.cc | 288 ++++-------------- .../neteq/mock/mock_delay_manager.h | 20 +- .../neteq/relative_arrival_delay_tracker.cc | 76 +++++ .../neteq/relative_arrival_delay_tracker.h | 57 ++++ ...relative_arrival_delay_tracker_unittest.cc | 81 +++++ .../audio_coding/neteq/underrun_optimizer.cc | 71 +++++ .../audio_coding/neteq/underrun_optimizer.h | 50 +++ .../neteq/underrun_optimizer_unittest.cc | 42 +++ 13 files changed, 561 insertions(+), 458 deletions(-) create mode 100644 modules/audio_coding/neteq/relative_arrival_delay_tracker.cc create mode 100644 modules/audio_coding/neteq/relative_arrival_delay_tracker.h create mode 100644 modules/audio_coding/neteq/relative_arrival_delay_tracker_unittest.cc create mode 100644 modules/audio_coding/neteq/underrun_optimizer.cc create mode 100644 modules/audio_coding/neteq/underrun_optimizer.h create mode 100644 modules/audio_coding/neteq/underrun_optimizer_unittest.cc diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn index 79148063b9..d5fb1a4de1 100644 --- a/modules/audio_coding/BUILD.gn +++ b/modules/audio_coding/BUILD.gn @@ -974,6 +974,8 @@ rtc_library("neteq") { "neteq/random_vector.h", "neteq/red_payload_splitter.cc", "neteq/red_payload_splitter.h", + "neteq/relative_arrival_delay_tracker.cc", + "neteq/relative_arrival_delay_tracker.h", "neteq/statistics_calculator.cc", "neteq/statistics_calculator.h", "neteq/sync_buffer.cc", @@ -982,6 +984,8 @@ rtc_library("neteq") { "neteq/time_stretch.h", "neteq/timestamp_scaler.cc", "neteq/timestamp_scaler.h", + "neteq/underrun_optimizer.cc", + "neteq/underrun_optimizer.h", ] deps = [ @@ -2011,12 +2015,14 @@ if (rtc_include_tests) { "neteq/post_decode_vad_unittest.cc", "neteq/random_vector_unittest.cc", "neteq/red_payload_splitter_unittest.cc", + "neteq/relative_arrival_delay_tracker_unittest.cc", "neteq/statistics_calculator_unittest.cc", "neteq/sync_buffer_unittest.cc", "neteq/time_stretch_unittest.cc", "neteq/timestamp_scaler_unittest.cc", "neteq/tools/input_audio_file_unittest.cc", "neteq/tools/packet_unittest.cc", + "neteq/underrun_optimizer_unittest.cc", ] deps = [ diff --git a/modules/audio_coding/neteq/decision_logic.cc b/modules/audio_coding/neteq/decision_logic.cc index ceefe50510..30463fcc49 100644 --- a/modules/audio_coding/neteq/decision_logic.cc +++ b/modules/audio_coding/neteq/decision_logic.cc @@ -22,21 +22,28 @@ #include "rtc_base/numerics/safe_conversions.h" #include "system_wrappers/include/field_trial.h" +namespace webrtc { + namespace { constexpr int kPostponeDecodingLevel = 50; constexpr int kDefaultTargetLevelWindowMs = 100; constexpr int kDecelerationTargetLevelOffsetMs = 85; -} // namespace +std::unique_ptr CreateDelayManager( + const NetEqController::Config& neteq_config) { + DelayManager::Config config; + config.max_packets_in_buffer = neteq_config.max_packets_in_buffer; + config.base_minimum_delay_ms = neteq_config.base_min_delay_ms; + config.Log(); + return std::make_unique(config, neteq_config.tick_timer); +} -namespace webrtc { +} // namespace DecisionLogic::DecisionLogic(NetEqController::Config config) : DecisionLogic(config, - DelayManager::Create(config.max_packets_in_buffer, - config.base_min_delay_ms, - config.tick_timer), + CreateDelayManager(config), std::make_unique()) {} DecisionLogic::DecisionLogic( diff --git a/modules/audio_coding/neteq/decision_logic_unittest.cc b/modules/audio_coding/neteq/decision_logic_unittest.cc index fc580350d2..b82165389f 100644 --- a/modules/audio_coding/neteq/decision_logic_unittest.cc +++ b/modules/audio_coding/neteq/decision_logic_unittest.cc @@ -61,11 +61,8 @@ class DecisionLogicTest : public ::testing::Test { NetEqController::Config config; config.tick_timer = &tick_timer_; config.allow_time_stretching = true; - std::unique_ptr histogram = - std::make_unique(200, 12345, 2); auto delay_manager = std::make_unique( - 200, 0, 12300, absl::nullopt, 2000, config.tick_timer, - std::move(histogram)); + DelayManager::Config(), config.tick_timer); mock_delay_manager_ = delay_manager.get(); auto buffer_level_filter = std::make_unique(); mock_buffer_level_filter_ = buffer_level_filter.get(); diff --git a/modules/audio_coding/neteq/delay_manager.cc b/modules/audio_coding/neteq/delay_manager.cc index c2509773f1..9f7eebd0ee 100644 --- a/modules/audio_coding/neteq/delay_manager.cc +++ b/modules/audio_coding/neteq/delay_manager.cc @@ -18,10 +18,8 @@ #include #include -#include "modules/audio_coding/neteq/histogram.h" #include "modules/include/module_common_types_public.h" #include "rtc_base/checks.h" -#include "rtc_base/experiments/struct_parameters_parser.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/numerics/safe_minmax.h" @@ -32,157 +30,94 @@ namespace { constexpr int kMinBaseMinimumDelayMs = 0; constexpr int kMaxBaseMinimumDelayMs = 10000; -constexpr int kDelayBuckets = 100; -constexpr int kBucketSizeMs = 20; constexpr int kStartDelayMs = 80; -struct DelayManagerConfig { - double quantile = 0.97; - double forget_factor = 0.9993; - absl::optional start_forget_weight = 2; - absl::optional resample_interval_ms; - int max_history_ms = 2000; - - std::unique_ptr Parser() { - return webrtc::StructParametersParser::Create( // - "quantile", &quantile, // - "forget_factor", &forget_factor, // - "start_forget_weight", &start_forget_weight, // - "resample_interval_ms", &resample_interval_ms, // - "max_history_ms", &max_history_ms); - } - - // TODO(jakobi): remove legacy field trial. - void MaybeUpdateFromLegacyFieldTrial() { - constexpr char kDelayHistogramFieldTrial[] = - "WebRTC-Audio-NetEqDelayHistogram"; - if (!webrtc::field_trial::IsEnabled(kDelayHistogramFieldTrial)) { - return; - } - const auto field_trial_string = - webrtc::field_trial::FindFullName(kDelayHistogramFieldTrial); - double percentile = -1.0; - double forget_factor = -1.0; - double start_forget_weight = -1.0; - if (sscanf(field_trial_string.c_str(), "Enabled-%lf-%lf-%lf", &percentile, - &forget_factor, &start_forget_weight) >= 2 && - percentile >= 0.0 && percentile <= 100.0 && forget_factor >= 0.0 && - forget_factor <= 1.0) { - this->quantile = percentile / 100; - this->forget_factor = forget_factor; - this->start_forget_weight = start_forget_weight >= 1 - ? absl::make_optional(start_forget_weight) - : absl::nullopt; - } - } - - explicit DelayManagerConfig() { - Parser()->Parse(webrtc::field_trial::FindFullName( - "WebRTC-Audio-NetEqDelayManagerConfig")); - MaybeUpdateFromLegacyFieldTrial(); - RTC_LOG(LS_INFO) << "Delay manager config:" - " quantile=" - << quantile << " forget_factor=" << forget_factor - << " start_forget_weight=" - << start_forget_weight.value_or(0) - << " resample_interval_ms=" - << resample_interval_ms.value_or(0) - << " max_history_ms=" << max_history_ms; - } -}; - } // namespace -DelayManager::DelayManager(int max_packets_in_buffer, - int base_minimum_delay_ms, - int histogram_quantile, - absl::optional resample_interval_ms, - int max_history_ms, - const TickTimer* tick_timer, - std::unique_ptr histogram) - : max_packets_in_buffer_(max_packets_in_buffer), - histogram_(std::move(histogram)), - histogram_quantile_(histogram_quantile), - tick_timer_(tick_timer), - resample_interval_ms_(resample_interval_ms), - max_history_ms_(max_history_ms), - base_minimum_delay_ms_(base_minimum_delay_ms), - effective_minimum_delay_ms_(base_minimum_delay_ms), +DelayManager::Config::Config() { + Parser()->Parse(webrtc::field_trial::FindFullName( + "WebRTC-Audio-NetEqDelayManagerConfig")); + MaybeUpdateFromLegacyFieldTrial(); +} + +void DelayManager::Config::Log() { + RTC_LOG(LS_INFO) << "Delay manager config:" + " quantile=" + << quantile << " forget_factor=" << forget_factor + << " start_forget_weight=" << start_forget_weight.value_or(0) + << " resample_interval_ms=" + << resample_interval_ms.value_or(0) + << " max_history_ms=" << max_history_ms; +} + +std::unique_ptr DelayManager::Config::Parser() { + return StructParametersParser::Create( // + "quantile", &quantile, // + "forget_factor", &forget_factor, // + "start_forget_weight", &start_forget_weight, // + "resample_interval_ms", &resample_interval_ms, // + "max_history_ms", &max_history_ms); +} + +// TODO(jakobi): remove legacy field trial. +void DelayManager::Config::MaybeUpdateFromLegacyFieldTrial() { + constexpr char kDelayHistogramFieldTrial[] = + "WebRTC-Audio-NetEqDelayHistogram"; + if (!webrtc::field_trial::IsEnabled(kDelayHistogramFieldTrial)) { + return; + } + const auto field_trial_string = + webrtc::field_trial::FindFullName(kDelayHistogramFieldTrial); + double percentile = -1.0; + double forget_factor = -1.0; + double start_forget_weight = -1.0; + if (sscanf(field_trial_string.c_str(), "Enabled-%lf-%lf-%lf", &percentile, + &forget_factor, &start_forget_weight) >= 2 && + percentile >= 0.0 && percentile <= 100.0 && forget_factor >= 0.0 && + forget_factor <= 1.0) { + this->quantile = percentile / 100; + this->forget_factor = forget_factor; + this->start_forget_weight = start_forget_weight >= 1 + ? absl::make_optional(start_forget_weight) + : absl::nullopt; + } +} + +DelayManager::DelayManager(const Config& config, const TickTimer* tick_timer) + : max_packets_in_buffer_(config.max_packets_in_buffer), + underrun_optimizer_(tick_timer, + (1 << 30) * config.quantile, + (1 << 15) * config.forget_factor, + config.start_forget_weight, + config.resample_interval_ms), + relative_arrival_delay_tracker_(tick_timer, config.max_history_ms), + base_minimum_delay_ms_(config.base_minimum_delay_ms), + effective_minimum_delay_ms_(config.base_minimum_delay_ms), minimum_delay_ms_(0), maximum_delay_ms_(0), target_level_ms_(kStartDelayMs) { - RTC_CHECK(histogram_); RTC_DCHECK_GE(base_minimum_delay_ms_, 0); Reset(); } -std::unique_ptr DelayManager::Create( - int max_packets_in_buffer, - int base_minimum_delay_ms, - const TickTimer* tick_timer) { - DelayManagerConfig config; - int forget_factor_q15 = (1 << 15) * config.forget_factor; - int quantile_q30 = (1 << 30) * config.quantile; - std::unique_ptr histogram = std::make_unique( - kDelayBuckets, forget_factor_q15, config.start_forget_weight); - return std::make_unique( - max_packets_in_buffer, base_minimum_delay_ms, quantile_q30, - config.resample_interval_ms, config.max_history_ms, tick_timer, - std::move(histogram)); -} - DelayManager::~DelayManager() {} absl::optional DelayManager::Update(uint32_t timestamp, int sample_rate_hz, bool reset) { - if (sample_rate_hz <= 0) { + if (reset) { + relative_arrival_delay_tracker_.Reset(); + } + absl::optional relative_delay = + relative_arrival_delay_tracker_.Update(timestamp, sample_rate_hz); + if (!relative_delay) { return absl::nullopt; } - if (!last_timestamp_ || reset) { - // Restart relative delay esimation from this packet. - delay_history_.clear(); - packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); - last_timestamp_ = timestamp; - resample_stopwatch_ = tick_timer_->GetNewStopwatch(); - max_delay_in_interval_ms_ = 0; - return absl::nullopt; - } - - const int expected_iat_ms = - 1000ll * static_cast(timestamp - *last_timestamp_) / - sample_rate_hz; - const int iat_ms = packet_iat_stopwatch_->ElapsedMs(); - const int iat_delay_ms = iat_ms - expected_iat_ms; - UpdateDelayHistory(iat_delay_ms, timestamp, sample_rate_hz); - int relative_delay = CalculateRelativePacketArrivalDelay(); - - absl::optional histogram_update; - if (resample_interval_ms_) { - if (static_cast(resample_stopwatch_->ElapsedMs()) > - *resample_interval_ms_) { - histogram_update = max_delay_in_interval_ms_; - resample_stopwatch_ = tick_timer_->GetNewStopwatch(); - max_delay_in_interval_ms_ = 0; - } - max_delay_in_interval_ms_ = - std::max(max_delay_in_interval_ms_, relative_delay); - } else { - histogram_update = relative_delay; - } - if (histogram_update) { - const int index = *histogram_update / kBucketSizeMs; - if (index < histogram_->NumBuckets()) { - // Maximum delay to register is 2000 ms. - histogram_->Add(index); - } - } - - // Calculate new `target_level_ms_` based on updated statistics. - int bucket_index = histogram_->Quantile(histogram_quantile_); - target_level_ms_ = (1 + bucket_index) * kBucketSizeMs; + underrun_optimizer_.Update(*relative_delay); + target_level_ms_ = + underrun_optimizer_.GetOptimalDelayMs().value_or(kStartDelayMs); target_level_ms_ = std::max(target_level_ms_, effective_minimum_delay_ms_); if (maximum_delay_ms_ > 0) { target_level_ms_ = std::min(target_level_ms_, maximum_delay_ms_); @@ -195,37 +130,9 @@ absl::optional DelayManager::Update(uint32_t timestamp, target_level_ms_, 3 * max_packets_in_buffer_ * packet_len_ms_ / 4); } - // Prepare for next packet arrival. - packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); - last_timestamp_ = timestamp; return relative_delay; } -void DelayManager::UpdateDelayHistory(int iat_delay_ms, - uint32_t timestamp, - int sample_rate_hz) { - PacketDelay delay; - delay.iat_delay_ms = iat_delay_ms; - delay.timestamp = timestamp; - delay_history_.push_back(delay); - while (static_cast(timestamp - delay_history_.front().timestamp) > - max_history_ms_ * sample_rate_hz / 1000) { - delay_history_.pop_front(); - } -} - -int DelayManager::CalculateRelativePacketArrivalDelay() const { - // This effectively calculates arrival delay of a packet relative to the - // packet preceding the history window. If the arrival delay ever becomes - // smaller than zero, it means the reference packet is invalid, and we - // move the reference. - int relative_delay = 0; - for (const PacketDelay& delay : delay_history_) { - relative_delay += delay.iat_delay_ms; - relative_delay = std::max(relative_delay, 0); - } - return relative_delay; -} int DelayManager::SetPacketAudioLength(int length_ms) { if (length_ms <= 0) { @@ -238,13 +145,9 @@ int DelayManager::SetPacketAudioLength(int length_ms) { void DelayManager::Reset() { packet_len_ms_ = 0; - histogram_->Reset(); - delay_history_.clear(); + underrun_optimizer_.Reset(); + relative_arrival_delay_tracker_.Reset(); target_level_ms_ = kStartDelayMs; - packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); - last_timestamp_ = absl::nullopt; - resample_stopwatch_ = tick_timer_->GetNewStopwatch(); - max_delay_in_interval_ms_ = 0; } int DelayManager::TargetDelayMs() const { diff --git a/modules/audio_coding/neteq/delay_manager.h b/modules/audio_coding/neteq/delay_manager.h index cc9820393c..c751836a6a 100644 --- a/modules/audio_coding/neteq/delay_manager.h +++ b/modules/audio_coding/neteq/delay_manager.h @@ -19,28 +19,38 @@ #include "absl/types/optional.h" #include "api/neteq/tick_timer.h" #include "modules/audio_coding/neteq/histogram.h" +#include "modules/audio_coding/neteq/relative_arrival_delay_tracker.h" +#include "modules/audio_coding/neteq/underrun_optimizer.h" #include "rtc_base/constructor_magic.h" +#include "rtc_base/experiments/struct_parameters_parser.h" namespace webrtc { class DelayManager { public: - DelayManager(int max_packets_in_buffer, - int base_minimum_delay_ms, - int histogram_quantile, - absl::optional resample_interval_ms, - int max_history_ms, - const TickTimer* tick_timer, - std::unique_ptr histogram); + struct Config { + Config(); + void Log(); - // Create a DelayManager object. Notify the delay manager that the packet - // buffer can hold no more than `max_packets_in_buffer` packets (i.e., this - // is the number of packet slots in the buffer) and that the target delay - // should be greater than or equal to `base_minimum_delay_ms`. Supply a - // PeakDetector object to the DelayManager. - static std::unique_ptr Create(int max_packets_in_buffer, - int base_minimum_delay_ms, - const TickTimer* tick_timer); + // Options that can be configured via field trial. + double quantile = 0.97; + double forget_factor = 0.9993; + absl::optional start_forget_weight = 2; + absl::optional resample_interval_ms; + int max_history_ms = 2000; + + // Options that are externally populated. + int max_packets_in_buffer = 200; + int base_minimum_delay_ms = 0; + + private: + std::unique_ptr Parser(); + + // TODO(jakobi): remove legacy field trial. + void MaybeUpdateFromLegacyFieldTrial(); + }; + + DelayManager(const Config& config, const TickTimer* tick_timer); virtual ~DelayManager(); @@ -73,22 +83,12 @@ class DelayManager { int effective_minimum_delay_ms_for_test() const { return effective_minimum_delay_ms_; } - int histogram_quantile() const { return histogram_quantile_; } - Histogram* histogram() const { return histogram_.get(); } private: // Provides value which minimum delay can't exceed based on current buffer // size and given `maximum_delay_ms_`. Lower bound is a constant 0. int MinimumDelayUpperBound() const; - // Updates `delay_history_`. - void UpdateDelayHistory(int iat_delay_ms, - uint32_t timestamp, - int sample_rate_hz); - - // Calculate relative packet arrival delay from `delay_history_`. - int CalculateRelativePacketArrivalDelay() const; - // Updates `effective_minimum_delay_ms_` delay based on current // `minimum_delay_ms_`, `base_minimum_delay_ms_` and `maximum_delay_ms_` // and buffer size. @@ -103,11 +103,8 @@ class DelayManager { // TODO(jakobi): set maximum buffer delay instead of number of packets. const int max_packets_in_buffer_; - std::unique_ptr histogram_; - const int histogram_quantile_; - const TickTimer* tick_timer_; - const absl::optional resample_interval_ms_; - const int max_history_ms_; + UnderrunOptimizer underrun_optimizer_; + RelativeArrivalDelayTracker relative_arrival_delay_tracker_; int base_minimum_delay_ms_; int effective_minimum_delay_ms_; // Used as lower bound for target delay. @@ -115,19 +112,7 @@ class DelayManager { int maximum_delay_ms_; // Externally set maximum allowed delay. int packet_len_ms_ = 0; - std::unique_ptr - packet_iat_stopwatch_; // Time elapsed since last packet. int target_level_ms_; // Currently preferred buffer level. - absl::optional - last_timestamp_; // Timestamp for the last received packet. - int max_delay_in_interval_ms_ = 0; - std::unique_ptr resample_stopwatch_; - - struct PacketDelay { - int iat_delay_ms; - uint32_t timestamp; - }; - std::deque delay_history_; RTC_DISALLOW_COPY_AND_ASSIGN(DelayManager); }; diff --git a/modules/audio_coding/neteq/delay_manager_unittest.cc b/modules/audio_coding/neteq/delay_manager_unittest.cc index 427386e06d..fc4e0cb4cd 100644 --- a/modules/audio_coding/neteq/delay_manager_unittest.cc +++ b/modules/audio_coding/neteq/delay_manager_unittest.cc @@ -28,60 +28,36 @@ namespace webrtc { namespace { -constexpr int kMaxNumberOfPackets = 240; -constexpr int kMinDelayMs = 0; -constexpr int kMaxHistoryMs = 2000; +constexpr int kMaxNumberOfPackets = 200; constexpr int kTimeStepMs = 10; constexpr int kFs = 8000; constexpr int kFrameSizeMs = 20; constexpr int kTsIncrement = kFrameSizeMs * kFs / 1000; constexpr int kMaxBufferSizeMs = kMaxNumberOfPackets * kFrameSizeMs; -constexpr int kDefaultHistogramQuantile = 1020054733; -constexpr int kNumBuckets = 100; -constexpr int kForgetFactor = 32745; + } // namespace class DelayManagerTest : public ::testing::Test { protected: DelayManagerTest(); virtual void SetUp(); - void RecreateDelayManager(); absl::optional InsertNextPacket(); void IncreaseTime(int inc_ms); - std::unique_ptr dm_; + DelayManager dm_; TickTimer tick_timer_; - MockStatisticsCalculator stats_; - MockHistogram* mock_histogram_; uint32_t ts_; - bool use_mock_histogram_ = false; - absl::optional resample_interval_ms_; }; DelayManagerTest::DelayManagerTest() - : dm_(nullptr), - ts_(0x12345678) {} + : dm_(DelayManager::Config(), &tick_timer_), ts_(0x12345678) {} void DelayManagerTest::SetUp() { - RecreateDelayManager(); -} - -void DelayManagerTest::RecreateDelayManager() { - if (use_mock_histogram_) { - mock_histogram_ = new MockHistogram(kNumBuckets, kForgetFactor); - std::unique_ptr histogram(mock_histogram_); - dm_ = std::make_unique(kMaxNumberOfPackets, kMinDelayMs, - kDefaultHistogramQuantile, - resample_interval_ms_, kMaxHistoryMs, - &tick_timer_, std::move(histogram)); - } else { - dm_ = DelayManager::Create(kMaxNumberOfPackets, kMinDelayMs, &tick_timer_); - } - dm_->SetPacketAudioLength(kFrameSizeMs); + dm_.SetPacketAudioLength(kFrameSizeMs); } absl::optional DelayManagerTest::InsertNextPacket() { - auto relative_delay = dm_->Update(ts_, kFs); + auto relative_delay = dm_.Update(ts_, kFs); ts_ += kTsIncrement; return relative_delay; } @@ -104,7 +80,7 @@ TEST_F(DelayManagerTest, UpdateNormal) { IncreaseTime(kFrameSizeMs); // Second packet arrival. InsertNextPacket(); - EXPECT_EQ(20, dm_->TargetDelayMs()); + EXPECT_EQ(20, dm_.TargetDelayMs()); } TEST_F(DelayManagerTest, UpdateLongInterArrivalTime) { @@ -114,7 +90,7 @@ TEST_F(DelayManagerTest, UpdateLongInterArrivalTime) { IncreaseTime(2 * kFrameSizeMs); // Second packet arrival. InsertNextPacket(); - EXPECT_EQ(40, dm_->TargetDelayMs()); + EXPECT_EQ(40, dm_.TargetDelayMs()); } TEST_F(DelayManagerTest, MaxDelay) { @@ -126,16 +102,16 @@ TEST_F(DelayManagerTest, MaxDelay) { InsertNextPacket(); // No limit is set. - EXPECT_EQ(kExpectedTarget, dm_->TargetDelayMs()); + EXPECT_EQ(kExpectedTarget, dm_.TargetDelayMs()); const int kMaxDelayMs = 3 * kFrameSizeMs; - EXPECT_TRUE(dm_->SetMaximumDelay(kMaxDelayMs)); + EXPECT_TRUE(dm_.SetMaximumDelay(kMaxDelayMs)); IncreaseTime(kFrameSizeMs); InsertNextPacket(); - EXPECT_EQ(kMaxDelayMs, dm_->TargetDelayMs()); + EXPECT_EQ(kMaxDelayMs, dm_.TargetDelayMs()); // Target level at least should be one packet. - EXPECT_FALSE(dm_->SetMaximumDelay(kFrameSizeMs - 1)); + EXPECT_FALSE(dm_.SetMaximumDelay(kFrameSizeMs - 1)); } TEST_F(DelayManagerTest, MinDelay) { @@ -147,23 +123,23 @@ TEST_F(DelayManagerTest, MinDelay) { InsertNextPacket(); // No limit is applied. - EXPECT_EQ(kExpectedTarget, dm_->TargetDelayMs()); + EXPECT_EQ(kExpectedTarget, dm_.TargetDelayMs()); int kMinDelayMs = 7 * kFrameSizeMs; - dm_->SetMinimumDelay(kMinDelayMs); + dm_.SetMinimumDelay(kMinDelayMs); IncreaseTime(kFrameSizeMs); InsertNextPacket(); - EXPECT_EQ(kMinDelayMs, dm_->TargetDelayMs()); + EXPECT_EQ(kMinDelayMs, dm_.TargetDelayMs()); } TEST_F(DelayManagerTest, BaseMinimumDelayCheckValidRange) { // Base minimum delay should be between [0, 10000] milliseconds. - EXPECT_FALSE(dm_->SetBaseMinimumDelay(-1)); - EXPECT_FALSE(dm_->SetBaseMinimumDelay(10001)); - EXPECT_EQ(dm_->GetBaseMinimumDelay(), 0); + EXPECT_FALSE(dm_.SetBaseMinimumDelay(-1)); + EXPECT_FALSE(dm_.SetBaseMinimumDelay(10001)); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), 0); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(7999)); - EXPECT_EQ(dm_->GetBaseMinimumDelay(), 7999); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(7999)); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), 7999); } TEST_F(DelayManagerTest, BaseMinimumDelayLowerThanMinimumDelay) { @@ -174,9 +150,9 @@ TEST_F(DelayManagerTest, BaseMinimumDelayLowerThanMinimumDelay) { // minimum delay is lower than minimum delay we use minimum delay. RTC_DCHECK_LT(kBaseMinimumDelayMs, kMinimumDelayMs); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); - EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); - EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMinimumDelayMs); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMinimumDelayMs); } TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanMinimumDelay) { @@ -187,9 +163,9 @@ TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanMinimumDelay) { // minimum delay is greater than minimum delay we use base minimum delay. RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); - EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); - EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMs); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMs); } TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanBufferSize) { @@ -198,7 +174,7 @@ TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanBufferSize) { constexpr int kMaximumDelayMs = 20; constexpr int kMaxBufferSizeMsQ75 = 3 * kMaxBufferSizeMs / 4; - EXPECT_TRUE(dm_->SetMaximumDelay(kMaximumDelayMs)); + EXPECT_TRUE(dm_.SetMaximumDelay(kMaximumDelayMs)); // Base minimum delay is greater than minimum delay, that is why we clamp // it to current the highest possible value which is maximum delay. @@ -207,15 +183,15 @@ TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanBufferSize) { RTC_DCHECK_GT(kBaseMinimumDelayMs, kMaximumDelayMs); RTC_DCHECK_LT(kMaximumDelayMs, kMaxBufferSizeMsQ75); - EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); // Unset maximum value. - EXPECT_TRUE(dm_->SetMaximumDelay(0)); + EXPECT_TRUE(dm_.SetMaximumDelay(0)); // With maximum value unset, the highest possible value now is 75% of // currently possible maximum buffer size. - EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMaxBufferSizeMsQ75); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMaxBufferSizeMsQ75); } TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanMaximumDelay) { @@ -229,10 +205,10 @@ TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanMaximumDelay) { RTC_DCHECK_GT(kBaseMinimumDelayMs, kMaximumDelayMs); RTC_DCHECK_LT(kMaximumDelayMs, kMaxBufferSizeMs); - EXPECT_TRUE(dm_->SetMaximumDelay(kMaximumDelayMs)); - EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); - EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMaximumDelayMs); + EXPECT_TRUE(dm_.SetMaximumDelay(kMaximumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMaximumDelayMs); } TEST_F(DelayManagerTest, BaseMinimumDelayLowerThanMaxSize) { @@ -245,10 +221,10 @@ TEST_F(DelayManagerTest, BaseMinimumDelayLowerThanMaxSize) { RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); RTC_DCHECK_LT(kBaseMinimumDelayMs, kMaximumDelayMs); - EXPECT_TRUE(dm_->SetMaximumDelay(kMaximumDelayMs)); - EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); - EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMs); + EXPECT_TRUE(dm_.SetMaximumDelay(kMaximumDelayMs)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMs); } TEST_F(DelayManagerTest, MinimumDelayMemorization) { @@ -260,19 +236,18 @@ TEST_F(DelayManagerTest, MinimumDelayMemorization) { constexpr int kMinimumDelayMs = 20; constexpr int kBaseMinimumDelayMsHigh = 30; - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMsLow)); - EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMsLow)); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); // Minimum delay is used as it is higher than base minimum delay. - EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMinimumDelayMs); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMinimumDelayMs); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMsHigh)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMsHigh)); // Base minimum delay is used as it is now higher than minimum delay. - EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), - kBaseMinimumDelayMsHigh); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMsHigh); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMsLow)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMsLow)); // Check that minimum delay is memorized and is used again. - EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMinimumDelayMs); + EXPECT_EQ(dm_.effective_minimum_delay_ms_for_test(), kMinimumDelayMs); } TEST_F(DelayManagerTest, BaseMinimumDelay) { @@ -284,16 +259,16 @@ TEST_F(DelayManagerTest, BaseMinimumDelay) { InsertNextPacket(); // No limit is applied. - EXPECT_EQ(kExpectedTarget, dm_->TargetDelayMs()); + EXPECT_EQ(kExpectedTarget, dm_.TargetDelayMs()); constexpr int kBaseMinimumDelayMs = 7 * kFrameSizeMs; - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); - EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinimumDelayMs); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), kBaseMinimumDelayMs); IncreaseTime(kFrameSizeMs); InsertNextPacket(); - EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinimumDelayMs); - EXPECT_EQ(kBaseMinimumDelayMs, dm_->TargetDelayMs()); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), kBaseMinimumDelayMs); + EXPECT_EQ(kBaseMinimumDelayMs, dm_.TargetDelayMs()); } TEST_F(DelayManagerTest, BaseMinimumDelayAffectsTargetDelay) { @@ -306,7 +281,7 @@ TEST_F(DelayManagerTest, BaseMinimumDelayAffectsTargetDelay) { InsertNextPacket(); // No limit is applied. - EXPECT_EQ(kTimeIncrement, dm_->TargetDelayMs()); + EXPECT_EQ(kTimeIncrement, dm_.TargetDelayMs()); // Minimum delay is lower than base minimum delay, that is why base minimum // delay is used to calculate target level. @@ -317,137 +292,31 @@ TEST_F(DelayManagerTest, BaseMinimumDelayAffectsTargetDelay) { constexpr int kBaseMinimumDelayMs = kBaseMinimumDelayPackets * kFrameSizeMs; EXPECT_TRUE(kMinimumDelayMs < kBaseMinimumDelayMs); - EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); - EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinimumDelayMs); + EXPECT_TRUE(dm_.SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_.SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), kBaseMinimumDelayMs); IncreaseTime(kFrameSizeMs); InsertNextPacket(); - EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinimumDelayMs); - EXPECT_EQ(kBaseMinimumDelayMs, dm_->TargetDelayMs()); + EXPECT_EQ(dm_.GetBaseMinimumDelay(), kBaseMinimumDelayMs); + EXPECT_EQ(kBaseMinimumDelayMs, dm_.TargetDelayMs()); } TEST_F(DelayManagerTest, Failures) { // Wrong sample rate. - EXPECT_EQ(absl::nullopt, dm_->Update(0, -1)); + EXPECT_EQ(absl::nullopt, dm_.Update(0, -1)); // Wrong packet size. - EXPECT_EQ(-1, dm_->SetPacketAudioLength(0)); - EXPECT_EQ(-1, dm_->SetPacketAudioLength(-1)); + EXPECT_EQ(-1, dm_.SetPacketAudioLength(0)); + EXPECT_EQ(-1, dm_.SetPacketAudioLength(-1)); // Minimum delay higher than a maximum delay is not accepted. - EXPECT_TRUE(dm_->SetMaximumDelay(20)); - EXPECT_FALSE(dm_->SetMinimumDelay(40)); + EXPECT_TRUE(dm_.SetMaximumDelay(20)); + EXPECT_FALSE(dm_.SetMinimumDelay(40)); // Maximum delay less than minimum delay is not accepted. - EXPECT_TRUE(dm_->SetMaximumDelay(100)); - EXPECT_TRUE(dm_->SetMinimumDelay(80)); - EXPECT_FALSE(dm_->SetMaximumDelay(60)); -} - -TEST_F(DelayManagerTest, DelayHistogramFieldTrial) { - { - test::ScopedFieldTrials field_trial( - "WebRTC-Audio-NetEqDelayHistogram/Enabled-96-0.998/"); - RecreateDelayManager(); - EXPECT_EQ(1030792151, dm_->histogram_quantile()); // 0.96 in Q30. - EXPECT_EQ( - 32702, - dm_->histogram()->base_forget_factor_for_testing()); // 0.998 in Q15. - EXPECT_FALSE(dm_->histogram()->start_forget_weight_for_testing()); - } - { - test::ScopedFieldTrials field_trial( - "WebRTC-Audio-NetEqDelayHistogram/Enabled-97.5-0.998/"); - RecreateDelayManager(); - EXPECT_EQ(1046898278, dm_->histogram_quantile()); // 0.975 in Q30. - EXPECT_EQ( - 32702, - dm_->histogram()->base_forget_factor_for_testing()); // 0.998 in Q15. - EXPECT_FALSE(dm_->histogram()->start_forget_weight_for_testing()); - } - // Test parameter for new call start adaptation. - { - test::ScopedFieldTrials field_trial( - "WebRTC-Audio-NetEqDelayHistogram/Enabled-96-0.998-1/"); - RecreateDelayManager(); - EXPECT_EQ(dm_->histogram()->start_forget_weight_for_testing().value(), 1.0); - } - { - test::ScopedFieldTrials field_trial( - "WebRTC-Audio-NetEqDelayHistogram/Enabled-96-0.998-1.5/"); - RecreateDelayManager(); - EXPECT_EQ(dm_->histogram()->start_forget_weight_for_testing().value(), 1.5); - } - { - test::ScopedFieldTrials field_trial( - "WebRTC-Audio-NetEqDelayHistogram/Enabled-96-0.998-0.5/"); - RecreateDelayManager(); - EXPECT_FALSE(dm_->histogram()->start_forget_weight_for_testing()); - } -} - -TEST_F(DelayManagerTest, RelativeArrivalDelay) { - use_mock_histogram_ = true; - RecreateDelayManager(); - - InsertNextPacket(); - - IncreaseTime(kFrameSizeMs); - EXPECT_CALL(*mock_histogram_, Add(0)); // Not delayed. - InsertNextPacket(); - - IncreaseTime(2 * kFrameSizeMs); - EXPECT_CALL(*mock_histogram_, Add(1)); // 20ms delayed. - dm_->Update(ts_, kFs); - - EXPECT_CALL(*mock_histogram_, Add(3)); // Reordered, 60ms delayed. - dm_->Update(ts_ - 2 * kTsIncrement, kFs); - - IncreaseTime(2 * kFrameSizeMs); - EXPECT_CALL(*mock_histogram_, Add(2)); // 40ms delayed. - dm_->Update(ts_ + kTsIncrement, kFs); -} - -TEST_F(DelayManagerTest, ReorderedPackets) { - use_mock_histogram_ = true; - RecreateDelayManager(); - - // Insert first packet. - InsertNextPacket(); - - // Insert reordered packet. - EXPECT_CALL(*mock_histogram_, Add(4)); - dm_->Update(ts_ - 5 * kTsIncrement, kFs); - - // Insert another reordered packet. - EXPECT_CALL(*mock_histogram_, Add(1)); - dm_->Update(ts_ - 2 * kTsIncrement, kFs); - - // Insert the next packet in order and verify that the relative delay is - // estimated based on the first inserted packet. - IncreaseTime(4 * kFrameSizeMs); - EXPECT_CALL(*mock_histogram_, Add(3)); - InsertNextPacket(); -} - -TEST_F(DelayManagerTest, MaxDelayHistory) { - use_mock_histogram_ = true; - RecreateDelayManager(); - - InsertNextPacket(); - - // Insert 20 ms iat delay in the delay history. - IncreaseTime(2 * kFrameSizeMs); - EXPECT_CALL(*mock_histogram_, Add(1)); // 20ms delayed. - InsertNextPacket(); - - // Insert next packet with a timestamp difference larger than maximum history - // size. This removes the previously inserted iat delay from the history. - constexpr int kMaxHistoryMs = 2000; - IncreaseTime(kMaxHistoryMs + kFrameSizeMs); - ts_ += kFs * kMaxHistoryMs / 1000; - EXPECT_CALL(*mock_histogram_, Add(0)); // Not delayed. - dm_->Update(ts_, kFs); + EXPECT_TRUE(dm_.SetMaximumDelay(100)); + EXPECT_TRUE(dm_.SetMinimumDelay(80)); + EXPECT_FALSE(dm_.SetMaximumDelay(60)); } TEST_F(DelayManagerTest, RelativeArrivalDelayStatistic) { @@ -459,31 +328,4 @@ TEST_F(DelayManagerTest, RelativeArrivalDelayStatistic) { EXPECT_EQ(20, InsertNextPacket()); } -TEST_F(DelayManagerTest, ResamplePacketDelays) { - use_mock_histogram_ = true; - resample_interval_ms_ = 500; - RecreateDelayManager(); - - // The histogram should be updated once with the maximum delay observed for - // the following sequence of packets. - EXPECT_CALL(*mock_histogram_, Add(5)).Times(1); - - EXPECT_EQ(absl::nullopt, InsertNextPacket()); - - IncreaseTime(kFrameSizeMs); - EXPECT_EQ(0, InsertNextPacket()); - IncreaseTime(3 * kFrameSizeMs); - EXPECT_EQ(2 * kFrameSizeMs, InsertNextPacket()); - IncreaseTime(4 * kFrameSizeMs); - EXPECT_EQ(5 * kFrameSizeMs, InsertNextPacket()); - - for (int i = 4; i >= 0; --i) { - EXPECT_EQ(i * kFrameSizeMs, InsertNextPacket()); - } - for (int i = 0; i < *resample_interval_ms_ / kFrameSizeMs; ++i) { - IncreaseTime(kFrameSizeMs); - EXPECT_EQ(0, InsertNextPacket()); - } -} - } // namespace webrtc diff --git a/modules/audio_coding/neteq/mock/mock_delay_manager.h b/modules/audio_coding/neteq/mock/mock_delay_manager.h index 5b5133eceb..d783f8743b 100644 --- a/modules/audio_coding/neteq/mock/mock_delay_manager.h +++ b/modules/audio_coding/neteq/mock/mock_delay_manager.h @@ -11,9 +11,6 @@ #ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ #define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ -#include -#include - #include "api/neteq/tick_timer.h" #include "modules/audio_coding/neteq/delay_manager.h" #include "test/gmock.h" @@ -22,20 +19,9 @@ namespace webrtc { class MockDelayManager : public DelayManager { public: - MockDelayManager(size_t max_packets_in_buffer, - int base_minimum_delay_ms, - int histogram_quantile, - absl::optional resample_interval_ms, - int max_history_ms, - const TickTimer* tick_timer, - std::unique_ptr histogram) - : DelayManager(max_packets_in_buffer, - base_minimum_delay_ms, - histogram_quantile, - resample_interval_ms, - max_history_ms, - tick_timer, - std::move(histogram)) {} + MockDelayManager(const MockDelayManager::Config& config, + const TickTimer* tick_timer) + : DelayManager(config, tick_timer) {} MOCK_METHOD(int, TargetDelayMs, (), (const)); }; diff --git a/modules/audio_coding/neteq/relative_arrival_delay_tracker.cc b/modules/audio_coding/neteq/relative_arrival_delay_tracker.cc new file mode 100644 index 0000000000..5b9580c127 --- /dev/null +++ b/modules/audio_coding/neteq/relative_arrival_delay_tracker.cc @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/relative_arrival_delay_tracker.h" + +#include + +namespace webrtc { + +absl::optional RelativeArrivalDelayTracker::Update(uint32_t timestamp, + int sample_rate_hz) { + if (sample_rate_hz <= 0) { + return absl::nullopt; + } + if (!last_timestamp_) { + // Restart relative delay esimation from this packet. + delay_history_.clear(); + packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + last_timestamp_ = timestamp; + return absl::nullopt; + } + + const int expected_iat_ms = + 1000ll * static_cast(timestamp - *last_timestamp_) / + sample_rate_hz; + const int iat_ms = packet_iat_stopwatch_->ElapsedMs(); + const int iat_delay_ms = iat_ms - expected_iat_ms; + UpdateDelayHistory(iat_delay_ms, timestamp, sample_rate_hz); + int relative_delay = CalculateRelativePacketArrivalDelay(); + + packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + last_timestamp_ = timestamp; + + return relative_delay; +} + +void RelativeArrivalDelayTracker::Reset() { + delay_history_.clear(); + packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); + last_timestamp_ = absl::nullopt; +} + +void RelativeArrivalDelayTracker::UpdateDelayHistory(int iat_delay_ms, + uint32_t timestamp, + int sample_rate_hz) { + PacketDelay delay; + delay.iat_delay_ms = iat_delay_ms; + delay.timestamp = timestamp; + delay_history_.push_back(delay); + while (static_cast(timestamp - delay_history_.front().timestamp) > + max_history_ms_ * sample_rate_hz / 1000) { + delay_history_.pop_front(); + } +} + +int RelativeArrivalDelayTracker::CalculateRelativePacketArrivalDelay() const { + // This effectively calculates arrival delay of a packet relative to the + // packet preceding the history window. If the arrival delay ever becomes + // smaller than zero, it means the reference packet is invalid, and we + // move the reference. + int relative_delay = 0; + for (const PacketDelay& delay : delay_history_) { + relative_delay += delay.iat_delay_ms; + relative_delay = std::max(relative_delay, 0); + } + return relative_delay; +} + +} // namespace webrtc diff --git a/modules/audio_coding/neteq/relative_arrival_delay_tracker.h b/modules/audio_coding/neteq/relative_arrival_delay_tracker.h new file mode 100644 index 0000000000..da7f1218d9 --- /dev/null +++ b/modules/audio_coding/neteq/relative_arrival_delay_tracker.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_RELATIVE_ARRIVAL_DELAY_TRACKER_H_ +#define MODULES_AUDIO_CODING_NETEQ_RELATIVE_ARRIVAL_DELAY_TRACKER_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/neteq/tick_timer.h" + +namespace webrtc { + +class RelativeArrivalDelayTracker { + public: + RelativeArrivalDelayTracker(const TickTimer* tick_timer, int max_history_ms) + : tick_timer_(tick_timer), max_history_ms_(max_history_ms) {} + + absl::optional Update(uint32_t timestamp, int sample_rate_hz); + + void Reset(); + + private: + // Updates `delay_history_`. + void UpdateDelayHistory(int iat_delay_ms, + uint32_t timestamp, + int sample_rate_hz); + + // Calculate relative packet arrival delay from `delay_history_`. + int CalculateRelativePacketArrivalDelay() const; + + const TickTimer* tick_timer_; + const int max_history_ms_; + + struct PacketDelay { + int iat_delay_ms; + uint32_t timestamp; + }; + std::deque delay_history_; + + absl::optional + last_timestamp_; // Timestamp for the last received packet. + + std::unique_ptr + packet_iat_stopwatch_; // Time elapsed since last packet. +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_RELATIVE_ARRIVAL_DELAY_TRACKER_H_ diff --git a/modules/audio_coding/neteq/relative_arrival_delay_tracker_unittest.cc b/modules/audio_coding/neteq/relative_arrival_delay_tracker_unittest.cc new file mode 100644 index 0000000000..f764b85820 --- /dev/null +++ b/modules/audio_coding/neteq/relative_arrival_delay_tracker_unittest.cc @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/relative_arrival_delay_tracker.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr int kMaxHistoryMs = 2000; +constexpr int kFs = 8000; +constexpr int kFrameSizeMs = 20; +constexpr int kTsIncrement = kFrameSizeMs * kFs / 1000; +constexpr uint32_t kTs = 0x12345678; +} // namespace + +TEST(RelativeArrivalDelayTrackerTest, RelativeArrivalDelay) { + TickTimer tick_timer; + RelativeArrivalDelayTracker tracker(&tick_timer, kMaxHistoryMs); + + EXPECT_FALSE(tracker.Update(kTs, kFs)); + + tick_timer.Increment(kFrameSizeMs / tick_timer.ms_per_tick()); + EXPECT_EQ(tracker.Update(kTs + kTsIncrement, kFs), 0); + + tick_timer.Increment(2 * kFrameSizeMs / tick_timer.ms_per_tick()); + EXPECT_EQ(tracker.Update(kTs + 2 * kTsIncrement, kFs), 20); + + EXPECT_EQ(tracker.Update(kTs, kFs), 60); // Reordered, 60ms delayed. + + tick_timer.Increment(2 * kFrameSizeMs / tick_timer.ms_per_tick()); + EXPECT_EQ(tracker.Update(kTs + 3 * kTsIncrement, kFs), 40); +} + +TEST(RelativeArrivalDelayTrackerTest, ReorderedPackets) { + TickTimer tick_timer; + RelativeArrivalDelayTracker tracker(&tick_timer, kMaxHistoryMs); + + // Insert first packet. + EXPECT_FALSE(tracker.Update(kTs, kFs)); + + // Insert reordered packet. + EXPECT_EQ(tracker.Update(kTs - 4 * kTsIncrement, kFs), 80); + + // Insert another reordered packet. + EXPECT_EQ(tracker.Update(kTs - kTsIncrement, kFs), 20); + + // Insert the next packet in order and verify that the relative delay is + // estimated based on the first inserted packet. + tick_timer.Increment(4 * kFrameSizeMs / tick_timer.ms_per_tick()); + EXPECT_EQ(tracker.Update(kTs + kTsIncrement, kFs), 60); +} + +TEST(RelativeArrivalDelayTrackerTest, MaxDelayHistory) { + TickTimer tick_timer; + RelativeArrivalDelayTracker tracker(&tick_timer, kMaxHistoryMs); + + EXPECT_FALSE(tracker.Update(kTs, kFs)); + + // Insert 20 ms iat delay in the delay history. + tick_timer.Increment(2 * kFrameSizeMs / tick_timer.ms_per_tick()); + EXPECT_EQ(tracker.Update(kTs + kTsIncrement, kFs), 20); + + // Insert next packet with a timestamp difference larger than maximum history + // size. This removes the previously inserted iat delay from the history. + tick_timer.Increment((kMaxHistoryMs + kFrameSizeMs) / + tick_timer.ms_per_tick()); + EXPECT_EQ( + tracker.Update(kTs + 2 * kTsIncrement + kFs * kMaxHistoryMs / 1000, kFs), + 0); +} + +} // namespace webrtc diff --git a/modules/audio_coding/neteq/underrun_optimizer.cc b/modules/audio_coding/neteq/underrun_optimizer.cc new file mode 100644 index 0000000000..dad0424c48 --- /dev/null +++ b/modules/audio_coding/neteq/underrun_optimizer.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/underrun_optimizer.h" + +#include + +namespace webrtc { + +namespace { + +constexpr int kDelayBuckets = 100; +constexpr int kBucketSizeMs = 20; + +} // namespace + +UnderrunOptimizer::UnderrunOptimizer(const TickTimer* tick_timer, + int histogram_quantile, + int forget_factor, + absl::optional start_forget_weight, + absl::optional resample_interval_ms) + : tick_timer_(tick_timer), + histogram_(kDelayBuckets, forget_factor, start_forget_weight), + histogram_quantile_(histogram_quantile), + resample_interval_ms_(resample_interval_ms) {} + +void UnderrunOptimizer::Update(int relative_delay_ms) { + absl::optional histogram_update; + if (resample_interval_ms_) { + if (!resample_stopwatch_) { + resample_stopwatch_ = tick_timer_->GetNewStopwatch(); + } + if (static_cast(resample_stopwatch_->ElapsedMs()) > + *resample_interval_ms_) { + histogram_update = max_delay_in_interval_ms_; + resample_stopwatch_ = tick_timer_->GetNewStopwatch(); + max_delay_in_interval_ms_ = 0; + } + max_delay_in_interval_ms_ = + std::max(max_delay_in_interval_ms_, relative_delay_ms); + } else { + histogram_update = relative_delay_ms; + } + if (!histogram_update) { + return; + } + + const int index = *histogram_update / kBucketSizeMs; + if (index < histogram_.NumBuckets()) { + // Maximum delay to register is 2000 ms. + histogram_.Add(index); + } + int bucket_index = histogram_.Quantile(histogram_quantile_); + optimal_delay_ms_ = (1 + bucket_index) * kBucketSizeMs; +} + +void UnderrunOptimizer::Reset() { + histogram_.Reset(); + resample_stopwatch_ = tick_timer_->GetNewStopwatch(); + max_delay_in_interval_ms_ = 0; + optimal_delay_ms_.reset(); +} + +} // namespace webrtc diff --git a/modules/audio_coding/neteq/underrun_optimizer.h b/modules/audio_coding/neteq/underrun_optimizer.h new file mode 100644 index 0000000000..b37ce18795 --- /dev/null +++ b/modules/audio_coding/neteq/underrun_optimizer.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_UNDERRUN_OPTIMIZER_H_ +#define MODULES_AUDIO_CODING_NETEQ_UNDERRUN_OPTIMIZER_H_ + +#include + +#include "absl/types/optional.h" +#include "api/neteq/tick_timer.h" +#include "modules/audio_coding/neteq/histogram.h" + +namespace webrtc { + +// Estimates probability of buffer underrun due to late packet arrival. +// The optimal delay is decided such that the probability of underrun is lower +// than 1 - `histogram_quantile`. +class UnderrunOptimizer { + public: + UnderrunOptimizer(const TickTimer* tick_timer, + int histogram_quantile, + int forget_factor, + absl::optional start_forget_weight, + absl::optional resample_interval_ms); + + void Update(int relative_delay_ms); + + absl::optional GetOptimalDelayMs() const { return optimal_delay_ms_; } + + void Reset(); + + private: + const TickTimer* tick_timer_; + Histogram histogram_; + const int histogram_quantile_; // In Q30. + const absl::optional resample_interval_ms_; + std::unique_ptr resample_stopwatch_; + int max_delay_in_interval_ms_ = 0; + absl::optional optimal_delay_ms_; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_UNDERRUN_OPTIMIZER_H_ diff --git a/modules/audio_coding/neteq/underrun_optimizer_unittest.cc b/modules/audio_coding/neteq/underrun_optimizer_unittest.cc new file mode 100644 index 0000000000..a86e9cf107 --- /dev/null +++ b/modules/audio_coding/neteq/underrun_optimizer_unittest.cc @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/neteq/underrun_optimizer.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +constexpr int kDefaultHistogramQuantile = 1020054733; // 0.95 in Q30. +constexpr int kForgetFactor = 32745; // 0.9993 in Q15. + +} // namespace + +TEST(UnderrunOptimizerTest, ResamplePacketDelays) { + TickTimer tick_timer; + constexpr int kResampleIntervalMs = 500; + UnderrunOptimizer underrun_optimizer(&tick_timer, kDefaultHistogramQuantile, + kForgetFactor, absl::nullopt, + kResampleIntervalMs); + + // The histogram should be updated once with the maximum delay observed for + // the following sequence of updates. + for (int i = 0; i < 500; i += 20) { + underrun_optimizer.Update(i); + EXPECT_FALSE(underrun_optimizer.GetOptimalDelayMs()); + } + tick_timer.Increment(kResampleIntervalMs / tick_timer.ms_per_tick() + 1); + underrun_optimizer.Update(0); + EXPECT_EQ(underrun_optimizer.GetOptimalDelayMs(), 500); +} + +} // namespace webrtc