diff --git a/modules/audio_coding/neteq/delay_manager.cc b/modules/audio_coding/neteq/delay_manager.cc index c97d5356d4..2f397d57d6 100644 --- a/modules/audio_coding/neteq/delay_manager.cc +++ b/modules/audio_coding/neteq/delay_manager.cc @@ -22,6 +22,7 @@ #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/numerics/safe_minmax.h" #include "system_wrappers/include/field_trial.h" namespace { @@ -31,6 +32,8 @@ constexpr int kLimitProbabilityStreaming = 536871; // 1/2000 in Q30. constexpr int kMaxStreamingPeakPeriodMs = 600000; // 10 minutes in ms. constexpr int kCumulativeSumDrift = 2; // Drift term for cumulative sum // |iat_cumulative_sum_|. +constexpr int kMinBaseMinimumDelayMs = 0; +constexpr int kMaxBaseMinimumDelayMs = 10000; // Steady-state forgetting factor for |iat_vector_|, 0.9993 in Q15. constexpr int kIatFactor_ = 32745; constexpr int kMaxIat = 64; // Max inter-arrival time to register. @@ -64,7 +67,7 @@ absl::optional GetForcedLimitProbability() { namespace webrtc { DelayManager::DelayManager(size_t max_packets_in_buffer, - int base_min_target_delay_ms, + int base_minimum_delay_ms, bool enable_rtx_handling, DelayPeakDetector* peak_detector, const TickTimer* tick_timer) @@ -73,14 +76,15 @@ DelayManager::DelayManager(size_t max_packets_in_buffer, iat_vector_(kMaxIat + 1, 0), iat_factor_(0), tick_timer_(tick_timer), - base_min_target_delay_ms_(base_min_target_delay_ms), + base_minimum_delay_ms_(base_minimum_delay_ms), + effective_minimum_delay_ms_(base_minimum_delay_ms), base_target_level_(4), // In Q0 domain. target_level_(base_target_level_ << 8), // In Q8 domain. packet_len_ms_(0), streaming_mode_(false), last_seq_no_(0), last_timestamp_(0), - minimum_delay_ms_(base_min_target_delay_ms_), + minimum_delay_ms_(base_minimum_delay_ms_), maximum_delay_ms_(target_level_), iat_cumulative_sum_(0), max_iat_cumulative_sum_(0), @@ -91,7 +95,7 @@ DelayManager::DelayManager(size_t max_packets_in_buffer, forced_limit_probability_(GetForcedLimitProbability()), enable_rtx_handling_(enable_rtx_handling) { assert(peak_detector); // Should never be NULL. - RTC_DCHECK_GE(base_min_target_delay_ms_, 0); + RTC_DCHECK_GE(base_minimum_delay_ms_, 0); RTC_DCHECK_LE(minimum_delay_ms_, maximum_delay_ms_); Reset(); @@ -283,12 +287,13 @@ void DelayManager::UpdateHistogram(size_t iat_packets) { // |maximum_delay_ms_| in packets. Note that in practice, if no // |maximum_delay_ms_| is specified, this does not have any impact, since the // target level is far below the buffer capacity in all reasonable cases. -// The lower limit is equivalent of |minimum_delay_ms_| in packets. We update -// |least_required_level_| while the above limits are applied. +// The lower limit is equivalent of |effective_minimum_delay_ms_| in packets. +// We update |least_required_level_| while the above limits are applied. // TODO(hlundin): Move this check to the buffer logistics class. void DelayManager::LimitTargetLevel() { - if (packet_len_ms_ > 0 && minimum_delay_ms_ > 0) { - int minimum_delay_packet_q8 = (minimum_delay_ms_ << 8) / packet_len_ms_; + if (packet_len_ms_ > 0 && effective_minimum_delay_ms_ > 0) { + int minimum_delay_packet_q8 = + (effective_minimum_delay_ms_ << 8) / packet_len_ms_; target_level_ = std::max(target_level_, minimum_delay_packet_q8); } @@ -492,49 +497,51 @@ DelayManager::IATVector DelayManager::ScaleHistogram(const IATVector& histogram, return new_histogram; } -bool DelayManager::IsValidMinimumDelay(int delay_ms) { - const int q75 = - rtc::dchecked_cast(3 * max_packets_in_buffer_ * packet_len_ms_ / 4); - return delay_ms >= 0 && - (maximum_delay_ms_ <= 0 || delay_ms <= maximum_delay_ms_) && - (packet_len_ms_ <= 0 || delay_ms <= q75); +bool DelayManager::IsValidMinimumDelay(int delay_ms) const { + return 0 <= delay_ms && delay_ms <= MinimumDelayUpperBound(); +} + +bool DelayManager::IsValidBaseMinimumDelay(int delay_ms) const { + return kMinBaseMinimumDelayMs <= delay_ms && + delay_ms <= kMaxBaseMinimumDelayMs; } bool DelayManager::SetMinimumDelay(int delay_ms) { if (!IsValidMinimumDelay(delay_ms)) { return false; } - // Ensure that minimum_delay_ms is no bigger than base_min_target_delay_ms_ - minimum_delay_ms_ = std::max(delay_ms, base_min_target_delay_ms_); + + minimum_delay_ms_ = delay_ms; + UpdateEffectiveMinimumDelay(); return true; } bool DelayManager::SetMaximumDelay(int delay_ms) { - if (delay_ms == 0) { - // Zero input unsets the maximum delay. - maximum_delay_ms_ = 0; - return true; - } else if (delay_ms < minimum_delay_ms_ || delay_ms < packet_len_ms_) { + // If |delay_ms| is zero then it unsets the maximum delay and target level is + // unconstrained by maximum delay. + if (delay_ms != 0 && + (delay_ms < minimum_delay_ms_ || delay_ms < packet_len_ms_)) { // Maximum delay shouldn't be less than minimum delay or less than a packet. return false; } + maximum_delay_ms_ = delay_ms; + UpdateEffectiveMinimumDelay(); return true; } bool DelayManager::SetBaseMinimumDelay(int delay_ms) { - if (!IsValidMinimumDelay(delay_ms)) { + if (!IsValidBaseMinimumDelay(delay_ms)) { return false; } - base_min_target_delay_ms_ = delay_ms; - // Ensure that minimum_delay_ms is no bigger than base_min_target_delay_ms_ - minimum_delay_ms_ = std::max(delay_ms, base_min_target_delay_ms_); + base_minimum_delay_ms_ = delay_ms; + UpdateEffectiveMinimumDelay(); return true; } int DelayManager::GetBaseMinimumDelay() const { - return base_min_target_delay_ms_; + return base_minimum_delay_ms_; } int DelayManager::base_target_level() const { @@ -550,4 +557,28 @@ int DelayManager::last_pack_cng_or_dtmf() const { void DelayManager::set_last_pack_cng_or_dtmf(int value) { last_pack_cng_or_dtmf_ = value; } + +void DelayManager::UpdateEffectiveMinimumDelay() { + // Clamp |base_minimum_delay_ms_| into the range which can be effectively + // used. + const int base_minimum_delay_ms = + rtc::SafeClamp(base_minimum_delay_ms_, 0, MinimumDelayUpperBound()); + effective_minimum_delay_ms_ = + std::max(minimum_delay_ms_, base_minimum_delay_ms); +} + +int DelayManager::MinimumDelayUpperBound() const { + // Choose the lowest possible bound discarding 0 cases which mean the value + // is not set and unconstrained. + int q75 = MaxBufferTimeQ75(); + q75 = q75 > 0 ? q75 : kMaxBaseMinimumDelayMs; + const int maximum_delay_ms = + maximum_delay_ms_ > 0 ? maximum_delay_ms_ : kMaxBaseMinimumDelayMs; + return std::min(maximum_delay_ms, q75); +} + +int DelayManager::MaxBufferTimeQ75() const { + const int max_buffer_time = max_packets_in_buffer_ * packet_len_ms_; + return rtc::dchecked_cast(3 * max_buffer_time / 4); +} } // namespace webrtc diff --git a/modules/audio_coding/neteq/delay_manager.h b/modules/audio_coding/neteq/delay_manager.h index 652a7736b7..59f9d379f1 100644 --- a/modules/audio_coding/neteq/delay_manager.h +++ b/modules/audio_coding/neteq/delay_manager.h @@ -32,10 +32,10 @@ class DelayManager { // Create a DelayManager object. Notify the delay manager that the packet // buffer can hold no more than |max_packets_in_buffer| packets (i.e., this // is the number of packet slots in the buffer) and that the target delay - // should be greater than or equal to |base_min_target_delay_ms|. Supply a + // should be greater than or equal to |base_minimum_delay_ms|. Supply a // PeakDetector object to the DelayManager. DelayManager(size_t max_packets_in_buffer, - int base_min_target_delay_ms, + int base_minimum_delay_ms, bool enable_rtx_handling, DelayPeakDetector* peak_detector, const TickTimer* tick_timer); @@ -125,7 +125,19 @@ class DelayManager { return forced_limit_probability_; } + // This accessor is only intended for testing purposes. + int effective_minimum_delay_ms_for_test() const { + return effective_minimum_delay_ms_; + } + private: + // Provides value which minimum delay can't exceed based on current buffer + // size and given |maximum_delay_ms_|. Lower bound is a constant 0. + int MinimumDelayUpperBound() const; + + // Provides 75% of currently possible maximum buffer size in milliseconds. + int MaxBufferTimeQ75() const; + // Sets |iat_vector_| to the default start distribution and sets the // |base_target_level_| and |target_level_| to the corresponding values. void ResetHistogram(); @@ -134,6 +146,11 @@ class DelayManager { // used by the streaming mode.) This method is called by Update(). void UpdateCumulativeSums(int packet_len_ms, uint16_t sequence_number); + // Updates |effective_minimum_delay_ms_| delay based on current + // |minimum_delay_ms_|, |base_minimum_delay_ms_| and |maximum_delay_ms_| + // and buffer size. + void UpdateEffectiveMinimumDelay(); + // Updates the histogram |iat_vector_|. The probability for inter-arrival time // equal to |iat_packets| (in integer packets) is increased slightly, while // all other entries are decreased. This method is called by Update(). @@ -147,15 +164,20 @@ class DelayManager { // Makes sure that |delay_ms| is less than maximum delay, if any maximum // is set. Also, if possible check |delay_ms| to be less than 75% of // |max_packets_in_buffer_|. - bool IsValidMinimumDelay(int delay_ms); + bool IsValidMinimumDelay(int delay_ms) const; + + bool IsValidBaseMinimumDelay(int delay_ms) const; bool first_packet_received_; const size_t max_packets_in_buffer_; // Capacity of the packet buffer. IATVector iat_vector_; // Histogram of inter-arrival times. int iat_factor_; // Forgetting factor for updating the IAT histogram (Q15). const TickTimer* tick_timer_; - int base_min_target_delay_ms_; // Lower bound for target_level_ and - // minimum_delay_ms_. + int base_minimum_delay_ms_; + // Provides delay which is used by LimitTargetLevel as lower bound on target + // delay. + int effective_minimum_delay_ms_; + // Time elapsed since last packet. std::unique_ptr packet_iat_stopwatch_; int base_target_level_; // Currently preferred buffer level before peak diff --git a/modules/audio_coding/neteq/delay_manager_unittest.cc b/modules/audio_coding/neteq/delay_manager_unittest.cc index e33b2aa894..f2656b817e 100644 --- a/modules/audio_coding/neteq/delay_manager_unittest.cc +++ b/modules/audio_coding/neteq/delay_manager_unittest.cc @@ -15,6 +15,7 @@ #include #include "modules/audio_coding/neteq/mock/mock_delay_peak_detector.h" +#include "rtc_base/checks.h" #include "test/field_trial.h" #include "test/gmock.h" #include "test/gtest.h" @@ -32,6 +33,7 @@ class DelayManagerTest : public ::testing::Test { static const int kFs = 8000; static const int kFrameSizeMs = 20; static const int kTsIncrement = kFrameSizeMs * kFs / 1000; + static const int kMaxBufferSizeMs = kMaxNumberOfPackets * kFrameSizeMs; DelayManagerTest(); virtual void SetUp(); @@ -268,7 +270,130 @@ TEST_F(DelayManagerTest, MinDelay) { EXPECT_EQ(kMinDelayPackets << 8, dm_->TargetLevel()); } -TEST_F(DelayManagerTest, BaseMinDelay) { +TEST_F(DelayManagerTest, BaseMinimumDelayCheckValidRange) { + SetPacketAudioLength(kFrameSizeMs); + + // Base minimum delay should be between [0, 10000] milliseconds. + EXPECT_FALSE(dm_->SetBaseMinimumDelay(-1)); + EXPECT_FALSE(dm_->SetBaseMinimumDelay(10001)); + EXPECT_EQ(dm_->GetBaseMinimumDelay(), 0); + + EXPECT_TRUE(dm_->SetBaseMinimumDelay(7999)); + EXPECT_EQ(dm_->GetBaseMinimumDelay(), 7999); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayLowerThanMinimumDelay) { + SetPacketAudioLength(kFrameSizeMs); + constexpr int kBaseMinimumDelayMs = 100; + constexpr int kMinimumDelayMs = 200; + + // Base minimum delay sets lower bound on minimum. That is why when base + // minimum delay is lower than minimum delay we use minimum delay. + RTC_DCHECK_LT(kBaseMinimumDelayMs, kMinimumDelayMs); + + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); + EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMinimumDelayMs); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanMinimumDelay) { + SetPacketAudioLength(kFrameSizeMs); + constexpr int kBaseMinimumDelayMs = 70; + constexpr int kMinimumDelayMs = 30; + + // Base minimum delay sets lower bound on minimum. That is why when base + // minimum delay is greater than minimum delay we use base minimum delay. + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); + + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); + EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMs); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanBufferSize) { + SetPacketAudioLength(kFrameSizeMs); + constexpr int kBaseMinimumDelayMs = kMaxBufferSizeMs + 1; + constexpr int kMinimumDelayMs = 12; + constexpr int kMaxBufferSizeMsQ75 = 3 * kMaxBufferSizeMs / 4; + + // Base minimum delay is greater than minimum delay, that is why we clamp + // it to current the highest possible value which is maximum delay. + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMaxBufferSizeMs); + + EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); + + // Unset maximum value. + EXPECT_TRUE(dm_->SetMaximumDelay(0)); + + // With maximum value unset, the highest possible value now is 75% of + // currently possible maximum buffer size. + EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMaxBufferSizeMsQ75); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayGreaterThanMaximumDelay) { + SetPacketAudioLength(kFrameSizeMs); + constexpr int kMaximumDelayMs = 400; + constexpr int kBaseMinimumDelayMs = kMaximumDelayMs + 1; + constexpr int kMinimumDelayMs = 20; + + // Base minimum delay is greater than minimum delay, that is why we clamp + // it to current the highest possible value which is kMaximumDelayMs. + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMaximumDelayMs); + RTC_DCHECK_LT(kMaximumDelayMs, kMaxBufferSizeMs); + + EXPECT_TRUE(dm_->SetMaximumDelay(kMaximumDelayMs)); + EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMaximumDelayMs); +} + +TEST_F(DelayManagerTest, BaseMinimumDelayLowerThanMaxSize) { + SetPacketAudioLength(kFrameSizeMs); + constexpr int kMaximumDelayMs = 400; + constexpr int kBaseMinimumDelayMs = kMaximumDelayMs - 1; + constexpr int kMinimumDelayMs = 20; + + // Base minimum delay is greater than minimum delay, and lower than maximum + // delays that is why it is used. + RTC_DCHECK_GT(kBaseMinimumDelayMs, kMinimumDelayMs); + RTC_DCHECK_LT(kBaseMinimumDelayMs, kMaximumDelayMs); + + EXPECT_TRUE(dm_->SetMaximumDelay(kMaximumDelayMs)); + EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kBaseMinimumDelayMs); +} + +TEST_F(DelayManagerTest, MinimumDelayMemorization) { + // Check that when we increase base minimum delay to value higher than + // minimum delay then minimum delay is still memorized. This allows to + // restore effective minimum delay to memorized minimum delay value when we + // decrease base minimum delay. + SetPacketAudioLength(kFrameSizeMs); + + constexpr int kBaseMinimumDelayMsLow = 10; + constexpr int kMinimumDelayMs = 20; + constexpr int kBaseMinimumDelayMsHigh = 30; + + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMsLow)); + EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); + // Minimum delay is used as it is higher than base minimum delay. + EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMinimumDelayMs); + + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMsHigh)); + // Base minimum delay is used as it is now higher than minimum delay. + EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), + kBaseMinimumDelayMsHigh); + + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMsLow)); + // Check that minimum delay is memorized and is used again. + EXPECT_EQ(dm_->effective_minimum_delay_ms_for_test(), kMinimumDelayMs); +} + +TEST_F(DelayManagerTest, BaseMinimumDelay) { const int kExpectedTarget = 5; const int kTimeIncrement = kExpectedTarget * kFrameSizeMs; SetPacketAudioLength(kFrameSizeMs); @@ -287,29 +412,53 @@ TEST_F(DelayManagerTest, BaseMinDelay) { // No limit is applied. EXPECT_EQ(kExpectedTarget << 8, dm_->TargetLevel()); - int kBaseMinDelayPackets = kExpectedTarget + 2; - int kBaseMinDelayMs = kBaseMinDelayPackets * kFrameSizeMs; - EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinDelayMs)); - EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinDelayMs); + constexpr int kBaseMinimumDelayPackets = kExpectedTarget + 2; + constexpr int kBaseMinimumDelayMs = kBaseMinimumDelayPackets * kFrameSizeMs; + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinimumDelayMs); IncreaseTime(kTimeIncrement); InsertNextPacket(); - EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinDelayMs); - EXPECT_EQ(kBaseMinDelayPackets << 8, dm_->TargetLevel()); + EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinimumDelayMs); + EXPECT_EQ(kBaseMinimumDelayPackets << 8, dm_->TargetLevel()); } -TEST_F(DelayManagerTest, BaseMinDelayGreaterThanMaxDelayIsInvalid) { - int kMaxDelayMs = 2 * kFrameSizeMs; - int kBaseMinDelayMs = 4 * kFrameSizeMs; - EXPECT_TRUE(dm_->SetMaximumDelay(kMaxDelayMs)); - EXPECT_FALSE(dm_->SetBaseMinimumDelay(kBaseMinDelayMs)); -} +TEST_F(DelayManagerTest, BaseMinimumDealyAffectTargetLevel) { + const int kExpectedTarget = 5; + const int kTimeIncrement = kExpectedTarget * kFrameSizeMs; + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to |kExpectedTarget|. Return true to indicate peaks found. + EXPECT_CALL(detector_, Update(kExpectedTarget, false, _)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(detector_, MaxPeakHeight()) + .WillRepeatedly(Return(kExpectedTarget)); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); -TEST_F(DelayManagerTest, BaseMinDelayGreaterThanQ75MaxPacketsIsInvalid) { - // .75 of |max_packets_in_buffer|, + 1 to ensure that |kBaseMinDelayMs| is - // greater. - int kBaseMinDelayMs = (3 * kMaxNumberOfPackets * kFrameSizeMs / 4) + 1; - EXPECT_FALSE(dm_->SetBaseMinimumDelay(kBaseMinDelayMs)); + // No limit is applied. + EXPECT_EQ(kExpectedTarget << 8, dm_->TargetLevel()); + + // Minimum delay is lower than base minimum delay, that is why base minimum + // delay is used to calculate target level. + constexpr int kMinimumDelayPackets = kExpectedTarget + 1; + constexpr int kBaseMinimumDelayPackets = kExpectedTarget + 2; + + constexpr int kMinimumDelayMs = kMinimumDelayPackets * kFrameSizeMs; + constexpr int kBaseMinimumDelayMs = kBaseMinimumDelayPackets * kFrameSizeMs; + + EXPECT_TRUE(kMinimumDelayMs < kBaseMinimumDelayMs); + EXPECT_TRUE(dm_->SetMinimumDelay(kMinimumDelayMs)); + EXPECT_TRUE(dm_->SetBaseMinimumDelay(kBaseMinimumDelayMs)); + EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinimumDelayMs); + + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + EXPECT_EQ(dm_->GetBaseMinimumDelay(), kBaseMinimumDelayMs); + EXPECT_EQ(kBaseMinimumDelayPackets << 8, dm_->TargetLevel()); } TEST_F(DelayManagerTest, UpdateReorderedPacket) {