Include packet waiting time in concealment decision.

This is to be more robust to packet loss during DTX and paused streams.

Without it, we can wait to decode an available packet when in CNG or
PLC mode until more packets arrive, which for DTX and paused streams
can take a long time.

We already include the waiting time if the last packet in the buffer
is a DTX packet.

Bug: webrtc:13322
Change-Id: Iaf5b3894500140d6f83377ba2cd65b44e0cdac05
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/299009
Reviewed-by: Henrik Lundin <henrik.lundin@webrtc.org>
Commit-Queue: Jakob Ivarsson‎ <jakobi@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39667}

api/neteq/neteq_controller.h

@@ -81,7 +81,7 @@ class NetEqController {
     bool dtx_or_cng;
     size_t num_samples;
     size_t span_samples;
-    size_t span_samples_no_dtx;
+    size_t span_samples_wait_time;
     size_t num_packets;
   };
 

modules/audio_coding/neteq/decision_logic.cc

@@ -357,8 +357,11 @@ NetEq::Operation DecisionLogic::FuturePacketAvailable(
   // Check if we should continue with an ongoing concealment because the new
   // packet is too far into the future.
   if (config_.combine_concealment_decision || IsCng(status.last_mode)) {
-    const int buffer_delay_ms =
+    const int buffer_delay_samples =
-        status.packet_buffer_info.span_samples / sample_rate_khz_;
+        config_.combine_concealment_decision
+            ? status.packet_buffer_info.span_samples_wait_time
+            : status.packet_buffer_info.span_samples;
+    const int buffer_delay_ms = buffer_delay_samples / sample_rate_khz_;
     const bool above_target_delay = buffer_delay_ms > HighThresholdCng();
     const bool below_target_delay = buffer_delay_ms < LowThresholdCng();
     if ((PacketTooEarly(status) && !above_target_delay) ||
@@ -370,8 +373,7 @@ NetEq::Operation DecisionLogic::FuturePacketAvailable(
     if (config_.combine_concealment_decision) {
       if (timestamp_leap != status.generated_noise_samples) {
         // The delay was adjusted, reinitialize the buffer level filter.
-        buffer_level_filter_->SetFilteredBufferLevel(
+        buffer_level_filter_->SetFilteredBufferLevel(buffer_delay_samples);
-            status.packet_buffer_info.span_samples);
       }
     } else {
       time_stretched_cn_samples_ =
@@ -405,7 +407,11 @@ bool DecisionLogic::PostponeDecode(NetEqController::NetEqStatus status) const {
   // running out of data right away again.
   const size_t min_buffer_level_samples =
       TargetLevelMs() * sample_rate_khz_ * kPostponeDecodingLevel / 100;
-  if (status.packet_buffer_info.span_samples >= min_buffer_level_samples) {
+  const size_t buffer_level_samples =
+      config_.combine_concealment_decision
+          ? status.packet_buffer_info.span_samples_wait_time
+          : status.packet_buffer_info.span_samples;
+  if (buffer_level_samples >= min_buffer_level_samples) {
     return false;
   }
   // Don't postpone decoding if there is a future DTX packet in the packet

modules/audio_coding/neteq/decision_logic_unittest.cc

@@ -40,7 +40,7 @@ NetEqController::NetEqStatus CreateNetEqStatus(NetEq::Mode last_mode,
   status.expand_mutefactor = 0;
   status.packet_buffer_info.num_samples = current_delay_ms * kSamplesPerMs;
   status.packet_buffer_info.span_samples = current_delay_ms * kSamplesPerMs;
-  status.packet_buffer_info.span_samples_no_dtx =
+  status.packet_buffer_info.span_samples_wait_time =
       current_delay_ms * kSamplesPerMs;
   status.packet_buffer_info.dtx_or_cng = false;
   status.next_packet = {status.target_timestamp, false, false};

modules/audio_coding/neteq/neteq_impl.cc

@@ -1100,10 +1100,10 @@ int NetEqImpl::GetDecision(Operation* operation,
   status.packet_buffer_info.num_samples =
       packet_buffer_->NumSamplesInBuffer(decoder_frame_length_);
   status.packet_buffer_info.span_samples = packet_buffer_->GetSpanSamples(
-      decoder_frame_length_, last_output_sample_rate_hz_, true);
+      decoder_frame_length_, last_output_sample_rate_hz_, false);
-  status.packet_buffer_info.span_samples_no_dtx =
+  status.packet_buffer_info.span_samples_wait_time =
       packet_buffer_->GetSpanSamples(decoder_frame_length_,
-                                     last_output_sample_rate_hz_, false);
+                                     last_output_sample_rate_hz_, true);
   status.packet_buffer_info.num_packets = packet_buffer_->NumPacketsInBuffer();
   status.target_timestamp = sync_buffer_->end_timestamp();
   status.expand_mutefactor = expand_->MuteFactor(0);

modules/audio_coding/neteq/packet_buffer.cc

@@ -119,7 +119,7 @@ void PacketBuffer::PartialFlush(int target_level_ms,
   // We should avoid flushing to very low levels.
   target_level_samples = std::max(
       target_level_samples, smart_flushing_config_->target_level_threshold_ms);
-  while (GetSpanSamples(last_decoded_length, sample_rate, true) >
+  while (GetSpanSamples(last_decoded_length, sample_rate, false) >
              static_cast<size_t>(target_level_samples) ||
          buffer_.size() > max_number_of_packets_ / 2) {
     LogPacketDiscarded(PeekNextPacket()->priority.codec_level, stats);
@@ -160,7 +160,7 @@ int PacketBuffer::InsertPacket(Packet&& packet,
           : 0;
   const bool smart_flush =
       smart_flushing_config_.has_value() &&
-      GetSpanSamples(last_decoded_length, sample_rate, true) >= span_threshold;
+      GetSpanSamples(last_decoded_length, sample_rate, false) >= span_threshold;
   if (buffer_.size() >= max_number_of_packets_ || smart_flush) {
     size_t buffer_size_before_flush = buffer_.size();
     if (smart_flushing_config_.has_value()) {
@@ -370,17 +370,19 @@ size_t PacketBuffer::NumSamplesInBuffer(size_t last_decoded_length) const {
 
 size_t PacketBuffer::GetSpanSamples(size_t last_decoded_length,
                                     size_t sample_rate,
-                                    bool count_dtx_waiting_time) const {
+                                    bool count_waiting_time) const {
   if (buffer_.size() == 0) {
     return 0;
   }
 
   size_t span = buffer_.back().timestamp - buffer_.front().timestamp;
-  if (buffer_.back().frame && buffer_.back().frame->Duration() > 0) {
-    size_t duration = buffer_.back().frame->Duration();
-    if (count_dtx_waiting_time && buffer_.back().frame->IsDtxPacket()) {
   size_t waiting_time_samples = rtc::dchecked_cast<size_t>(
       buffer_.back().waiting_time->ElapsedMs() * (sample_rate / 1000));
+  if (count_waiting_time) {
+    span += waiting_time_samples;
+  } else if (buffer_.back().frame && buffer_.back().frame->Duration() > 0) {
+    size_t duration = buffer_.back().frame->Duration();
+    if (buffer_.back().frame->IsDtxPacket()) {
       duration = std::max(duration, waiting_time_samples);
     }
     span += duration;

modules/audio_coding/neteq/packet_buffer.h

@@ -150,7 +150,7 @@ class PacketBuffer {
   // across.
   virtual size_t GetSpanSamples(size_t last_decoded_length,
                                 size_t sample_rate,
-                                bool count_dtx_waiting_time) const;
+                                bool count_waiting_time) const;
 
   // Returns true if the packet buffer contains any DTX or CNG packets.
   virtual bool ContainsDtxOrCngPacket(

modules/audio_coding/neteq/packet_buffer_unittest.cc

@@ -871,7 +871,7 @@ TEST(PacketBuffer, GetSpanSamples) {
   constexpr int kPayloadSizeBytes = 1;  // Does not matter to this test;
   constexpr uint32_t kStartTimeStamp = 0xFFFFFFFE;  // Close to wrap around.
   constexpr int kSampleRateHz = 48000;
-  constexpr bool KCountDtxWaitingTime = false;
+  constexpr bool kCountWaitingTime = false;
   TickTimer tick_timer;
   PacketBuffer buffer(3, &tick_timer);
   PacketGenerator gen(0, kStartTimeStamp, 0, kFrameSizeSamples);
@@ -903,7 +903,7 @@ TEST(PacketBuffer, GetSpanSamples) {
   // input.
   EXPECT_EQ(kLastDecodedSizeSamples,
             buffer.GetSpanSamples(kLastDecodedSizeSamples, kSampleRateHz,
-                                  KCountDtxWaitingTime));
+                                  kCountWaitingTime));
 
   EXPECT_EQ(PacketBuffer::kOK,
             buffer.InsertPacket(/*packet=*/std::move(packet_2),
@@ -914,13 +914,46 @@ TEST(PacketBuffer, GetSpanSamples) {
                                 /*decoder_database=*/decoder_database));
 
   EXPECT_EQ(kFrameSizeSamples * 2,
-            buffer.GetSpanSamples(0, kSampleRateHz, KCountDtxWaitingTime));
+            buffer.GetSpanSamples(0, kSampleRateHz, kCountWaitingTime));
 
   // packet_2 has access to duration, and ignores last decoded duration as
   // input.
   EXPECT_EQ(kFrameSizeSamples * 2,
             buffer.GetSpanSamples(kLastDecodedSizeSamples, kSampleRateHz,
-                                  KCountDtxWaitingTime));
+                                  kCountWaitingTime));
+}
+
+TEST(PacketBuffer, GetSpanSamplesCountWaitingTime) {
+  constexpr size_t kFrameSizeSamples = 10;
+  constexpr int kPayloadSizeBytes = 1;  // Does not matter to this test;
+  constexpr uint32_t kStartTimeStamp = 0xFFFFFFFE;  // Close to wrap around.
+  constexpr int kSampleRateHz = 48000;
+  constexpr bool kCountWaitingTime = true;
+  constexpr size_t kLastDecodedSizeSamples = 0;
+  TickTimer tick_timer;
+  PacketBuffer buffer(3, &tick_timer);
+  PacketGenerator gen(0, kStartTimeStamp, 0, kFrameSizeSamples);
+  StrictMock<MockStatisticsCalculator> mock_stats;
+  MockDecoderDatabase decoder_database;
+
+  Packet packet = gen.NextPacket(kPayloadSizeBytes, nullptr);
+
+  EXPECT_EQ(PacketBuffer::kOK,
+            buffer.InsertPacket(/*packet=*/std::move(packet),
+                                /*stats=*/&mock_stats,
+                                /*last_decoded_length=*/kFrameSizeSamples,
+                                /*sample_rate=*/kSampleRateHz,
+                                /*target_level_ms=*/60,
+                                /*decoder_database=*/decoder_database));
+
+  EXPECT_EQ(0u, buffer.GetSpanSamples(kLastDecodedSizeSamples, kSampleRateHz,
+                                      kCountWaitingTime));
+
+  tick_timer.Increment();
+  EXPECT_EQ(480u, buffer.GetSpanSamples(0, kSampleRateHz, kCountWaitingTime));
+
+  tick_timer.Increment();
+  EXPECT_EQ(960u, buffer.GetSpanSamples(0, kSampleRateHz, kCountWaitingTime));
 }
 
 namespace {