From b3f1c5d2fe00ac2201e533345f55eee220d66876 Mon Sep 17 00:00:00 2001 From: "henrik.lundin" Date: Mon, 22 Aug 2016 15:39:53 -0700 Subject: [PATCH] Add NetEq::FilteredCurrentDelayMs() and use it in VoiceEngine The new method returns the current total delay (packet buffer and sync buffer) in ms, with smoothing applied to even out short-time fluctuations due to jitter. The packet buffer part of the delay is not updated during DTX/CNG periods. This CL also pipes the new metric through ACM and uses it in VoiceEngine. It replaces the previous method of estimating the buffer delay (where an inserted packet's RTP timestamp was compared with the last played timestamp from NetEq). The new method works better under periods of DTX/CNG. Review-Url: https://codereview.webrtc.org/2262203002 Cr-Commit-Position: refs/heads/master@{#13855} --- webrtc/modules/audio_coding/acm2/acm_receiver.cc | 4 ++++ webrtc/modules/audio_coding/acm2/acm_receiver.h | 7 +++++++ .../audio_coding/acm2/audio_coding_module.cc | 6 ++++++ .../audio_coding/include/audio_coding_module.h | 9 +++++++++ .../modules/audio_coding/neteq/include/neteq.h | 5 +++++ webrtc/modules/audio_coding/neteq/neteq_impl.cc | 16 ++++++++++++++++ webrtc/modules/audio_coding/neteq/neteq_impl.h | 2 ++ webrtc/voice_engine/channel.cc | 9 ++++----- 8 files changed, 53 insertions(+), 5 deletions(-) diff --git a/webrtc/modules/audio_coding/acm2/acm_receiver.cc b/webrtc/modules/audio_coding/acm2/acm_receiver.cc index bad484cda4..9a882aac37 100644 --- a/webrtc/modules/audio_coding/acm2/acm_receiver.cc +++ b/webrtc/modules/audio_coding/acm2/acm_receiver.cc @@ -309,6 +309,10 @@ rtc::Optional AcmReceiver::GetPlayoutTimestamp() { return neteq_->GetPlayoutTimestamp(); } +int AcmReceiver::FilteredCurrentDelayMs() const { + return neteq_->FilteredCurrentDelayMs(); +} + int AcmReceiver::LastAudioCodec(CodecInst* codec) const { rtc::CritScope lock(&crit_sect_); if (!last_audio_decoder_) { diff --git a/webrtc/modules/audio_coding/acm2/acm_receiver.h b/webrtc/modules/audio_coding/acm2/acm_receiver.h index f37212c067..e62e7144d6 100644 --- a/webrtc/modules/audio_coding/acm2/acm_receiver.h +++ b/webrtc/modules/audio_coding/acm2/acm_receiver.h @@ -201,6 +201,13 @@ class AcmReceiver { // The return value will be empty if no valid timestamp is available. rtc::Optional GetPlayoutTimestamp(); + // Returns the current total delay from NetEq (packet buffer and sync buffer) + // in ms, with smoothing applied to even out short-time fluctuations due to + // jitter. The packet buffer part of the delay is not updated during DTX/CNG + // periods. + // + int FilteredCurrentDelayMs() const; + // // Get the audio codec associated with the last non-CNG/non-DTMF received // payload. If no non-CNG/non-DTMF packet is received -1 is returned, diff --git a/webrtc/modules/audio_coding/acm2/audio_coding_module.cc b/webrtc/modules/audio_coding/acm2/audio_coding_module.cc index 3f404f736f..f719b6c59c 100644 --- a/webrtc/modules/audio_coding/acm2/audio_coding_module.cc +++ b/webrtc/modules/audio_coding/acm2/audio_coding_module.cc @@ -159,6 +159,8 @@ class AudioCodingModuleImpl final : public AudioCodingModule { rtc::Optional PlayoutTimestamp() override; + int FilteredCurrentDelayMs() const override; + // Get 10 milliseconds of raw audio data to play out, and // automatic resample to the requested frequency if > 0. int PlayoutData10Ms(int desired_freq_hz, @@ -1225,6 +1227,10 @@ rtc::Optional AudioCodingModuleImpl::PlayoutTimestamp() { return receiver_.GetPlayoutTimestamp(); } +int AudioCodingModuleImpl::FilteredCurrentDelayMs() const { + return receiver_.FilteredCurrentDelayMs(); +} + bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const { if (!encoder_stack_) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceAudioCoding, id_, diff --git a/webrtc/modules/audio_coding/include/audio_coding_module.h b/webrtc/modules/audio_coding/include/audio_coding_module.h index 30a17f72ea..5adbe60d00 100644 --- a/webrtc/modules/audio_coding/include/audio_coding_module.h +++ b/webrtc/modules/audio_coding/include/audio_coding_module.h @@ -678,6 +678,15 @@ class AudioCodingModule { // virtual rtc::Optional PlayoutTimestamp() = 0; + /////////////////////////////////////////////////////////////////////////// + // int FilteredCurrentDelayMs() + // Returns the current total delay from NetEq (packet buffer and sync buffer) + // in ms, with smoothing applied to even out short-time fluctuations due to + // jitter. The packet buffer part of the delay is not updated during DTX/CNG + // periods. + // + virtual int FilteredCurrentDelayMs() const = 0; + /////////////////////////////////////////////////////////////////////////// // int32_t PlayoutData10Ms( // Get 10 milliseconds of raw audio data for playout, at the given sampling diff --git a/webrtc/modules/audio_coding/neteq/include/neteq.h b/webrtc/modules/audio_coding/neteq/include/neteq.h index b03cff7765..1ae7664077 100644 --- a/webrtc/modules/audio_coding/neteq/include/neteq.h +++ b/webrtc/modules/audio_coding/neteq/include/neteq.h @@ -223,6 +223,11 @@ class NetEq { // Returns the current total delay (packet buffer and sync buffer) in ms. virtual int CurrentDelayMs() const = 0; + // Returns the current total delay (packet buffer and sync buffer) in ms, + // with smoothing applied to even out short-time fluctuations due to jitter. + // The packet buffer part of the delay is not updated during DTX/CNG periods. + virtual int FilteredCurrentDelayMs() const = 0; + // Sets the playout mode to |mode|. // Deprecated. Set the mode in the Config struct passed to the constructor. // TODO(henrik.lundin) Delete. diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc index 024c73d23d..9cd73b6409 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -353,6 +353,22 @@ int NetEqImpl::CurrentDelayMs() const { return delay_ms; } +int NetEqImpl::FilteredCurrentDelayMs() const { + rtc::CritScope lock(&crit_sect_); + // Calculate the filtered packet buffer level in samples. The value from + // |buffer_level_filter_| is in number of packets, represented in Q8. + const size_t packet_buffer_samples = + (buffer_level_filter_->filtered_current_level() * + decoder_frame_length_) >> + 8; + // Sum up the filtered packet buffer level with the future length of the sync + // buffer, and divide the sum by the sample rate. + const size_t delay_samples = + packet_buffer_samples + sync_buffer_->FutureLength(); + // The division below will truncate. The return value is in ms. + return static_cast(delay_samples) / rtc::CheckedDivExact(fs_hz_, 1000); +} + // Deprecated. // TODO(henrik.lundin) Delete. void NetEqImpl::SetPlayoutMode(NetEqPlayoutMode mode) { diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.h b/webrtc/modules/audio_coding/neteq/neteq_impl.h index e24dfb9e9b..91770c5995 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.h +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.h @@ -146,6 +146,8 @@ class NetEqImpl : public webrtc::NetEq { int CurrentDelayMs() const override; + int FilteredCurrentDelayMs() const override; + // Sets the playout mode to |mode|. // Deprecated. // TODO(henrik.lundin) Delete. diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc index bbc63895ee..c2e0891555 100644 --- a/webrtc/voice_engine/channel.cc +++ b/webrtc/voice_engine/channel.cc @@ -3173,11 +3173,7 @@ void Channel::GetDecodingCallStatistics(AudioDecodingCallStats* stats) const { bool Channel::GetDelayEstimate(int* jitter_buffer_delay_ms, int* playout_buffer_delay_ms) const { rtc::CritScope lock(&video_sync_lock_); - if (_average_jitter_buffer_delay_us == 0) { - return false; - } - *jitter_buffer_delay_ms = - (_average_jitter_buffer_delay_us + 500) / 1000 + _recPacketDelayMs; + *jitter_buffer_delay_ms = audio_coding_->FilteredCurrentDelayMs(); *playout_buffer_delay_ms = playout_delay_ms_; return true; } @@ -3390,6 +3386,9 @@ void Channel::UpdatePlayoutTimestamp(bool rtcp) { } // Called for incoming RTP packets after successful RTP header parsing. +// TODO(henrik.lundin): Clean out this method. With the introduction of +// AudioCoding::FilteredCurrentDelayMs() most (if not all) of this method can +// be deleted, along with a few member variables. (WebRTC issue 6237.) void Channel::UpdatePacketDelay(uint32_t rtp_timestamp, uint16_t sequence_number) { WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),