From 7ff6ca584499d078df9a00ee20c9b11e90e64d66 Mon Sep 17 00:00:00 2001 From: Alex Narest Date: Wed, 7 Feb 2018 18:46:33 +0100 Subject: [PATCH] Adds voice concealment periods reporting to neteq_rtpplay. Change-Id: Ie5a89eacef8c1cf7d5a6220b045d2c331fef199e Bug: webrtc:8847 Change-Id: Ie5a89eacef8c1cf7d5a6220b045d2c331fef199e Reviewed-on: https://webrtc-review.googlesource.com/48100 Commit-Queue: Alex Narest Reviewed-by: Henrik Lundin Cr-Commit-Position: refs/heads/master@{#21950} --- modules/audio_coding/neteq/include/neteq.h | 2 + .../neteq/statistics_calculator.cc | 21 +++++-- .../neteq/statistics_calculator.h | 3 +- .../audio_coding/neteq/tools/neteq_rtpplay.cc | 59 ++++++++++++++++++- .../audio_coding/neteq/tools/neteq_test.cc | 4 ++ modules/audio_coding/neteq/tools/neteq_test.h | 1 + 6 files changed, 83 insertions(+), 7 deletions(-) diff --git a/modules/audio_coding/neteq/include/neteq.h b/modules/audio_coding/neteq/include/neteq.h index 0d47f21a06..f5bd8cd91c 100644 --- a/modules/audio_coding/neteq/include/neteq.h +++ b/modules/audio_coding/neteq/include/neteq.h @@ -69,6 +69,8 @@ struct NetEqLifetimeStatistics { uint64_t concealed_samples = 0; uint64_t concealment_events = 0; uint64_t jitter_buffer_delay_ms = 0; + // Below stat is not part of the spec. + uint64_t voice_concealed_samples = 0; }; enum NetEqPlayoutMode { diff --git a/modules/audio_coding/neteq/statistics_calculator.cc b/modules/audio_coding/neteq/statistics_calculator.cc index 7228a5778e..c698790b8a 100644 --- a/modules/audio_coding/neteq/statistics_calculator.cc +++ b/modules/audio_coding/neteq/statistics_calculator.cc @@ -153,34 +153,38 @@ void StatisticsCalculator::ResetMcu() { void StatisticsCalculator::ExpandedVoiceSamples(size_t num_samples, bool is_new_concealment_event) { expanded_speech_samples_ += num_samples; - ConcealedSamplesCorrection(rtc::dchecked_cast(num_samples)); + ConcealedSamplesCorrection(rtc::dchecked_cast(num_samples), true); lifetime_stats_.concealment_events += is_new_concealment_event; } void StatisticsCalculator::ExpandedNoiseSamples(size_t num_samples, bool is_new_concealment_event) { expanded_noise_samples_ += num_samples; - ConcealedSamplesCorrection(rtc::dchecked_cast(num_samples)); + ConcealedSamplesCorrection(rtc::dchecked_cast(num_samples), false); lifetime_stats_.concealment_events += is_new_concealment_event; } void StatisticsCalculator::ExpandedVoiceSamplesCorrection(int num_samples) { expanded_speech_samples_ = AddIntToSizeTWithLowerCap(num_samples, expanded_speech_samples_); - ConcealedSamplesCorrection(num_samples); + ConcealedSamplesCorrection(num_samples, true); } void StatisticsCalculator::ExpandedNoiseSamplesCorrection(int num_samples) { expanded_noise_samples_ = AddIntToSizeTWithLowerCap(num_samples, expanded_noise_samples_); - ConcealedSamplesCorrection(num_samples); + ConcealedSamplesCorrection(num_samples, false); } -void StatisticsCalculator::ConcealedSamplesCorrection(int num_samples) { +void StatisticsCalculator::ConcealedSamplesCorrection(int num_samples, + bool is_voice) { if (num_samples < 0) { // Store negative correction to subtract from future positive additions. // See also the function comment in the header file. concealed_samples_correction_ -= num_samples; + if (is_voice) { + voice_concealed_samples_correction_ -= num_samples; + } return; } @@ -188,6 +192,13 @@ void StatisticsCalculator::ConcealedSamplesCorrection(int num_samples) { std::min(static_cast(num_samples), concealed_samples_correction_); concealed_samples_correction_ -= canceled_out; lifetime_stats_.concealed_samples += num_samples - canceled_out; + + if (is_voice) { + const size_t voice_canceled_out = std::min( + static_cast(num_samples), voice_concealed_samples_correction_); + voice_concealed_samples_correction_ -= voice_canceled_out; + lifetime_stats_.voice_concealed_samples += num_samples - voice_canceled_out; + } } void StatisticsCalculator::PreemptiveExpandedSamples(size_t num_samples) { diff --git a/modules/audio_coding/neteq/statistics_calculator.h b/modules/audio_coding/neteq/statistics_calculator.h index c3d5c86cff..a06ddfba62 100644 --- a/modules/audio_coding/neteq/statistics_calculator.h +++ b/modules/audio_coding/neteq/statistics_calculator.h @@ -173,13 +173,14 @@ class StatisticsCalculator { // If the correction is negative, it is cached and will be subtracted against // future additions to the counter. This is meant to be called from // Expanded{Voice,Noise}Samples{Correction}. - void ConcealedSamplesCorrection(int num_samples); + void ConcealedSamplesCorrection(int num_samples, bool is_voice); // Calculates numerator / denominator, and returns the value in Q14. static uint16_t CalculateQ14Ratio(size_t numerator, uint32_t denominator); NetEqLifetimeStatistics lifetime_stats_; size_t concealed_samples_correction_ = 0; + size_t voice_concealed_samples_correction_ = 0; size_t preemptive_samples_; size_t accelerate_samples_; size_t added_zero_samples_; diff --git a/modules/audio_coding/neteq/tools/neteq_rtpplay.cc b/modules/audio_coding/neteq/tools/neteq_rtpplay.cc index de7ff2ea99..8c1fa38c47 100644 --- a/modules/audio_coding/neteq/tools/neteq_rtpplay.cc +++ b/modules/audio_coding/neteq/tools/neteq_rtpplay.cc @@ -129,6 +129,7 @@ DEFINE_bool(pythonplot, false, "Generates a python script for plotting the delay profile"); DEFINE_bool(help, false, "Prints this message"); +DEFINE_bool(concealment_events, false, "Prints concealment events"); // Maps a codec type to a printable name string. std::string CodecName(NetEqDecoder codec) { @@ -331,6 +332,21 @@ class StatsGetter : public NetEqGetAudioCallback { double max_waiting_time_ms = 0.0; }; + struct ConcealmentEvent { + uint64_t duration_ms; + size_t concealment_event_number; + int64_t time_from_previous_event_end_ms; + + friend std::ostream& operator<<(std::ostream& stream, + const ConcealmentEvent& concealment_event) { + stream << "ConcealmentEvent duration_ms:" << concealment_event.duration_ms + << " event_number:" << concealment_event.concealment_event_number + << " time_from_previous_event_end_ms:" + << concealment_event.time_from_previous_event_end_ms << "\n"; + return stream; + } + }; + // Takes a pointer to another callback object, which will be invoked after // this object finishes. This does not transfer ownership, and null is a // valid value. @@ -353,6 +369,31 @@ class StatsGetter : public NetEqGetAudioCallback { RTC_CHECK_EQ(neteq->NetworkStatistics(&stats), 0); stats_.push_back(stats); } + const auto lifetime_stat = neteq->GetLifetimeStatistics(); + if (current_concealment_event_ != lifetime_stat.concealment_events) { + if (last_event_end_time_ms_ > 0) { + // Do not account for the first event to avoid start of the call + // skewing. + ConcealmentEvent concealment_event; + uint64_t last_event_voice_concealed_samples = + lifetime_stat.voice_concealed_samples - + voice_concealed_samples_until_last_event_; + RTC_CHECK_GT(last_event_voice_concealed_samples, 0); + concealment_event.duration_ms = last_event_voice_concealed_samples / + (audio_frame.sample_rate_hz_ / 1000); + concealment_event.concealment_event_number = current_concealment_event_; + concealment_event.time_from_previous_event_end_ms = + time_now_ms - last_event_end_time_ms_; + concealment_events_.emplace_back(concealment_event); + voice_concealed_samples_until_last_event_ = + lifetime_stat.voice_concealed_samples; + } + last_event_end_time_ms_ = time_now_ms; + voice_concealed_samples_until_last_event_ = + lifetime_stat.voice_concealed_samples; + current_concealment_event_ = lifetime_stat.concealment_events; + } + if (other_callback_) { other_callback_->AfterGetAudio(time_now_ms, audio_frame, muted, neteq); } @@ -367,6 +408,12 @@ class StatsGetter : public NetEqGetAudioCallback { return sum_speech_expand / 16384.0 / stats_.size(); } + const std::vector& concealment_events() { + // Do not account for the last concealment event to avoid potential end + // call skewing. + return concealment_events_; + } + Stats AverageStats() const { Stats sum_stats = std::accumulate( stats_.begin(), stats_.end(), Stats(), @@ -416,6 +463,10 @@ class StatsGetter : public NetEqGetAudioCallback { NetEqGetAudioCallback* other_callback_; size_t counter_ = 0; std::vector stats_; + size_t current_concealment_event_ = 1; + uint64_t voice_concealed_samples_until_last_event_ = 0; + std::vector concealment_events_; + int64_t last_event_end_time_ms_ = 0; }; int RunTest(int argc, char* argv[]) { @@ -668,7 +719,13 @@ int RunTest(int argc, char* argv[]) { printf(" max_waiting_time_ms: %f ms\n", stats.max_waiting_time_ms); printf(" current_buffer_size_ms: %f ms\n", stats.current_buffer_size_ms); printf(" preferred_buffer_size_ms: %f ms\n", stats.preferred_buffer_size_ms); - + if (FLAG_concealment_events) { + std::cout << " concealment_events_ms:" + << "\n"; + for (auto concealment_event : stats_getter.concealment_events()) + std::cout << concealment_event; + std::cout << " end of concealment_events_ms\n"; + } return 0; } diff --git a/modules/audio_coding/neteq/tools/neteq_test.cc b/modules/audio_coding/neteq/tools/neteq_test.cc index 39d9549577..e6dd114e38 100644 --- a/modules/audio_coding/neteq/tools/neteq_test.cc +++ b/modules/audio_coding/neteq/tools/neteq_test.cc @@ -111,6 +111,10 @@ NetEqNetworkStatistics NetEqTest::SimulationStats() { return stats; } +NetEqLifetimeStatistics NetEqTest::LifetimeStats() const { + return neteq_->GetLifetimeStatistics(); +} + void NetEqTest::RegisterDecoders(const DecoderMap& codecs) { for (const auto& c : codecs) { RTC_CHECK_EQ( diff --git a/modules/audio_coding/neteq/tools/neteq_test.h b/modules/audio_coding/neteq/tools/neteq_test.h index 2c0a07caad..e645e42814 100644 --- a/modules/audio_coding/neteq/tools/neteq_test.h +++ b/modules/audio_coding/neteq/tools/neteq_test.h @@ -89,6 +89,7 @@ class NetEqTest { // Returns the statistics from NetEq. NetEqNetworkStatistics SimulationStats(); + NetEqLifetimeStatistics LifetimeStats() const; private: void RegisterDecoders(const DecoderMap& codecs);