From bef77e234fa53a52b830b5833948711f75ab8bbb Mon Sep 17 00:00:00 2001 From: Henrik Lundin Date: Tue, 18 Aug 2015 14:58:09 +0200 Subject: [PATCH] NetEq: Implement logging of Delayed Packet Outage Events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures the duration of each packet loss concealment (a.k.a. expand) event that is not followed by a merge operation. Having decoded and played packet m−1, the next expected packet is m. If packet m arrives after some time of packet loss concealment, we have a delayed packet outage event. However, if instead packet n>m arrives, we have a lost packet outage event. In NetEq, the two outage types results in different operations. Both types start with expand operations to generate audio to play while the buffer is empty. When a lost packet outage happens, the expand operation(s) are followed by one merge operation. For delayed packet outages, merge is not done, and the expand operations are immediately followed by normal operations. This change also includes unit tests for the new statistics. BUG=webrtc:4915, chromium:488124 R=minyue@webrtc.org Review URL: https://codereview.webrtc.org/1290113002 . Cr-Commit-Position: refs/heads/master@{#9725} --- webrtc/modules/audio_coding/neteq/expand.cc | 17 ++- webrtc/modules/audio_coding/neteq/expand.h | 11 +- .../audio_coding/neteq/expand_unittest.cc | 134 +++++++++++++++++- .../audio_coding/neteq/merge_unittest.cc | 4 +- .../audio_coding/neteq/mock/mock_expand.h | 12 +- .../modules/audio_coding/neteq/neteq_impl.cc | 2 +- .../audio_coding/neteq/normal_unittest.cc | 12 +- .../neteq/statistics_calculator.cc | 7 + .../neteq/statistics_calculator.h | 5 + .../neteq/tools/input_audio_file.cc | 21 +++ .../neteq/tools/input_audio_file.h | 6 + 11 files changed, 215 insertions(+), 16 deletions(-) diff --git a/webrtc/modules/audio_coding/neteq/expand.cc b/webrtc/modules/audio_coding/neteq/expand.cc index ae12e50461..d01465ab60 100644 --- a/webrtc/modules/audio_coding/neteq/expand.cc +++ b/webrtc/modules/audio_coding/neteq/expand.cc @@ -16,10 +16,12 @@ #include // min, max #include // numeric_limits +#include "webrtc/base/safe_conversions.h" #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" #include "webrtc/modules/audio_coding/neteq/background_noise.h" #include "webrtc/modules/audio_coding/neteq/dsp_helper.h" #include "webrtc/modules/audio_coding/neteq/random_vector.h" +#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" namespace webrtc { @@ -27,6 +29,7 @@ namespace webrtc { Expand::Expand(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels) : random_vector_(random_vector), @@ -36,10 +39,12 @@ Expand::Expand(BackgroundNoise* background_noise, num_channels_(num_channels), consecutive_expands_(0), background_noise_(background_noise), + statistics_(statistics), overlap_length_(5 * fs / 8000), lag_index_direction_(0), current_lag_index_(0), stop_muting_(false), + expand_duration_samples_(0), channel_parameters_(new ChannelParameters[num_channels_]) { assert(fs == 8000 || fs == 16000 || fs == 32000 || fs == 48000); assert(fs <= kMaxSampleRate); // Should not be possible. @@ -78,6 +83,7 @@ int Expand::Process(AudioMultiVector* output) { // Perform initial setup if this is the first expansion since last reset. AnalyzeSignal(random_vector); first_expand_ = false; + expand_duration_samples_ = 0; } else { // This is not the first expansion, parameters are already estimated. // Extract a noise segment. @@ -298,6 +304,10 @@ int Expand::Process(AudioMultiVector* output) { // Increase call number and cap it. consecutive_expands_ = consecutive_expands_ >= kMaxConsecutiveExpands ? kMaxConsecutiveExpands : consecutive_expands_ + 1; + expand_duration_samples_ += output->Size(); + // Clamp the duration counter at 2 seconds. + expand_duration_samples_ = + std::min(expand_duration_samples_, rtc::checked_cast(fs_hz_ * 2)); return 0; } @@ -305,6 +315,8 @@ void Expand::SetParametersForNormalAfterExpand() { current_lag_index_ = 0; lag_index_direction_ = 0; stop_muting_ = true; // Do not mute signal any more. + statistics_->LogDelayedPacketOutageEvent( + rtc::checked_cast(expand_duration_samples_) / (fs_hz_ / 1000)); } void Expand::SetParametersForMergeAfterExpand() { @@ -833,10 +845,11 @@ void Expand::UpdateLagIndex() { Expand* ExpandFactory::Create(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels) const { - return new Expand(background_noise, sync_buffer, random_vector, fs, - num_channels); + return new Expand(background_noise, sync_buffer, random_vector, statistics, + fs, num_channels); } // TODO(turajs): This can be moved to BackgroundNoise class. diff --git a/webrtc/modules/audio_coding/neteq/expand.h b/webrtc/modules/audio_coding/neteq/expand.h index 5fb117d519..3fbafdb97f 100644 --- a/webrtc/modules/audio_coding/neteq/expand.h +++ b/webrtc/modules/audio_coding/neteq/expand.h @@ -23,6 +23,7 @@ namespace webrtc { // Forward declarations. class BackgroundNoise; class RandomVector; +class StatisticsCalculator; class SyncBuffer; // This class handles extrapolation of audio data from the sync_buffer to @@ -34,6 +35,7 @@ class Expand { Expand(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels); @@ -86,8 +88,8 @@ class Expand { // necessary to produce concealment data. void AnalyzeSignal(int16_t* random_vector); - RandomVector* random_vector_; - SyncBuffer* sync_buffer_; + RandomVector* const random_vector_; + SyncBuffer* const sync_buffer_; bool first_expand_; const int fs_hz_; const size_t num_channels_; @@ -127,13 +129,15 @@ class Expand { void UpdateLagIndex(); - BackgroundNoise* background_noise_; + BackgroundNoise* const background_noise_; + StatisticsCalculator* const statistics_; const size_t overlap_length_; int16_t max_lag_; size_t expand_lags_[kNumLags]; int lag_index_direction_; int current_lag_index_; bool stop_muting_; + size_t expand_duration_samples_; rtc::scoped_ptr channel_parameters_; DISALLOW_COPY_AND_ASSIGN(Expand); @@ -146,6 +150,7 @@ struct ExpandFactory { virtual Expand* Create(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels) const; }; diff --git a/webrtc/modules/audio_coding/neteq/expand_unittest.cc b/webrtc/modules/audio_coding/neteq/expand_unittest.cc index 68b4f60f15..1441704102 100644 --- a/webrtc/modules/audio_coding/neteq/expand_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/expand_unittest.cc @@ -13,9 +13,14 @@ #include "webrtc/modules/audio_coding/neteq/expand.h" #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/safe_conversions.h" +#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" #include "webrtc/modules/audio_coding/neteq/background_noise.h" #include "webrtc/modules/audio_coding/neteq/random_vector.h" +#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" +#include "webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "webrtc/test/testsupport/fileutils.h" namespace webrtc { @@ -25,7 +30,8 @@ TEST(Expand, CreateAndDestroy) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; - Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); } TEST(Expand, CreateUsingFactory) { @@ -34,13 +40,135 @@ TEST(Expand, CreateUsingFactory) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; + StatisticsCalculator statistics; ExpandFactory expand_factory; - Expand* expand = - expand_factory.Create(&bgn, &sync_buffer, &random_vector, fs, channels); + Expand* expand = expand_factory.Create(&bgn, &sync_buffer, &random_vector, + &statistics, fs, channels); EXPECT_TRUE(expand != NULL); delete expand; } +namespace { +class FakeStatisticsCalculator : public StatisticsCalculator { + public: + void LogDelayedPacketOutageEvent(int outage_duration_ms) override { + last_outage_duration_ms_ = outage_duration_ms; + } + + int last_outage_duration_ms() const { return last_outage_duration_ms_; } + + private: + int last_outage_duration_ms_ = 0; +}; + +// This is the same size that is given to the SyncBuffer object in NetEq. +const size_t kNetEqSyncBufferLengthMs = 720; +} // namespace + +class ExpandTest : public ::testing::Test { + protected: + ExpandTest() + : input_file_(test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000), + test_sample_rate_hz_(32000), + num_channels_(1), + background_noise_(num_channels_), + sync_buffer_(num_channels_, + kNetEqSyncBufferLengthMs * test_sample_rate_hz_ / 1000), + expand_(&background_noise_, + &sync_buffer_, + &random_vector_, + &statistics_, + test_sample_rate_hz_, + num_channels_) { + WebRtcSpl_Init(); + input_file_.set_output_rate_hz(test_sample_rate_hz_); + } + + void SetUp() override { + // Fast-forward the input file until there is speech (about 1.1 second into + // the file). + const size_t speech_start_samples = + static_cast(test_sample_rate_hz_ * 1.1f); + ASSERT_TRUE(input_file_.Seek(speech_start_samples)); + + // Pre-load the sync buffer with speech data. + ASSERT_TRUE( + input_file_.Read(sync_buffer_.Size(), &sync_buffer_.Channel(0)[0])); + ASSERT_EQ(1u, num_channels_) << "Fix: Must populate all channels."; + } + + test::ResampleInputAudioFile input_file_; + int test_sample_rate_hz_; + size_t num_channels_; + BackgroundNoise background_noise_; + SyncBuffer sync_buffer_; + RandomVector random_vector_; + FakeStatisticsCalculator statistics_; + Expand expand_; +}; + +// This test calls the expand object to produce concealment data a few times, +// and then ends by calling SetParametersForNormalAfterExpand. This simulates +// the situation where the packet next up for decoding was just delayed, not +// lost. +TEST_F(ExpandTest, DelayedPacketOutage) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForNormalAfterExpand(); + // Convert |sum_output_len_samples| to milliseconds. + EXPECT_EQ(rtc::checked_cast(sum_output_len_samples / + (test_sample_rate_hz_ / 1000)), + statistics_.last_outage_duration_ms()); +} + +// This test is similar to DelayedPacketOutage, but ends by calling +// SetParametersForMergeAfterExpand. This simulates the situation where the +// packet next up for decoding was actually lost (or at least a later packet +// arrived before it). +TEST_F(ExpandTest, LostPacketOutage) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForMergeAfterExpand(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); +} + +// This test is similar to the DelayedPacketOutage test above, but with the +// difference that Expand::Reset() is called after 5 calls to Expand::Process(). +// This should reset the statistics, and will in the end lead to an outage of +// 5 periods instead of 10. +TEST_F(ExpandTest, CheckOutageStatsAfterReset) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + if (i == 5) { + expand_.Reset(); + sum_output_len_samples = 0; + } + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForNormalAfterExpand(); + // Convert |sum_output_len_samples| to milliseconds. + EXPECT_EQ(rtc::checked_cast(sum_output_len_samples / + (test_sample_rate_hz_ / 1000)), + statistics_.last_outage_duration_ms()); +} + // TODO(hlundin): Write more tests. } // namespace webrtc diff --git a/webrtc/modules/audio_coding/neteq/merge_unittest.cc b/webrtc/modules/audio_coding/neteq/merge_unittest.cc index bdcbbb8a9b..ddb0e16ddf 100644 --- a/webrtc/modules/audio_coding/neteq/merge_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/merge_unittest.cc @@ -18,6 +18,7 @@ #include "webrtc/modules/audio_coding/neteq/background_noise.h" #include "webrtc/modules/audio_coding/neteq/expand.h" #include "webrtc/modules/audio_coding/neteq/random_vector.h" +#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" namespace webrtc { @@ -28,7 +29,8 @@ TEST(Merge, CreateAndDestroy) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; - Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); Merge merge(fs, channels, &expand, &sync_buffer); } diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_expand.h b/webrtc/modules/audio_coding/neteq/mock/mock_expand.h index 45e3239f61..f5ca077531 100644 --- a/webrtc/modules/audio_coding/neteq/mock/mock_expand.h +++ b/webrtc/modules/audio_coding/neteq/mock/mock_expand.h @@ -22,10 +22,15 @@ class MockExpand : public Expand { MockExpand(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels) - : Expand(background_noise, sync_buffer, random_vector, fs, num_channels) { - } + : Expand(background_noise, + sync_buffer, + random_vector, + statistics, + fs, + num_channels) {} virtual ~MockExpand() { Die(); } MOCK_METHOD0(Die, void()); MOCK_METHOD0(Reset, @@ -46,10 +51,11 @@ namespace webrtc { class MockExpandFactory : public ExpandFactory { public: - MOCK_CONST_METHOD5(Create, + MOCK_CONST_METHOD6(Create, Expand*(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels)); }; diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc index 3b81999dbb..636ae87632 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -1874,7 +1874,7 @@ void NetEqImpl::UpdatePlcComponents(int fs_hz, size_t channels) { // Delete objects and create new ones. expand_.reset(expand_factory_->Create(background_noise_.get(), sync_buffer_.get(), &random_vector_, - fs_hz, channels)); + &stats_, fs_hz, channels)); merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get())); } diff --git a/webrtc/modules/audio_coding/neteq/normal_unittest.cc b/webrtc/modules/audio_coding/neteq/normal_unittest.cc index 796409b25d..1ac32f46a7 100644 --- a/webrtc/modules/audio_coding/neteq/normal_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/normal_unittest.cc @@ -23,6 +23,7 @@ #include "webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h" #include "webrtc/modules/audio_coding/neteq/mock/mock_expand.h" #include "webrtc/modules/audio_coding/neteq/random_vector.h" +#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" using ::testing::_; @@ -36,7 +37,8 @@ TEST(Normal, CreateAndDestroy) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; - Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); Normal normal(fs, &db, bgn, &expand); EXPECT_CALL(db, Die()); // Called when |db| goes out of scope. } @@ -49,7 +51,9 @@ TEST(Normal, AvoidDivideByZero) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; - MockExpand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, + channels); Normal normal(fs, &db, bgn, &expand); int16_t input[1000] = {0}; @@ -93,7 +97,9 @@ TEST(Normal, InputLengthAndChannelsDoNotMatch) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(channels, 1000); RandomVector random_vector; - MockExpand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, + channels); Normal normal(fs, &db, bgn, &expand); int16_t input[1000] = {0}; diff --git a/webrtc/modules/audio_coding/neteq/statistics_calculator.cc b/webrtc/modules/audio_coding/neteq/statistics_calculator.cc index ce800dd065..37a0d50946 100644 --- a/webrtc/modules/audio_coding/neteq/statistics_calculator.cc +++ b/webrtc/modules/audio_coding/neteq/statistics_calculator.cc @@ -15,6 +15,7 @@ #include "webrtc/modules/audio_coding/neteq/decision_logic.h" #include "webrtc/modules/audio_coding/neteq/delay_manager.h" +#include "webrtc/system_wrappers/interface/metrics.h" namespace webrtc { @@ -96,6 +97,12 @@ void StatisticsCalculator::SecondaryDecodedSamples(int num_samples) { secondary_decoded_samples_ += num_samples; } +void StatisticsCalculator::LogDelayedPacketOutageEvent(int outage_duration_ms) { + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.DelayedPacketOutageEventMs", + outage_duration_ms, 1 /* min */, 2000 /* max */, + 100 /* bucket count */); +} + void StatisticsCalculator::StoreWaitingTime(int waiting_time_ms) { assert(next_waiting_time_index_ < kLenWaitingTimes); waiting_times_[next_waiting_time_index_] = waiting_time_ms; diff --git a/webrtc/modules/audio_coding/neteq/statistics_calculator.h b/webrtc/modules/audio_coding/neteq/statistics_calculator.h index a2cd9be6ed..513322f882 100644 --- a/webrtc/modules/audio_coding/neteq/statistics_calculator.h +++ b/webrtc/modules/audio_coding/neteq/statistics_calculator.h @@ -73,6 +73,11 @@ class StatisticsCalculator { // Reports that |num_samples| samples were decoded from secondary packets. void SecondaryDecodedSamples(int num_samples); + // Logs a delayed packet outage event of |outage_duration_ms|. A delayed + // packet outage event is defined as an expand period caused not by an actual + // packet loss, but by a delayed packet. + virtual void LogDelayedPacketOutageEvent(int outage_duration_ms); + // Returns the current network statistics in |stats|. The current sample rate // is |fs_hz|, the total number of samples in packet buffer and sync buffer // yet to play out is |num_samples_in_buffers|, and the number of samples per diff --git a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc index 6bbb3286e4..e2ec419b24 100644 --- a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc +++ b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc @@ -10,6 +10,8 @@ #include "webrtc/modules/audio_coding/neteq/tools/input_audio_file.h" +#include "webrtc/base/checks.h" + namespace webrtc { namespace test { @@ -37,6 +39,25 @@ bool InputAudioFile::Read(size_t samples, int16_t* destination) { return true; } +bool InputAudioFile::Seek(int samples) { + if (!fp_) { + return false; + } + // Find file boundaries. + const long current_pos = ftell(fp_); + CHECK_NE(EOF, current_pos) << "Error returned when getting file position."; + CHECK_EQ(0, fseek(fp_, 0, SEEK_END)); // Move to end of file. + const long file_size = ftell(fp_); + CHECK_NE(EOF, file_size) << "Error returned when getting file position."; + // Find new position. + long new_pos = current_pos + sizeof(int16_t) * samples; // Samples to bytes. + CHECK_GE(new_pos, 0) << "Trying to move to before the beginning of the file"; + new_pos = new_pos % file_size; // Wrap around the end of the file. + // Move to new position relative to the beginning of the file. + CHECK_EQ(0, fseek(fp_, new_pos, SEEK_SET)); + return true; +} + void InputAudioFile::DuplicateInterleaved(const int16_t* source, size_t samples, size_t channels, int16_t* destination) { diff --git a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h index 075b5d33b5..fae55733f4 100644 --- a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h +++ b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h @@ -34,6 +34,12 @@ class InputAudioFile { // The output |destination| must have the capacity to hold |samples| elements. virtual bool Read(size_t samples, int16_t* destination); + // Fast-forwards (|samples| > 0) or -backwards (|samples| < 0) the file by the + // indicated number of samples. Just like Read(), Seek() starts over at the + // beginning of the file if the end is reached. However, seeking backwards + // past the beginning of the file is not possible. + virtual bool Seek(int samples); + // Creates a multi-channel signal from a mono signal. Each sample is repeated // |channels| times to create an interleaved multi-channel signal where all // channels are identical. The output |destination| must have the capacity to