diff --git a/webrtc/modules/audio_coding/neteq/expand.cc b/webrtc/modules/audio_coding/neteq/expand.cc index ae12e50461..d01465ab60 100644 --- a/webrtc/modules/audio_coding/neteq/expand.cc +++ b/webrtc/modules/audio_coding/neteq/expand.cc @@ -16,10 +16,12 @@ #include // min, max #include // numeric_limits +#include "webrtc/base/safe_conversions.h" #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" #include "webrtc/modules/audio_coding/neteq/background_noise.h" #include "webrtc/modules/audio_coding/neteq/dsp_helper.h" #include "webrtc/modules/audio_coding/neteq/random_vector.h" +#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" namespace webrtc { @@ -27,6 +29,7 @@ namespace webrtc { Expand::Expand(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels) : random_vector_(random_vector), @@ -36,10 +39,12 @@ Expand::Expand(BackgroundNoise* background_noise, num_channels_(num_channels), consecutive_expands_(0), background_noise_(background_noise), + statistics_(statistics), overlap_length_(5 * fs / 8000), lag_index_direction_(0), current_lag_index_(0), stop_muting_(false), + expand_duration_samples_(0), channel_parameters_(new ChannelParameters[num_channels_]) { assert(fs == 8000 || fs == 16000 || fs == 32000 || fs == 48000); assert(fs <= kMaxSampleRate); // Should not be possible. @@ -78,6 +83,7 @@ int Expand::Process(AudioMultiVector* output) { // Perform initial setup if this is the first expansion since last reset. AnalyzeSignal(random_vector); first_expand_ = false; + expand_duration_samples_ = 0; } else { // This is not the first expansion, parameters are already estimated. // Extract a noise segment. @@ -298,6 +304,10 @@ int Expand::Process(AudioMultiVector* output) { // Increase call number and cap it. consecutive_expands_ = consecutive_expands_ >= kMaxConsecutiveExpands ? kMaxConsecutiveExpands : consecutive_expands_ + 1; + expand_duration_samples_ += output->Size(); + // Clamp the duration counter at 2 seconds. + expand_duration_samples_ = + std::min(expand_duration_samples_, rtc::checked_cast(fs_hz_ * 2)); return 0; } @@ -305,6 +315,8 @@ void Expand::SetParametersForNormalAfterExpand() { current_lag_index_ = 0; lag_index_direction_ = 0; stop_muting_ = true; // Do not mute signal any more. + statistics_->LogDelayedPacketOutageEvent( + rtc::checked_cast(expand_duration_samples_) / (fs_hz_ / 1000)); } void Expand::SetParametersForMergeAfterExpand() { @@ -833,10 +845,11 @@ void Expand::UpdateLagIndex() { Expand* ExpandFactory::Create(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels) const { - return new Expand(background_noise, sync_buffer, random_vector, fs, - num_channels); + return new Expand(background_noise, sync_buffer, random_vector, statistics, + fs, num_channels); } // TODO(turajs): This can be moved to BackgroundNoise class. diff --git a/webrtc/modules/audio_coding/neteq/expand.h b/webrtc/modules/audio_coding/neteq/expand.h index 5fb117d519..3fbafdb97f 100644 --- a/webrtc/modules/audio_coding/neteq/expand.h +++ b/webrtc/modules/audio_coding/neteq/expand.h @@ -23,6 +23,7 @@ namespace webrtc { // Forward declarations. class BackgroundNoise; class RandomVector; +class StatisticsCalculator; class SyncBuffer; // This class handles extrapolation of audio data from the sync_buffer to @@ -34,6 +35,7 @@ class Expand { Expand(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels); @@ -86,8 +88,8 @@ class Expand { // necessary to produce concealment data. void AnalyzeSignal(int16_t* random_vector); - RandomVector* random_vector_; - SyncBuffer* sync_buffer_; + RandomVector* const random_vector_; + SyncBuffer* const sync_buffer_; bool first_expand_; const int fs_hz_; const size_t num_channels_; @@ -127,13 +129,15 @@ class Expand { void UpdateLagIndex(); - BackgroundNoise* background_noise_; + BackgroundNoise* const background_noise_; + StatisticsCalculator* const statistics_; const size_t overlap_length_; int16_t max_lag_; size_t expand_lags_[kNumLags]; int lag_index_direction_; int current_lag_index_; bool stop_muting_; + size_t expand_duration_samples_; rtc::scoped_ptr channel_parameters_; DISALLOW_COPY_AND_ASSIGN(Expand); @@ -146,6 +150,7 @@ struct ExpandFactory { virtual Expand* Create(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels) const; }; diff --git a/webrtc/modules/audio_coding/neteq/expand_unittest.cc b/webrtc/modules/audio_coding/neteq/expand_unittest.cc index 68b4f60f15..1441704102 100644 --- a/webrtc/modules/audio_coding/neteq/expand_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/expand_unittest.cc @@ -13,9 +13,14 @@ #include "webrtc/modules/audio_coding/neteq/expand.h" #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/safe_conversions.h" +#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" #include "webrtc/modules/audio_coding/neteq/background_noise.h" #include "webrtc/modules/audio_coding/neteq/random_vector.h" +#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" +#include "webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h" +#include "webrtc/test/testsupport/fileutils.h" namespace webrtc { @@ -25,7 +30,8 @@ TEST(Expand, CreateAndDestroy) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; - Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); } TEST(Expand, CreateUsingFactory) { @@ -34,13 +40,135 @@ TEST(Expand, CreateUsingFactory) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; + StatisticsCalculator statistics; ExpandFactory expand_factory; - Expand* expand = - expand_factory.Create(&bgn, &sync_buffer, &random_vector, fs, channels); + Expand* expand = expand_factory.Create(&bgn, &sync_buffer, &random_vector, + &statistics, fs, channels); EXPECT_TRUE(expand != NULL); delete expand; } +namespace { +class FakeStatisticsCalculator : public StatisticsCalculator { + public: + void LogDelayedPacketOutageEvent(int outage_duration_ms) override { + last_outage_duration_ms_ = outage_duration_ms; + } + + int last_outage_duration_ms() const { return last_outage_duration_ms_; } + + private: + int last_outage_duration_ms_ = 0; +}; + +// This is the same size that is given to the SyncBuffer object in NetEq. +const size_t kNetEqSyncBufferLengthMs = 720; +} // namespace + +class ExpandTest : public ::testing::Test { + protected: + ExpandTest() + : input_file_(test::ResourcePath("audio_coding/testfile32kHz", "pcm"), + 32000), + test_sample_rate_hz_(32000), + num_channels_(1), + background_noise_(num_channels_), + sync_buffer_(num_channels_, + kNetEqSyncBufferLengthMs * test_sample_rate_hz_ / 1000), + expand_(&background_noise_, + &sync_buffer_, + &random_vector_, + &statistics_, + test_sample_rate_hz_, + num_channels_) { + WebRtcSpl_Init(); + input_file_.set_output_rate_hz(test_sample_rate_hz_); + } + + void SetUp() override { + // Fast-forward the input file until there is speech (about 1.1 second into + // the file). + const size_t speech_start_samples = + static_cast(test_sample_rate_hz_ * 1.1f); + ASSERT_TRUE(input_file_.Seek(speech_start_samples)); + + // Pre-load the sync buffer with speech data. + ASSERT_TRUE( + input_file_.Read(sync_buffer_.Size(), &sync_buffer_.Channel(0)[0])); + ASSERT_EQ(1u, num_channels_) << "Fix: Must populate all channels."; + } + + test::ResampleInputAudioFile input_file_; + int test_sample_rate_hz_; + size_t num_channels_; + BackgroundNoise background_noise_; + SyncBuffer sync_buffer_; + RandomVector random_vector_; + FakeStatisticsCalculator statistics_; + Expand expand_; +}; + +// This test calls the expand object to produce concealment data a few times, +// and then ends by calling SetParametersForNormalAfterExpand. This simulates +// the situation where the packet next up for decoding was just delayed, not +// lost. +TEST_F(ExpandTest, DelayedPacketOutage) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForNormalAfterExpand(); + // Convert |sum_output_len_samples| to milliseconds. + EXPECT_EQ(rtc::checked_cast(sum_output_len_samples / + (test_sample_rate_hz_ / 1000)), + statistics_.last_outage_duration_ms()); +} + +// This test is similar to DelayedPacketOutage, but ends by calling +// SetParametersForMergeAfterExpand. This simulates the situation where the +// packet next up for decoding was actually lost (or at least a later packet +// arrived before it). +TEST_F(ExpandTest, LostPacketOutage) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForMergeAfterExpand(); + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); +} + +// This test is similar to the DelayedPacketOutage test above, but with the +// difference that Expand::Reset() is called after 5 calls to Expand::Process(). +// This should reset the statistics, and will in the end lead to an outage of +// 5 periods instead of 10. +TEST_F(ExpandTest, CheckOutageStatsAfterReset) { + AudioMultiVector output(num_channels_); + size_t sum_output_len_samples = 0; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(0, expand_.Process(&output)); + EXPECT_GT(output.Size(), 0u); + sum_output_len_samples += output.Size(); + if (i == 5) { + expand_.Reset(); + sum_output_len_samples = 0; + } + EXPECT_EQ(0, statistics_.last_outage_duration_ms()); + } + expand_.SetParametersForNormalAfterExpand(); + // Convert |sum_output_len_samples| to milliseconds. + EXPECT_EQ(rtc::checked_cast(sum_output_len_samples / + (test_sample_rate_hz_ / 1000)), + statistics_.last_outage_duration_ms()); +} + // TODO(hlundin): Write more tests. } // namespace webrtc diff --git a/webrtc/modules/audio_coding/neteq/merge_unittest.cc b/webrtc/modules/audio_coding/neteq/merge_unittest.cc index bdcbbb8a9b..ddb0e16ddf 100644 --- a/webrtc/modules/audio_coding/neteq/merge_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/merge_unittest.cc @@ -18,6 +18,7 @@ #include "webrtc/modules/audio_coding/neteq/background_noise.h" #include "webrtc/modules/audio_coding/neteq/expand.h" #include "webrtc/modules/audio_coding/neteq/random_vector.h" +#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" namespace webrtc { @@ -28,7 +29,8 @@ TEST(Merge, CreateAndDestroy) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; - Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); Merge merge(fs, channels, &expand, &sync_buffer); } diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_expand.h b/webrtc/modules/audio_coding/neteq/mock/mock_expand.h index 45e3239f61..f5ca077531 100644 --- a/webrtc/modules/audio_coding/neteq/mock/mock_expand.h +++ b/webrtc/modules/audio_coding/neteq/mock/mock_expand.h @@ -22,10 +22,15 @@ class MockExpand : public Expand { MockExpand(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels) - : Expand(background_noise, sync_buffer, random_vector, fs, num_channels) { - } + : Expand(background_noise, + sync_buffer, + random_vector, + statistics, + fs, + num_channels) {} virtual ~MockExpand() { Die(); } MOCK_METHOD0(Die, void()); MOCK_METHOD0(Reset, @@ -46,10 +51,11 @@ namespace webrtc { class MockExpandFactory : public ExpandFactory { public: - MOCK_CONST_METHOD5(Create, + MOCK_CONST_METHOD6(Create, Expand*(BackgroundNoise* background_noise, SyncBuffer* sync_buffer, RandomVector* random_vector, + StatisticsCalculator* statistics, int fs, size_t num_channels)); }; diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc index 3b81999dbb..636ae87632 100644 --- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc +++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -1874,7 +1874,7 @@ void NetEqImpl::UpdatePlcComponents(int fs_hz, size_t channels) { // Delete objects and create new ones. expand_.reset(expand_factory_->Create(background_noise_.get(), sync_buffer_.get(), &random_vector_, - fs_hz, channels)); + &stats_, fs_hz, channels)); merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get())); } diff --git a/webrtc/modules/audio_coding/neteq/normal_unittest.cc b/webrtc/modules/audio_coding/neteq/normal_unittest.cc index 796409b25d..1ac32f46a7 100644 --- a/webrtc/modules/audio_coding/neteq/normal_unittest.cc +++ b/webrtc/modules/audio_coding/neteq/normal_unittest.cc @@ -23,6 +23,7 @@ #include "webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h" #include "webrtc/modules/audio_coding/neteq/mock/mock_expand.h" #include "webrtc/modules/audio_coding/neteq/random_vector.h" +#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h" #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" using ::testing::_; @@ -36,7 +37,8 @@ TEST(Normal, CreateAndDestroy) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; - Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels); Normal normal(fs, &db, bgn, &expand); EXPECT_CALL(db, Die()); // Called when |db| goes out of scope. } @@ -49,7 +51,9 @@ TEST(Normal, AvoidDivideByZero) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(1, 1000); RandomVector random_vector; - MockExpand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, + channels); Normal normal(fs, &db, bgn, &expand); int16_t input[1000] = {0}; @@ -93,7 +97,9 @@ TEST(Normal, InputLengthAndChannelsDoNotMatch) { BackgroundNoise bgn(channels); SyncBuffer sync_buffer(channels, 1000); RandomVector random_vector; - MockExpand expand(&bgn, &sync_buffer, &random_vector, fs, channels); + StatisticsCalculator statistics; + MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, + channels); Normal normal(fs, &db, bgn, &expand); int16_t input[1000] = {0}; diff --git a/webrtc/modules/audio_coding/neteq/statistics_calculator.cc b/webrtc/modules/audio_coding/neteq/statistics_calculator.cc index ce800dd065..37a0d50946 100644 --- a/webrtc/modules/audio_coding/neteq/statistics_calculator.cc +++ b/webrtc/modules/audio_coding/neteq/statistics_calculator.cc @@ -15,6 +15,7 @@ #include "webrtc/modules/audio_coding/neteq/decision_logic.h" #include "webrtc/modules/audio_coding/neteq/delay_manager.h" +#include "webrtc/system_wrappers/interface/metrics.h" namespace webrtc { @@ -96,6 +97,12 @@ void StatisticsCalculator::SecondaryDecodedSamples(int num_samples) { secondary_decoded_samples_ += num_samples; } +void StatisticsCalculator::LogDelayedPacketOutageEvent(int outage_duration_ms) { + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.DelayedPacketOutageEventMs", + outage_duration_ms, 1 /* min */, 2000 /* max */, + 100 /* bucket count */); +} + void StatisticsCalculator::StoreWaitingTime(int waiting_time_ms) { assert(next_waiting_time_index_ < kLenWaitingTimes); waiting_times_[next_waiting_time_index_] = waiting_time_ms; diff --git a/webrtc/modules/audio_coding/neteq/statistics_calculator.h b/webrtc/modules/audio_coding/neteq/statistics_calculator.h index a2cd9be6ed..513322f882 100644 --- a/webrtc/modules/audio_coding/neteq/statistics_calculator.h +++ b/webrtc/modules/audio_coding/neteq/statistics_calculator.h @@ -73,6 +73,11 @@ class StatisticsCalculator { // Reports that |num_samples| samples were decoded from secondary packets. void SecondaryDecodedSamples(int num_samples); + // Logs a delayed packet outage event of |outage_duration_ms|. A delayed + // packet outage event is defined as an expand period caused not by an actual + // packet loss, but by a delayed packet. + virtual void LogDelayedPacketOutageEvent(int outage_duration_ms); + // Returns the current network statistics in |stats|. The current sample rate // is |fs_hz|, the total number of samples in packet buffer and sync buffer // yet to play out is |num_samples_in_buffers|, and the number of samples per diff --git a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc index 6bbb3286e4..e2ec419b24 100644 --- a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc +++ b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc @@ -10,6 +10,8 @@ #include "webrtc/modules/audio_coding/neteq/tools/input_audio_file.h" +#include "webrtc/base/checks.h" + namespace webrtc { namespace test { @@ -37,6 +39,25 @@ bool InputAudioFile::Read(size_t samples, int16_t* destination) { return true; } +bool InputAudioFile::Seek(int samples) { + if (!fp_) { + return false; + } + // Find file boundaries. + const long current_pos = ftell(fp_); + CHECK_NE(EOF, current_pos) << "Error returned when getting file position."; + CHECK_EQ(0, fseek(fp_, 0, SEEK_END)); // Move to end of file. + const long file_size = ftell(fp_); + CHECK_NE(EOF, file_size) << "Error returned when getting file position."; + // Find new position. + long new_pos = current_pos + sizeof(int16_t) * samples; // Samples to bytes. + CHECK_GE(new_pos, 0) << "Trying to move to before the beginning of the file"; + new_pos = new_pos % file_size; // Wrap around the end of the file. + // Move to new position relative to the beginning of the file. + CHECK_EQ(0, fseek(fp_, new_pos, SEEK_SET)); + return true; +} + void InputAudioFile::DuplicateInterleaved(const int16_t* source, size_t samples, size_t channels, int16_t* destination) { diff --git a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h index 075b5d33b5..fae55733f4 100644 --- a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h +++ b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h @@ -34,6 +34,12 @@ class InputAudioFile { // The output |destination| must have the capacity to hold |samples| elements. virtual bool Read(size_t samples, int16_t* destination); + // Fast-forwards (|samples| > 0) or -backwards (|samples| < 0) the file by the + // indicated number of samples. Just like Read(), Seek() starts over at the + // beginning of the file if the end is reached. However, seeking backwards + // past the beginning of the file is not possible. + virtual bool Seek(int samples); + // Creates a multi-channel signal from a mono signal. Each sample is repeated // |channels| times to create an interleaved multi-channel signal where all // channels are identical. The output |destination| must have the capacity to