diff --git a/modules/audio_coding/neteq/include/neteq.h b/modules/audio_coding/neteq/include/neteq.h index f376b0568c..918b006ddf 100644 --- a/modules/audio_coding/neteq/include/neteq.h +++ b/modules/audio_coding/neteq/include/neteq.h @@ -74,6 +74,21 @@ struct NetEqLifetimeStatistics { uint64_t voice_concealed_samples = 0; }; +// Metrics that describe the operations performed in NetEq, and the internal +// state. +struct NetEqOperationsAndState { + // These sample counters are cumulative, and don't reset. As a reference, the + // total number of output samples can be found in + // NetEqLifetimeStatistics::total_samples_received. + uint64_t preemptive_samples = 0; + uint64_t accelerate_samples = 0; + // The statistics below are not cumulative. + // The waiting time of the last decoded packet. + uint64_t last_waiting_time_ms = 0; + // The sum of the packet and jitter buffer size in ms. + uint64_t current_buffer_size_ms = 0; +}; + // This is the interface class for NetEq. class NetEq { public: @@ -206,6 +221,10 @@ class NetEq { // never reset. virtual NetEqLifetimeStatistics GetLifetimeStatistics() const = 0; + // Returns statistics about the performed operations and internal state. These + // statistics are never reset. + virtual NetEqOperationsAndState GetOperationsAndState() const = 0; + // Writes the current RTCP statistics to |stats|. The statistics are reset // and a new report period is started with the call. virtual void GetRtcpStatistics(RtcpStatistics* stats) = 0; diff --git a/modules/audio_coding/neteq/neteq_impl.cc b/modules/audio_coding/neteq/neteq_impl.cc index 3b2bd834b5..98c2372f68 100644 --- a/modules/audio_coding/neteq/neteq_impl.cc +++ b/modules/audio_coding/neteq/neteq_impl.cc @@ -371,6 +371,16 @@ NetEqLifetimeStatistics NetEqImpl::GetLifetimeStatistics() const { return stats_.GetLifetimeStatistics(); } +NetEqOperationsAndState NetEqImpl::GetOperationsAndState() const { + rtc::CritScope lock(&crit_sect_); + auto result = stats_.GetOperationsAndState(); + result.current_buffer_size_ms = + (packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) + + sync_buffer_->FutureLength()) * + 1000 / fs_hz_; + return result; +} + void NetEqImpl::GetRtcpStatistics(RtcpStatistics* stats) { rtc::CritScope lock(&crit_sect_); if (stats) { diff --git a/modules/audio_coding/neteq/neteq_impl.h b/modules/audio_coding/neteq/neteq_impl.h index dcb931eeb1..077426b2c3 100644 --- a/modules/audio_coding/neteq/neteq_impl.h +++ b/modules/audio_coding/neteq/neteq_impl.h @@ -176,6 +176,8 @@ class NetEqImpl : public webrtc::NetEq { NetEqLifetimeStatistics GetLifetimeStatistics() const override; + NetEqOperationsAndState GetOperationsAndState() const override; + // Same as RtcpStatistics(), but does not reset anything. void GetRtcpStatisticsNoReset(RtcpStatistics* stats) override; diff --git a/modules/audio_coding/neteq/statistics_calculator.cc b/modules/audio_coding/neteq/statistics_calculator.cc index 3d5744cb29..7ce156ef9a 100644 --- a/modules/audio_coding/neteq/statistics_calculator.cc +++ b/modules/audio_coding/neteq/statistics_calculator.cc @@ -200,10 +200,12 @@ void StatisticsCalculator::ConcealedSamplesCorrection(int num_samples, void StatisticsCalculator::PreemptiveExpandedSamples(size_t num_samples) { preemptive_samples_ += num_samples; + operations_and_state_.preemptive_samples += num_samples; } void StatisticsCalculator::AcceleratedSamples(size_t num_samples) { accelerate_samples_ += num_samples; + operations_and_state_.accelerate_samples += num_samples; } void StatisticsCalculator::AddZeros(size_t num_samples) { @@ -261,6 +263,7 @@ void StatisticsCalculator::StoreWaitingTime(int waiting_time_ms) { waiting_times_.pop_front(); } waiting_times_.push_back(waiting_time_ms); + operations_and_state_.last_waiting_time_ms = waiting_time_ms; } void StatisticsCalculator::GetNetworkStatistics(int fs_hz, @@ -345,6 +348,10 @@ NetEqLifetimeStatistics StatisticsCalculator::GetLifetimeStatistics() const { return lifetime_stats_; } +NetEqOperationsAndState StatisticsCalculator::GetOperationsAndState() const { + return operations_and_state_; +} + uint16_t StatisticsCalculator::CalculateQ14Ratio(size_t numerator, uint32_t denominator) { if (numerator == 0) { diff --git a/modules/audio_coding/neteq/statistics_calculator.h b/modules/audio_coding/neteq/statistics_calculator.h index 5f7d06a23b..37540fdbc5 100644 --- a/modules/audio_coding/neteq/statistics_calculator.h +++ b/modules/audio_coding/neteq/statistics_calculator.h @@ -111,6 +111,8 @@ class StatisticsCalculator { // never reset. NetEqLifetimeStatistics GetLifetimeStatistics() const; + NetEqOperationsAndState GetOperationsAndState() const; + private: static const int kMaxReportPeriod = 60; // Seconds before auto-reset. static const size_t kLenWaitingTimes = 100; @@ -178,6 +180,7 @@ class StatisticsCalculator { static uint16_t CalculateQ14Ratio(size_t numerator, uint32_t denominator); NetEqLifetimeStatistics lifetime_stats_; + NetEqOperationsAndState operations_and_state_; size_t concealed_samples_correction_ = 0; size_t voice_concealed_samples_correction_ = 0; size_t preemptive_samples_; diff --git a/modules/audio_coding/neteq/tools/neteq_test.cc b/modules/audio_coding/neteq/tools/neteq_test.cc index e51ee479ef..fdd1b24bd6 100644 --- a/modules/audio_coding/neteq/tools/neteq_test.cc +++ b/modules/audio_coding/neteq/tools/neteq_test.cc @@ -153,25 +153,35 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { input_->AdvanceOutputEvent(); result.simulation_step_ms = input_->NextEventTime().value_or(time_now_ms) - start_time_ms; - const auto network_stats = SimulationStats(); - current_state_.current_delay_ms = network_stats.current_buffer_size_ms; - current_state_.packet_loss_occurred = network_stats.packet_loss_rate > 0; - int scaling_factor = std::max(1 << 14, network_stats.accelerate_rate + - network_stats.expand_rate + - network_stats.preemptive_rate); - // TODO(ivoc): Improve the accuracy of these numbers by adding a new API - // to NetEq. - result.action_times_ms[Action::kAccelerate] = - (10 * network_stats.accelerate_rate) / scaling_factor; - result.action_times_ms[Action::kExpand] = - (10 * network_stats.expand_rate) / scaling_factor; - result.action_times_ms[Action::kPreemptiveExpand] = - (10 * network_stats.preemptive_rate) / scaling_factor; - result.action_times_ms[Action::kNormal] = - 10 - result.action_times_ms[Action::kAccelerate] - - result.action_times_ms[Action::kExpand] - - result.action_times_ms[Action::kPreemptiveExpand]; + const auto operations_state = neteq_->GetOperationsAndState(); + current_state_.current_delay_ms = operations_state.current_buffer_size_ms; + // TODO(ivoc): Add more accurate reporting by tracking the origin of + // samples in the sync buffer. + result.action_times_ms[Action::kExpand] = 0; + result.action_times_ms[Action::kAccelerate] = 0; + result.action_times_ms[Action::kPreemptiveExpand] = 0; + result.action_times_ms[Action::kNormal] = 0; + + if (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC || + out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG) { + // Consider the whole frame to be the result of expansion. + result.action_times_ms[Action::kExpand] = 10; + } else if (operations_state.accelerate_samples - + prev_ops_state_.accelerate_samples > + 0) { + // Consider the whole frame to be the result of acceleration. + result.action_times_ms[Action::kAccelerate] = 10; + } else if (operations_state.preemptive_samples - + prev_ops_state_.preemptive_samples > + 0) { + // Consider the whole frame to be the result of preemptive expansion. + result.action_times_ms[Action::kPreemptiveExpand] = 10; + } else { + // Consider the whole frame to be the result of normal playout. + result.action_times_ms[Action::kNormal] = 10; + } result.is_simulation_finished = input_->ended(); + prev_ops_state_ = operations_state; return result; } } diff --git a/modules/audio_coding/neteq/tools/neteq_test.h b/modules/audio_coding/neteq/tools/neteq_test.h index 787d507504..23d7c22aaa 100644 --- a/modules/audio_coding/neteq/tools/neteq_test.h +++ b/modules/audio_coding/neteq/tools/neteq_test.h @@ -120,6 +120,7 @@ class NetEqTest : public NetEqSimulator { Callbacks callbacks_; int sample_rate_hz_; NetEqState current_state_; + NetEqOperationsAndState prev_ops_state_; }; } // namespace test