diff --git a/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc b/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc index cf310d1efb..ec6ade9c11 100644 --- a/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc +++ b/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc @@ -127,6 +127,10 @@ class LossyInput : public NetEqInput { return input_->NextOutputEventTime(); } + absl::optional NextSetMinimumDelayInfo() const override { + return input_->NextSetMinimumDelayInfo(); + } + std::unique_ptr PopPacket() override { if (loss_cadence_ != 0 && (++count_ % loss_cadence_) == 0) { // Pop `burst_length_` packets to create the loss. @@ -141,6 +145,10 @@ class LossyInput : public NetEqInput { void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); } + void AdvanceSetMinimumDelay() override { + return input_->AdvanceSetMinimumDelay(); + } + bool ended() const override { return input_->ended(); } absl::optional NextHeader() const override { diff --git a/modules/audio_coding/neteq/tools/encode_neteq_input.h b/modules/audio_coding/neteq/tools/encode_neteq_input.h index caa9ac76f4..f2ed4b1cf5 100644 --- a/modules/audio_coding/neteq/tools/encode_neteq_input.h +++ b/modules/audio_coding/neteq/tools/encode_neteq_input.h @@ -41,10 +41,16 @@ class EncodeNetEqInput : public NetEqInput { absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override { + return absl::nullopt; + } + std::unique_ptr PopPacket() override; void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override {} + bool ended() const override; absl::optional NextHeader() const override; diff --git a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc index 3c33aabf1c..763078eed2 100644 --- a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc +++ b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc @@ -37,6 +37,11 @@ absl::optional InitialPacketInserterNetEqInput::NextOutputEventTime() return source_->NextOutputEventTime(); } +absl::optional +InitialPacketInserterNetEqInput::NextSetMinimumDelayInfo() const { + return source_->NextSetMinimumDelayInfo(); +} + std::unique_ptr InitialPacketInserterNetEqInput::PopPacket() { if (!first_packet_) { @@ -63,6 +68,10 @@ InitialPacketInserterNetEqInput::PopPacket() { return source_->PopPacket(); } +void InitialPacketInserterNetEqInput::AdvanceSetMinimumDelay() { + source_->AdvanceSetMinimumDelay(); +} + void InitialPacketInserterNetEqInput::AdvanceOutputEvent() { source_->AdvanceOutputEvent(); } diff --git a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h index bd20a7aecf..7246949956 100644 --- a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h +++ b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h @@ -29,8 +29,10 @@ class InitialPacketInserterNetEqInput final : public NetEqInput { int sample_rate_hz); absl::optional NextPacketTime() const override; absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override; std::unique_ptr PopPacket() override; void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override; bool ended() const override; absl::optional NextHeader() const override; diff --git a/modules/audio_coding/neteq/tools/neteq_event_log_input.cc b/modules/audio_coding/neteq/tools/neteq_event_log_input.cc index 0c1f27799a..26255b7771 100644 --- a/modules/audio_coding/neteq/tools/neteq_event_log_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_event_log_input.cc @@ -46,6 +46,11 @@ absl::optional NetEqEventLogInput::NextOutputEventTime() const { return next_output_event_ms_; } +absl::optional +NetEqEventLogInput::NextSetMinimumDelayInfo() const { + return next_minimum_delay_event_info_; +} + void NetEqEventLogInput::AdvanceOutputEvent() { next_output_event_ms_ = source_->NextAudioOutputEventMs(); if (*next_output_event_ms_ == std::numeric_limits::max()) { @@ -53,6 +58,10 @@ void NetEqEventLogInput::AdvanceOutputEvent() { } } +void NetEqEventLogInput::AdvanceSetMinimumDelay() { + next_minimum_delay_event_info_ = source_->NextSetMinimumDelayEvent(); +} + PacketSource* NetEqEventLogInput::source() { return source_.get(); } @@ -62,6 +71,7 @@ NetEqEventLogInput::NetEqEventLogInput( : source_(std::move(source)) { LoadNextPacket(); AdvanceOutputEvent(); + AdvanceSetMinimumDelay(); } } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_event_log_input.h b/modules/audio_coding/neteq/tools/neteq_event_log_input.h index c947ee1fc0..1e64cc59a4 100644 --- a/modules/audio_coding/neteq/tools/neteq_event_log_input.h +++ b/modules/audio_coding/neteq/tools/neteq_event_log_input.h @@ -36,7 +36,9 @@ class NetEqEventLogInput final : public NetEqPacketSourceInput { absl::optional ssrc_filter); absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override; void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override; protected: PacketSource* source() override; @@ -44,6 +46,7 @@ class NetEqEventLogInput final : public NetEqPacketSourceInput { private: NetEqEventLogInput(std::unique_ptr source); std::unique_ptr source_; + absl::optional next_minimum_delay_event_info_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_input.cc b/modules/audio_coding/neteq/tools/neteq_input.cc index de416348f1..57160b9969 100644 --- a/modules/audio_coding/neteq/tools/neteq_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_input.cc @@ -58,6 +58,11 @@ absl::optional TimeLimitedNetEqInput::NextOutputEventTime() const { return ended_ ? absl::nullopt : input_->NextOutputEventTime(); } +absl::optional +TimeLimitedNetEqInput::NextSetMinimumDelayInfo() const { + return ended_ ? absl::nullopt : input_->NextSetMinimumDelayInfo(); +} + std::unique_ptr TimeLimitedNetEqInput::PopPacket() { if (ended_) { return std::unique_ptr(); @@ -74,6 +79,13 @@ void TimeLimitedNetEqInput::AdvanceOutputEvent() { } } +void TimeLimitedNetEqInput::AdvanceSetMinimumDelay() { + if (!ended_) { + input_->AdvanceSetMinimumDelay(); + MaybeSetEnded(); + } +} + bool TimeLimitedNetEqInput::ended() const { return ended_ || input_->ended(); } diff --git a/modules/audio_coding/neteq/tools/neteq_input.h b/modules/audio_coding/neteq/tools/neteq_input.h index 3a66264043..56b0212add 100644 --- a/modules/audio_coding/neteq/tools/neteq_input.h +++ b/modules/audio_coding/neteq/tools/neteq_input.h @@ -36,6 +36,13 @@ class NetEqInput { int64_t time_ms; }; + struct SetMinimumDelayInfo { + SetMinimumDelayInfo(int64_t timestamp_ms_in, int delay_ms_in) + : timestamp_ms(timestamp_ms_in), delay_ms(delay_ms_in) {} + int64_t timestamp_ms; + int delay_ms; + }; + virtual ~NetEqInput() = default; // Returns at what time (in ms) NetEq::InsertPacket should be called next, or @@ -46,16 +53,31 @@ class NetEqInput { // empty if no more output events are available. virtual absl::optional NextOutputEventTime() const = 0; - // Returns the time (in ms) for the next event from either NextPacketTime() - // or NextOutputEventTime(), or empty if both are out of events. + // Returns the information related to the next NetEq set minimum delay event + // if available. + virtual absl::optional NextSetMinimumDelayInfo() + const = 0; + + // Returns the time (in ms) for the next event (packet, output or set minimum + // delay event) or empty if there are no more events. absl::optional NextEventTime() const { - const auto a = NextPacketTime(); - const auto b = NextOutputEventTime(); + absl::optional next_event_time = NextPacketTime(); + const auto next_output_time = NextOutputEventTime(); // Return the minimum of non-empty `a` and `b`, or empty if both are empty. - if (a) { - return b ? std::min(*a, *b) : a; + if (next_output_time) { + next_event_time = next_event_time ? std::min(next_event_time.value(), + next_output_time.value()) + : next_output_time; } - return b ? b : absl::nullopt; + const auto next_neteq_minimum_delay = NextSetMinimumDelayInfo(); + if (next_neteq_minimum_delay) { + next_event_time = + next_event_time + ? std::min(next_event_time.value(), + next_neteq_minimum_delay.value().timestamp_ms) + : next_neteq_minimum_delay.value().timestamp_ms; + } + return next_event_time; } // Returns the next packet to be inserted into NetEq. The packet following the @@ -69,6 +91,10 @@ class NetEqInput { // time). virtual void AdvanceOutputEvent() = 0; + // Move to the next NetEq set minimum delay. This will make + // `NextSetMinimumDelayInfo` return a new value. + virtual void AdvanceSetMinimumDelay() = 0; + // Returns true if the source has come to an end. An implementation must // eventually return true from this method, or the test will end up in an // infinite loop. @@ -88,8 +114,10 @@ class TimeLimitedNetEqInput : public NetEqInput { ~TimeLimitedNetEqInput() override; absl::optional NextPacketTime() const override; absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override; std::unique_ptr PopPacket() override; void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override; bool ended() const override; absl::optional NextHeader() const override; diff --git a/modules/audio_coding/neteq/tools/neteq_packet_source_input.h b/modules/audio_coding/neteq/tools/neteq_packet_source_input.h index 407fa491b1..1b48b1f8cf 100644 --- a/modules/audio_coding/neteq/tools/neteq_packet_source_input.h +++ b/modules/audio_coding/neteq/tools/neteq_packet_source_input.h @@ -54,7 +54,11 @@ class NetEqRtpDumpInput final : public NetEqPacketSourceInput { absl::optional ssrc_filter); absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override { + return absl::nullopt; + } void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override {} protected: PacketSource* source() override; diff --git a/modules/audio_coding/neteq/tools/neteq_replacement_input.cc b/modules/audio_coding/neteq/tools/neteq_replacement_input.cc index ffd114ae5b..9436b68ac9 100644 --- a/modules/audio_coding/neteq/tools/neteq_replacement_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_replacement_input.cc @@ -40,6 +40,11 @@ absl::optional NetEqReplacementInput::NextOutputEventTime() const { return source_->NextOutputEventTime(); } +absl::optional +NetEqReplacementInput::NextSetMinimumDelayInfo() const { + return source_->NextSetMinimumDelayInfo(); +} + std::unique_ptr NetEqReplacementInput::PopPacket() { std::unique_ptr to_return = std::move(packet_); while (true) { @@ -59,6 +64,10 @@ void NetEqReplacementInput::AdvanceOutputEvent() { source_->AdvanceOutputEvent(); } +void NetEqReplacementInput::AdvanceSetMinimumDelay() { + source_->AdvanceSetMinimumDelay(); +} + bool NetEqReplacementInput::ended() const { return source_->ended(); } diff --git a/modules/audio_coding/neteq/tools/neteq_replacement_input.h b/modules/audio_coding/neteq/tools/neteq_replacement_input.h index 9ce9b9dc63..23e4beae84 100644 --- a/modules/audio_coding/neteq/tools/neteq_replacement_input.h +++ b/modules/audio_coding/neteq/tools/neteq_replacement_input.h @@ -30,8 +30,10 @@ class NetEqReplacementInput : public NetEqInput { absl::optional NextPacketTime() const override; absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override; std::unique_ptr PopPacket() override; void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override; bool ended() const override; absl::optional NextHeader() const override; diff --git a/modules/audio_coding/neteq/tools/neteq_test.cc b/modules/audio_coding/neteq/tools/neteq_test.cc index a567efe2de..125d087157 100644 --- a/modules/audio_coding/neteq/tools/neteq_test.cc +++ b/modules/audio_coding/neteq/tools/neteq_test.cc @@ -163,6 +163,13 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { absl::make_optional(packet_data->header.timestamp); } + if (input_->NextSetMinimumDelayInfo().has_value() && + time_now_ms >= input_->NextSetMinimumDelayInfo().value().timestamp_ms) { + neteq_->SetBaseMinimumDelayMs( + input_->NextSetMinimumDelayInfo().value().delay_ms); + input_->AdvanceSetMinimumDelay(); + } + // Check if it is time to get output audio. if (input_->NextOutputEventTime() && time_now_ms >= *input_->NextOutputEventTime()) { diff --git a/modules/audio_coding/neteq/tools/rtc_event_log_source.cc b/modules/audio_coding/neteq/tools/rtc_event_log_source.cc index 1407aab5f2..68acfb7f62 100644 --- a/modules/audio_coding/neteq/tools/rtc_event_log_source.cc +++ b/modules/audio_coding/neteq/tools/rtc_event_log_source.cc @@ -20,6 +20,7 @@ #include "absl/strings/string_view.h" #include "logging/rtc_event_log/rtc_event_processor.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" #include "modules/audio_coding/neteq/tools/packet.h" #include "rtc_base/checks.h" @@ -94,6 +95,14 @@ int64_t RtcEventLogSource::NextAudioOutputEventMs() { return output_time_ms; } +absl::optional +RtcEventLogSource::NextSetMinimumDelayEvent() { + if (minimum_delay_index_ >= minimum_delay_.size()) { + return absl::nullopt; + } + return minimum_delay_[minimum_delay_index_++]; +} + RtcEventLogSource::RtcEventLogSource() : PacketSource() {} bool RtcEventLogSource::Initialize(const ParsedRtcEventLog& parsed_log, @@ -130,6 +139,17 @@ bool RtcEventLogSource::Initialize(const ParsedRtcEventLog& parsed_log, } }; + auto handle_neteq_set_minimum_delay = + [this, first_log_end_time_us, &packet_ssrcs]( + const webrtc::LoggedNetEqSetMinimumDelayEvent minimum_delay_event) { + if (minimum_delay_event.log_time_us() < first_log_end_time_us) { + if (packet_ssrcs.count(minimum_delay_event.remote_ssrc) > 0) { + minimum_delay_.emplace_back(minimum_delay_event.log_time_ms(), + minimum_delay_event.minimum_delay_ms); + } + } + }; + // This wouldn't be needed if we knew that there was at most one audio stream. webrtc::RtcEventProcessor event_processor; for (const auto& rtp_packets : parsed_log.incoming_rtp_packets_by_ssrc()) { @@ -152,6 +172,16 @@ bool RtcEventLogSource::Initialize(const ParsedRtcEventLog& parsed_log, event_processor.AddEvents(audio_playouts.second, handle_audio_playout); } + for (const auto& neteq_set_minimum_delay_event : + parsed_log.neteq_set_minimum_delay_events()) { + if (ssrc_filter.has_value() && + neteq_set_minimum_delay_event.first != *ssrc_filter) { + continue; + } + event_processor.AddEvents(neteq_set_minimum_delay_event.second, + handle_neteq_set_minimum_delay); + } + // Fills in rtp_packets_ and audio_outputs_. event_processor.ProcessEventsInOrder(); diff --git a/modules/audio_coding/neteq/tools/rtc_event_log_source.h b/modules/audio_coding/neteq/tools/rtc_event_log_source.h index c67912a67d..1445314247 100644 --- a/modules/audio_coding/neteq/tools/rtc_event_log_source.h +++ b/modules/audio_coding/neteq/tools/rtc_event_log_source.h @@ -18,6 +18,7 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "logging/rtc_event_log/rtc_event_log_parser.h" +#include "modules/audio_coding/neteq/tools/neteq_input.h" #include "modules/audio_coding/neteq/tools/packet_source.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" @@ -53,6 +54,9 @@ class RtcEventLogSource : public PacketSource { // events available. int64_t NextAudioOutputEventMs(); + // Returns the next NetEq set minimum delay event if available. + absl::optional NextSetMinimumDelayEvent(); + private: RtcEventLogSource(); @@ -63,6 +67,8 @@ class RtcEventLogSource : public PacketSource { size_t rtp_packet_index_ = 0; std::vector audio_outputs_; size_t audio_output_index_ = 0; + std::vector minimum_delay_; + size_t minimum_delay_index_ = 0; }; } // namespace test diff --git a/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc b/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc index 1ebb6030b6..e2b4ecfe8d 100644 --- a/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc +++ b/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc @@ -182,11 +182,17 @@ class NetEqStreamInput : public test::NetEqInput { // that outlive the one constructed. NetEqStreamInput(const std::vector* packet_stream, const std::vector* output_events, + const std::vector* + neteq_set_minimum_delay_events, absl::optional end_time_ms) : packet_stream_(*packet_stream), packet_stream_it_(packet_stream_.begin()), output_events_it_(output_events->begin()), output_events_end_(output_events->end()), + neteq_set_minimum_delay_events_it_( + neteq_set_minimum_delay_events->begin()), + neteq_set_minimum_delay_events_end_( + neteq_set_minimum_delay_events->end()), end_time_ms_(end_time_ms) { RTC_DCHECK(packet_stream); RTC_DCHECK(output_events); @@ -212,6 +218,20 @@ class NetEqStreamInput : public test::NetEqInput { return output_events_it_->log_time_ms(); } + absl::optional NextSetMinimumDelayInfo() const override { + if (neteq_set_minimum_delay_events_it_ == + neteq_set_minimum_delay_events_end_) { + return absl::nullopt; + } + if (end_time_ms_ && + neteq_set_minimum_delay_events_it_->log_time_ms() > *end_time_ms_) { + return absl::nullopt; + } + return SetMinimumDelayInfo( + neteq_set_minimum_delay_events_it_->log_time_ms(), + neteq_set_minimum_delay_events_it_->minimum_delay_ms); + } + std::unique_ptr PopPacket() override { if (packet_stream_it_ == packet_stream_.end()) { return std::unique_ptr(); @@ -236,6 +256,13 @@ class NetEqStreamInput : public test::NetEqInput { } } + void AdvanceSetMinimumDelay() override { + if (neteq_set_minimum_delay_events_it_ != + neteq_set_minimum_delay_events_end_) { + ++neteq_set_minimum_delay_events_it_; + } + } + bool ended() const override { return !NextEventTime(); } absl::optional NextHeader() const override { @@ -250,6 +277,10 @@ class NetEqStreamInput : public test::NetEqInput { std::vector::const_iterator packet_stream_it_; std::vector::const_iterator output_events_it_; const std::vector::const_iterator output_events_end_; + std::vector::const_iterator + neteq_set_minimum_delay_events_it_; + const std::vector::const_iterator + neteq_set_minimum_delay_events_end_; const absl::optional end_time_ms_; }; @@ -294,11 +325,14 @@ class ReplacementAudioDecoderFactory : public AudioDecoderFactory { std::unique_ptr CreateNetEqTestAndRun( const std::vector* packet_stream, const std::vector* output_events, + const std::vector* + neteq_set_minimum_delay_events, absl::optional end_time_ms, const std::string& replacement_file_name, int file_sample_rate_hz) { std::unique_ptr input( - new NetEqStreamInput(packet_stream, output_events, end_time_ms)); + new NetEqStreamInput(packet_stream, output_events, + neteq_set_minimum_delay_events, end_time_ms)); constexpr int kReplacementPt = 127; std::set cn_types; @@ -361,11 +395,23 @@ NetEqStatsGetterMap SimulateNetEq(const ParsedRtcEventLog& parsed_log, output_events_it = parsed_log.audio_playout_events().cbegin(); } + const auto neteq_set_minimum_delay_events_it = + parsed_log.neteq_set_minimum_delay_events().find(ssrc); + std::vector + empty_neteq_set_minimum_delay_event; + const std::vector& + neteq_set_minimum_delay_events = + neteq_set_minimum_delay_events_it == + parsed_log.neteq_set_minimum_delay_events().cend() + ? empty_neteq_set_minimum_delay_event + : neteq_set_minimum_delay_events_it->second; + int64_t end_time_ms = parsed_log.first_log_segment().stop_time_ms(); - neteq_stats[ssrc] = CreateNetEqTestAndRun( - audio_packets, &output_events_it->second, end_time_ms, - replacement_file_name, file_sample_rate_hz); + neteq_stats[ssrc] = + CreateNetEqTestAndRun(audio_packets, &output_events_it->second, + &neteq_set_minimum_delay_events, end_time_ms, + replacement_file_name, file_sample_rate_hz); } return neteq_stats; diff --git a/test/fuzzers/neteq_rtp_fuzzer.cc b/test/fuzzers/neteq_rtp_fuzzer.cc index 348c84f040..3caa5fe5de 100644 --- a/test/fuzzers/neteq_rtp_fuzzer.cc +++ b/test/fuzzers/neteq_rtp_fuzzer.cc @@ -77,6 +77,10 @@ class FuzzRtpInput : public NetEqInput { return input_->NextOutputEventTime(); } + absl::optional NextSetMinimumDelayInfo() const override { + return input_->NextSetMinimumDelayInfo(); + } + std::unique_ptr PopPacket() override { RTC_DCHECK(packet_); std::unique_ptr packet_to_return = std::move(packet_); @@ -88,6 +92,10 @@ class FuzzRtpInput : public NetEqInput { void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); } + void AdvanceSetMinimumDelay() override { + return input_->AdvanceSetMinimumDelay(); + } + bool ended() const override { return ended_; } absl::optional NextHeader() const override { diff --git a/test/fuzzers/neteq_signal_fuzzer.cc b/test/fuzzers/neteq_signal_fuzzer.cc index 8653f137a2..485c38085e 100644 --- a/test/fuzzers/neteq_signal_fuzzer.cc +++ b/test/fuzzers/neteq_signal_fuzzer.cc @@ -97,6 +97,10 @@ class FuzzSignalInput : public NetEqInput { return next_output_event_ms_; } + absl::optional NextSetMinimumDelayInfo() const override { + return input_->NextSetMinimumDelayInfo(); + } + std::unique_ptr PopPacket() override { RTC_DCHECK(packet_); std::unique_ptr packet_to_return = std::move(packet_); @@ -123,6 +127,10 @@ class FuzzSignalInput : public NetEqInput { next_output_event_ms_ += output_event_period_ms_; } + void AdvanceSetMinimumDelay() override { + return input_->AdvanceSetMinimumDelay(); + } + bool ended() const override { return ended_; } absl::optional NextHeader() const override {