diff --git a/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc b/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc index ec6ade9c11..8ed64ffdec 100644 --- a/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc +++ b/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc @@ -119,34 +119,26 @@ class LossyInput : public NetEqInput { burst_length_(burst_length), input_(std::move(input)) {} - absl::optional NextPacketTime() const override { - return input_->NextPacketTime(); + absl::optional NextEventTime() const { + return input_->NextEventTime(); } - absl::optional NextOutputEventTime() const override { - return input_->NextOutputEventTime(); - } - - absl::optional NextSetMinimumDelayInfo() const override { - return input_->NextSetMinimumDelayInfo(); - } - - std::unique_ptr PopPacket() override { - if (loss_cadence_ != 0 && (++count_ % loss_cadence_) == 0) { - // Pop `burst_length_` packets to create the loss. - auto packet_to_return = input_->PopPacket(); - for (int i = 0; i < burst_length_; i++) { - input_->PopPacket(); - } - return packet_to_return; + std::unique_ptr PopEvent() override { + std::unique_ptr event = input_->PopEvent(); + if (event == nullptr || event->type() != Event::Type::kPacketData) { + return event; } - return input_->PopPacket(); - } - - void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); } - - void AdvanceSetMinimumDelay() override { - return input_->AdvanceSetMinimumDelay(); + if (loss_cadence_ != 0 && remaining_losses_ == 0 && + (++count_ % loss_cadence_) == 0) { + // Pop `burst_length_` packets to create the loss. + remaining_losses_ = burst_length_; + } else { + if (remaining_losses_ != 0) { + remaining_losses_--; + return PopEvent(); + } + } + return event; } bool ended() const override { return input_->ended(); } @@ -158,6 +150,7 @@ class LossyInput : public NetEqInput { private: const int loss_cadence_; const int burst_length_; + int remaining_losses_ = 0; int count_ = 0; const std::unique_ptr input_; }; diff --git a/modules/audio_coding/neteq/neteq_unittest.cc b/modules/audio_coding/neteq/neteq_unittest.cc index fff14bf830..4d7b0b9cf0 100644 --- a/modules/audio_coding/neteq/neteq_unittest.cc +++ b/modules/audio_coding/neteq/neteq_unittest.cc @@ -988,9 +988,9 @@ TEST(NetEqNoTimeStretchingMode, RunTest) { {5, kRtpExtensionTransportSequenceNumber}, {7, kRtpExtensionVideoContentType}, {8, kRtpExtensionVideoTiming}}; - std::unique_ptr input(new NetEqRtpDumpInput( + std::unique_ptr input = std::make_unique( webrtc::test::ResourcePath("audio_coding/neteq_universal_new", "rtp"), - rtp_ext_map, absl::nullopt /*No SSRC filter*/)); + rtp_ext_map, absl::nullopt /*No SSRC filter*/); std::unique_ptr input_time_limit( new TimeLimitedNetEqInput(std::move(input), 20000)); std::unique_ptr output(new VoidAudioSink); diff --git a/modules/audio_coding/neteq/tools/encode_neteq_input.cc b/modules/audio_coding/neteq/tools/encode_neteq_input.cc index 87b987ddb6..f09aa5b006 100644 --- a/modules/audio_coding/neteq/tools/encode_neteq_input.cc +++ b/modules/audio_coding/neteq/tools/encode_neteq_input.cc @@ -24,36 +24,19 @@ EncodeNetEqInput::EncodeNetEqInput(std::unique_ptr generator, : generator_(std::move(generator)), encoder_(std::move(encoder)), input_duration_ms_(input_duration_ms) { - CreatePacket(); + event_ = GetNextEvent(); } EncodeNetEqInput::~EncodeNetEqInput() = default; -absl::optional EncodeNetEqInput::NextPacketTime() const { - RTC_DCHECK(packet_data_); - return static_cast(packet_data_->time_ms); -} - -absl::optional EncodeNetEqInput::NextOutputEventTime() const { - return next_output_event_ms_; -} - -std::unique_ptr EncodeNetEqInput::PopPacket() { - RTC_DCHECK(packet_data_); - // Grab the packet to return... - std::unique_ptr packet_to_return = std::move(packet_data_); - // ... and line up the next packet for future use. - CreatePacket(); - - return packet_to_return; -} - -void EncodeNetEqInput::AdvanceOutputEvent() { - next_output_event_ms_ += kOutputPeriodMs; +std::unique_ptr EncodeNetEqInput::PopEvent() { + std::unique_ptr event_to_return = std::move(event_); + event_ = GetNextEvent(); + return event_to_return; } bool EncodeNetEqInput::ended() const { - return next_output_event_ms_ > input_duration_ms_; + return next_output_event_ms_ > input_duration_ms_ + kOutputPeriodMs; } absl::optional EncodeNetEqInput::NextHeader() const { @@ -61,9 +44,23 @@ absl::optional EncodeNetEqInput::NextHeader() const { return packet_data_->header; } +std::unique_ptr EncodeNetEqInput::GetNextEvent() { + std::unique_ptr event; + if (packet_data_ == nullptr) { + CreatePacket(); + } + if (next_output_event_ms_ < packet_data_->timestamp_ms_) { + event = std::make_unique(next_output_event_ms_); + next_output_event_ms_ += kOutputPeriodMs; + return event; + } + event = std::move(packet_data_); + CreatePacket(); + return event; +} + void EncodeNetEqInput::CreatePacket() { // Create a new PacketData object. - RTC_DCHECK(!packet_data_); packet_data_.reset(new NetEqInput::PacketData); RTC_DCHECK_EQ(packet_data_->payload.size(), 0); @@ -86,7 +83,7 @@ void EncodeNetEqInput::CreatePacket() { packet_data_->header.timestamp = info.encoded_timestamp; packet_data_->header.payloadType = info.payload_type; packet_data_->header.sequenceNumber = sequence_number_++; - packet_data_->time_ms = next_packet_time_ms_; + packet_data_->timestamp_ms_ = next_packet_time_ms_; next_packet_time_ms_ += num_blocks * kOutputPeriodMs; } diff --git a/modules/audio_coding/neteq/tools/encode_neteq_input.h b/modules/audio_coding/neteq/tools/encode_neteq_input.h index f2ed4b1cf5..f028e4f20a 100644 --- a/modules/audio_coding/neteq/tools/encode_neteq_input.h +++ b/modules/audio_coding/neteq/tools/encode_neteq_input.h @@ -37,28 +37,24 @@ class EncodeNetEqInput : public NetEqInput { int64_t input_duration_ms); ~EncodeNetEqInput() override; - absl::optional NextPacketTime() const override; - - absl::optional NextOutputEventTime() const override; - - absl::optional NextSetMinimumDelayInfo() const override { - return absl::nullopt; - } - - std::unique_ptr PopPacket() override; - - void AdvanceOutputEvent() override; - - void AdvanceSetMinimumDelay() override {} + std::unique_ptr PopEvent() override; bool ended() const override; absl::optional NextHeader() const override; + absl::optional NextEventTime() const override { + if (event_) { + return event_->timestamp_ms(); + } + return absl::nullopt; + } + private: static constexpr int64_t kOutputPeriodMs = 10; void CreatePacket(); + std::unique_ptr GetNextEvent(); std::unique_ptr generator_; std::unique_ptr encoder_; @@ -68,6 +64,7 @@ class EncodeNetEqInput : public NetEqInput { int64_t next_packet_time_ms_ = 0; int64_t next_output_event_ms_ = 0; const int64_t input_duration_ms_; + std::unique_ptr event_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc index 763078eed2..cd5955b9f7 100644 --- a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc +++ b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc @@ -27,37 +27,27 @@ InitialPacketInserterNetEqInput::InitialPacketInserterNetEqInput( packets_to_insert_(number_of_initial_packets), sample_rate_hz_(sample_rate_hz) {} -absl::optional InitialPacketInserterNetEqInput::NextPacketTime() - const { - return source_->NextPacketTime(); +absl::optional InitialPacketInserterNetEqInput::NextEventTime() const { + return source_->NextEventTime(); } -absl::optional InitialPacketInserterNetEqInput::NextOutputEventTime() - const { - return source_->NextOutputEventTime(); -} - -absl::optional -InitialPacketInserterNetEqInput::NextSetMinimumDelayInfo() const { - return source_->NextSetMinimumDelayInfo(); -} - -std::unique_ptr -InitialPacketInserterNetEqInput::PopPacket() { +std::unique_ptr InitialPacketInserterNetEqInput::PopEvent() { + std::unique_ptr event; if (!first_packet_) { - first_packet_ = source_->PopPacket(); - if (!first_packet_) { - // The source has no packets, so we should not insert any dummy packets. - packets_to_insert_ = 0; + event = source_->PopEvent(); + if (event == nullptr || event->type() != Event::Type::kPacketData) { + return event; } + first_packet_ = std::move(event); } if (packets_to_insert_ > 0) { RTC_CHECK(first_packet_); - auto dummy_packet = std::unique_ptr(new PacketData()); - dummy_packet->header = first_packet_->header; - dummy_packet->payload = rtc::Buffer(first_packet_->payload.data(), - first_packet_->payload.size()); - dummy_packet->time_ms = first_packet_->time_ms; + std::unique_ptr dummy_packet = std::make_unique(); + PacketData& first_packet = static_cast(*first_packet_); + dummy_packet->header = first_packet.header; + dummy_packet->payload = + rtc::Buffer(first_packet.payload.data(), first_packet.payload.size()); + dummy_packet->timestamp_ms_ = first_packet.timestamp_ms_; dummy_packet->header.sequenceNumber -= packets_to_insert_; // This assumes 20ms per packet. dummy_packet->header.timestamp -= @@ -65,15 +55,7 @@ InitialPacketInserterNetEqInput::PopPacket() { packets_to_insert_--; return dummy_packet; } - return source_->PopPacket(); -} - -void InitialPacketInserterNetEqInput::AdvanceSetMinimumDelay() { - source_->AdvanceSetMinimumDelay(); -} - -void InitialPacketInserterNetEqInput::AdvanceOutputEvent() { - source_->AdvanceOutputEvent(); + return source_->PopEvent(); } bool InitialPacketInserterNetEqInput::ended() const { diff --git a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h index 7246949956..f45c6950d8 100644 --- a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h +++ b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h @@ -27,20 +27,16 @@ class InitialPacketInserterNetEqInput final : public NetEqInput { InitialPacketInserterNetEqInput(std::unique_ptr source, int number_of_initial_packets, int sample_rate_hz); - absl::optional NextPacketTime() const override; - absl::optional NextOutputEventTime() const override; - absl::optional NextSetMinimumDelayInfo() const override; - std::unique_ptr PopPacket() override; - void AdvanceOutputEvent() override; - void AdvanceSetMinimumDelay() override; + std::unique_ptr PopEvent() override; bool ended() const override; absl::optional NextHeader() const override; + absl::optional NextEventTime() const override; private: const std::unique_ptr source_; int packets_to_insert_; const int sample_rate_hz_; - std::unique_ptr first_packet_; + std::unique_ptr first_packet_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc index 9e77457775..5e2cf5d95e 100644 --- a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc +++ b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc @@ -87,8 +87,8 @@ void PrintDelays(const NetEqDelayAnalyzer::Delays& delays, void NetEqDelayAnalyzer::AfterInsertPacket( const test::NetEqInput::PacketData& packet, NetEq* neteq) { - data_.insert( - std::make_pair(packet.header.timestamp, TimingData(packet.time_ms))); + data_.insert(std::make_pair(packet.header.timestamp, + TimingData(packet.timestamp_ms()))); ssrcs_.insert(packet.header.ssrc); payload_types_.insert(packet.header.payloadType); } diff --git a/modules/audio_coding/neteq/tools/neteq_event_log_input.cc b/modules/audio_coding/neteq/tools/neteq_event_log_input.cc index 26255b7771..c187ba61c4 100644 --- a/modules/audio_coding/neteq/tools/neteq_event_log_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_event_log_input.cc @@ -42,36 +42,88 @@ NetEqEventLogInput* NetEqEventLogInput::CreateFromString( return new NetEqEventLogInput(std::move(event_log_src)); } -absl::optional NetEqEventLogInput::NextOutputEventTime() const { - return next_output_event_ms_; +std::unique_ptr NetEqEventLogInput::PopEvent() { + std::unique_ptr event_to_return = std::move(event_); + event_ = GetNextEvent(); + return event_to_return; } -absl::optional -NetEqEventLogInput::NextSetMinimumDelayInfo() const { - return next_minimum_delay_event_info_; +absl::optional NetEqEventLogInput::NextHeader() const { + return packet_ ? absl::optional(packet_->header()) : absl::nullopt; } -void NetEqEventLogInput::AdvanceOutputEvent() { - next_output_event_ms_ = source_->NextAudioOutputEventMs(); - if (*next_output_event_ms_ == std::numeric_limits::max()) { - next_output_event_ms_ = absl::nullopt; +absl::optional NetEqEventLogInput::NextOutputEventTime() { + absl::optional next_output_event_ms; + next_output_event_ms = source_->NextAudioOutputEventMs(); + if (*next_output_event_ms == std::numeric_limits::max()) { + next_output_event_ms = absl::nullopt; } + return next_output_event_ms; } -void NetEqEventLogInput::AdvanceSetMinimumDelay() { - next_minimum_delay_event_info_ = source_->NextSetMinimumDelayEvent(); +std::unique_ptr NetEqEventLogInput::CreatePacketEvent() { + std::unique_ptr packet_data = std::make_unique(); + packet_data->header = packet_->header(); + if (packet_->payload_length_bytes() == 0 && + packet_->virtual_payload_length_bytes() > 0) { + // This is a header-only "dummy" packet. Set the payload to all zeros, with + // length according to the virtual length. + packet_data->payload.SetSize(packet_->virtual_payload_length_bytes()); + std::fill_n(packet_data->payload.data(), packet_data->payload.size(), 0); + } else { + packet_data->payload.SetData(packet_->payload(), + packet_->payload_length_bytes()); + } + packet_data->timestamp_ms_ = packet_->time_ms(); + packet_ = source_->NextPacket(); + return packet_data; } -PacketSource* NetEqEventLogInput::source() { - return source_.get(); +std::unique_ptr NetEqEventLogInput::CreateOutputEvent() { + std::unique_ptr event = + std::make_unique(next_output_event_ms_.value()); + next_output_event_ms_ = NextOutputEventTime(); + return event; +} + +std::unique_ptr +NetEqEventLogInput::CreateSetMinimumDelayEvent() { + std::unique_ptr event = + std::make_unique( + next_minimum_delay_event_.value()); + next_minimum_delay_event_ = source_->NextSetMinimumDelayEvent(); + return event; +} + +std::unique_ptr NetEqEventLogInput::GetNextEvent() { + std::unique_ptr event; + int64_t packet_time_ms = packet_ ? static_cast(packet_->time_ms()) + : std::numeric_limits::max(); + int64_t output_time_ms = next_output_event_ms_.has_value() + ? next_output_event_ms_.value() + : std::numeric_limits::max(); + int64_t minimum_delay_ms = next_minimum_delay_event_.has_value() + ? next_minimum_delay_event_->timestamp_ms() + : std::numeric_limits::max(); + if (packet_ && packet_time_ms <= minimum_delay_ms && + packet_time_ms <= output_time_ms) { + event = CreatePacketEvent(); + } else if (next_minimum_delay_event_.has_value() && + minimum_delay_ms <= output_time_ms) { + event = CreateSetMinimumDelayEvent(); + } else if (next_output_event_ms_.has_value()) { + event = CreateOutputEvent(); + } + return event; } NetEqEventLogInput::NetEqEventLogInput( std::unique_ptr source) : source_(std::move(source)) { - LoadNextPacket(); - AdvanceOutputEvent(); - AdvanceSetMinimumDelay(); + packet_ = source_->NextPacket(); + next_minimum_delay_event_ = source_->NextSetMinimumDelayEvent(); + next_output_event_ms_ = NextOutputEventTime(); + event_ = GetNextEvent(); } } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_event_log_input.h b/modules/audio_coding/neteq/tools/neteq_event_log_input.h index 1e64cc59a4..0ab09f4811 100644 --- a/modules/audio_coding/neteq/tools/neteq_event_log_input.h +++ b/modules/audio_coding/neteq/tools/neteq_event_log_input.h @@ -24,9 +24,8 @@ namespace test { class RtcEventLogSource; -// Implementation of NetEqPacketSourceInput to be used with an -// RtcEventLogSource. -class NetEqEventLogInput final : public NetEqPacketSourceInput { +// Implementation of a NetEqInput from an RtcEventLogSource. +class NetEqEventLogInput final : public NetEqInput { public: static NetEqEventLogInput* CreateFromFile( absl::string_view file_name, @@ -35,18 +34,31 @@ class NetEqEventLogInput final : public NetEqPacketSourceInput { absl::string_view file_contents, absl::optional ssrc_filter); - absl::optional NextOutputEventTime() const override; - absl::optional NextSetMinimumDelayInfo() const override; - void AdvanceOutputEvent() override; - void AdvanceSetMinimumDelay() override; - - protected: - PacketSource* source() override; + absl::optional NextEventTime() const override { + if (event_) { + return event_->timestamp_ms(); + } + return absl::nullopt; + } + std::unique_ptr PopEvent() override; + absl::optional NextHeader() const override; + bool ended() const override { + return !next_output_event_ms_ || packet_ == nullptr; + } private: NetEqEventLogInput(std::unique_ptr source); + std::unique_ptr GetNextEvent(); + std::unique_ptr CreatePacketEvent(); + std::unique_ptr CreateOutputEvent(); + std::unique_ptr CreateSetMinimumDelayEvent(); + absl::optional NextOutputEventTime(); + std::unique_ptr source_; - absl::optional next_minimum_delay_event_info_; + std::unique_ptr packet_; + absl::optional next_output_event_ms_; + absl::optional next_minimum_delay_event_; + std::unique_ptr event_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_input.cc b/modules/audio_coding/neteq/tools/neteq_input.cc index 57160b9969..d47a7d0ab7 100644 --- a/modules/audio_coding/neteq/tools/neteq_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_input.cc @@ -22,7 +22,7 @@ std::string NetEqInput::PacketData::ToString() const { rtc::StringBuilder ss; ss << "{" "time_ms: " - << static_cast(time_ms) + << static_cast(timestamp_ms_) << ", " "header: {" "pt: " @@ -50,46 +50,24 @@ TimeLimitedNetEqInput::TimeLimitedNetEqInput(std::unique_ptr input, TimeLimitedNetEqInput::~TimeLimitedNetEqInput() = default; -absl::optional TimeLimitedNetEqInput::NextPacketTime() const { - return ended_ ? absl::nullopt : input_->NextPacketTime(); -} - -absl::optional TimeLimitedNetEqInput::NextOutputEventTime() const { - return ended_ ? absl::nullopt : input_->NextOutputEventTime(); -} - -absl::optional -TimeLimitedNetEqInput::NextSetMinimumDelayInfo() const { - return ended_ ? absl::nullopt : input_->NextSetMinimumDelayInfo(); -} - -std::unique_ptr TimeLimitedNetEqInput::PopPacket() { +std::unique_ptr TimeLimitedNetEqInput::PopEvent() { + std::unique_ptr event; if (ended_) { - return std::unique_ptr(); + return event; } - auto packet = input_->PopPacket(); + event = input_->PopEvent(); MaybeSetEnded(); - return packet; -} - -void TimeLimitedNetEqInput::AdvanceOutputEvent() { - if (!ended_) { - input_->AdvanceOutputEvent(); - MaybeSetEnded(); - } -} - -void TimeLimitedNetEqInput::AdvanceSetMinimumDelay() { - if (!ended_) { - input_->AdvanceSetMinimumDelay(); - MaybeSetEnded(); - } + return event; } bool TimeLimitedNetEqInput::ended() const { return ended_ || input_->ended(); } +absl::optional TimeLimitedNetEqInput::NextEventTime() const { + return input_->NextEventTime(); +} + absl::optional TimeLimitedNetEqInput::NextHeader() const { return ended_ ? absl::nullopt : input_->NextHeader(); } diff --git a/modules/audio_coding/neteq/tools/neteq_input.h b/modules/audio_coding/neteq/tools/neteq_input.h index 56b0212add..8c6b1c33f3 100644 --- a/modules/audio_coding/neteq/tools/neteq_input.h +++ b/modules/audio_coding/neteq/tools/neteq_input.h @@ -26,74 +26,53 @@ namespace test { // Interface class for input to the NetEqTest class. class NetEqInput { public: - struct PacketData { - PacketData(); - ~PacketData(); - std::string ToString() const; - - RTPHeader header; - rtc::Buffer payload; - int64_t time_ms; + class Event { + public: + enum class Type { kPacketData, kGetAudio, kSetMinimumDelay }; + virtual Type type() = 0; + virtual int64_t timestamp_ms() const = 0; + virtual ~Event() = default; }; - struct SetMinimumDelayInfo { - SetMinimumDelayInfo(int64_t timestamp_ms_in, int delay_ms_in) - : timestamp_ms(timestamp_ms_in), delay_ms(delay_ms_in) {} - int64_t timestamp_ms; - int delay_ms; + class PacketData : public Event { + public: + PacketData(); + ~PacketData(); + Type type() override { return Type::kPacketData; } + int64_t timestamp_ms() const override { return timestamp_ms_; } + std::string ToString() const; + RTPHeader header; + rtc::Buffer payload; + int64_t timestamp_ms_; + }; + + class SetMinimumDelay : public Event { + public: + SetMinimumDelay(int64_t timestamp_ms_in, int delay_ms_in) + : timestamp_ms_(timestamp_ms_in), delay_ms_(delay_ms_in) {} + Type type() override { return Type::kSetMinimumDelay; } + int64_t timestamp_ms() const override { return timestamp_ms_; } + int delay_ms() { return delay_ms_; } + + private: + int64_t timestamp_ms_; + int delay_ms_; + }; + + class GetAudio : public Event { + public: + explicit GetAudio(int64_t timestamp_ms_in) + : timestamp_ms_(timestamp_ms_in) {} + Type type() override { return Type::kGetAudio; } + int64_t timestamp_ms() const override { return timestamp_ms_; } + + private: + int64_t timestamp_ms_; }; virtual ~NetEqInput() = default; - // Returns at what time (in ms) NetEq::InsertPacket should be called next, or - // empty if the source is out of packets. - virtual absl::optional NextPacketTime() const = 0; - - // Returns at what time (in ms) NetEq::GetAudio should be called next, or - // empty if no more output events are available. - virtual absl::optional NextOutputEventTime() const = 0; - - // Returns the information related to the next NetEq set minimum delay event - // if available. - virtual absl::optional NextSetMinimumDelayInfo() - const = 0; - - // Returns the time (in ms) for the next event (packet, output or set minimum - // delay event) or empty if there are no more events. - absl::optional NextEventTime() const { - absl::optional next_event_time = NextPacketTime(); - const auto next_output_time = NextOutputEventTime(); - // Return the minimum of non-empty `a` and `b`, or empty if both are empty. - if (next_output_time) { - next_event_time = next_event_time ? std::min(next_event_time.value(), - next_output_time.value()) - : next_output_time; - } - const auto next_neteq_minimum_delay = NextSetMinimumDelayInfo(); - if (next_neteq_minimum_delay) { - next_event_time = - next_event_time - ? std::min(next_event_time.value(), - next_neteq_minimum_delay.value().timestamp_ms) - : next_neteq_minimum_delay.value().timestamp_ms; - } - return next_event_time; - } - - // Returns the next packet to be inserted into NetEq. The packet following the - // returned one is pre-fetched in the NetEqInput object, such that future - // calls to NextPacketTime() or NextHeader() will return information from that - // packet. - virtual std::unique_ptr PopPacket() = 0; - - // Move to the next output event. This will make NextOutputEventTime() return - // a new value (potentially the same if several output events share the same - // time). - virtual void AdvanceOutputEvent() = 0; - - // Move to the next NetEq set minimum delay. This will make - // `NextSetMinimumDelayInfo` return a new value. - virtual void AdvanceSetMinimumDelay() = 0; + virtual std::unique_ptr PopEvent() = 0; // Returns true if the source has come to an end. An implementation must // eventually return true from this method, or the test will end up in an @@ -103,6 +82,10 @@ class NetEqInput { // Returns the RTP header for the next packet, i.e., the packet that will be // delivered next by PopPacket(). virtual absl::optional NextHeader() const = 0; + + // Returns the time (in ms) for the next event, or empty if both are out of + // events. + virtual absl::optional NextEventTime() const = 0; }; // Wrapper class to impose a time limit on a NetEqInput object, typically @@ -112,12 +95,8 @@ class TimeLimitedNetEqInput : public NetEqInput { public: TimeLimitedNetEqInput(std::unique_ptr input, int64_t duration_ms); ~TimeLimitedNetEqInput() override; - absl::optional NextPacketTime() const override; - absl::optional NextOutputEventTime() const override; - absl::optional NextSetMinimumDelayInfo() const override; - std::unique_ptr PopPacket() override; - void AdvanceOutputEvent() override; - void AdvanceSetMinimumDelay() override; + absl::optional NextEventTime() const override; + std::unique_ptr PopEvent() override; bool ended() const override; absl::optional NextHeader() const override; diff --git a/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc b/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc index 55a5653238..f5b5e73152 100644 --- a/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc @@ -20,27 +20,45 @@ namespace webrtc { namespace test { -NetEqPacketSourceInput::NetEqPacketSourceInput() : next_output_event_ms_(0) {} +NetEqPacketSourceInput::NetEqPacketSourceInput( + absl::string_view file_name, + const NetEqPacketSourceInput::RtpHeaderExtensionMap& hdr_ext_map, + absl::optional ssrc_filter) + : next_output_event_ms_(0) { + std::unique_ptr source( + RtpFileSource::Create(file_name, ssrc_filter)); + for (const auto& ext_pair : hdr_ext_map) { + source->RegisterRtpHeaderExtension(ext_pair.second, ext_pair.first); + } + packet_source_ = std::move(source); + packet_ = packet_source_->NextPacket(); + event_ = GetNextEvent(); +} -absl::optional NetEqPacketSourceInput::NextPacketTime() const { - return packet_ - ? absl::optional(static_cast(packet_->time_ms())) - : absl::nullopt; +std::unique_ptr NetEqPacketSourceInput::PopEvent() { + std::unique_ptr event_to_return = std::move(event_); + event_ = GetNextEvent(); + return event_to_return; } absl::optional NetEqPacketSourceInput::NextHeader() const { - return packet_ ? absl::optional(packet_->header()) : absl::nullopt; -} - -void NetEqPacketSourceInput::LoadNextPacket() { - packet_ = source()->NextPacket(); -} - -std::unique_ptr NetEqPacketSourceInput::PopPacket() { - if (!packet_) { - return std::unique_ptr(); + if (packet_) { + return packet_->header(); } - std::unique_ptr packet_data(new PacketData); + return absl::nullopt; +} + +std::unique_ptr NetEqPacketSourceInput::GetNextEvent() { + if (!packet_) { + return nullptr; + } + if (packet_->time_ms() > next_output_event_ms_) { + std::unique_ptr event = + std::make_unique(next_output_event_ms_); + next_output_event_ms_ += kOutputPeriodMs; + return event; + } + std::unique_ptr packet_data = std::make_unique(); packet_data->header = packet_->header(); if (packet_->payload_length_bytes() == 0 && packet_->virtual_payload_length_bytes() > 0) { @@ -52,39 +70,10 @@ std::unique_ptr NetEqPacketSourceInput::PopPacket() { packet_data->payload.SetData(packet_->payload(), packet_->payload_length_bytes()); } - packet_data->time_ms = packet_->time_ms(); - - LoadNextPacket(); - + packet_data->timestamp_ms_ = packet_->time_ms(); + packet_ = packet_source_->NextPacket(); return packet_data; } -NetEqRtpDumpInput::NetEqRtpDumpInput(absl::string_view file_name, - const RtpHeaderExtensionMap& hdr_ext_map, - absl::optional ssrc_filter) - : source_(RtpFileSource::Create(file_name, ssrc_filter)) { - for (const auto& ext_pair : hdr_ext_map) { - source_->RegisterRtpHeaderExtension(ext_pair.second, ext_pair.first); - } - LoadNextPacket(); -} - -absl::optional NetEqRtpDumpInput::NextOutputEventTime() const { - return next_output_event_ms_; -} - -void NetEqRtpDumpInput::AdvanceOutputEvent() { - if (next_output_event_ms_) { - *next_output_event_ms_ += kOutputPeriodMs; - } - if (!NextPacketTime()) { - next_output_event_ms_ = absl::nullopt; - } -} - -PacketSource* NetEqRtpDumpInput::source() { - return source_.get(); -} - } // namespace test } // namespace webrtc diff --git a/modules/audio_coding/neteq/tools/neteq_packet_source_input.h b/modules/audio_coding/neteq/tools/neteq_packet_source_input.h index 1b48b1f8cf..6c78850f09 100644 --- a/modules/audio_coding/neteq/tools/neteq_packet_source_input.h +++ b/modules/audio_coding/neteq/tools/neteq_packet_source_input.h @@ -30,43 +30,29 @@ class NetEqPacketSourceInput : public NetEqInput { public: using RtpHeaderExtensionMap = std::map; - NetEqPacketSourceInput(); - absl::optional NextPacketTime() const override; - std::unique_ptr PopPacket() override; - absl::optional NextHeader() const override; - bool ended() const override { return !next_output_event_ms_; } + NetEqPacketSourceInput( + absl::string_view file_name, + const NetEqPacketSourceInput::RtpHeaderExtensionMap& hdr_ext_map, + absl::optional ssrc_filter); - protected: - virtual PacketSource* source() = 0; - void LoadNextPacket(); - - absl::optional next_output_event_ms_; - - private: - std::unique_ptr packet_; -}; - -// Implementation of NetEqPacketSourceInput to be used with an RtpFileSource. -class NetEqRtpDumpInput final : public NetEqPacketSourceInput { - public: - NetEqRtpDumpInput(absl::string_view file_name, - const RtpHeaderExtensionMap& hdr_ext_map, - absl::optional ssrc_filter); - - absl::optional NextOutputEventTime() const override; - absl::optional NextSetMinimumDelayInfo() const override { + absl::optional NextEventTime() const override { + if (event_) { + return event_->timestamp_ms(); + } return absl::nullopt; } - void AdvanceOutputEvent() override; - void AdvanceSetMinimumDelay() override {} - - protected: - PacketSource* source() override; + std::unique_ptr PopEvent() override; + absl::optional NextHeader() const override; + bool ended() const override { return event_ == nullptr; } private: static constexpr int64_t kOutputPeriodMs = 10; + std::unique_ptr GetNextEvent(); - std::unique_ptr source_; + std::unique_ptr packet_; + std::unique_ptr packet_source_; + int64_t next_output_event_ms_; + std::unique_ptr event_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_replacement_input.cc b/modules/audio_coding/neteq/tools/neteq_replacement_input.cc index 9436b68ac9..1a2cff2a3b 100644 --- a/modules/audio_coding/neteq/tools/neteq_replacement_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_replacement_input.cc @@ -26,46 +26,25 @@ NetEqReplacementInput::NetEqReplacementInput( comfort_noise_types_(comfort_noise_types), forbidden_types_(forbidden_types) { RTC_CHECK(source_); - packet_ = source_->PopPacket(); - ReplacePacket(); + event_ = source_->PopEvent(); + ReplaceIfPacketEvent(); } -absl::optional NetEqReplacementInput::NextPacketTime() const { - return packet_ - ? absl::optional(static_cast(packet_->time_ms)) - : absl::nullopt; -} - -absl::optional NetEqReplacementInput::NextOutputEventTime() const { - return source_->NextOutputEventTime(); -} - -absl::optional -NetEqReplacementInput::NextSetMinimumDelayInfo() const { - return source_->NextSetMinimumDelayInfo(); -} - -std::unique_ptr NetEqReplacementInput::PopPacket() { - std::unique_ptr to_return = std::move(packet_); +std::unique_ptr NetEqReplacementInput::PopEvent() { + std::unique_ptr event_to_return = std::move(event_); while (true) { - packet_ = source_->PopPacket(); - if (!packet_) + event_ = source_->PopEvent(); + if (event_ == nullptr || event_->type() != Event::Type::kPacketData) { break; - if (packet_->payload.size() > packet_->header.paddingLength) { + } + PacketData& packet = static_cast(*event_); + if (packet.payload.size() > packet.header.paddingLength) { // Not padding only. Good to go. Skip this packet otherwise. break; } } - ReplacePacket(); - return to_return; -} - -void NetEqReplacementInput::AdvanceOutputEvent() { - source_->AdvanceOutputEvent(); -} - -void NetEqReplacementInput::AdvanceSetMinimumDelay() { - source_->AdvanceSetMinimumDelay(); + ReplaceIfPacketEvent(); + return event_to_return; } bool NetEqReplacementInput::ended() const { @@ -76,36 +55,38 @@ absl::optional NetEqReplacementInput::NextHeader() const { return source_->NextHeader(); } -void NetEqReplacementInput::ReplacePacket() { - if (!source_->NextPacketTime()) { - // End of input. Cannot do proper replacement on the very last packet, so we - // delete it instead. - packet_.reset(); +void NetEqReplacementInput::ReplaceIfPacketEvent() { + if (event_ == nullptr || event_->type() != Event::Type::kPacketData) { return; } - RTC_DCHECK(packet_); + PacketData& packet = static_cast(*event_); - RTC_CHECK_EQ(forbidden_types_.count(packet_->header.payloadType), 0) - << "Payload type " << static_cast(packet_->header.payloadType) + RTC_CHECK_EQ(forbidden_types_.count(packet.header.payloadType), 0) + << "Payload type " << static_cast(packet.header.payloadType) << " is forbidden."; // Check if this packet is comfort noise. - if (comfort_noise_types_.count(packet_->header.payloadType) != 0) { + if (comfort_noise_types_.count(packet.header.payloadType) != 0) { // If CNG, simply insert a zero-energy one-byte payload. uint8_t cng_payload[1] = {127}; // Max attenuation of CNG. - packet_->payload.SetData(cng_payload); + packet.payload.SetData(cng_payload); return; } absl::optional next_hdr = source_->NextHeader(); + if (!next_hdr) { + // End of input. Cannot do proper replacement on the very last packet, so we + // delete it instead. + event_ = nullptr; + return; + } RTC_DCHECK(next_hdr); uint8_t payload[12]; RTC_DCHECK_LE(last_frame_size_timestamps_, 120 * 48); uint32_t input_frame_size_timestamps = last_frame_size_timestamps_; - const uint32_t timestamp_diff = - next_hdr->timestamp - packet_->header.timestamp; - if (next_hdr->sequenceNumber == packet_->header.sequenceNumber + 1 && + const uint32_t timestamp_diff = next_hdr->timestamp - packet.header.timestamp; + if (next_hdr->sequenceNumber == packet.header.sequenceNumber + 1 && timestamp_diff <= 120 * 48) { // Packets are in order and the timestamp diff is less than 5760 samples. // Accept the timestamp diff as a valid frame size. @@ -113,11 +94,11 @@ void NetEqReplacementInput::ReplacePacket() { last_frame_size_timestamps_ = input_frame_size_timestamps; } RTC_DCHECK_LE(input_frame_size_timestamps, 120 * 48); - FakeDecodeFromFile::PrepareEncoded(packet_->header.timestamp, + FakeDecodeFromFile::PrepareEncoded(packet.header.timestamp, input_frame_size_timestamps, - packet_->payload.size(), payload); - packet_->payload.SetData(payload); - packet_->header.payloadType = replacement_payload_type_; + packet.payload.size(), payload); + packet.payload.SetData(payload); + packet.header.payloadType = replacement_payload_type_; return; } diff --git a/modules/audio_coding/neteq/tools/neteq_replacement_input.h b/modules/audio_coding/neteq/tools/neteq_replacement_input.h index 23e4beae84..8f76494461 100644 --- a/modules/audio_coding/neteq/tools/neteq_replacement_input.h +++ b/modules/audio_coding/neteq/tools/neteq_replacement_input.h @@ -28,23 +28,24 @@ class NetEqReplacementInput : public NetEqInput { const std::set& comfort_noise_types, const std::set& forbidden_types); - absl::optional NextPacketTime() const override; - absl::optional NextOutputEventTime() const override; - absl::optional NextSetMinimumDelayInfo() const override; - std::unique_ptr PopPacket() override; - void AdvanceOutputEvent() override; - void AdvanceSetMinimumDelay() override; + absl::optional NextEventTime() const override { + if (event_) { + return event_->timestamp_ms(); + } + return absl::nullopt; + } + std::unique_ptr PopEvent() override; bool ended() const override; absl::optional NextHeader() const override; private: - void ReplacePacket(); + void ReplaceIfPacketEvent(); std::unique_ptr source_; const uint8_t replacement_payload_type_; const std::set comfort_noise_types_; const std::set forbidden_types_; - std::unique_ptr packet_; // The next packet to deliver. + std::unique_ptr event_; // The next event to deliver. uint32_t last_frame_size_timestamps_ = 960; // Initial guess: 20 ms @ 48 kHz. }; diff --git a/modules/audio_coding/neteq/tools/neteq_test.cc b/modules/audio_coding/neteq/tools/neteq_test.cc index 125d087157..3b3f9f54db 100644 --- a/modules/audio_coding/neteq/tools/neteq_test.cc +++ b/modules/audio_coding/neteq/tools/neteq_test.cc @@ -107,25 +107,27 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { RTC_DCHECK(input_->NextEventTime()); clock_.AdvanceTimeMilliseconds(*input_->NextEventTime() - time_now_ms); time_now_ms = *input_->NextEventTime(); + std::unique_ptr event = input_->PopEvent(); // Check if it is time to insert packet. - if (input_->NextPacketTime() && time_now_ms >= *input_->NextPacketTime()) { - std::unique_ptr packet_data = input_->PopPacket(); - RTC_CHECK(packet_data); + RTC_CHECK(event); + if (event->type() == NetEqInput::Event::Type::kPacketData) { + NetEqInput::PacketData& packet_data = + static_cast(*event); const size_t payload_data_length = - packet_data->payload.size() - packet_data->header.paddingLength; + packet_data.payload.size() - packet_data.header.paddingLength; if (payload_data_length != 0) { int error = neteq_->InsertPacket( - packet_data->header, - rtc::ArrayView(packet_data->payload)); + packet_data.header, + rtc::ArrayView(packet_data.payload)); if (error != NetEq::kOK && callbacks_.error_callback) { - callbacks_.error_callback->OnInsertPacketError(*packet_data); + callbacks_.error_callback->OnInsertPacketError(packet_data); } if (callbacks_.post_insert_packet) { - callbacks_.post_insert_packet->AfterInsertPacket(*packet_data, + callbacks_.post_insert_packet->AfterInsertPacket(packet_data, neteq_.get()); } } else { - neteq_->InsertEmptyPacket(packet_data->header); + neteq_->InsertEmptyPacket(packet_data.header); } if (last_packet_time_ms_) { current_state_.packet_iat_ms.push_back(time_now_ms - @@ -137,20 +139,20 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { last_packet_time_ms_ ? (time_now_ms - *last_packet_time_ms_) : -1; const auto delta_timestamp = last_packet_timestamp_ - ? (static_cast(packet_data->header.timestamp) - + ? (static_cast(packet_data.header.timestamp) - *last_packet_timestamp_) * 1000 / sample_rate_hz_ : -1; const auto packet_size_bytes = - packet_data->payload.size() == 12 + packet_data.payload.size() == 12 ? ByteReader::ReadLittleEndian( - &packet_data->payload[8]) + &packet_data.payload[8]) : -1; *text_log_ << "Packet - wallclock: " << std::setw(5) << time_now_ms << ", delta wc: " << std::setw(4) << delta_wallclock - << ", seq_no: " << packet_data->header.sequenceNumber + << ", seq_no: " << packet_data.header.sequenceNumber << ", timestamp: " << std::setw(10) - << packet_data->header.timestamp + << packet_data.header.timestamp << ", delta ts: " << std::setw(4) << delta_timestamp << ", size: " << std::setw(5) << packet_size_bytes << ", frame size: " << std::setw(3) @@ -160,19 +162,16 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { } last_packet_time_ms_ = absl::make_optional(time_now_ms); last_packet_timestamp_ = - absl::make_optional(packet_data->header.timestamp); + absl::make_optional(packet_data.header.timestamp); } - if (input_->NextSetMinimumDelayInfo().has_value() && - time_now_ms >= input_->NextSetMinimumDelayInfo().value().timestamp_ms) { + if (event->type() == NetEqInput::Event::Type::kSetMinimumDelay) { neteq_->SetBaseMinimumDelayMs( - input_->NextSetMinimumDelayInfo().value().delay_ms); - input_->AdvanceSetMinimumDelay(); + static_cast(*event).delay_ms()); } // Check if it is time to get output audio. - if (input_->NextOutputEventTime() && - time_now_ms >= *input_->NextOutputEventTime()) { + if (event->type() == NetEqInput::Event::Type::kGetAudio) { if (callbacks_.get_audio_callback) { callbacks_.get_audio_callback->BeforeGetAudio(neteq_.get()); } @@ -200,7 +199,6 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { out_frame.samples_per_channel_ * out_frame.num_channels_)); } - input_->AdvanceOutputEvent(); result.simulation_step_ms = input_->NextEventTime().value_or(time_now_ms) - start_time_ms; const auto operations_state = neteq_->GetOperationsAndState(); @@ -271,8 +269,8 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { } } prev_lifetime_stats_ = lifetime_stats; - const bool no_more_packets_to_decode = - !input_->NextPacketTime() && !operations_state.next_packet_available; + const bool no_more_events = + !input_->NextEventTime() && !operations_state.next_packet_available; // End the simulation if the gap is too large. This indicates an issue // with the event log file. const bool simulation_step_too_large = result.simulation_step_ms > 1000; @@ -281,9 +279,8 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { // the simulation time, which can be a large distortion. result.simulation_step_ms = 10; } - result.is_simulation_finished = simulation_step_too_large || - no_more_packets_to_decode || - input_->ended(); + result.is_simulation_finished = + simulation_step_too_large || no_more_events || input_->ended(); prev_ops_state_ = operations_state; return result; } diff --git a/modules/audio_coding/neteq/tools/neteq_test_factory.cc b/modules/audio_coding/neteq/tools/neteq_test_factory.cc index 6cd371406c..faf098ccef 100644 --- a/modules/audio_coding/neteq/tools/neteq_test_factory.cc +++ b/modules/audio_coding/neteq/tools/neteq_test_factory.cc @@ -136,8 +136,8 @@ std::unique_ptr NetEqTestFactory::InitializeTestFromFile( std::unique_ptr input; if (RtpFileSource::ValidRtpDump(input_file_name) || RtpFileSource::ValidPcap(input_file_name)) { - input.reset(new NetEqRtpDumpInput(input_file_name, rtp_ext_map, - config.ssrc_filter)); + input = std::make_unique( + input_file_name, rtp_ext_map, config.ssrc_filter); } else { input.reset(NetEqEventLogInput::CreateFromFile(input_file_name, config.ssrc_filter)); @@ -169,21 +169,17 @@ std::unique_ptr NetEqTestFactory::InitializeTest( if (config.skip_get_audio_events > 0) { std::cout << "Skipping " << config.skip_get_audio_events << " get_audio events" << std::endl; - if (!input->NextPacketTime() || !input->NextOutputEventTime()) { + if (!input->NextEventTime()) { std::cerr << "No events found" << std::endl; return nullptr; } for (int i = 0; i < config.skip_get_audio_events; i++) { - input->AdvanceOutputEvent(); - if (!input->NextOutputEventTime()) { - std::cerr << "Not enough get_audio events found" << std::endl; - return nullptr; + std::unique_ptr event = input->PopEvent(); + while (event && event->type() != NetEqInput::Event::Type::kGetAudio) { + event = input->PopEvent(); } - } - while (*input->NextPacketTime() < *input->NextOutputEventTime()) { - input->PopPacket(); - if (!input->NextPacketTime()) { - std::cerr << "Not enough incoming packets found" << std::endl; + if (event == nullptr) { + std::cerr << "Not enough events found" << std::endl; return nullptr; } } @@ -212,7 +208,10 @@ std::unique_ptr NetEqTestFactory::InitializeTest( // types and SSRCs. discarded_pt_and_ssrc.emplace(first_rtp_header->payloadType, first_rtp_header->ssrc); - input->PopPacket(); + std::unique_ptr event = input->PopEvent(); + while (event && event->type() != NetEqInput::Event::Type::kPacketData) { + event = input->PopEvent(); + } } if (!discarded_pt_and_ssrc.empty()) { std::cout << "Discarded initial packets with the following payload types " diff --git a/modules/audio_coding/neteq/tools/rtc_event_log_source.cc b/modules/audio_coding/neteq/tools/rtc_event_log_source.cc index 68acfb7f62..64ae891274 100644 --- a/modules/audio_coding/neteq/tools/rtc_event_log_source.cc +++ b/modules/audio_coding/neteq/tools/rtc_event_log_source.cc @@ -95,7 +95,7 @@ int64_t RtcEventLogSource::NextAudioOutputEventMs() { return output_time_ms; } -absl::optional +absl::optional RtcEventLogSource::NextSetMinimumDelayEvent() { if (minimum_delay_index_ >= minimum_delay_.size()) { return absl::nullopt; diff --git a/modules/audio_coding/neteq/tools/rtc_event_log_source.h b/modules/audio_coding/neteq/tools/rtc_event_log_source.h index 1445314247..f8c021c313 100644 --- a/modules/audio_coding/neteq/tools/rtc_event_log_source.h +++ b/modules/audio_coding/neteq/tools/rtc_event_log_source.h @@ -55,7 +55,7 @@ class RtcEventLogSource : public PacketSource { int64_t NextAudioOutputEventMs(); // Returns the next NetEq set minimum delay event if available. - absl::optional NextSetMinimumDelayEvent(); + absl::optional NextSetMinimumDelayEvent(); private: RtcEventLogSource(); @@ -67,7 +67,7 @@ class RtcEventLogSource : public PacketSource { size_t rtp_packet_index_ = 0; std::vector audio_outputs_; size_t audio_output_index_ = 0; - std::vector minimum_delay_; + std::vector minimum_delay_; size_t minimum_delay_index_ = 0; }; diff --git a/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc b/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc index e2b4ecfe8d..38cb55425e 100644 --- a/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc +++ b/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc @@ -196,9 +196,52 @@ class NetEqStreamInput : public test::NetEqInput { end_time_ms_(end_time_ms) { RTC_DCHECK(packet_stream); RTC_DCHECK(output_events); + event_ = GetNextEvent(); } - absl::optional NextPacketTime() const override { + absl::optional NextEventTime() const override { + if (event_) { + return event_->timestamp_ms(); + } + return absl::nullopt; + } + + std::unique_ptr PopEvent() override { + std::unique_ptr return_event = std::move(event_); + event_ = GetNextEvent(); + return return_event; + } + + std::unique_ptr GetNextEvent() { + absl::optional next_time; + absl::optional next_packet_time = NextPacketTime(); + absl::optional next_output_time = NextOutputEventTime(); + absl::optional next_set_minimum_delay_time = + NextSetMinimumDelayTime(); + for (auto time : + {next_packet_time, next_output_time, next_set_minimum_delay_time}) { + if (time) { + if (next_time) { + next_time = time.value() < next_time.value() ? time : next_time; + } else { + next_time = time; + } + } + } + if (next_packet_time && next_packet_time.value() == next_time.value()) { + return PopPacket(); + } + if (next_output_time && next_output_time.value() == next_time.value()) { + return PopGetAudio(); + } + if (next_set_minimum_delay_time && + next_set_minimum_delay_time.value() == next_time.value()) { + return PopNetEqSetMinimumDelay(); + } + return nullptr; + } + + absl::optional NextPacketTime() const { if (packet_stream_it_ == packet_stream_.end()) { return absl::nullopt; } @@ -208,7 +251,7 @@ class NetEqStreamInput : public test::NetEqInput { return packet_stream_it_->rtp.log_time_ms(); } - absl::optional NextOutputEventTime() const override { + absl::optional NextOutputEventTime() const { if (output_events_it_ == output_events_end_) { return absl::nullopt; } @@ -218,7 +261,7 @@ class NetEqStreamInput : public test::NetEqInput { return output_events_it_->log_time_ms(); } - absl::optional NextSetMinimumDelayInfo() const override { + absl::optional NextSetMinimumDelayTime() const { if (neteq_set_minimum_delay_events_it_ == neteq_set_minimum_delay_events_end_) { return absl::nullopt; @@ -227,18 +270,16 @@ class NetEqStreamInput : public test::NetEqInput { neteq_set_minimum_delay_events_it_->log_time_ms() > *end_time_ms_) { return absl::nullopt; } - return SetMinimumDelayInfo( - neteq_set_minimum_delay_events_it_->log_time_ms(), - neteq_set_minimum_delay_events_it_->minimum_delay_ms); + return neteq_set_minimum_delay_events_it_->log_time_ms(); } - std::unique_ptr PopPacket() override { + std::unique_ptr PopPacket() { if (packet_stream_it_ == packet_stream_.end()) { return std::unique_ptr(); } std::unique_ptr packet_data(new PacketData()); packet_data->header = packet_stream_it_->rtp.header; - packet_data->time_ms = packet_stream_it_->rtp.log_time_ms(); + packet_data->timestamp_ms_ = packet_stream_it_->rtp.log_time_ms(); // This is a header-only "dummy" packet. Set the payload to all zeros, with // length according to the virtual length. @@ -250,17 +291,26 @@ class NetEqStreamInput : public test::NetEqInput { return packet_data; } - void AdvanceOutputEvent() override { + std::unique_ptr PopGetAudio() { + std::unique_ptr get_audio; if (output_events_it_ != output_events_end_) { + get_audio = std::make_unique(output_events_it_->log_time_ms()); ++output_events_it_; } + return get_audio; } - void AdvanceSetMinimumDelay() override { + std::unique_ptr PopNetEqSetMinimumDelay() { + std::unique_ptr net_eq_set_minimum_delay; if (neteq_set_minimum_delay_events_it_ != neteq_set_minimum_delay_events_end_) { + net_eq_set_minimum_delay = std::make_unique( + neteq_set_minimum_delay_events_it_->log_time_ms(), + neteq_set_minimum_delay_events_it_->minimum_delay_ms); + ++neteq_set_minimum_delay_events_it_; } + return net_eq_set_minimum_delay; } bool ended() const override { return !NextEventTime(); } @@ -282,6 +332,7 @@ class NetEqStreamInput : public test::NetEqInput { const std::vector::const_iterator neteq_set_minimum_delay_events_end_; const absl::optional end_time_ms_; + std::unique_ptr event_; }; namespace { diff --git a/test/fuzzers/neteq_rtp_fuzzer.cc b/test/fuzzers/neteq_rtp_fuzzer.cc index 3caa5fe5de..afde2a87c7 100644 --- a/test/fuzzers/neteq_rtp_fuzzer.cc +++ b/test/fuzzers/neteq_rtp_fuzzer.cc @@ -64,71 +64,65 @@ class FuzzRtpInput : public NetEqInput { new SineGenerator(config.sample_rate_hz)); input_.reset(new EncodeNetEqInput(std::move(generator), std::move(encoder), std::numeric_limits::max())); - packet_ = input_->PopPacket(); - FuzzHeader(); - MaybeFuzzPayload(); + // We pop the first event so we have information about the timing of such + // first event. + event_ = input_->PopEvent(); + if (event_ && event_->type() == Event::Type::kPacketData) { + FuzzPacket(static_cast(event_.get())); + } } - absl::optional NextPacketTime() const override { - return packet_->time_ms; - } - - absl::optional NextOutputEventTime() const override { - return input_->NextOutputEventTime(); - } - - absl::optional NextSetMinimumDelayInfo() const override { - return input_->NextSetMinimumDelayInfo(); - } - - std::unique_ptr PopPacket() override { - RTC_DCHECK(packet_); - std::unique_ptr packet_to_return = std::move(packet_); - packet_ = input_->PopPacket(); - FuzzHeader(); - MaybeFuzzPayload(); - return packet_to_return; - } - - void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); } - - void AdvanceSetMinimumDelay() override { - return input_->AdvanceSetMinimumDelay(); + std::unique_ptr PopEvent() override { + std::unique_ptr event_to_return = std::move(event_); + event_ = input_->PopEvent(); + if (event_ && event_->type() == Event::Type::kPacketData) { + FuzzPacket(static_cast(event_.get())); + } + return event_to_return; } bool ended() const override { return ended_; } absl::optional NextHeader() const override { - RTC_DCHECK(packet_); - return packet_->header; + RTC_DCHECK(event_ && event_->type() == Event::Type::kPacketData); + return static_cast(*event_).header; + } + + absl::optional NextEventTime() const override { + if (event_) { + return event_->timestamp_ms(); + } + return absl::nullopt; } private: - void FuzzHeader() { + void FuzzPacket(PacketData* packet_data) { + FuzzHeader(packet_data->header); + MaybeFuzzPayload(packet_data->payload); + } + + void FuzzHeader(RTPHeader& header) { constexpr size_t kNumBytesToFuzz = 11; if (data_ix_ + kNumBytesToFuzz > data_.size()) { ended_ = true; return; } - RTC_DCHECK(packet_); const size_t start_ix = data_ix_; - packet_->header.payloadType = + header.payloadType = ByteReader::ReadLittleEndian(&data_[data_ix_]); - packet_->header.payloadType &= 0x7F; + header.payloadType &= 0x7F; data_ix_ += sizeof(uint8_t); - packet_->header.sequenceNumber = + header.sequenceNumber = ByteReader::ReadLittleEndian(&data_[data_ix_]); data_ix_ += sizeof(uint16_t); - packet_->header.timestamp = - ByteReader::ReadLittleEndian(&data_[data_ix_]); + header.timestamp = ByteReader::ReadLittleEndian(&data_[data_ix_]); data_ix_ += sizeof(uint32_t); - packet_->header.ssrc = - ByteReader::ReadLittleEndian(&data_[data_ix_]); + header.ssrc = ByteReader::ReadLittleEndian(&data_[data_ix_]); data_ix_ += sizeof(uint32_t); RTC_CHECK_EQ(data_ix_ - start_ix, kNumBytesToFuzz); } - void MaybeFuzzPayload() { + void MaybeFuzzPayload(rtc::Buffer& payload) { // Read one byte of fuzz data to determine how many payload bytes to fuzz. if (data_ix_ + 1 > data_.size()) { ended_ = true; @@ -138,7 +132,7 @@ class FuzzRtpInput : public NetEqInput { // Restrict number of bytes to fuzz to 16; a reasonably low number enough to // cover a few RED headers. Also don't write outside the payload length. - bytes_to_fuzz = std::min(bytes_to_fuzz % 16, packet_->payload.size()); + bytes_to_fuzz = std::min(bytes_to_fuzz % 16, payload.size()); if (bytes_to_fuzz == 0) return; @@ -148,7 +142,7 @@ class FuzzRtpInput : public NetEqInput { return; } - std::memcpy(packet_->payload.data(), &data_[data_ix_], bytes_to_fuzz); + std::memcpy(payload.data(), &data_[data_ix_], bytes_to_fuzz); data_ix_ += bytes_to_fuzz; } @@ -156,7 +150,7 @@ class FuzzRtpInput : public NetEqInput { rtc::ArrayView data_; size_t data_ix_ = 0; std::unique_ptr input_; - std::unique_ptr packet_; + std::unique_ptr event_; }; } // namespace diff --git a/test/fuzzers/neteq_signal_fuzzer.cc b/test/fuzzers/neteq_signal_fuzzer.cc index 485c38085e..6ca8c7dfaf 100644 --- a/test/fuzzers/neteq_signal_fuzzer.cc +++ b/test/fuzzers/neteq_signal_fuzzer.cc @@ -80,8 +80,10 @@ class FuzzSignalInput : public NetEqInput { new SineAndNoiseGenerator(config.sample_rate_hz, fuzz_data)); input_.reset(new EncodeNetEqInput(std::move(generator), std::move(encoder), std::numeric_limits::max())); - packet_ = input_->PopPacket(); - + // We pop the first event so we have information about the timing of such + // first event. + PopPacket(); + PopEvent(); // Select an output event period. This is how long time we wait between each // call to NetEq::GetAudio. 10 ms is nominal, 9 and 11 ms will both lead to // clock drift (in different directions). @@ -89,32 +91,45 @@ class FuzzSignalInput : public NetEqInput { output_event_period_ms_ = fuzz_data_.SelectOneOf(output_event_periods); } - absl::optional NextPacketTime() const override { - return packet_->time_ms; + std::unique_ptr PopEvent() override { + std::unique_ptr event_to_return = std::move(event_); + if (packet_ && packet_->timestamp_ms() <= next_output_event_ms_) { + event_ = PopPacket(); + } else { + event_ = std::make_unique(next_output_event_ms_); + next_output_event_ms_ += output_event_period_ms_; + } + return event_to_return; } - absl::optional NextOutputEventTime() const override { - return next_output_event_ms_; + absl::optional NextEventTime() const override { + if (event_) { + return event_->timestamp_ms(); + } + return absl::nullopt; } - absl::optional NextSetMinimumDelayInfo() const override { - return input_->NextSetMinimumDelayInfo(); - } - - std::unique_ptr PopPacket() override { - RTC_DCHECK(packet_); - std::unique_ptr packet_to_return = std::move(packet_); + std::unique_ptr PopPacket() { + std::unique_ptr packet_to_return = std::move(packet_); + int64_t packet_to_return_time_ms = + packet_to_return ? packet_to_return->timestamp_ms() : 0; + PacketData* packet_data = nullptr; do { - packet_ = input_->PopPacket(); + std::unique_ptr event = input_->PopEvent(); + while (event && event->type() != Event::Type::kPacketData) { + event = input_->PopEvent(); + } + packet_data = static_cast(event.get()); // If the next value from the fuzzer input is 0, the packet is discarded // and the next one is pulled from the source. } while (fuzz_data_.CanReadBytes(1) && fuzz_data_.Read() == 0); - if (fuzz_data_.CanReadBytes(1)) { + if (fuzz_data_.CanReadBytes(1) && packet_data) { // Generate jitter by setting an offset for the arrival time. const int8_t arrival_time_offset_ms = fuzz_data_.Read(); // The arrival time can not be before the previous packets. - packet_->time_ms = std::max(packet_to_return->time_ms, - packet_->time_ms + arrival_time_offset_ms); + packet_data->timestamp_ms_ = + std::max(packet_to_return_time_ms, + packet_data->timestamp_ms_ + arrival_time_offset_ms); } else { // Mark that we are at the end of the test. However, the current packet is // still valid (but it may not have been fuzzed as expected). @@ -123,28 +138,21 @@ class FuzzSignalInput : public NetEqInput { return packet_to_return; } - void AdvanceOutputEvent() override { - next_output_event_ms_ += output_event_period_ms_; - } - - void AdvanceSetMinimumDelay() override { - return input_->AdvanceSetMinimumDelay(); - } - bool ended() const override { return ended_; } absl::optional NextHeader() const override { RTC_DCHECK(packet_); - return packet_->header; + return static_cast(*packet_).header; } private: bool ended_ = false; FuzzDataHelper& fuzz_data_; std::unique_ptr input_; - std::unique_ptr packet_; + std::unique_ptr packet_; int64_t next_output_event_ms_ = 0; - int64_t output_event_period_ms_ = 10; + int64_t output_event_period_ms_; + std::unique_ptr event_; }; template