From 4ddd93102339244167e22cc5f470975a22e1bced Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Tue, 30 Jul 2024 10:33:51 -0700 Subject: [PATCH] video_replay: add pcapng support as documented on https://pcapng.com/ Nanosec resolution for timestamps is assumed as described in https://gitlab.com/wireshark/wireshark/-/blob/master/writecap/pcapio.c BUG=webrtc:351327754 Change-Id: Ieec601a33c131908e30e7f7e41ddc89ddc1c36b7 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/356461 Reviewed-by: Mirko Bonadei Commit-Queue: Mirko Bonadei Reviewed-by: Harald Alvestrand Cr-Commit-Position: refs/heads/main@{#42699} --- test/BUILD.gn | 1 + test/rtp_file_reader.cc | 328 +++++++++++++++++++++++++++------------- 2 files changed, 224 insertions(+), 105 deletions(-) diff --git a/test/BUILD.gn b/test/BUILD.gn index c16d0316be..67ff523aad 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -305,6 +305,7 @@ rtc_library("rtp_test_utils") { "../rtc_base:criticalsection", "../rtc_base:logging", "../rtc_base:macromagic", + "../rtc_base:timeutils", "../rtc_base/synchronization:mutex", "../rtc_base/system:arch", "//third_party/abseil-cpp/absl/strings:string_view", diff --git a/test/rtp_file_reader.cc b/test/rtp_file_reader.cc index b6f3cbbe5b..244f183ae5 100644 --- a/test/rtp_file_reader.cc +++ b/test/rtp_file_reader.cc @@ -10,8 +10,6 @@ #include "test/rtp_file_reader.h" -#include - #include #include #include @@ -21,12 +19,44 @@ #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/system/arch.h" +#include "rtc_base/time_utils.h" -namespace webrtc { -namespace test { +namespace { +constexpr size_t kRtpDumpFirstLineLength = 80; +constexpr uint16_t kRtpDumpPacketHeaderSize = 8; -static const size_t kFirstLineLength = 80; -static uint16_t kPacketHeaderSize = 8; +enum { + kResultFail = -1, + kResultSuccess = 0, + kResultSkip = 1, +}; + +enum { + kPcapVersionMajor = 2, + kPcapVersionMinor = 4, + kLinktypeNull = 0, + kLinktypeEthernet = 1, + kBsdNullLoopback1 = 0x00000002, + kBsdNullLoopback2 = 0x02000000, + kEthernetIIHeaderMacSkip = 12, + kEthertypeIp = 0x0800, + kIpVersion4 = 4, + kMinIpHeaderLength = 20, + kFragmentOffsetClear = 0x0000, + kFragmentOffsetDoNotFragment = 0x4000, + kProtocolTcp = 0x06, + kProtocolUdp = 0x11, + kUdpHeaderLength = 8, +}; + +constexpr size_t kMaxReadBufferSize = 4096; +constexpr uint32_t kPcapBOMSwapOrder = 0xd4c3b2a1UL; +constexpr uint32_t kPcapBOMNoSwapOrder = 0xa1b2c3d4UL; +constexpr uint32_t kPcapNgBOMLittleEndian = 0x4d3c2b1aUL; + +constexpr uint32_t kPcapNgSectionHeaderBlock = 0x0a0d0d0aUL; +constexpr uint32_t kPcapNgInterfaceDescriptionBlock = 0x00000001LU; +constexpr uint32_t kPcapNgPacketBlock = 0x00000006LU; #define TRY(expr) \ do { \ @@ -36,6 +66,17 @@ static uint16_t kPacketHeaderSize = 8; } \ } while (0) +#define TRY_PCAP(expr) \ + do { \ + int r = (expr); \ + if (r == kResultFail) { \ + RTC_LOG(LS_INFO) << "FAIL at " << __FILE__ << ":" << __LINE__; \ + return kResultFail; \ + } else if (r == kResultSkip) { \ + return kResultSkip; \ + } \ + } while (0) + bool ReadUint32(uint32_t* out, FILE* file) { *out = 0; for (size_t i = 0; i < 4; ++i) { @@ -60,6 +101,11 @@ bool ReadUint16(uint16_t* out, FILE* file) { return true; } +} // namespace + +namespace webrtc { +namespace test { + class RtpFileReaderImpl : public RtpFileReader { public: virtual bool Init(FILE* file, const std::set& ssrc_filter) = 0; @@ -123,8 +169,8 @@ class RtpDumpReader : public RtpFileReaderImpl { bool Init(FILE* file, const std::set& ssrc_filter) override { file_ = file; - char firstline[kFirstLineLength + 1] = {0}; - if (fgets(firstline, kFirstLineLength, file_) == nullptr) { + char firstline[kRtpDumpFirstLineLength + 1] = {0}; + if (fgets(firstline, kRtpDumpFirstLineLength, file_) == nullptr) { RTC_LOG(LS_INFO) << "Can't read from file"; return false; } @@ -139,7 +185,8 @@ class RtpDumpReader : public RtpFileReaderImpl { return false; } } else { - RTC_LOG(LS_INFO) << "Wrong file format of input file"; + RTC_LOG(LS_INFO) + << "Input file is neither in rtpplay nor RTPencode format"; return false; } @@ -169,7 +216,7 @@ class RtpDumpReader : public RtpFileReaderImpl { TRY(ReadUint32(&offset, file_)); // Use 'len' here because a 'plen' of 0 specifies rtcp. - len -= kPacketHeaderSize; + len -= kRtpDumpPacketHeaderSize; if (packet->length < len) { RTC_LOG(LS_ERROR) << "Packet is too large to fit: " << len << " bytes vs " << packet->length @@ -191,45 +238,10 @@ class RtpDumpReader : public RtpFileReaderImpl { FILE* file_; }; -enum { - kResultFail = -1, - kResultSuccess = 0, - kResultSkip = 1, - - kPcapVersionMajor = 2, - kPcapVersionMinor = 4, - kLinktypeNull = 0, - kLinktypeEthernet = 1, - kBsdNullLoopback1 = 0x00000002, - kBsdNullLoopback2 = 0x02000000, - kEthernetIIHeaderMacSkip = 12, - kEthertypeIp = 0x0800, - kIpVersion4 = 4, - kMinIpHeaderLength = 20, - kFragmentOffsetClear = 0x0000, - kFragmentOffsetDoNotFragment = 0x4000, - kProtocolTcp = 0x06, - kProtocolUdp = 0x11, - kUdpHeaderLength = 8, - kMaxReadBufferSize = 4096 -}; - -const uint32_t kPcapBOMSwapOrder = 0xd4c3b2a1UL; -const uint32_t kPcapBOMNoSwapOrder = 0xa1b2c3d4UL; - -#define TRY_PCAP(expr) \ - do { \ - int r = (expr); \ - if (r == kResultFail) { \ - RTC_LOG(LS_INFO) << "FAIL at " << __FILE__ << ":" << __LINE__; \ - return kResultFail; \ - } else if (r == kResultSkip) { \ - return kResultSkip; \ - } \ - } while (0) - // Read RTP packets from file in tcpdump/libpcap format, as documented at: // http://wiki.wireshark.org/Development/LibpcapFileFormat +// Transparently supports PCAPNG as described at +// https://pcapng.com/ class PcapReader : public RtpFileReaderImpl { public: PcapReader() @@ -240,6 +252,7 @@ class PcapReader : public RtpFileReaderImpl { #else swap_network_byte_order_(true), #endif + pcapng_(false), read_buffer_(), packets_by_ssrc_(), packets_(), @@ -263,42 +276,30 @@ class PcapReader : public RtpFileReaderImpl { int Initialize(FILE* file, const std::set& ssrc_filter) { file_ = file; + size_t total_packet_count = 0; if (ReadGlobalHeader() < 0) { return kResultFail; } - - int total_packet_count = 0; - uint32_t stream_start_ms = 0; - int32_t next_packet_pos = ftell(file_); - for (;;) { - TRY_PCAP(fseek(file_, next_packet_pos, SEEK_SET)); - int result = ReadPacket(&next_packet_pos, stream_start_ms, - ++total_packet_count, ssrc_filter); - if (result == kResultFail) { - break; - } else if (result == kResultSuccess && packets_.size() == 1) { - RTC_DCHECK_EQ(stream_start_ms, 0); - PacketIterator it = packets_.begin(); - stream_start_ms = it->time_offset_ms; - it->time_offset_ms = 0; - } + int result; + if (!pcapng_) { + result = ReadPcap(ssrc_filter, total_packet_count); + } else { + result = ReadPcapNg(ssrc_filter, total_packet_count); } - - if (feof(file_) == 0) { - printf("Failed reading file!\n"); + if (result == kResultFail) { return kResultFail; } - printf("Total packets in file: %d\n", total_packet_count); - printf("Total RTP/RTCP packets: %zu\n", packets_.size()); + RTC_LOG(LS_INFO) << "Total packets in file: " << total_packet_count; + RTC_LOG(LS_INFO) << "Total RTP/RTCP packets: " << packets_.size(); for (SsrcMapIterator mit = packets_by_ssrc_.begin(); mit != packets_by_ssrc_.end(); ++mit) { uint32_t ssrc = mit->first; const std::vector& packet_indices = mit->second; int pt = packets_[packet_indices[0]].payload_type; - printf("SSRC: %08x, %zu packets, pt=%d\n", ssrc, packet_indices.size(), - pt); + RTC_LOG(LS_INFO) << "SSRC: " << ssrc << ", " << packet_indices.size() + << " packets, pt=" << pt << "."; } // TODO(solenberg): Better validation of identified SSRC streams. @@ -321,6 +322,56 @@ class PcapReader : public RtpFileReaderImpl { return kResultSuccess; } + int ReadPcap(const std::set& ssrc_filter, + size_t& total_packet_count) { + uint32_t stream_start_ms = 0; + int32_t next_packet_pos = ftell(file_); + for (;;) { + TRY_PCAP(fseek(file_, next_packet_pos, SEEK_SET)); + int result = ReadPacket(&next_packet_pos, stream_start_ms, ssrc_filter); + if (result == kResultFail) { + break; + } else if (result == kResultSuccess && packets_.size() == 1) { + RTC_DCHECK_EQ(stream_start_ms, 0); + PacketIterator it = packets_.begin(); + stream_start_ms = it->time_offset_ms; + it->time_offset_ms = 0; + } + total_packet_count++; + } + + if (feof(file_) == 0) { + RTC_LOG(LS_ERROR) << "Failed reading file!"; + return kResultFail; + } + return kResultSuccess; + } + + int ReadPcapNg(const std::set& ssrc_filter, + size_t& total_packet_count) { + uint32_t stream_start_ms = 0; + int next_packet_pos = 0; + for (;;) { + TRY_PCAP(fseek(file_, next_packet_pos, SEEK_SET)); + int result = ReadPacketNg(&next_packet_pos, stream_start_ms, ssrc_filter); + + if (result == kResultFail) { + break; + } else if (result == kResultSuccess && packets_.size() == 1) { + RTC_DCHECK_EQ(stream_start_ms, 0); + PacketIterator it = packets_.begin(); + stream_start_ms = it->time_offset_ms; + it->time_offset_ms = 0; + } + total_packet_count++; + } + if (feof(file_) == 0) { + RTC_LOG(LS_ERROR) << "Failed reading file!"; + return kResultFail; + } + return kResultSuccess; + } + bool NextPacket(RtpPacket* packet) override { uint32_t length = RtpPacket::kMaxPacketBufferSize; if (NextPcap(packet->data, &length, &packet->time_ms) != kResultSuccess) @@ -353,7 +404,6 @@ class PcapReader : public RtpFileReaderImpl { private: // A marker of an RTP packet within the file. struct RtpPacketMarker { - uint32_t packet_number; // One-based index (like in WireShark) uint32_t time_offset_ms; uint32_t source_ip; uint32_t dest_ip; @@ -377,6 +427,10 @@ class PcapReader : public RtpFileReaderImpl { swap_pcap_byte_order_ = true; } else if (magic == kPcapBOMNoSwapOrder) { swap_pcap_byte_order_ = false; + } else if (magic == kPcapNgSectionHeaderBlock) { + pcapng_ = true; + RTC_LOG(LS_INFO) << "PCAPNG detected, support is experimental"; + return kResultSuccess; } else { return kResultFail; } @@ -408,36 +462,9 @@ class PcapReader : public RtpFileReaderImpl { return kResultSuccess; } - int ReadPacket(int32_t* next_packet_pos, - uint32_t stream_start_ms, - uint32_t number, - const std::set& ssrc_filter) { - RTC_DCHECK(next_packet_pos); - - uint32_t ts_sec; // Timestamp seconds. - uint32_t ts_usec; // Timestamp microseconds. - uint32_t incl_len; // Number of octets of packet saved in file. - uint32_t orig_len; // Actual length of packet. - TRY_PCAP(Read(&ts_sec, false)); - TRY_PCAP(Read(&ts_usec, false)); - TRY_PCAP(Read(&incl_len, false)); - TRY_PCAP(Read(&orig_len, false)); - - *next_packet_pos = ftell(file_) + incl_len; - - RtpPacketMarker marker = {0}; - marker.packet_number = number; - marker.time_offset_ms = CalcTimeDelta(ts_sec, ts_usec, stream_start_ms); - TRY_PCAP(ReadPacketHeader(&marker)); - marker.pos_in_file = ftell(file_); - - if (marker.payload_length > sizeof(read_buffer_)) { - printf("Packet too large!\n"); - return kResultFail; - } - TRY_PCAP(Read(read_buffer_, marker.payload_length)); - - rtc::ArrayView packet(read_buffer_, marker.payload_length); + int ProcessPacket(RtpPacketMarker& marker, + const std::set& ssrc_filter, + rtc::ArrayView packet) { if (IsRtcpPacket(packet)) { marker.payload_type = packet[1]; packets_.push_back(marker); @@ -459,6 +486,98 @@ class PcapReader : public RtpFileReaderImpl { return kResultSuccess; } + int ReadPacket(int32_t* next_packet_pos, + uint32_t stream_start_ms, + const std::set& ssrc_filter) { + RTC_DCHECK(next_packet_pos); + + uint32_t ts_sec; // Timestamp seconds. + uint32_t ts_usec; // Timestamp microseconds. + uint32_t incl_len; // Number of octets of packet saved in file. + uint32_t orig_len; // Actual length of packet. + TRY_PCAP(Read(&ts_sec, false)); + TRY_PCAP(Read(&ts_usec, false)); + TRY_PCAP(Read(&incl_len, false)); + TRY_PCAP(Read(&orig_len, false)); + + *next_packet_pos = ftell(file_) + incl_len; + + RtpPacketMarker marker = {0}; + marker.time_offset_ms = CalcTimeDelta(ts_sec, ts_usec, stream_start_ms); + TRY_PCAP(ReadPacketHeader(&marker)); + marker.pos_in_file = ftell(file_); + + if (marker.payload_length > sizeof(read_buffer_)) { + RTC_LOG(LS_ERROR) << "Packet too large!"; + return kResultFail; + } + TRY_PCAP(Read(read_buffer_, marker.payload_length)); + return ProcessPacket(marker, ssrc_filter, + {read_buffer_, marker.payload_length}); + } + + int ReadPacketNg(int32_t* next_packet_pos, + uint32_t stream_start_ms, + const std::set& ssrc_filter) { + uint32_t block_type; + uint32_t block_length; + TRY_PCAP(Read(&block_type, false)); + TRY_PCAP(Read(&block_length, false)); + if (block_length == 0) { + RTC_LOG(LS_ERROR) << "Empty PCAPNG block"; + return kResultFail; + } + + *next_packet_pos += block_length; + switch (block_type) { + case kPcapNgSectionHeaderBlock: { + // TODO: https://issues.webrtc.org/issues/351327754 - interpret more of + // this block, in particular the if_tsresol option. + uint32_t byte_order_magic; + TRY_PCAP(Read(&byte_order_magic, false)); + swap_pcap_byte_order_ = (byte_order_magic == kPcapNgBOMLittleEndian); + } break; + case kPcapNgInterfaceDescriptionBlock: + break; + case kPcapNgPacketBlock: { + uint32_t interface; // Interface ID. Unused. + uint32_t ts_upper; // Upper 32 bits of timestamp. + uint32_t ts_lower; // Lower 32 bits of timestamp. + uint32_t incl_len; // Number of octets of packet saved in file. + uint32_t orig_len; // Actual length of packet. + TRY_PCAP(Read(&interface, false)); + TRY_PCAP(Read(&ts_upper, false)); + TRY_PCAP(Read(&ts_lower, false)); + TRY_PCAP(Read(&incl_len, false)); + TRY_PCAP(Read(&orig_len, false)); + + RtpPacketMarker marker = {0}; + // Note: Wireshark writes nanoseconds most of the time, see comments in + // it's pcapio.c. We are only interesting in the time difference so + // truncating to uint32_t is ok. + uint64_t timestamp_ms = + ((static_cast(ts_upper) << 32) | ts_lower) / + rtc::kNumMicrosecsPerSec; + marker.time_offset_ms = + static_cast(timestamp_ms) - stream_start_ms; + TRY_PCAP(ReadPacketHeader(&marker)); + marker.pos_in_file = ftell(file_); + if (marker.payload_length > sizeof(read_buffer_)) { + RTC_LOG(LS_ERROR) << "Packet too large!"; + return kResultFail; + } + TRY_PCAP(Read(read_buffer_, marker.payload_length)); + if (ProcessPacket(marker, ssrc_filter, + {read_buffer_, marker.payload_length}) != + kResultSuccess) { + return kResultFail; + } + return kResultSuccess; + } break; + } + return kResultSkip; + } + int ReadPacketHeader(RtpPacketMarker* marker) { int32_t file_pos = ftell(file_); @@ -470,7 +589,6 @@ class PcapReader : public RtpFileReaderImpl { TRY_PCAP(Read(&protocol, true)); if (protocol == kBsdNullLoopback1 || protocol == kBsdNullLoopback2) { int result = ReadXxpIpHeader(marker); - RTC_LOG(LS_INFO) << "Recognized loopback frame"; if (result != kResultSkip) { return result; } @@ -484,7 +602,6 @@ class PcapReader : public RtpFileReaderImpl { TRY_PCAP(Read(&type, true)); if (type == kEthertypeIp) { int result = ReadXxpIpHeader(marker); - RTC_LOG(LS_INFO) << "Recognized ethernet 2 frame"; if (result != kResultSkip) { return result; } @@ -617,6 +734,7 @@ class PcapReader : public RtpFileReaderImpl { FILE* file_; bool swap_pcap_byte_order_; const bool swap_network_byte_order_; + bool pcapng_; uint8_t read_buffer_[kMaxReadBufferSize]; SsrcMap packets_by_ssrc_; @@ -648,7 +766,7 @@ RtpFileReader* RtpFileReader::Create(FileFormat format, FILE* file = tmpfile(); if (file == nullptr) { - printf("ERROR: Can't open file from memory buffer\n"); + RTC_LOG(LS_ERROR) << "ERROR: Can't open file from memory buffer."; return nullptr; } @@ -671,7 +789,7 @@ RtpFileReader* RtpFileReader::Create(FileFormat format, std::string filename_str = std::string(filename); FILE* file = fopen(filename_str.c_str(), "rb"); if (file == nullptr) { - printf("ERROR: Can't open file: %s\n", filename_str.c_str()); + RTC_LOG(LS_ERROR) << "ERROR: Can't open file: '" << filename_str << "'."; return nullptr; }