admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

video_replay: add pcapng support

Browse Source 
as documented on
  https://pcapng.com/

Nanosec resolution for timestamps is assumed as described in
  https://gitlab.com/wireshark/wireshark/-/blob/master/writecap/pcapio.c

BUG=webrtc:351327754

Change-Id: Ieec601a33c131908e30e7f7e41ddc89ddc1c36b7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/356461
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42699}
This commit is contained in:
Philipp Hancke 2024-07-30 10:33:51 -07:00 committed by WebRTC LUCI CQ
parent 96c1b9c5ea
commit 4ddd931023
2 changed files with 224 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/BUILD.gn
View File

@ -305,6 +305,7 @@ rtc_library("rtp_test_utils") {
"../rtc_base:criticalsection", "../rtc_base:criticalsection",
"../rtc_base:logging", "../rtc_base:logging",
"../rtc_base:macromagic", "../rtc_base:macromagic",
"../rtc_base:timeutils",
"../rtc_base/synchronization:mutex", "../rtc_base/synchronization:mutex",
"../rtc_base/system:arch", "../rtc_base/system:arch",
"//third_party/abseil-cpp/absl/strings:string_view", "//third_party/abseil-cpp/absl/strings:string_view",

328
test/rtp_file_reader.cc
View File

@ -10,8 +10,6 @@
#include "test/rtp_file_reader.h" #include "test/rtp_file_reader.h"
#include <stdio.h>
#include <map> #include <map>
#include <string> #include <string>
#include <vector> #include <vector>
@ -21,12 +19,44 @@
#include "rtc_base/checks.h" #include "rtc_base/checks.h"
#include "rtc_base/logging.h" #include "rtc_base/logging.h"
#include "rtc_base/system/arch.h" #include "rtc_base/system/arch.h"
#include "rtc_base/time_utils.h"
namespace webrtc { namespace {
namespace test { constexpr size_t kRtpDumpFirstLineLength = 80;
constexpr uint16_t kRtpDumpPacketHeaderSize = 8;
static const size_t kFirstLineLength = 80; enum {
static uint16_t kPacketHeaderSize = 8; kResultFail = -1,
kResultSuccess = 0,
kResultSkip = 1,
};
enum {
kPcapVersionMajor = 2,
kPcapVersionMinor = 4,
kLinktypeNull = 0,
kLinktypeEthernet = 1,
kBsdNullLoopback1 = 0x00000002,
kBsdNullLoopback2 = 0x02000000,
kEthernetIIHeaderMacSkip = 12,
kEthertypeIp = 0x0800,
kIpVersion4 = 4,
kMinIpHeaderLength = 20,
kFragmentOffsetClear = 0x0000,
kFragmentOffsetDoNotFragment = 0x4000,
kProtocolTcp = 0x06,
kProtocolUdp = 0x11,
kUdpHeaderLength = 8,
};
constexpr size_t kMaxReadBufferSize = 4096;
constexpr uint32_t kPcapBOMSwapOrder = 0xd4c3b2a1UL;
constexpr uint32_t kPcapBOMNoSwapOrder = 0xa1b2c3d4UL;
constexpr uint32_t kPcapNgBOMLittleEndian = 0x4d3c2b1aUL;
constexpr uint32_t kPcapNgSectionHeaderBlock = 0x0a0d0d0aUL;
constexpr uint32_t kPcapNgInterfaceDescriptionBlock = 0x00000001LU;
constexpr uint32_t kPcapNgPacketBlock = 0x00000006LU;
#define TRY(expr) \ #define TRY(expr) \
do { \ do { \
@ -36,6 +66,17 @@ static uint16_t kPacketHeaderSize = 8;
} \ } \
} while (0) } while (0)
#define TRY_PCAP(expr) \
do { \
int r = (expr); \
if (r == kResultFail) { \
RTC_LOG(LS_INFO) << "FAIL at " << __FILE__ << ":" << __LINE__; \
return kResultFail; \
} else if (r == kResultSkip) { \
return kResultSkip; \
} \
} while (0)
bool ReadUint32(uint32_t* out, FILE* file) { bool ReadUint32(uint32_t* out, FILE* file) {
*out = 0; *out = 0;
for (size_t i = 0; i < 4; ++i) { for (size_t i = 0; i < 4; ++i) {
@ -60,6 +101,11 @@ bool ReadUint16(uint16_t* out, FILE* file) {
return true; return true;
} }
} // namespace
namespace webrtc {
namespace test {
class RtpFileReaderImpl : public RtpFileReader { class RtpFileReaderImpl : public RtpFileReader {
public: public:
virtual bool Init(FILE* file, const std::set<uint32_t>& ssrc_filter) = 0; virtual bool Init(FILE* file, const std::set<uint32_t>& ssrc_filter) = 0;
@ -123,8 +169,8 @@ class RtpDumpReader : public RtpFileReaderImpl {
bool Init(FILE* file, const std::set<uint32_t>& ssrc_filter) override { bool Init(FILE* file, const std::set<uint32_t>& ssrc_filter) override {
file_ = file; file_ = file;
char firstline[kFirstLineLength + 1] = {0}; char firstline[kRtpDumpFirstLineLength + 1] = {0};
if (fgets(firstline, kFirstLineLength, file_) == nullptr) { if (fgets(firstline, kRtpDumpFirstLineLength, file_) == nullptr) {
RTC_LOG(LS_INFO) << "Can't read from file"; RTC_LOG(LS_INFO) << "Can't read from file";
return false; return false;
} }
@ -139,7 +185,8 @@ class RtpDumpReader : public RtpFileReaderImpl {
return false; return false;
} }
} else { } else {
RTC_LOG(LS_INFO) << "Wrong file format of input file"; RTC_LOG(LS_INFO)
<< "Input file is neither in rtpplay nor RTPencode format";
return false; return false;
} }
@ -169,7 +216,7 @@ class RtpDumpReader : public RtpFileReaderImpl {
TRY(ReadUint32(&offset, file_)); TRY(ReadUint32(&offset, file_));
// Use 'len' here because a 'plen' of 0 specifies rtcp. // Use 'len' here because a 'plen' of 0 specifies rtcp.
len -= kPacketHeaderSize; len -= kRtpDumpPacketHeaderSize;
if (packet->length < len) { if (packet->length < len) {
RTC_LOG(LS_ERROR) << "Packet is too large to fit: " << len << " bytes vs " RTC_LOG(LS_ERROR) << "Packet is too large to fit: " << len << " bytes vs "
<< packet->length << packet->length
@ -191,45 +238,10 @@ class RtpDumpReader : public RtpFileReaderImpl {
FILE* file_; FILE* file_;
}; };
enum {
kResultFail = -1,
kResultSuccess = 0,
kResultSkip = 1,
kPcapVersionMajor = 2,
kPcapVersionMinor = 4,
kLinktypeNull = 0,
kLinktypeEthernet = 1,
kBsdNullLoopback1 = 0x00000002,
kBsdNullLoopback2 = 0x02000000,
kEthernetIIHeaderMacSkip = 12,
kEthertypeIp = 0x0800,
kIpVersion4 = 4,
kMinIpHeaderLength = 20,
kFragmentOffsetClear = 0x0000,
kFragmentOffsetDoNotFragment = 0x4000,
kProtocolTcp = 0x06,
kProtocolUdp = 0x11,
kUdpHeaderLength = 8,
kMaxReadBufferSize = 4096
};
const uint32_t kPcapBOMSwapOrder = 0xd4c3b2a1UL;
const uint32_t kPcapBOMNoSwapOrder = 0xa1b2c3d4UL;
#define TRY_PCAP(expr) \
do { \
int r = (expr); \
if (r == kResultFail) { \
RTC_LOG(LS_INFO) << "FAIL at " << __FILE__ << ":" << __LINE__; \
return kResultFail; \
} else if (r == kResultSkip) { \
return kResultSkip; \
} \
} while (0)
// Read RTP packets from file in tcpdump/libpcap format, as documented at: // Read RTP packets from file in tcpdump/libpcap format, as documented at:
// http://wiki.wireshark.org/Development/LibpcapFileFormat // http://wiki.wireshark.org/Development/LibpcapFileFormat
// Transparently supports PCAPNG as described at
// https://pcapng.com/
class PcapReader : public RtpFileReaderImpl { class PcapReader : public RtpFileReaderImpl {
public: public:
PcapReader() PcapReader()
@ -240,6 +252,7 @@ class PcapReader : public RtpFileReaderImpl {
#else #else
swap_network_byte_order_(true), swap_network_byte_order_(true),
#endif #endif
pcapng_(false),
read_buffer_(), read_buffer_(),
packets_by_ssrc_(), packets_by_ssrc_(),
packets_(), packets_(),
@ -263,42 +276,30 @@ class PcapReader : public RtpFileReaderImpl {
int Initialize(FILE* file, const std::set<uint32_t>& ssrc_filter) { int Initialize(FILE* file, const std::set<uint32_t>& ssrc_filter) {
file_ = file; file_ = file;
size_t total_packet_count = 0;
if (ReadGlobalHeader() < 0) { if (ReadGlobalHeader() < 0) {
return kResultFail; return kResultFail;
} }
int result;
int total_packet_count = 0; if (!pcapng_) {
uint32_t stream_start_ms = 0; result = ReadPcap(ssrc_filter, total_packet_count);
int32_t next_packet_pos = ftell(file_); } else {
for (;;) { result = ReadPcapNg(ssrc_filter, total_packet_count);
TRY_PCAP(fseek(file_, next_packet_pos, SEEK_SET));
int result = ReadPacket(&next_packet_pos, stream_start_ms,
++total_packet_count, ssrc_filter);
if (result == kResultFail) {
break;
} else if (result == kResultSuccess && packets_.size() == 1) {
RTC_DCHECK_EQ(stream_start_ms, 0);
PacketIterator it = packets_.begin();
stream_start_ms = it->time_offset_ms;
it->time_offset_ms = 0;
}
} }
if (result == kResultFail) {
if (feof(file_) == 0) {
printf("Failed reading file!\n");
return kResultFail; return kResultFail;
} }
printf("Total packets in file: %d\n", total_packet_count); RTC_LOG(LS_INFO) << "Total packets in file: " << total_packet_count;
printf("Total RTP/RTCP packets: %zu\n", packets_.size()); RTC_LOG(LS_INFO) << "Total RTP/RTCP packets: " << packets_.size();
for (SsrcMapIterator mit = packets_by_ssrc_.begin(); for (SsrcMapIterator mit = packets_by_ssrc_.begin();
mit != packets_by_ssrc_.end(); ++mit) { mit != packets_by_ssrc_.end(); ++mit) {
uint32_t ssrc = mit->first; uint32_t ssrc = mit->first;
const std::vector<uint32_t>& packet_indices = mit->second; const std::vector<uint32_t>& packet_indices = mit->second;
int pt = packets_[packet_indices[0]].payload_type; int pt = packets_[packet_indices[0]].payload_type;
printf("SSRC: %08x, %zu packets, pt=%d\n", ssrc, packet_indices.size(), RTC_LOG(LS_INFO) << "SSRC: " << ssrc << ", " << packet_indices.size()
pt); << " packets, pt=" << pt << ".";
} }
// TODO(solenberg): Better validation of identified SSRC streams. // TODO(solenberg): Better validation of identified SSRC streams.
@ -321,6 +322,56 @@ class PcapReader : public RtpFileReaderImpl {
return kResultSuccess; return kResultSuccess;
} }
int ReadPcap(const std::set<uint32_t>& ssrc_filter,
size_t& total_packet_count) {
uint32_t stream_start_ms = 0;
int32_t next_packet_pos = ftell(file_);
for (;;) {
TRY_PCAP(fseek(file_, next_packet_pos, SEEK_SET));
int result = ReadPacket(&next_packet_pos, stream_start_ms, ssrc_filter);
if (result == kResultFail) {
break;
} else if (result == kResultSuccess && packets_.size() == 1) {
RTC_DCHECK_EQ(stream_start_ms, 0);
PacketIterator it = packets_.begin();
stream_start_ms = it->time_offset_ms;
it->time_offset_ms = 0;
}
total_packet_count++;
}
if (feof(file_) == 0) {
RTC_LOG(LS_ERROR) << "Failed reading file!";
return kResultFail;
}
return kResultSuccess;
}
int ReadPcapNg(const std::set<uint32_t>& ssrc_filter,
size_t& total_packet_count) {
uint32_t stream_start_ms = 0;
int next_packet_pos = 0;
for (;;) {
TRY_PCAP(fseek(file_, next_packet_pos, SEEK_SET));
int result = ReadPacketNg(&next_packet_pos, stream_start_ms, ssrc_filter);
if (result == kResultFail) {
break;
} else if (result == kResultSuccess && packets_.size() == 1) {
RTC_DCHECK_EQ(stream_start_ms, 0);
PacketIterator it = packets_.begin();
stream_start_ms = it->time_offset_ms;
it->time_offset_ms = 0;
}
total_packet_count++;
}
if (feof(file_) == 0) {
RTC_LOG(LS_ERROR) << "Failed reading file!";
return kResultFail;
}
return kResultSuccess;
}
bool NextPacket(RtpPacket* packet) override { bool NextPacket(RtpPacket* packet) override {
uint32_t length = RtpPacket::kMaxPacketBufferSize; uint32_t length = RtpPacket::kMaxPacketBufferSize;
if (NextPcap(packet->data, &length, &packet->time_ms) != kResultSuccess) if (NextPcap(packet->data, &length, &packet->time_ms) != kResultSuccess)
@ -353,7 +404,6 @@ class PcapReader : public RtpFileReaderImpl {
private: private:
// A marker of an RTP packet within the file. // A marker of an RTP packet within the file.
struct RtpPacketMarker { struct RtpPacketMarker {
uint32_t packet_number; // One-based index (like in WireShark)
uint32_t time_offset_ms; uint32_t time_offset_ms;
uint32_t source_ip; uint32_t source_ip;
uint32_t dest_ip; uint32_t dest_ip;
@ -377,6 +427,10 @@ class PcapReader : public RtpFileReaderImpl {
swap_pcap_byte_order_ = true; swap_pcap_byte_order_ = true;
} else if (magic == kPcapBOMNoSwapOrder) { } else if (magic == kPcapBOMNoSwapOrder) {
swap_pcap_byte_order_ = false; swap_pcap_byte_order_ = false;
} else if (magic == kPcapNgSectionHeaderBlock) {
pcapng_ = true;
RTC_LOG(LS_INFO) << "PCAPNG detected, support is experimental";
return kResultSuccess;
} else { } else {
return kResultFail; return kResultFail;
} }
@ -408,36 +462,9 @@ class PcapReader : public RtpFileReaderImpl {
return kResultSuccess; return kResultSuccess;
} }
int ReadPacket(int32_t* next_packet_pos, int ProcessPacket(RtpPacketMarker& marker,
uint32_t stream_start_ms, const std::set<uint32_t>& ssrc_filter,
uint32_t number, rtc::ArrayView<const uint8_t> packet) {
const std::set<uint32_t>& ssrc_filter) {
RTC_DCHECK(next_packet_pos);
uint32_t ts_sec; // Timestamp seconds.
uint32_t ts_usec; // Timestamp microseconds.
uint32_t incl_len; // Number of octets of packet saved in file.
uint32_t orig_len; // Actual length of packet.
TRY_PCAP(Read(&ts_sec, false));
TRY_PCAP(Read(&ts_usec, false));
TRY_PCAP(Read(&incl_len, false));
TRY_PCAP(Read(&orig_len, false));
*next_packet_pos = ftell(file_) + incl_len;
RtpPacketMarker marker = {0};
marker.packet_number = number;
marker.time_offset_ms = CalcTimeDelta(ts_sec, ts_usec, stream_start_ms);
TRY_PCAP(ReadPacketHeader(&marker));
marker.pos_in_file = ftell(file_);
if (marker.payload_length > sizeof(read_buffer_)) {
printf("Packet too large!\n");
return kResultFail;
}
TRY_PCAP(Read(read_buffer_, marker.payload_length));
rtc::ArrayView<const uint8_t> packet(read_buffer_, marker.payload_length);
if (IsRtcpPacket(packet)) { if (IsRtcpPacket(packet)) {
marker.payload_type = packet[1]; marker.payload_type = packet[1];
packets_.push_back(marker); packets_.push_back(marker);
@ -459,6 +486,98 @@ class PcapReader : public RtpFileReaderImpl {
return kResultSuccess; return kResultSuccess;
} }
int ReadPacket(int32_t* next_packet_pos,
uint32_t stream_start_ms,
const std::set<uint32_t>& ssrc_filter) {
RTC_DCHECK(next_packet_pos);
uint32_t ts_sec; // Timestamp seconds.
uint32_t ts_usec; // Timestamp microseconds.
uint32_t incl_len; // Number of octets of packet saved in file.
uint32_t orig_len; // Actual length of packet.
TRY_PCAP(Read(&ts_sec, false));
TRY_PCAP(Read(&ts_usec, false));
TRY_PCAP(Read(&incl_len, false));
TRY_PCAP(Read(&orig_len, false));
*next_packet_pos = ftell(file_) + incl_len;
RtpPacketMarker marker = {0};
marker.time_offset_ms = CalcTimeDelta(ts_sec, ts_usec, stream_start_ms);
TRY_PCAP(ReadPacketHeader(&marker));
marker.pos_in_file = ftell(file_);
if (marker.payload_length > sizeof(read_buffer_)) {
RTC_LOG(LS_ERROR) << "Packet too large!";
return kResultFail;
}
TRY_PCAP(Read(read_buffer_, marker.payload_length));
return ProcessPacket(marker, ssrc_filter,
{read_buffer_, marker.payload_length});
}
int ReadPacketNg(int32_t* next_packet_pos,
uint32_t stream_start_ms,
const std::set<uint32_t>& ssrc_filter) {
uint32_t block_type;
uint32_t block_length;
TRY_PCAP(Read(&block_type, false));
TRY_PCAP(Read(&block_length, false));
if (block_length == 0) {
RTC_LOG(LS_ERROR) << "Empty PCAPNG block";
return kResultFail;
}
*next_packet_pos += block_length;
switch (block_type) {
case kPcapNgSectionHeaderBlock: {
// TODO: https://issues.webrtc.org/issues/351327754 - interpret more of
// this block, in particular the if_tsresol option.
uint32_t byte_order_magic;
TRY_PCAP(Read(&byte_order_magic, false));
swap_pcap_byte_order_ = (byte_order_magic == kPcapNgBOMLittleEndian);
} break;
case kPcapNgInterfaceDescriptionBlock:
break;
case kPcapNgPacketBlock: {
uint32_t interface; // Interface ID. Unused.
uint32_t ts_upper; // Upper 32 bits of timestamp.
uint32_t ts_lower; // Lower 32 bits of timestamp.
uint32_t incl_len; // Number of octets of packet saved in file.
uint32_t orig_len; // Actual length of packet.
TRY_PCAP(Read(&interface, false));
TRY_PCAP(Read(&ts_upper, false));
TRY_PCAP(Read(&ts_lower, false));
TRY_PCAP(Read(&incl_len, false));
TRY_PCAP(Read(&orig_len, false));
RtpPacketMarker marker = {0};
// Note: Wireshark writes nanoseconds most of the time, see comments in
// it's pcapio.c. We are only interesting in the time difference so
// truncating to uint32_t is ok.
uint64_t timestamp_ms =
((static_cast<uint64_t>(ts_upper) << 32) | ts_lower) /
rtc::kNumMicrosecsPerSec;
marker.time_offset_ms =
static_cast<uint32_t>(timestamp_ms) - stream_start_ms;
TRY_PCAP(ReadPacketHeader(&marker));
marker.pos_in_file = ftell(file_);
if (marker.payload_length > sizeof(read_buffer_)) {
RTC_LOG(LS_ERROR) << "Packet too large!";
return kResultFail;
}
TRY_PCAP(Read(read_buffer_, marker.payload_length));
if (ProcessPacket(marker, ssrc_filter,
{read_buffer_, marker.payload_length}) !=
kResultSuccess) {
return kResultFail;
}
return kResultSuccess;
} break;
}
return kResultSkip;
}
int ReadPacketHeader(RtpPacketMarker* marker) { int ReadPacketHeader(RtpPacketMarker* marker) {
int32_t file_pos = ftell(file_); int32_t file_pos = ftell(file_);
@ -470,7 +589,6 @@ class PcapReader : public RtpFileReaderImpl {
TRY_PCAP(Read(&protocol, true)); TRY_PCAP(Read(&protocol, true));
if (protocol == kBsdNullLoopback1 || protocol == kBsdNullLoopback2) { if (protocol == kBsdNullLoopback1 || protocol == kBsdNullLoopback2) {
int result = ReadXxpIpHeader(marker); int result = ReadXxpIpHeader(marker);
RTC_LOG(LS_INFO) << "Recognized loopback frame";
if (result != kResultSkip) { if (result != kResultSkip) {
return result; return result;
} }
@ -484,7 +602,6 @@ class PcapReader : public RtpFileReaderImpl {
TRY_PCAP(Read(&type, true)); TRY_PCAP(Read(&type, true));
if (type == kEthertypeIp) { if (type == kEthertypeIp) {
int result = ReadXxpIpHeader(marker); int result = ReadXxpIpHeader(marker);
RTC_LOG(LS_INFO) << "Recognized ethernet 2 frame";
if (result != kResultSkip) { if (result != kResultSkip) {
return result; return result;
} }
@ -617,6 +734,7 @@ class PcapReader : public RtpFileReaderImpl {
FILE* file_; FILE* file_;
bool swap_pcap_byte_order_; bool swap_pcap_byte_order_;
const bool swap_network_byte_order_; const bool swap_network_byte_order_;
bool pcapng_;
uint8_t read_buffer_[kMaxReadBufferSize]; uint8_t read_buffer_[kMaxReadBufferSize];
SsrcMap packets_by_ssrc_; SsrcMap packets_by_ssrc_;
@ -648,7 +766,7 @@ RtpFileReader* RtpFileReader::Create(FileFormat format,
FILE* file = tmpfile(); FILE* file = tmpfile();
if (file == nullptr) { if (file == nullptr) {
printf("ERROR: Can't open file from memory buffer\n"); RTC_LOG(LS_ERROR) << "ERROR: Can't open file from memory buffer.";
return nullptr; return nullptr;
} }
@ -671,7 +789,7 @@ RtpFileReader* RtpFileReader::Create(FileFormat format,
std::string filename_str = std::string(filename); std::string filename_str = std::string(filename);
FILE* file = fopen(filename_str.c_str(), "rb"); FILE* file = fopen(filename_str.c_str(), "rb");
if (file == nullptr) { if (file == nullptr) {
printf("ERROR: Can't open file: %s\n", filename_str.c_str()); RTC_LOG(LS_ERROR) << "ERROR: Can't open file: '" << filename_str << "'.";
return nullptr; return nullptr;
} }