admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add writing and parsing of the abs-capture-time RTP header extension.

Browse Source 
This change adds the writing and parsing of the `abs-capture-time` RTP header extension defined at:

  http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time

We are still missing the code to:

- Negotiate the header extension.
- Collect capture time for audio and video and have the info sent with the header extension.
- Receive the header extension and use its info.

Bug: webrtc:10739
Change-Id: I75af492e994367f45a5bdc110af199900327b126
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/144221
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Commit-Queue: Chen Xing <chxg@google.com>
Cr-Commit-Position: refs/heads/master@{#28468}
This commit is contained in:
Chen Xing 2019-07-01 10:56:51 +02:00 committed by Commit Bot
parent 53d45baa50
commit cd8a6e2f38
14 changed files with 250 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
api/rtp_headers.h
View File

@ -38,6 +38,46 @@ struct FeedbackRequest {
int sequence_count;
};
// The Absolute Capture Time extension is used to stamp RTP packets with a NTP
// timestamp showing when the first audio or video frame in a packet was
// originally captured. The intent of this extension is to provide a way to
// accomplish audio-to-video synchronization when RTCP-terminating intermediate
// systems (e.g. mixers) are involved. See:
// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time
struct AbsoluteCaptureTime {
// Absolute capture timestamp is the NTP timestamp of when the first frame in
// a packet was originally captured. This timestamp MUST be based on the same
// clock as the clock used to generate NTP timestamps for RTCP sender reports
// on the capture system.
//
// It’s not always possible to do an NTP clock readout at the exact moment of
// when a media frame is captured. A capture system MAY postpone the readout
// until a more convenient time. A capture system SHOULD have known delays
// (e.g. from hardware buffers) subtracted from the readout to make the final
// timestamp as close to the actual capture time as possible.
//
// This field is encoded as a 64-bit unsigned fixed-point number with the high
// 32 bits for the timestamp in seconds and low 32 bits for the fractional
// part. This is also known as the UQ32.32 format and is what the RTP
// specification defines as the canonical format to represent NTP timestamps.
uint64_t absolute_capture_timestamp;
// Estimated capture clock offset is the sender’s estimate of the offset
// between its own NTP clock and the capture system’s NTP clock. The sender is
// here defined as the system that owns the NTP clock used to generate the NTP
// timestamps for the RTCP sender reports on this stream. The sender system is
// typically either the capture system or a mixer.
//
// This field is encoded as a 64-bit two’s complement signed fixed-point
// number with the high 32 bits for the seconds and low 32 bits for the
// fractional part. It’s intended to make it easy for a receiver, that knows
// how to estimate the sender system’s NTP clock, to also estimate the capture
// system’s NTP clock:
//
// Capture NTP Clock = Sender NTP Clock + Capture Clock Offset
absl::optional<int64_t> estimated_capture_clock_offset;
};
struct RTPHeaderExtension {
RTPHeaderExtension();
RTPHeaderExtension(const RTPHeaderExtension& other);
@ -56,6 +96,7 @@ struct RTPHeaderExtension {
int32_t transmissionTimeOffset;
bool hasAbsoluteSendTime;
uint32_t absoluteSendTime;
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
bool hasTransportSequenceNumber;
uint16_t transportSequenceNumber;
absl::optional<FeedbackRequest> feedback_request;

8
api/rtp_parameters.cc
View File

@ -100,6 +100,9 @@ const char RtpExtension::kTimestampOffsetUri[] =
const char RtpExtension::kAbsSendTimeUri[] =
"http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time";
const char RtpExtension::kAbsoluteCaptureTimeUri[] =
"http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time";
const char RtpExtension::kVideoRotationUri[] = "urn:3gpp:video-orientation";
const char RtpExtension::kTransportSequenceNumberUri[] =
@ -152,6 +155,8 @@ constexpr int RtpExtension::kOneByteHeaderExtensionMaxValueSize;
bool RtpExtension::IsSupportedForAudio(const std::string& uri) {
return uri == webrtc::RtpExtension::kAudioLevelUri ||
// TODO(bugs.webrtc.org/10739): Uncomment once the audio impl is ready.
// uri == webrtc::RtpExtension::kAbsoluteCaptureTimeUri ||
uri == webrtc::RtpExtension::kTransportSequenceNumberUri ||
uri == webrtc::RtpExtension::kTransportSequenceNumberV2Uri ||
uri == webrtc::RtpExtension::kMidUri ||
@ -162,6 +167,8 @@ bool RtpExtension::IsSupportedForAudio(const std::string& uri) {
bool RtpExtension::IsSupportedForVideo(const std::string& uri) {
return uri == webrtc::RtpExtension::kTimestampOffsetUri ||
uri == webrtc::RtpExtension::kAbsSendTimeUri ||
// TODO(bugs.webrtc.org/10739): Uncomment once the video impl is ready.
// uri == webrtc::RtpExtension::kAbsoluteCaptureTimeUri ||
uri == webrtc::RtpExtension::kVideoRotationUri ||
uri == webrtc::RtpExtension::kTransportSequenceNumberUri ||
uri == webrtc::RtpExtension::kTransportSequenceNumberV2Uri ||
@ -188,6 +195,7 @@ bool RtpExtension::IsEncryptionSupported(const std::string& uri) {
// encrypted (which can't be done by Chromium).
uri == webrtc::RtpExtension::kAbsSendTimeUri ||
#endif
uri == webrtc::RtpExtension::kAbsoluteCaptureTimeUri ||
uri == webrtc::RtpExtension::kVideoRotationUri ||
uri == webrtc::RtpExtension::kTransportSequenceNumberUri ||
uri == webrtc::RtpExtension::kTransportSequenceNumberV2Uri ||

4
api/rtp_parameters.h
View File

@ -267,6 +267,10 @@ struct RtpExtension {
// http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
static const char kAbsSendTimeUri[];
// Header extension for absolute capture time, see url for details:
// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time
static const char kAbsoluteCaptureTimeUri[];
// Header extension for coordination of video orientation, see url for
// details:
// http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf

1
modules/rtp_rtcp/include/rtp_rtcp_defines.h
View File

@ -56,6 +56,7 @@ enum RTPExtensionType : int {
kRtpExtensionTransmissionTimeOffset,
kRtpExtensionAudioLevel,
kRtpExtensionAbsoluteSendTime,
kRtpExtensionAbsoluteCaptureTime,
kRtpExtensionVideoRotation,
kRtpExtensionTransportSequenceNumber,
kRtpExtensionTransportSequenceNumber02,

1
modules/rtp_rtcp/source/rtp_header_extension_map.cc
View File

@ -34,6 +34,7 @@ constexpr ExtensionInfo kExtensions[] = {
CreateExtensionInfo<TransmissionOffset>(),
CreateExtensionInfo<AudioLevel>(),
CreateExtensionInfo<AbsoluteSendTime>(),
CreateExtensionInfo<AbsoluteCaptureTimeExtension>(),
CreateExtensionInfo<VideoOrientation>(),
CreateExtensionInfo<TransportSequenceNumber>(),
CreateExtensionInfo<TransportSequenceNumberV2>(),

83
modules/rtp_rtcp/source/rtp_header_extensions.cc
View File

@ -56,6 +56,89 @@ bool AbsoluteSendTime::Write(rtc::ArrayView<uint8_t> data,
return true;
}
// Absolute Capture Time
//
// The Absolute Capture Time extension is used to stamp RTP packets with a NTP
// timestamp showing when the first audio or video frame in a packet was
// originally captured. The intent of this extension is to provide a way to
// accomplish audio-to-video synchronization when RTCP-terminating intermediate
// systems (e.g. mixers) are involved.
//
// Data layout of the shortened version of abs-capture-time:
//
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ID | len=7 | absolute capture timestamp (bit 0-23) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | absolute capture timestamp (bit 24-55) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ... (56-63) |
// +-+-+-+-+-+-+-+-+
//
// Data layout of the extended version of abs-capture-time:
//
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ID | len=15| absolute capture timestamp (bit 0-23) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | absolute capture timestamp (bit 24-55) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ... (56-63) | estimated capture clock offset (bit 0-23) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | estimated capture clock offset (bit 24-55) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ... (56-63) |
// +-+-+-+-+-+-+-+-+
constexpr RTPExtensionType AbsoluteCaptureTimeExtension::kId;
constexpr uint8_t AbsoluteCaptureTimeExtension::kValueSizeBytes;
constexpr uint8_t AbsoluteCaptureTimeExtension::
kValueSizeBytesWithoutEstimatedCaptureClockOffset;
constexpr const char AbsoluteCaptureTimeExtension::kUri[];
bool AbsoluteCaptureTimeExtension::Parse(rtc::ArrayView<const uint8_t> data,
AbsoluteCaptureTime* extension) {
if (data.size() != kValueSizeBytes &&
data.size() != kValueSizeBytesWithoutEstimatedCaptureClockOffset) {
return false;
}
extension->absolute_capture_timestamp =
ByteReader<uint64_t>::ReadBigEndian(data.data());
if (data.size() != kValueSizeBytesWithoutEstimatedCaptureClockOffset) {
extension->estimated_capture_clock_offset =
ByteReader<int64_t>::ReadBigEndian(data.data() + 8);
}
return true;
}
size_t AbsoluteCaptureTimeExtension::ValueSize(
const AbsoluteCaptureTime& extension) {
if (extension.estimated_capture_clock_offset != absl::nullopt) {
return kValueSizeBytes;
} else {
return kValueSizeBytesWithoutEstimatedCaptureClockOffset;
}
}
bool AbsoluteCaptureTimeExtension::Write(rtc::ArrayView<uint8_t> data,
const AbsoluteCaptureTime& extension) {
RTC_DCHECK_EQ(data.size(), ValueSize(extension));
ByteWriter<uint64_t>::WriteBigEndian(data.data(),
extension.absolute_capture_timestamp);
if (data.size() != kValueSizeBytesWithoutEstimatedCaptureClockOffset) {
ByteWriter<int64_t>::WriteBigEndian(
data.data() + 8, extension.estimated_capture_clock_offset.value());
}
return true;
}
// An RTP Header Extension for Client-to-Mixer Audio Level Indication
//
// https://datatracker.ietf.org/doc/draft-lennox-avt-rtp-audio-level-exthdr/

17
modules/rtp_rtcp/source/rtp_header_extensions.h
View File

@ -42,6 +42,23 @@ class AbsoluteSendTime {
}
};
class AbsoluteCaptureTimeExtension {
public:
using value_type = AbsoluteCaptureTime;
static constexpr RTPExtensionType kId = kRtpExtensionAbsoluteCaptureTime;
static constexpr uint8_t kValueSizeBytes = 16;
static constexpr uint8_t kValueSizeBytesWithoutEstimatedCaptureClockOffset =
8;
static constexpr const char kUri[] =
"http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time";
static bool Parse(rtc::ArrayView<const uint8_t> data,
AbsoluteCaptureTime* extension);
static size_t ValueSize(const AbsoluteCaptureTime& extension);
static bool Write(rtc::ArrayView<uint8_t> data,
const AbsoluteCaptureTime& extension);
};
class AudioLevel {
public:
static constexpr RTPExtensionType kId = kRtpExtensionAudioLevel;

1
modules/rtp_rtcp/source/rtp_packet.cc
View File

@ -183,6 +183,7 @@ void RtpPacket::CopyAndZeroMutableExtensions(
break;
}
case RTPExtensionType::kRtpExtensionAudioLevel:
case RTPExtensionType::kRtpExtensionAbsoluteCaptureTime:
case RTPExtensionType::kRtpExtensionColorSpace:
case RTPExtensionType::kRtpExtensionFrameMarking:
case RTPExtensionType::kRtpExtensionGenericFrameDescriptor00:

2
modules/rtp_rtcp/source/rtp_packet_received.cc
View File

@ -51,6 +51,8 @@ void RtpPacketReceived::GetHeader(RTPHeader* header) const {
&header->extension.transmissionTimeOffset);
header->extension.hasAbsoluteSendTime =
GetExtension<AbsoluteSendTime>(&header->extension.absoluteSendTime);
header->extension.absolute_capture_time =
GetExtension<AbsoluteCaptureTimeExtension>();
header->extension.hasTransportSequenceNumber =
GetExtension<TransportSequenceNumberV2>(
&header->extension.transportSequenceNumber,

59
modules/rtp_rtcp/source/rtp_packet_unittest.cc
View File

@ -827,6 +827,65 @@ TEST(RtpPacketTest, CreateAndParseColorSpaceExtensionWithoutHdrMetadata) {
TestCreateAndParseColorSpaceExtension(/*with_hdr_metadata=*/false);
}
TEST(RtpPacketTest, CreateAndParseAbsoluteCaptureTime) {
// Create a packet with absolute capture time extension populated.
RtpPacketToSend::ExtensionManager extensions;
constexpr int kExtensionId = 1;
extensions.Register<AbsoluteCaptureTimeExtension>(kExtensionId);
RtpPacketToSend send_packet(&extensions);
send_packet.SetPayloadType(kPayloadType);
send_packet.SetSequenceNumber(kSeqNum);
send_packet.SetTimestamp(kTimestamp);
send_packet.SetSsrc(kSsrc);
constexpr AbsoluteCaptureTime kAbsoluteCaptureTime{
/*absolute_capture_timestamp=*/9876543210123456789ULL,
/*estimated_capture_clock_offset=*/-1234567890987654321LL};
send_packet.SetExtension<AbsoluteCaptureTimeExtension>(kAbsoluteCaptureTime);
// Serialize the packet and then parse it again.
RtpPacketReceived receive_packet(&extensions);
EXPECT_TRUE(receive_packet.Parse(send_packet.Buffer()));
AbsoluteCaptureTime received_absolute_capture_time;
EXPECT_TRUE(receive_packet.GetExtension<AbsoluteCaptureTimeExtension>(
&received_absolute_capture_time));
EXPECT_EQ(kAbsoluteCaptureTime.absolute_capture_timestamp,
received_absolute_capture_time.absolute_capture_timestamp);
EXPECT_EQ(kAbsoluteCaptureTime.estimated_capture_clock_offset,
received_absolute_capture_time.estimated_capture_clock_offset);
}
TEST(RtpPacketTest,
CreateAndParseAbsoluteCaptureTimeWithoutEstimatedCaptureClockOffset) {
// Create a packet with absolute capture time extension populated.
RtpPacketToSend::ExtensionManager extensions;
constexpr int kExtensionId = 1;
extensions.Register<AbsoluteCaptureTimeExtension>(kExtensionId);
RtpPacketToSend send_packet(&extensions);
send_packet.SetPayloadType(kPayloadType);
send_packet.SetSequenceNumber(kSeqNum);
send_packet.SetTimestamp(kTimestamp);
send_packet.SetSsrc(kSsrc);
constexpr AbsoluteCaptureTime kAbsoluteCaptureTime{
/*absolute_capture_timestamp=*/9876543210123456789ULL,
/*estimated_capture_clock_offset=*/absl::nullopt};
send_packet.SetExtension<AbsoluteCaptureTimeExtension>(kAbsoluteCaptureTime);
// Serialize the packet and then parse it again.
RtpPacketReceived receive_packet(&extensions);
EXPECT_TRUE(receive_packet.Parse(send_packet.Buffer()));
AbsoluteCaptureTime received_absolute_capture_time;
EXPECT_TRUE(receive_packet.GetExtension<AbsoluteCaptureTimeExtension>(
&received_absolute_capture_time));
EXPECT_EQ(kAbsoluteCaptureTime.absolute_capture_timestamp,
received_absolute_capture_time.absolute_capture_timestamp);
EXPECT_EQ(kAbsoluteCaptureTime.estimated_capture_clock_offset,
received_absolute_capture_time.estimated_capture_clock_offset);
}
TEST(RtpPacketTest, CreateAndParseTransportSequenceNumber) {
// Create a packet with transport sequence number extension populated.
RtpPacketToSend::ExtensionManager extensions;

1
modules/rtp_rtcp/source/rtp_sender.cc
View File

@ -72,6 +72,7 @@ constexpr RtpExtensionSize kFecOrPaddingExtensionSizes[] = {
// Size info for header extensions that might be used in video packets.
constexpr RtpExtensionSize kVideoExtensionSizes[] = {
CreateExtensionSize<AbsoluteSendTime>(),
CreateExtensionSize<AbsoluteCaptureTimeExtension>(),
CreateExtensionSize<TransmissionOffset>(),
CreateExtensionSize<TransportSequenceNumber>(),
CreateExtensionSize<PlayoutDelayLimits>(),

11
modules/rtp_rtcp/source/rtp_utility.cc
View File

@ -400,6 +400,17 @@ void RtpHeaderParser::ParseOneByteExtensionHeader(
header->extension.hasAbsoluteSendTime = true;
break;
}
case kRtpExtensionAbsoluteCaptureTime: {
AbsoluteCaptureTime extension;
if (!AbsoluteCaptureTimeExtension::Parse(
rtc::MakeArrayView(ptr, len + 1), &extension)) {
RTC_LOG(LS_WARNING)
<< "Incorrect absolute capture time len: " << len;
return;
}
header->extension.absolute_capture_time = extension;
break;
}
case kRtpExtensionVideoRotation: {
if (len != 0) {
RTC_LOG(LS_WARNING)

19
modules/rtp_rtcp/source/rtp_utility_unittest.cc
View File

@ -149,23 +149,27 @@ TEST(RtpHeaderParser, ParseWithOverSizedExtension) {
EXPECT_EQ(sizeof(kPacket), header.headerLength);
}
TEST(RtpHeaderParser, ParseAll8Extensions) {
TEST(RtpHeaderParser, ParseAll9Extensions) {
const uint8_t kAudioLevel = 0x5a;
// clang-format off
const uint8_t kPacket[] = {
0x90, kPayloadType, 0x00, kSeqNum,
0x65, 0x43, 0x12, 0x78, // kTimestamp.
0x12, 0x34, 0x56, 0x78, // kSsrc.
0xbe, 0xde, 0x00, 0x08, // Extension of size 8x32bit words.
0xbe, 0xde, 0x00, 0x0c, // Extension of size 12x32bit words.
0x40, 0x80|kAudioLevel, // AudioLevel.
0x22, 0x01, 0x56, 0xce, // TransmissionOffset.
0x62, 0x12, 0x34, 0x56, // AbsoluteSendTime.
0x7f, 0x12, 0x34, 0x56, 0x78, // AbsoluteCaptureTime.
0x90, 0xab, 0xcd, 0xef, // AbsoluteCaptureTime. (cont.)
0x11, 0x22, 0x33, 0x44, // AbsoluteCaptureTime. (cont.)
0x55, 0x66, 0x77, 0x88, // AbsoluteCaptureTime. (cont.)
0x81, 0xce, 0xab, // TransportSequenceNumber.
0xa0, 0x03, // VideoRotation.
0xb2, 0x12, 0x48, 0x76, // PlayoutDelayLimits.
0xc2, 'r', 't', 'x', // RtpStreamId
0xd5, 's', 't', 'r', 'e', 'a', 'm', // RepairedRtpStreamId
0x00, 0x00, // Padding to 32bit boundary.
0x00, // Padding to 32bit boundary.
};
// clang-format on
ASSERT_EQ(sizeof(kPacket) % 4, 0u);
@ -174,6 +178,7 @@ TEST(RtpHeaderParser, ParseAll8Extensions) {
extensions.Register<TransmissionOffset>(2);
extensions.Register<AudioLevel>(4);
extensions.Register<AbsoluteSendTime>(6);
extensions.Register<AbsoluteCaptureTimeExtension>(7);
extensions.Register<TransportSequenceNumber>(8);
extensions.Register<VideoOrientation>(0xa);
extensions.Register<PlayoutDelayLimits>(0xb);
@ -194,6 +199,14 @@ TEST(RtpHeaderParser, ParseAll8Extensions) {
EXPECT_TRUE(header.extension.hasAbsoluteSendTime);
EXPECT_EQ(0x123456U, header.extension.absoluteSendTime);
ASSERT_TRUE(header.extension.absolute_capture_time.has_value());
EXPECT_EQ(0x1234567890abcdefULL,
header.extension.absolute_capture_time->absolute_capture_timestamp);
ASSERT_TRUE(header.extension.absolute_capture_time
->estimated_capture_clock_offset.has_value());
EXPECT_EQ(0x1122334455667788LL, header.extension.absolute_capture_time
->estimated_capture_clock_offset.value());
EXPECT_TRUE(header.extension.hasTransportSequenceNumber);
EXPECT_EQ(0xceab, header.extension.transportSequenceNumber);

5
test/fuzzers/rtp_packet_fuzzer.cc
View File

@ -79,6 +79,11 @@ void FuzzOneInput(const uint8_t* data, size_t size) {
uint32_t sendtime;
packet.GetExtension<AbsoluteSendTime>(&sendtime);
break;
case kRtpExtensionAbsoluteCaptureTime: {
AbsoluteCaptureTime extension;
packet.GetExtension<AbsoluteCaptureTimeExtension>(&extension);
break;
}
case kRtpExtensionVideoRotation:
uint8_t rotation;
packet.GetExtension<VideoOrientation>(&rotation);