From 4536289353cdcc315cc5e6218893e4843cf528e6 Mon Sep 17 00:00:00 2001 From: "guoweis@webrtc.org" Date: Wed, 4 Mar 2015 22:55:15 +0000 Subject: [PATCH] Add CVO support to RTP sender side. According to http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf, CVO byte should only be added in the last packet of each key frame or when the rotation changes. Currently, we're adding this byte in each frame to start with. BUG=4145 R=mflodman@webrtc.org, pthatcher@webrtc.org Review URL: https://webrtc-codereview.appspot.com/42439004 Cr-Commit-Position: refs/heads/master@{#8606} git-svn-id: http://webrtc.googlecode.com/svn/trunk@8606 4adac7df-926f-26a2-2b94-8c16560cd09d --- webrtc/common_types.h | 10 +- .../modules/interface/module_common_types.h | 4 + .../rtp_rtcp/interface/rtp_rtcp_defines.h | 12 +- .../rtp_rtcp/source/rtp_header_extension.h | 4 + .../modules/rtp_rtcp/source/rtp_rtcp_impl.cc | 12 +- .../rtp_rtcp/source/rtp_rtcp_impl_unittest.cc | 9 +- webrtc/modules/rtp_rtcp/source/rtp_sender.cc | 230 ++++++++++++++---- webrtc/modules/rtp_rtcp/source/rtp_sender.h | 30 ++- .../rtp_rtcp/source/rtp_sender_unittest.cc | 184 +++++++++++++- .../rtp_rtcp/source/rtp_sender_video.cc | 42 +++- .../rtp_rtcp/source/rtp_sender_video.h | 4 +- webrtc/modules/rtp_rtcp/source/rtp_utility.cc | 19 ++ 12 files changed, 462 insertions(+), 98 deletions(-) diff --git a/webrtc/common_types.h b/webrtc/common_types.h index bdba653e39..f3f84dee40 100644 --- a/webrtc/common_types.h +++ b/webrtc/common_types.h @@ -798,7 +798,9 @@ struct RTPHeaderExtension { hasAbsoluteSendTime(false), absoluteSendTime(0), hasAudioLevel(false), - audioLevel(0) {} + audioLevel(0), + hasVideoRotation(false), + videoRotation(0) {} bool hasTransmissionTimeOffset; int32_t transmissionTimeOffset; @@ -809,6 +811,12 @@ struct RTPHeaderExtension { // https://datatracker.ietf.org/doc/draft-lennox-avt-rtp-audio-level-exthdr/ bool hasAudioLevel; uint8_t audioLevel; + + // For Coordination of Video Orientation. See + // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ + // ts_126114v120700p.pdf + bool hasVideoRotation; + uint8_t videoRotation; }; struct RTPHeader { diff --git a/webrtc/modules/interface/module_common_types.h b/webrtc/modules/interface/module_common_types.h index 5cb2c56108..207680d4ee 100644 --- a/webrtc/modules/interface/module_common_types.h +++ b/webrtc/modules/interface/module_common_types.h @@ -18,6 +18,7 @@ #include "webrtc/base/constructormagic.h" #include "webrtc/common_types.h" +#include "webrtc/common_video/rotation.h" #include "webrtc/typedefs.h" namespace webrtc { @@ -76,9 +77,12 @@ enum RtpVideoCodecTypes { kRtpVideoVp8, kRtpVideoH264 }; +// Since RTPVideoHeader is used as a member of a union, it can't have a +// non-trivial default constructor. struct RTPVideoHeader { uint16_t width; // size uint16_t height; + VideoRotation rotation; bool isFirstPacket; // first packet in frame uint8_t simulcastIdx; // Index if the simulcast encoder creating diff --git a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h index 4938b5fefb..d15f31d2e3 100644 --- a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h +++ b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h @@ -74,12 +74,12 @@ enum StorageType { kAllowRetransmission }; -enum RTPExtensionType -{ - kRtpExtensionNone, - kRtpExtensionTransmissionTimeOffset, - kRtpExtensionAudioLevel, - kRtpExtensionAbsoluteSendTime +enum RTPExtensionType { + kRtpExtensionNone, + kRtpExtensionTransmissionTimeOffset, + kRtpExtensionAudioLevel, + kRtpExtensionAbsoluteSendTime, + kRtpExtensionVideoRotation }; enum RTCPAppSubTypes diff --git a/webrtc/modules/rtp_rtcp/source/rtp_header_extension.h b/webrtc/modules/rtp_rtcp/source/rtp_header_extension.h index 335d0a1b8f..36c2f73031 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_header_extension.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_header_extension.h @@ -24,6 +24,7 @@ const size_t kRtpOneByteHeaderLength = 4; const size_t kTransmissionTimeOffsetLength = 4; const size_t kAudioLevelLength = 4; const size_t kAbsoluteSendTimeLength = 4; +const size_t kVideoRotationLength = 4; struct HeaderExtension { HeaderExtension(RTPExtensionType extension_type) @@ -42,6 +43,9 @@ struct HeaderExtension { case kRtpExtensionAbsoluteSendTime: length = kAbsoluteSendTimeLength; break; + case kRtpExtensionVideoRotation: + length = kVideoRotationLength; + break; default: assert(false); } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 99d2df56e0..d483e76c78 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -401,15 +401,9 @@ int32_t ModuleRtpRtcpImpl::SendOutgoingData( if (rtcp_sender_.TimeToSendRTCPReport(kVideoFrameKey == frame_type)) { rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport); } - return rtp_sender_.SendOutgoingData(frame_type, - payload_type, - time_stamp, - capture_time_ms, - payload_data, - payload_size, - fragmentation, - NULL, - &(rtp_video_hdr->codecHeader)); + return rtp_sender_.SendOutgoingData( + frame_type, payload_type, time_stamp, capture_time_ms, payload_data, + payload_size, fragmentation, NULL, rtp_video_hdr); } bool ModuleRtpRtcpImpl::TimeToSendPacket(uint32_t ssrc, diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc index e7b58a2a60..3eb05a9b84 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc @@ -190,8 +190,13 @@ class RtpRtcpImplTest : public ::testing::Test { void SendFrame(const RtpRtcpModule* module, uint8_t tid) { RTPVideoHeaderVP8 vp8_header = {}; vp8_header.temporalIdx = tid; - RTPVideoHeader rtp_video_header = { - codec_.width, codec_.height, true, 0, kRtpVideoVp8, {vp8_header}}; + RTPVideoHeader rtp_video_header = {codec_.width, + codec_.height, + kVideoRotation_0, + true, + 0, + kRtpVideoVp8, + {vp8_header}}; const uint8_t payload[100] = {0}; EXPECT_EQ(0, module->impl_->SendOutgoingData(kVideoFrameKey, diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc index e6a912eb99..28c458ddb7 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc @@ -27,6 +27,8 @@ const int kSendSideDelayWindowMs = 1000; namespace { +const size_t kRtpHeaderLength = 12; + const char* FrameTypeToString(FrameType frame_type) { switch (frame_type) { case kFrameEmpty: return "empty"; @@ -124,6 +126,7 @@ RTPSender::RTPSender(int32_t id, rtp_header_extension_map_(), transmission_time_offset_(0), absolute_send_time_(0), + rotation_(kVideoRotation_0), // NACK. nack_byte_count_times_(), nack_byte_count_(), @@ -246,12 +249,22 @@ int32_t RTPSender::SetAbsoluteSendTime(uint32_t absolute_send_time) { return 0; } +void RTPSender::SetVideoRotation(VideoRotation rotation) { + CriticalSectionScoped cs(send_critsect_.get()); + rotation_ = rotation; +} + int32_t RTPSender::RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id) { CriticalSectionScoped cs(send_critsect_.get()); return rtp_header_extension_map_.Register(type, id); } +bool RTPSender::IsRtpHeaderExtensionRegistered(RTPExtensionType type) { + CriticalSectionScoped cs(send_critsect_.get()); + return rtp_header_extension_map_.IsRegistered(type); +} + int32_t RTPSender::DeregisterRtpHeaderExtension(RTPExtensionType type) { CriticalSectionScoped cs(send_critsect_.get()); return rtp_header_extension_map_.Deregister(type); @@ -440,6 +453,25 @@ int32_t RTPSender::CheckPayloadType(int8_t payload_type, return 0; } +// Please refer to http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/ +// 12.07.00_60/ts_126114v120700p.pdf Section 7.4.5. The rotation of a frame is +// the clockwise angle the frames must be rotated in order to display the frames +// correctly if the display is rotated in its natural orientation. +uint8_t RTPSender::ConvertToCVOByte(VideoRotation rotation) { + switch (rotation) { + case kVideoRotation_0: + return 0; + case kVideoRotation_90: + return 1; + case kVideoRotation_180: + return 2; + case kVideoRotation_270: + return 3; + } + assert(false); + return 0; +} + int32_t RTPSender::SendOutgoingData(FrameType frame_type, int8_t payload_type, uint32_t capture_timestamp, @@ -448,7 +480,7 @@ int32_t RTPSender::SendOutgoingData(FrameType frame_type, size_t payload_size, const RTPFragmentationHeader* fragmentation, VideoCodecInformation* codec_info, - const RTPVideoTypeHeader* rtp_type_hdr) { + const RTPVideoHeader* rtp_hdr) { uint32_t ssrc; { // Drop this packet if we're not sending media packets. @@ -481,12 +513,10 @@ int32_t RTPSender::SendOutgoingData(FrameType frame_type, if (frame_type == kFrameEmpty) return 0; - ret_val = video_->SendVideo(video_type, frame_type, payload_type, - capture_timestamp, capture_time_ms, - payload_data, payload_size, - fragmentation, codec_info, - rtp_type_hdr); - + ret_val = + video_->SendVideo(video_type, frame_type, payload_type, + capture_timestamp, capture_time_ms, payload_data, + payload_size, fragmentation, codec_info, rtp_hdr); } CriticalSectionScoped cs(statistics_crit_.get()); @@ -1040,7 +1070,7 @@ void RTPSender::ProcessBitrate() { size_t RTPSender::RTPHeaderLength() const { CriticalSectionScoped lock(send_critsect_.get()); - size_t rtp_header_length = 12; + size_t rtp_header_length = kRtpHeaderLength; rtp_header_length += sizeof(uint32_t) * csrcs_.size(); rtp_header_length += RtpHeaderExtensionTotalLength(); return rtp_header_length; @@ -1093,7 +1123,7 @@ size_t RTPSender::CreateRtpHeader(uint8_t* header, RtpUtility::AssignUWord16ToBuffer(header + 2, sequence_number); RtpUtility::AssignUWord32ToBuffer(header + 4, timestamp); RtpUtility::AssignUWord32ToBuffer(header + 8, ssrc); - int32_t rtp_header_length = 12; + int32_t rtp_header_length = kRtpHeaderLength; if (csrcs.size() > 0) { uint8_t *ptr = &header[rtp_header_length]; @@ -1107,7 +1137,8 @@ size_t RTPSender::CreateRtpHeader(uint8_t* header, rtp_header_length += sizeof(uint32_t) * csrcs.size(); } - uint16_t len = BuildRTPHeaderExtension(header + rtp_header_length); + uint16_t len = + BuildRTPHeaderExtension(header + rtp_header_length, marker_bit); if (len > 0) { header[0] |= 0x10; // Set extension bit. rtp_header_length += len; @@ -1141,7 +1172,8 @@ int32_t RTPSender::BuildRTPheader(uint8_t* data_buffer, timestamp_, sequence_number, csrcs_); } -uint16_t RTPSender::BuildRTPHeaderExtension(uint8_t* data_buffer) const { +uint16_t RTPSender::BuildRTPHeaderExtension(uint8_t* data_buffer, + bool marker_bit) const { if (rtp_header_extension_map_.Size() <= 0) { return 0; } @@ -1179,6 +1211,12 @@ uint16_t RTPSender::BuildRTPHeaderExtension(uint8_t* data_buffer) const { block_length = BuildAbsoluteSendTimeExtension( data_buffer + kHeaderLength + total_block_length); break; + case kRtpExtensionVideoRotation: + if (marker_bit) { + block_length = BuildVideoRotationExtension( + data_buffer + kHeaderLength + total_block_length); + } + break; default: assert(false); } @@ -1301,6 +1339,78 @@ uint8_t RTPSender::BuildAbsoluteSendTimeExtension(uint8_t* data_buffer) const { return kAbsoluteSendTimeLength; } +uint8_t RTPSender::BuildVideoRotationExtension(uint8_t* data_buffer) const { + // Coordination of Video Orientation in RTP streams. + // + // Coordination of Video Orientation consists in signalling of the current + // orientation of the image captured on the sender side to the receiver for + // appropriate rendering and displaying. + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | ID | len=0 |V|0 0 0 0 C F R R| 0x00 | 0x00 | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // + // Note that we always include 2 pad bytes, which will result in legal and + // correctly parsed RTP, but may be a bit wasteful if more short extensions + // are implemented. Right now the pad bytes would anyway be required at end + // of the extension block, so it makes no difference. + // + + // Get id defined by user. + uint8_t id; + if (rtp_header_extension_map_.GetId(kRtpExtensionVideoRotation, &id) != 0) { + // Not registered. + return 0; + } + size_t pos = 0; + const uint8_t len = 0; + data_buffer[pos++] = (id << 4) + len; + data_buffer[pos++] = ConvertToCVOByte(rotation_); + data_buffer[pos++] = 0; // padding + data_buffer[pos++] = 0; // padding + assert(pos == kVideoRotationLength); + return kVideoRotationLength; +} + +bool RTPSender::FindHeaderExtensionPosition(RTPExtensionType type, + const uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + size_t* position) const { + // Get length until start of header extension block. + int extension_block_pos = + rtp_header_extension_map_.GetLengthUntilBlockStartInBytes(type); + if (extension_block_pos < 0) { + LOG(LS_WARNING) << "Failed to find extension position for " << type + << " as it is not registered."; + return false; + } + + HeaderExtension header_extension(type); + + size_t block_pos = + kRtpHeaderLength + rtp_header.numCSRCs + extension_block_pos; + if (rtp_packet_length < block_pos + header_extension.length || + rtp_header.headerLength < block_pos + header_extension.length) { + LOG(LS_WARNING) << "Failed to find extension position for " << type + << " as the length is invalid."; + return false; + } + + // Verify that header contains extension. + if (!((rtp_packet[kRtpHeaderLength + rtp_header.numCSRCs] == 0xBE) && + (rtp_packet[kRtpHeaderLength + rtp_header.numCSRCs + 1] == 0xDE))) { + LOG(LS_WARNING) << "Failed to find extension position for " << type + << "as hdr extension not found."; + return false; + } + + *position = block_pos; + return true; +} + void RTPSender::UpdateTransmissionTimeOffset(uint8_t* rtp_packet, size_t rtp_packet_length, const RTPHeader& rtp_header, @@ -1313,30 +1423,15 @@ void RTPSender::UpdateTransmissionTimeOffset(uint8_t* rtp_packet, // Not registered. return; } - // Get length until start of header extension block. - int extension_block_pos = - rtp_header_extension_map_.GetLengthUntilBlockStartInBytes( - kRtpExtensionTransmissionTimeOffset); - if (extension_block_pos < 0) { - LOG(LS_WARNING) - << "Failed to update transmission time offset, not registered."; - return; - } - size_t block_pos = 12 + rtp_header.numCSRCs + extension_block_pos; - if (rtp_packet_length < block_pos + kTransmissionTimeOffsetLength || - rtp_header.headerLength < - block_pos + kTransmissionTimeOffsetLength) { - LOG(LS_WARNING) - << "Failed to update transmission time offset, invalid length."; - return; - } - // Verify that header contains extension. - if (!((rtp_packet[12 + rtp_header.numCSRCs] == 0xBE) && - (rtp_packet[12 + rtp_header.numCSRCs + 1] == 0xDE))) { - LOG(LS_WARNING) << "Failed to update transmission time offset, hdr " - "extension not found."; + + size_t block_pos = 0; + if (!FindHeaderExtensionPosition(kRtpExtensionTransmissionTimeOffset, + rtp_packet, rtp_packet_length, rtp_header, + &block_pos)) { + LOG(LS_WARNING) << "Failed to update transmission time offset."; return; } + // Verify first byte in block. const uint8_t first_block_byte = (id << 4) + 2; if (rtp_packet[block_pos] != first_block_byte) { @@ -1361,26 +1456,14 @@ bool RTPSender::UpdateAudioLevel(uint8_t* rtp_packet, // Not registered. return false; } - // Get length until start of header extension block. - int extension_block_pos = - rtp_header_extension_map_.GetLengthUntilBlockStartInBytes( - kRtpExtensionAudioLevel); - if (extension_block_pos < 0) { - // The feature is not enabled. - return false; - } - size_t block_pos = 12 + rtp_header.numCSRCs + extension_block_pos; - if (rtp_packet_length < block_pos + kAudioLevelLength || - rtp_header.headerLength < block_pos + kAudioLevelLength) { - LOG(LS_WARNING) << "Failed to update audio level, invalid length."; - return false; - } - // Verify that header contains extension. - if (!((rtp_packet[12 + rtp_header.numCSRCs] == 0xBE) && - (rtp_packet[12 + rtp_header.numCSRCs + 1] == 0xDE))) { - LOG(LS_WARNING) << "Failed to update audio level, hdr extension not found."; + + size_t block_pos = 0; + if (!FindHeaderExtensionPosition(kRtpExtensionAudioLevel, rtp_packet, + rtp_packet_length, rtp_header, &block_pos)) { + LOG(LS_WARNING) << "Failed to update audio level."; return false; } + // Verify first byte in block. const uint8_t first_block_byte = (id << 4) + 0; if (rtp_packet[block_pos] != first_block_byte) { @@ -1391,6 +1474,44 @@ bool RTPSender::UpdateAudioLevel(uint8_t* rtp_packet, return true; } +bool RTPSender::UpdateVideoRotation(uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + VideoRotation rotation) const { + CriticalSectionScoped cs(send_critsect_.get()); + + // Get id. + uint8_t id = 0; + if (rtp_header_extension_map_.GetId(kRtpExtensionVideoRotation, &id) != 0) { + // Not registered. + return false; + } + + size_t block_pos = 0; + if (!FindHeaderExtensionPosition(kRtpExtensionVideoRotation, rtp_packet, + rtp_packet_length, rtp_header, &block_pos)) { + LOG(LS_WARNING) << "Failed to update video rotation (CVO)."; + return false; + } + // Get length until start of header extension block. + int extension_block_pos = + rtp_header_extension_map_.GetLengthUntilBlockStartInBytes( + kRtpExtensionVideoRotation); + if (extension_block_pos < 0) { + // The feature is not enabled. + return false; + } + + // Verify first byte in block. + const uint8_t first_block_byte = (id << 4) + 0; + if (rtp_packet[block_pos] != first_block_byte) { + LOG(LS_WARNING) << "Failed to update CVO."; + return false; + } + rtp_packet[block_pos + 1] = ConvertToCVOByte(rotation); + return true; +} + void RTPSender::UpdateAbsoluteSendTime(uint8_t* rtp_packet, size_t rtp_packet_length, const RTPHeader& rtp_header, @@ -1412,15 +1533,16 @@ void RTPSender::UpdateAbsoluteSendTime(uint8_t* rtp_packet, // The feature is not enabled. return; } - size_t block_pos = 12 + rtp_header.numCSRCs + extension_block_pos; + size_t block_pos = + kRtpHeaderLength + rtp_header.numCSRCs + extension_block_pos; if (rtp_packet_length < block_pos + kAbsoluteSendTimeLength || rtp_header.headerLength < block_pos + kAbsoluteSendTimeLength) { LOG(LS_WARNING) << "Failed to update absolute send time, invalid length."; return; } // Verify that header contains extension. - if (!((rtp_packet[12 + rtp_header.numCSRCs] == 0xBE) && - (rtp_packet[12 + rtp_header.numCSRCs + 1] == 0xDE))) { + if (!((rtp_packet[kRtpHeaderLength + rtp_header.numCSRCs] == 0xBE) && + (rtp_packet[kRtpHeaderLength + rtp_header.numCSRCs + 1] == 0xDE))) { LOG(LS_WARNING) << "Failed to update absolute send time, hdr extension not found."; return; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.h b/webrtc/modules/rtp_rtcp/source/rtp_sender.h index 9342e5e5e4..67fb1feaf0 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.h @@ -64,6 +64,12 @@ class RTPSenderInterface { uint8_t *data_buffer, size_t payload_length, size_t rtp_header_length, int64_t capture_time_ms, StorageType storage, PacedSender::Priority priority) = 0; + + virtual bool UpdateVideoRotation(uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + VideoRotation rotation) const = 0; + virtual bool IsRtpHeaderExtensionRegistered(RTPExtensionType type) = 0; }; class RTPSender : public RTPSenderInterface { @@ -141,23 +147,25 @@ class RTPSender : public RTPSenderInterface { size_t payload_size, const RTPFragmentationHeader* fragmentation, VideoCodecInformation* codec_info = NULL, - const RTPVideoTypeHeader* rtp_type_hdr = NULL); + const RTPVideoHeader* rtp_hdr = NULL); // RTP header extension int32_t SetTransmissionTimeOffset(int32_t transmission_time_offset); int32_t SetAbsoluteSendTime(uint32_t absolute_send_time); + void SetVideoRotation(VideoRotation rotation); int32_t RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id); - + virtual bool IsRtpHeaderExtensionRegistered(RTPExtensionType type) override; int32_t DeregisterRtpHeaderExtension(RTPExtensionType type); size_t RtpHeaderExtensionTotalLength() const; - uint16_t BuildRTPHeaderExtension(uint8_t* data_buffer) const; + uint16_t BuildRTPHeaderExtension(uint8_t* data_buffer, bool marker_bit) const; uint8_t BuildTransmissionTimeOffsetExtension(uint8_t *data_buffer) const; uint8_t BuildAudioLevelExtension(uint8_t* data_buffer) const; uint8_t BuildAbsoluteSendTimeExtension(uint8_t* data_buffer) const; + uint8_t BuildVideoRotationExtension(uint8_t* data_buffer) const; bool UpdateAudioLevel(uint8_t* rtp_packet, size_t rtp_packet_length, @@ -165,6 +173,11 @@ class RTPSender : public RTPSenderInterface { bool is_voiced, uint8_t dBov) const; + virtual bool UpdateVideoRotation(uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + VideoRotation rotation) const override; + bool TimeToSendPacket(uint16_t sequence_number, int64_t capture_time_ms, bool retransmission); size_t TimeToSendPadding(size_t bytes); @@ -271,6 +284,8 @@ class RTPSender : public RTPSenderInterface { void SetRtxRtpState(const RtpState& rtp_state); RtpState GetRtxRtpState() const; + static uint8_t ConvertToCVOByte(VideoRotation rotation); + protected: int32_t CheckPayloadType(int8_t payload_type, RtpVideoCodecTypes* video_type); @@ -310,6 +325,14 @@ class RTPSender : public RTPSenderInterface { void UpdateDelayStatistics(int64_t capture_time_ms, int64_t now_ms); + // Find the byte position of the RTP extension as indicated by |type| in + // |rtp_packet|. Return false if such extension doesn't exist. + bool FindHeaderExtensionPosition(RTPExtensionType type, + const uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + size_t* position) const; + void UpdateTransmissionTimeOffset(uint8_t* rtp_packet, size_t rtp_packet_length, const RTPHeader& rtp_header, @@ -354,6 +377,7 @@ class RTPSender : public RTPSenderInterface { RtpHeaderExtensionMap rtp_header_extension_map_; int32_t transmission_time_offset_; uint32_t absolute_send_time_; + VideoRotation rotation_; // NACK uint32_t nack_byte_count_times_[NACK_BYTECOUNT_SIZE]; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc index f57ee27323..c6e8398658 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -14,6 +14,7 @@ #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/buffer.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/modules/pacing/include/mock/mock_paced_sender.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" @@ -21,6 +22,8 @@ #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h" #include "webrtc/modules/rtp_rtcp/source/rtp_header_extension.h" #include "webrtc/modules/rtp_rtcp/source/rtp_sender.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h" +#include "webrtc/system_wrappers/interface/stl_util.h" #include "webrtc/test/mock_transport.h" #include "webrtc/typedefs.h" @@ -40,7 +43,8 @@ const uint8_t kAudioLevelExtensionId = 9; const int kAudioPayload = 103; const uint64_t kStartTime = 123456789; const size_t kMaxPaddingSize = 224u; -} // namespace +const int kVideoRotationExtensionId = 5; +const VideoRotation kRotation = kVideoRotation_270; using testing::_; @@ -61,12 +65,21 @@ uint64_t ConvertMsToAbsSendTime(int64_t time_ms) { class LoopbackTransportTest : public webrtc::Transport { public: LoopbackTransportTest() - : packets_sent_(0), last_sent_packet_len_(0), total_bytes_sent_(0) {} - int SendPacket(int channel, const void* data, size_t len) override { + : packets_sent_(0), + last_sent_packet_len_(0), + total_bytes_sent_(0), + last_sent_packet_(NULL) {} + + ~LoopbackTransportTest() { + STLDeleteContainerPointers(sent_packets_.begin(), sent_packets_.end()); + } + int SendPacket(int channel, const void *data, size_t len) override { packets_sent_++; - memcpy(last_sent_packet_, data, len); + rtc::Buffer* buffer = new rtc::Buffer(data, len); + last_sent_packet_ = reinterpret_cast(buffer->data()); last_sent_packet_len_ = len; total_bytes_sent_ += len; + sent_packets_.push_back(buffer); return static_cast(len); } int SendRTCPPacket(int channel, const void* data, size_t len) override { @@ -75,9 +88,12 @@ class LoopbackTransportTest : public webrtc::Transport { int packets_sent_; size_t last_sent_packet_len_; size_t total_bytes_sent_; - uint8_t last_sent_packet_[kMaxPacketLength]; + uint8_t* last_sent_packet_; + std::vector sent_packets_; }; +} // namespace + class RtpSenderTest : public ::testing::Test { protected: RtpSenderTest() @@ -106,7 +122,11 @@ class RtpSenderTest : public ::testing::Test { uint8_t packet_[kMaxPacketLength]; void VerifyRTPHeaderCommon(const RTPHeader& rtp_header) { - EXPECT_EQ(kMarkerBit, rtp_header.markerBit); + VerifyRTPHeaderCommon(rtp_header, kMarkerBit); + } + + void VerifyRTPHeaderCommon(const RTPHeader& rtp_header, bool marker_bit) { + EXPECT_EQ(marker_bit, rtp_header.markerBit); EXPECT_EQ(payload_, rtp_header.payloadType); EXPECT_EQ(kSeqNum, rtp_header.sequenceNumber); EXPECT_EQ(kTimestamp, rtp_header.timestamp); @@ -134,6 +154,46 @@ class RtpSenderTest : public ::testing::Test { } }; +class RtpSenderVideoTest : public RtpSenderTest { + protected: + virtual void SetUp() override { + RtpSenderTest::SetUp(); + rtp_sender_video_.reset( + new RTPSenderVideo(&fake_clock_, rtp_sender_.get())); + } + rtc::scoped_ptr rtp_sender_video_; + + void VerifyCVOPacket(uint8_t* data, + size_t len, + bool expect_cvo, + RtpHeaderExtensionMap* map, + uint16_t seq_num, + VideoRotation rotation) { + webrtc::RtpUtility::RtpHeaderParser rtp_parser(data, len); + + webrtc::RTPHeader rtp_header; + size_t length = static_cast(rtp_sender_->BuildRTPheader( + packet_, kPayload, expect_cvo /* marker_bit */, kTimestamp, 0)); + if (expect_cvo) { + ASSERT_EQ(kRtpHeaderSize + rtp_sender_->RtpHeaderExtensionTotalLength(), + length); + } else { + ASSERT_EQ(kRtpHeaderSize, length); + } + ASSERT_TRUE(rtp_parser.Parse(rtp_header, map)); + ASSERT_FALSE(rtp_parser.RTCP()); + EXPECT_EQ(expect_cvo, rtp_header.markerBit); + EXPECT_EQ(payload_, rtp_header.payloadType); + EXPECT_EQ(seq_num, rtp_header.sequenceNumber); + EXPECT_EQ(kTimestamp, rtp_header.timestamp); + EXPECT_EQ(rtp_sender_->SSRC(), rtp_header.ssrc); + EXPECT_EQ(0, rtp_header.numCSRCs); + EXPECT_EQ(0U, rtp_header.paddingLength); + EXPECT_EQ(RTPSender::ConvertToCVOByte(rotation), + rtp_header.extension.videoRotation); + } +}; + TEST_F(RtpSenderTest, RegisterRtpTransmissionTimeOffsetHeaderExtension) { EXPECT_EQ(0u, rtp_sender_->RtpHeaderExtensionTotalLength()); EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( @@ -182,16 +242,40 @@ TEST_F(RtpSenderTest, RegisterRtpHeaderExtensions) { EXPECT_EQ(kRtpOneByteHeaderLength + kTransmissionTimeOffsetLength + kAbsoluteSendTimeLength + kAudioLevelLength, rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + EXPECT_EQ(kRtpOneByteHeaderLength + kTransmissionTimeOffsetLength + + kAbsoluteSendTimeLength + kAudioLevelLength + + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); + + // Deregister starts. EXPECT_EQ(0, rtp_sender_->DeregisterRtpHeaderExtension( kRtpExtensionTransmissionTimeOffset)); EXPECT_EQ(kRtpOneByteHeaderLength + kAbsoluteSendTimeLength + - kAudioLevelLength, rtp_sender_->RtpHeaderExtensionTotalLength()); + kAudioLevelLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); EXPECT_EQ(0, rtp_sender_->DeregisterRtpHeaderExtension( kRtpExtensionAbsoluteSendTime)); - EXPECT_EQ(kRtpOneByteHeaderLength + kAudioLevelLength, - rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ(kRtpOneByteHeaderLength + kAudioLevelLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); EXPECT_EQ(0, rtp_sender_->DeregisterRtpHeaderExtension( kRtpExtensionAudioLevel)); + EXPECT_EQ(kRtpOneByteHeaderLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ( + 0, rtp_sender_->DeregisterRtpHeaderExtension(kRtpExtensionVideoRotation)); + EXPECT_EQ(0u, rtp_sender_->RtpHeaderExtensionTotalLength()); +} + +TEST_F(RtpSenderTest, RegisterRtpVideoRotationHeaderExtension) { + EXPECT_EQ(0u, rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + EXPECT_EQ(kRtpOneByteHeaderLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ( + 0, rtp_sender_->DeregisterRtpHeaderExtension(kRtpExtensionVideoRotation)); EXPECT_EQ(0u, rtp_sender_->RtpHeaderExtensionTotalLength()); } @@ -216,6 +300,7 @@ TEST_F(RtpSenderTest, BuildRTPPacket) { EXPECT_EQ(0, rtp_header.extension.transmissionTimeOffset); EXPECT_EQ(0u, rtp_header.extension.absoluteSendTime); EXPECT_EQ(0u, rtp_header.extension.audioLevel); + EXPECT_EQ(0u, rtp_header.extension.videoRotation); } TEST_F(RtpSenderTest, BuildRTPPacketWithTransmissionOffsetExtension) { @@ -319,6 +404,57 @@ TEST_F(RtpSenderTest, BuildRTPPacketWithAbsoluteSendTimeExtension) { EXPECT_EQ(0u, rtp_header2.extension.absoluteSendTime); } +// Test CVO header extension is only set when marker bit is true. +TEST_F(RtpSenderTest, BuildRTPPacketWithVideoRotation_MarkerBit) { + rtp_sender_->SetVideoRotation(kRotation); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + + RtpHeaderExtensionMap map; + map.Register(kRtpExtensionVideoRotation, kVideoRotationExtensionId); + + size_t length = static_cast( + rtp_sender_->BuildRTPheader(packet_, kPayload, true, kTimestamp, 0)); + ASSERT_EQ(kRtpHeaderSize + rtp_sender_->RtpHeaderExtensionTotalLength(), + length); + + // Verify + webrtc::RtpUtility::RtpHeaderParser rtp_parser(packet_, length); + webrtc::RTPHeader rtp_header; + + ASSERT_TRUE(rtp_parser.Parse(rtp_header, &map)); + ASSERT_FALSE(rtp_parser.RTCP()); + VerifyRTPHeaderCommon(rtp_header); + EXPECT_EQ(length, rtp_header.headerLength); + EXPECT_TRUE(rtp_header.extension.hasVideoRotation); + EXPECT_EQ(RTPSender::ConvertToCVOByte(kRotation), + rtp_header.extension.videoRotation); +} + +// Test CVO header extension is not set when marker bit is false. +TEST_F(RtpSenderTest, BuildRTPPacketWithVideoRotation_NoMarkerBit) { + rtp_sender_->SetVideoRotation(kRotation); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + + RtpHeaderExtensionMap map; + map.Register(kRtpExtensionVideoRotation, kVideoRotationExtensionId); + + size_t length = static_cast( + rtp_sender_->BuildRTPheader(packet_, kPayload, false, kTimestamp, 0)); + ASSERT_EQ(kRtpHeaderSize, length); + + // Verify + webrtc::RtpUtility::RtpHeaderParser rtp_parser(packet_, length); + webrtc::RTPHeader rtp_header; + + ASSERT_TRUE(rtp_parser.Parse(rtp_header, &map)); + ASSERT_FALSE(rtp_parser.RTCP()); + VerifyRTPHeaderCommon(rtp_header, false); + EXPECT_EQ(length, rtp_header.headerLength); + EXPECT_FALSE(rtp_header.extension.hasVideoRotation); +} + TEST_F(RtpSenderTest, BuildRTPPacketWithAudioLevelExtension) { EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( kRtpExtensionAudioLevel, kAudioLevelExtensionId)); @@ -1168,4 +1304,34 @@ TEST_F(RtpSenderTest, BytesReportedCorrectly) { rtp_stats.transmitted.TotalBytes() + rtx_stats.transmitted.TotalBytes()); } + +// Verify that only the last packet of a frame has CVO byte set. +TEST_F(RtpSenderVideoTest, SendVideoWithCVO) { + RTPVideoHeader hdr = {0}; + hdr.rotation = kVideoRotation_90; + + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + EXPECT_EQ(kRtpOneByteHeaderLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); + + rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, kPayload, + kTimestamp, 0, packet_, sizeof(packet_), NULL, + NULL, &hdr); + + RtpHeaderExtensionMap map; + map.Register(kRtpExtensionVideoRotation, kVideoRotationExtensionId); + + // Verify that this packet doesn't have CVO byte. + VerifyCVOPacket( + reinterpret_cast(transport_.sent_packets_[0]->data()), + transport_.sent_packets_[0]->length(), false, &map, kSeqNum, + kVideoRotation_0); + + // Verify that this packet doesn't have CVO byte. + VerifyCVOPacket( + reinterpret_cast(transport_.sent_packets_[1]->data()), + transport_.sent_packets_[1]->length(), true, &map, kSeqNum + 1, + hdr.rotation); +} } // namespace webrtc diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc index bfcb5e29cf..c802bd5742 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -259,7 +259,7 @@ int32_t RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType, const size_t payloadSize, const RTPFragmentationHeader* fragmentation, VideoCodecInformation* codecInfo, - const RTPVideoTypeHeader* rtpTypeHdr) { + const RTPVideoHeader* rtpHdr) { if (payloadSize == 0) { return -1; } @@ -274,15 +274,8 @@ int32_t RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType, // Will be extracted in SendVP8 for VP8 codec; other codecs use 0 _numberFirstPartition = 0; - return Send(videoType, - frameType, - payloadType, - captureTimeStamp, - capture_time_ms, - payloadData, - payloadSize, - fragmentation, - rtpTypeHdr) + return Send(videoType, frameType, payloadType, captureTimeStamp, + capture_time_ms, payloadData, payloadSize, fragmentation, rtpHdr) ? 0 : -1; } @@ -307,14 +300,14 @@ bool RTPSenderVideo::Send(const RtpVideoCodecTypes videoType, const uint8_t* payloadData, const size_t payloadSize, const RTPFragmentationHeader* fragmentation, - const RTPVideoTypeHeader* rtpTypeHdr) { + const RTPVideoHeader* rtpHdr) { uint16_t rtp_header_length = _rtpSender.RTPHeaderLength(); size_t payload_bytes_to_send = payloadSize; const uint8_t* data = payloadData; size_t max_payload_length = _rtpSender.MaxDataPayloadLength(); rtc::scoped_ptr packetizer(RtpPacketizer::Create( - videoType, max_payload_length, rtpTypeHdr, frameType)); + videoType, max_payload_length, &(rtpHdr->codecHeader), frameType)); // TODO(changbin): we currently don't support to configure the codec to // output multiple partitions for VP8. Should remove below check after the @@ -337,6 +330,31 @@ bool RTPSenderVideo::Send(const RtpVideoCodecTypes videoType, // Set marker bit true if this is the last packet in frame. _rtpSender.BuildRTPheader( dataBuffer, payloadType, last, captureTimeStamp, capture_time_ms); + + // According to + // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ + // ts_126114v120700p.pdf Section 7.4.5: + // The MTSI client shall add the payload bytes as defined in this clause + // onto the last RTP packet in each group of packets which make up a key + // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265 + // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP + // packet in each group of packets which make up another type of frame + // (e.g. a P-Frame) only if the current value is different from the previous + // value sent. + // Here we are adding it to the last packet of every frame at this point. + if (!rtpHdr) { + assert(!_rtpSender.IsRtpHeaderExtensionRegistered( + kRtpExtensionVideoRotation)); + } else if (last) { + // Checking whether CVO header extension is registered will require taking + // a lock. It'll be a no-op if it's not registered. + size_t packetSize = payloadSize + rtp_header_length; + RtpUtility::RtpHeaderParser rtp_parser(dataBuffer, packetSize); + RTPHeader rtp_header; + rtp_parser.Parse(rtp_header); + _rtpSender.UpdateVideoRotation(dataBuffer, packetSize, rtp_header, + rtpHdr->rotation); + } if (SendVideoPacket(dataBuffer, payload_bytes_in_packet, rtp_header_length, diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h index 7338875c12..92c312f5e9 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h @@ -51,7 +51,7 @@ class RTPSenderVideo { const size_t payloadSize, const RTPFragmentationHeader* fragmentation, VideoCodecInformation* codecInfo, - const RTPVideoTypeHeader* rtpTypeHdr); + const RTPVideoHeader* rtpHdr); int32_t SendRTPIntraRequest(); @@ -101,7 +101,7 @@ class RTPSenderVideo { const uint8_t* payloadData, const size_t payloadSize, const RTPFragmentationHeader* fragmentation, - const RTPVideoTypeHeader* rtpTypeHdr); + const RTPVideoHeader* rtpHdr); private: RTPSenderInterface& _rtpSender; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_utility.cc b/webrtc/modules/rtp_rtcp/source/rtp_utility.cc index b12839e16e..d26ceda3cf 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_utility.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_utility.cc @@ -373,6 +373,10 @@ bool RtpHeaderParser::Parse(RTPHeader& header, header.extension.hasAudioLevel = false; header.extension.audioLevel = 0; + // May not be present in packet. + header.extension.hasVideoRotation = false; + header.extension.videoRotation = 0; + if (X) { /* RTP header extension, RFC 3550. 0 1 2 3 @@ -511,6 +515,21 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( header.extension.hasAbsoluteSendTime = true; break; } + case kRtpExtensionVideoRotation: { + if (len != 0) { + LOG(LS_WARNING) + << "Incorrect coordination of video coordination len: " << len; + return; + } + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | ID | len=0 |V|0 0 0 0 C F R R| 0x00 | 0x00 | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + header.extension.hasVideoRotation = true; + header.extension.videoRotation = ptr[0]; + break; + } default: { LOG(LS_WARNING) << "Extension type not implemented: " << type; return;