diff --git a/webrtc/common_types.h b/webrtc/common_types.h index bdba653e39..f3f84dee40 100644 --- a/webrtc/common_types.h +++ b/webrtc/common_types.h @@ -798,7 +798,9 @@ struct RTPHeaderExtension { hasAbsoluteSendTime(false), absoluteSendTime(0), hasAudioLevel(false), - audioLevel(0) {} + audioLevel(0), + hasVideoRotation(false), + videoRotation(0) {} bool hasTransmissionTimeOffset; int32_t transmissionTimeOffset; @@ -809,6 +811,12 @@ struct RTPHeaderExtension { // https://datatracker.ietf.org/doc/draft-lennox-avt-rtp-audio-level-exthdr/ bool hasAudioLevel; uint8_t audioLevel; + + // For Coordination of Video Orientation. See + // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ + // ts_126114v120700p.pdf + bool hasVideoRotation; + uint8_t videoRotation; }; struct RTPHeader { diff --git a/webrtc/modules/interface/module_common_types.h b/webrtc/modules/interface/module_common_types.h index 5cb2c56108..207680d4ee 100644 --- a/webrtc/modules/interface/module_common_types.h +++ b/webrtc/modules/interface/module_common_types.h @@ -18,6 +18,7 @@ #include "webrtc/base/constructormagic.h" #include "webrtc/common_types.h" +#include "webrtc/common_video/rotation.h" #include "webrtc/typedefs.h" namespace webrtc { @@ -76,9 +77,12 @@ enum RtpVideoCodecTypes { kRtpVideoVp8, kRtpVideoH264 }; +// Since RTPVideoHeader is used as a member of a union, it can't have a +// non-trivial default constructor. struct RTPVideoHeader { uint16_t width; // size uint16_t height; + VideoRotation rotation; bool isFirstPacket; // first packet in frame uint8_t simulcastIdx; // Index if the simulcast encoder creating diff --git a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h index 4938b5fefb..d15f31d2e3 100644 --- a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h +++ b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h @@ -74,12 +74,12 @@ enum StorageType { kAllowRetransmission }; -enum RTPExtensionType -{ - kRtpExtensionNone, - kRtpExtensionTransmissionTimeOffset, - kRtpExtensionAudioLevel, - kRtpExtensionAbsoluteSendTime +enum RTPExtensionType { + kRtpExtensionNone, + kRtpExtensionTransmissionTimeOffset, + kRtpExtensionAudioLevel, + kRtpExtensionAbsoluteSendTime, + kRtpExtensionVideoRotation }; enum RTCPAppSubTypes diff --git a/webrtc/modules/rtp_rtcp/source/rtp_header_extension.h b/webrtc/modules/rtp_rtcp/source/rtp_header_extension.h index 335d0a1b8f..36c2f73031 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_header_extension.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_header_extension.h @@ -24,6 +24,7 @@ const size_t kRtpOneByteHeaderLength = 4; const size_t kTransmissionTimeOffsetLength = 4; const size_t kAudioLevelLength = 4; const size_t kAbsoluteSendTimeLength = 4; +const size_t kVideoRotationLength = 4; struct HeaderExtension { HeaderExtension(RTPExtensionType extension_type) @@ -42,6 +43,9 @@ struct HeaderExtension { case kRtpExtensionAbsoluteSendTime: length = kAbsoluteSendTimeLength; break; + case kRtpExtensionVideoRotation: + length = kVideoRotationLength; + break; default: assert(false); } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 99d2df56e0..d483e76c78 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -401,15 +401,9 @@ int32_t ModuleRtpRtcpImpl::SendOutgoingData( if (rtcp_sender_.TimeToSendRTCPReport(kVideoFrameKey == frame_type)) { rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport); } - return rtp_sender_.SendOutgoingData(frame_type, - payload_type, - time_stamp, - capture_time_ms, - payload_data, - payload_size, - fragmentation, - NULL, - &(rtp_video_hdr->codecHeader)); + return rtp_sender_.SendOutgoingData( + frame_type, payload_type, time_stamp, capture_time_ms, payload_data, + payload_size, fragmentation, NULL, rtp_video_hdr); } bool ModuleRtpRtcpImpl::TimeToSendPacket(uint32_t ssrc, diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc index e7b58a2a60..3eb05a9b84 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc @@ -190,8 +190,13 @@ class RtpRtcpImplTest : public ::testing::Test { void SendFrame(const RtpRtcpModule* module, uint8_t tid) { RTPVideoHeaderVP8 vp8_header = {}; vp8_header.temporalIdx = tid; - RTPVideoHeader rtp_video_header = { - codec_.width, codec_.height, true, 0, kRtpVideoVp8, {vp8_header}}; + RTPVideoHeader rtp_video_header = {codec_.width, + codec_.height, + kVideoRotation_0, + true, + 0, + kRtpVideoVp8, + {vp8_header}}; const uint8_t payload[100] = {0}; EXPECT_EQ(0, module->impl_->SendOutgoingData(kVideoFrameKey, diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc index e6a912eb99..28c458ddb7 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc @@ -27,6 +27,8 @@ const int kSendSideDelayWindowMs = 1000; namespace { +const size_t kRtpHeaderLength = 12; + const char* FrameTypeToString(FrameType frame_type) { switch (frame_type) { case kFrameEmpty: return "empty"; @@ -124,6 +126,7 @@ RTPSender::RTPSender(int32_t id, rtp_header_extension_map_(), transmission_time_offset_(0), absolute_send_time_(0), + rotation_(kVideoRotation_0), // NACK. nack_byte_count_times_(), nack_byte_count_(), @@ -246,12 +249,22 @@ int32_t RTPSender::SetAbsoluteSendTime(uint32_t absolute_send_time) { return 0; } +void RTPSender::SetVideoRotation(VideoRotation rotation) { + CriticalSectionScoped cs(send_critsect_.get()); + rotation_ = rotation; +} + int32_t RTPSender::RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id) { CriticalSectionScoped cs(send_critsect_.get()); return rtp_header_extension_map_.Register(type, id); } +bool RTPSender::IsRtpHeaderExtensionRegistered(RTPExtensionType type) { + CriticalSectionScoped cs(send_critsect_.get()); + return rtp_header_extension_map_.IsRegistered(type); +} + int32_t RTPSender::DeregisterRtpHeaderExtension(RTPExtensionType type) { CriticalSectionScoped cs(send_critsect_.get()); return rtp_header_extension_map_.Deregister(type); @@ -440,6 +453,25 @@ int32_t RTPSender::CheckPayloadType(int8_t payload_type, return 0; } +// Please refer to http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/ +// 12.07.00_60/ts_126114v120700p.pdf Section 7.4.5. The rotation of a frame is +// the clockwise angle the frames must be rotated in order to display the frames +// correctly if the display is rotated in its natural orientation. +uint8_t RTPSender::ConvertToCVOByte(VideoRotation rotation) { + switch (rotation) { + case kVideoRotation_0: + return 0; + case kVideoRotation_90: + return 1; + case kVideoRotation_180: + return 2; + case kVideoRotation_270: + return 3; + } + assert(false); + return 0; +} + int32_t RTPSender::SendOutgoingData(FrameType frame_type, int8_t payload_type, uint32_t capture_timestamp, @@ -448,7 +480,7 @@ int32_t RTPSender::SendOutgoingData(FrameType frame_type, size_t payload_size, const RTPFragmentationHeader* fragmentation, VideoCodecInformation* codec_info, - const RTPVideoTypeHeader* rtp_type_hdr) { + const RTPVideoHeader* rtp_hdr) { uint32_t ssrc; { // Drop this packet if we're not sending media packets. @@ -481,12 +513,10 @@ int32_t RTPSender::SendOutgoingData(FrameType frame_type, if (frame_type == kFrameEmpty) return 0; - ret_val = video_->SendVideo(video_type, frame_type, payload_type, - capture_timestamp, capture_time_ms, - payload_data, payload_size, - fragmentation, codec_info, - rtp_type_hdr); - + ret_val = + video_->SendVideo(video_type, frame_type, payload_type, + capture_timestamp, capture_time_ms, payload_data, + payload_size, fragmentation, codec_info, rtp_hdr); } CriticalSectionScoped cs(statistics_crit_.get()); @@ -1040,7 +1070,7 @@ void RTPSender::ProcessBitrate() { size_t RTPSender::RTPHeaderLength() const { CriticalSectionScoped lock(send_critsect_.get()); - size_t rtp_header_length = 12; + size_t rtp_header_length = kRtpHeaderLength; rtp_header_length += sizeof(uint32_t) * csrcs_.size(); rtp_header_length += RtpHeaderExtensionTotalLength(); return rtp_header_length; @@ -1093,7 +1123,7 @@ size_t RTPSender::CreateRtpHeader(uint8_t* header, RtpUtility::AssignUWord16ToBuffer(header + 2, sequence_number); RtpUtility::AssignUWord32ToBuffer(header + 4, timestamp); RtpUtility::AssignUWord32ToBuffer(header + 8, ssrc); - int32_t rtp_header_length = 12; + int32_t rtp_header_length = kRtpHeaderLength; if (csrcs.size() > 0) { uint8_t *ptr = &header[rtp_header_length]; @@ -1107,7 +1137,8 @@ size_t RTPSender::CreateRtpHeader(uint8_t* header, rtp_header_length += sizeof(uint32_t) * csrcs.size(); } - uint16_t len = BuildRTPHeaderExtension(header + rtp_header_length); + uint16_t len = + BuildRTPHeaderExtension(header + rtp_header_length, marker_bit); if (len > 0) { header[0] |= 0x10; // Set extension bit. rtp_header_length += len; @@ -1141,7 +1172,8 @@ int32_t RTPSender::BuildRTPheader(uint8_t* data_buffer, timestamp_, sequence_number, csrcs_); } -uint16_t RTPSender::BuildRTPHeaderExtension(uint8_t* data_buffer) const { +uint16_t RTPSender::BuildRTPHeaderExtension(uint8_t* data_buffer, + bool marker_bit) const { if (rtp_header_extension_map_.Size() <= 0) { return 0; } @@ -1179,6 +1211,12 @@ uint16_t RTPSender::BuildRTPHeaderExtension(uint8_t* data_buffer) const { block_length = BuildAbsoluteSendTimeExtension( data_buffer + kHeaderLength + total_block_length); break; + case kRtpExtensionVideoRotation: + if (marker_bit) { + block_length = BuildVideoRotationExtension( + data_buffer + kHeaderLength + total_block_length); + } + break; default: assert(false); } @@ -1301,6 +1339,78 @@ uint8_t RTPSender::BuildAbsoluteSendTimeExtension(uint8_t* data_buffer) const { return kAbsoluteSendTimeLength; } +uint8_t RTPSender::BuildVideoRotationExtension(uint8_t* data_buffer) const { + // Coordination of Video Orientation in RTP streams. + // + // Coordination of Video Orientation consists in signalling of the current + // orientation of the image captured on the sender side to the receiver for + // appropriate rendering and displaying. + // + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | ID | len=0 |V|0 0 0 0 C F R R| 0x00 | 0x00 | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // + // Note that we always include 2 pad bytes, which will result in legal and + // correctly parsed RTP, but may be a bit wasteful if more short extensions + // are implemented. Right now the pad bytes would anyway be required at end + // of the extension block, so it makes no difference. + // + + // Get id defined by user. + uint8_t id; + if (rtp_header_extension_map_.GetId(kRtpExtensionVideoRotation, &id) != 0) { + // Not registered. + return 0; + } + size_t pos = 0; + const uint8_t len = 0; + data_buffer[pos++] = (id << 4) + len; + data_buffer[pos++] = ConvertToCVOByte(rotation_); + data_buffer[pos++] = 0; // padding + data_buffer[pos++] = 0; // padding + assert(pos == kVideoRotationLength); + return kVideoRotationLength; +} + +bool RTPSender::FindHeaderExtensionPosition(RTPExtensionType type, + const uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + size_t* position) const { + // Get length until start of header extension block. + int extension_block_pos = + rtp_header_extension_map_.GetLengthUntilBlockStartInBytes(type); + if (extension_block_pos < 0) { + LOG(LS_WARNING) << "Failed to find extension position for " << type + << " as it is not registered."; + return false; + } + + HeaderExtension header_extension(type); + + size_t block_pos = + kRtpHeaderLength + rtp_header.numCSRCs + extension_block_pos; + if (rtp_packet_length < block_pos + header_extension.length || + rtp_header.headerLength < block_pos + header_extension.length) { + LOG(LS_WARNING) << "Failed to find extension position for " << type + << " as the length is invalid."; + return false; + } + + // Verify that header contains extension. + if (!((rtp_packet[kRtpHeaderLength + rtp_header.numCSRCs] == 0xBE) && + (rtp_packet[kRtpHeaderLength + rtp_header.numCSRCs + 1] == 0xDE))) { + LOG(LS_WARNING) << "Failed to find extension position for " << type + << "as hdr extension not found."; + return false; + } + + *position = block_pos; + return true; +} + void RTPSender::UpdateTransmissionTimeOffset(uint8_t* rtp_packet, size_t rtp_packet_length, const RTPHeader& rtp_header, @@ -1313,30 +1423,15 @@ void RTPSender::UpdateTransmissionTimeOffset(uint8_t* rtp_packet, // Not registered. return; } - // Get length until start of header extension block. - int extension_block_pos = - rtp_header_extension_map_.GetLengthUntilBlockStartInBytes( - kRtpExtensionTransmissionTimeOffset); - if (extension_block_pos < 0) { - LOG(LS_WARNING) - << "Failed to update transmission time offset, not registered."; - return; - } - size_t block_pos = 12 + rtp_header.numCSRCs + extension_block_pos; - if (rtp_packet_length < block_pos + kTransmissionTimeOffsetLength || - rtp_header.headerLength < - block_pos + kTransmissionTimeOffsetLength) { - LOG(LS_WARNING) - << "Failed to update transmission time offset, invalid length."; - return; - } - // Verify that header contains extension. - if (!((rtp_packet[12 + rtp_header.numCSRCs] == 0xBE) && - (rtp_packet[12 + rtp_header.numCSRCs + 1] == 0xDE))) { - LOG(LS_WARNING) << "Failed to update transmission time offset, hdr " - "extension not found."; + + size_t block_pos = 0; + if (!FindHeaderExtensionPosition(kRtpExtensionTransmissionTimeOffset, + rtp_packet, rtp_packet_length, rtp_header, + &block_pos)) { + LOG(LS_WARNING) << "Failed to update transmission time offset."; return; } + // Verify first byte in block. const uint8_t first_block_byte = (id << 4) + 2; if (rtp_packet[block_pos] != first_block_byte) { @@ -1361,26 +1456,14 @@ bool RTPSender::UpdateAudioLevel(uint8_t* rtp_packet, // Not registered. return false; } - // Get length until start of header extension block. - int extension_block_pos = - rtp_header_extension_map_.GetLengthUntilBlockStartInBytes( - kRtpExtensionAudioLevel); - if (extension_block_pos < 0) { - // The feature is not enabled. - return false; - } - size_t block_pos = 12 + rtp_header.numCSRCs + extension_block_pos; - if (rtp_packet_length < block_pos + kAudioLevelLength || - rtp_header.headerLength < block_pos + kAudioLevelLength) { - LOG(LS_WARNING) << "Failed to update audio level, invalid length."; - return false; - } - // Verify that header contains extension. - if (!((rtp_packet[12 + rtp_header.numCSRCs] == 0xBE) && - (rtp_packet[12 + rtp_header.numCSRCs + 1] == 0xDE))) { - LOG(LS_WARNING) << "Failed to update audio level, hdr extension not found."; + + size_t block_pos = 0; + if (!FindHeaderExtensionPosition(kRtpExtensionAudioLevel, rtp_packet, + rtp_packet_length, rtp_header, &block_pos)) { + LOG(LS_WARNING) << "Failed to update audio level."; return false; } + // Verify first byte in block. const uint8_t first_block_byte = (id << 4) + 0; if (rtp_packet[block_pos] != first_block_byte) { @@ -1391,6 +1474,44 @@ bool RTPSender::UpdateAudioLevel(uint8_t* rtp_packet, return true; } +bool RTPSender::UpdateVideoRotation(uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + VideoRotation rotation) const { + CriticalSectionScoped cs(send_critsect_.get()); + + // Get id. + uint8_t id = 0; + if (rtp_header_extension_map_.GetId(kRtpExtensionVideoRotation, &id) != 0) { + // Not registered. + return false; + } + + size_t block_pos = 0; + if (!FindHeaderExtensionPosition(kRtpExtensionVideoRotation, rtp_packet, + rtp_packet_length, rtp_header, &block_pos)) { + LOG(LS_WARNING) << "Failed to update video rotation (CVO)."; + return false; + } + // Get length until start of header extension block. + int extension_block_pos = + rtp_header_extension_map_.GetLengthUntilBlockStartInBytes( + kRtpExtensionVideoRotation); + if (extension_block_pos < 0) { + // The feature is not enabled. + return false; + } + + // Verify first byte in block. + const uint8_t first_block_byte = (id << 4) + 0; + if (rtp_packet[block_pos] != first_block_byte) { + LOG(LS_WARNING) << "Failed to update CVO."; + return false; + } + rtp_packet[block_pos + 1] = ConvertToCVOByte(rotation); + return true; +} + void RTPSender::UpdateAbsoluteSendTime(uint8_t* rtp_packet, size_t rtp_packet_length, const RTPHeader& rtp_header, @@ -1412,15 +1533,16 @@ void RTPSender::UpdateAbsoluteSendTime(uint8_t* rtp_packet, // The feature is not enabled. return; } - size_t block_pos = 12 + rtp_header.numCSRCs + extension_block_pos; + size_t block_pos = + kRtpHeaderLength + rtp_header.numCSRCs + extension_block_pos; if (rtp_packet_length < block_pos + kAbsoluteSendTimeLength || rtp_header.headerLength < block_pos + kAbsoluteSendTimeLength) { LOG(LS_WARNING) << "Failed to update absolute send time, invalid length."; return; } // Verify that header contains extension. - if (!((rtp_packet[12 + rtp_header.numCSRCs] == 0xBE) && - (rtp_packet[12 + rtp_header.numCSRCs + 1] == 0xDE))) { + if (!((rtp_packet[kRtpHeaderLength + rtp_header.numCSRCs] == 0xBE) && + (rtp_packet[kRtpHeaderLength + rtp_header.numCSRCs + 1] == 0xDE))) { LOG(LS_WARNING) << "Failed to update absolute send time, hdr extension not found."; return; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.h b/webrtc/modules/rtp_rtcp/source/rtp_sender.h index 9342e5e5e4..67fb1feaf0 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.h @@ -64,6 +64,12 @@ class RTPSenderInterface { uint8_t *data_buffer, size_t payload_length, size_t rtp_header_length, int64_t capture_time_ms, StorageType storage, PacedSender::Priority priority) = 0; + + virtual bool UpdateVideoRotation(uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + VideoRotation rotation) const = 0; + virtual bool IsRtpHeaderExtensionRegistered(RTPExtensionType type) = 0; }; class RTPSender : public RTPSenderInterface { @@ -141,23 +147,25 @@ class RTPSender : public RTPSenderInterface { size_t payload_size, const RTPFragmentationHeader* fragmentation, VideoCodecInformation* codec_info = NULL, - const RTPVideoTypeHeader* rtp_type_hdr = NULL); + const RTPVideoHeader* rtp_hdr = NULL); // RTP header extension int32_t SetTransmissionTimeOffset(int32_t transmission_time_offset); int32_t SetAbsoluteSendTime(uint32_t absolute_send_time); + void SetVideoRotation(VideoRotation rotation); int32_t RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id); - + virtual bool IsRtpHeaderExtensionRegistered(RTPExtensionType type) override; int32_t DeregisterRtpHeaderExtension(RTPExtensionType type); size_t RtpHeaderExtensionTotalLength() const; - uint16_t BuildRTPHeaderExtension(uint8_t* data_buffer) const; + uint16_t BuildRTPHeaderExtension(uint8_t* data_buffer, bool marker_bit) const; uint8_t BuildTransmissionTimeOffsetExtension(uint8_t *data_buffer) const; uint8_t BuildAudioLevelExtension(uint8_t* data_buffer) const; uint8_t BuildAbsoluteSendTimeExtension(uint8_t* data_buffer) const; + uint8_t BuildVideoRotationExtension(uint8_t* data_buffer) const; bool UpdateAudioLevel(uint8_t* rtp_packet, size_t rtp_packet_length, @@ -165,6 +173,11 @@ class RTPSender : public RTPSenderInterface { bool is_voiced, uint8_t dBov) const; + virtual bool UpdateVideoRotation(uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + VideoRotation rotation) const override; + bool TimeToSendPacket(uint16_t sequence_number, int64_t capture_time_ms, bool retransmission); size_t TimeToSendPadding(size_t bytes); @@ -271,6 +284,8 @@ class RTPSender : public RTPSenderInterface { void SetRtxRtpState(const RtpState& rtp_state); RtpState GetRtxRtpState() const; + static uint8_t ConvertToCVOByte(VideoRotation rotation); + protected: int32_t CheckPayloadType(int8_t payload_type, RtpVideoCodecTypes* video_type); @@ -310,6 +325,14 @@ class RTPSender : public RTPSenderInterface { void UpdateDelayStatistics(int64_t capture_time_ms, int64_t now_ms); + // Find the byte position of the RTP extension as indicated by |type| in + // |rtp_packet|. Return false if such extension doesn't exist. + bool FindHeaderExtensionPosition(RTPExtensionType type, + const uint8_t* rtp_packet, + size_t rtp_packet_length, + const RTPHeader& rtp_header, + size_t* position) const; + void UpdateTransmissionTimeOffset(uint8_t* rtp_packet, size_t rtp_packet_length, const RTPHeader& rtp_header, @@ -354,6 +377,7 @@ class RTPSender : public RTPSenderInterface { RtpHeaderExtensionMap rtp_header_extension_map_; int32_t transmission_time_offset_; uint32_t absolute_send_time_; + VideoRotation rotation_; // NACK uint32_t nack_byte_count_times_[NACK_BYTECOUNT_SIZE]; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc index f57ee27323..c6e8398658 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -14,6 +14,7 @@ #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/buffer.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/modules/pacing/include/mock/mock_paced_sender.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" @@ -21,6 +22,8 @@ #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h" #include "webrtc/modules/rtp_rtcp/source/rtp_header_extension.h" #include "webrtc/modules/rtp_rtcp/source/rtp_sender.h" +#include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h" +#include "webrtc/system_wrappers/interface/stl_util.h" #include "webrtc/test/mock_transport.h" #include "webrtc/typedefs.h" @@ -40,7 +43,8 @@ const uint8_t kAudioLevelExtensionId = 9; const int kAudioPayload = 103; const uint64_t kStartTime = 123456789; const size_t kMaxPaddingSize = 224u; -} // namespace +const int kVideoRotationExtensionId = 5; +const VideoRotation kRotation = kVideoRotation_270; using testing::_; @@ -61,12 +65,21 @@ uint64_t ConvertMsToAbsSendTime(int64_t time_ms) { class LoopbackTransportTest : public webrtc::Transport { public: LoopbackTransportTest() - : packets_sent_(0), last_sent_packet_len_(0), total_bytes_sent_(0) {} - int SendPacket(int channel, const void* data, size_t len) override { + : packets_sent_(0), + last_sent_packet_len_(0), + total_bytes_sent_(0), + last_sent_packet_(NULL) {} + + ~LoopbackTransportTest() { + STLDeleteContainerPointers(sent_packets_.begin(), sent_packets_.end()); + } + int SendPacket(int channel, const void *data, size_t len) override { packets_sent_++; - memcpy(last_sent_packet_, data, len); + rtc::Buffer* buffer = new rtc::Buffer(data, len); + last_sent_packet_ = reinterpret_cast(buffer->data()); last_sent_packet_len_ = len; total_bytes_sent_ += len; + sent_packets_.push_back(buffer); return static_cast(len); } int SendRTCPPacket(int channel, const void* data, size_t len) override { @@ -75,9 +88,12 @@ class LoopbackTransportTest : public webrtc::Transport { int packets_sent_; size_t last_sent_packet_len_; size_t total_bytes_sent_; - uint8_t last_sent_packet_[kMaxPacketLength]; + uint8_t* last_sent_packet_; + std::vector sent_packets_; }; +} // namespace + class RtpSenderTest : public ::testing::Test { protected: RtpSenderTest() @@ -106,7 +122,11 @@ class RtpSenderTest : public ::testing::Test { uint8_t packet_[kMaxPacketLength]; void VerifyRTPHeaderCommon(const RTPHeader& rtp_header) { - EXPECT_EQ(kMarkerBit, rtp_header.markerBit); + VerifyRTPHeaderCommon(rtp_header, kMarkerBit); + } + + void VerifyRTPHeaderCommon(const RTPHeader& rtp_header, bool marker_bit) { + EXPECT_EQ(marker_bit, rtp_header.markerBit); EXPECT_EQ(payload_, rtp_header.payloadType); EXPECT_EQ(kSeqNum, rtp_header.sequenceNumber); EXPECT_EQ(kTimestamp, rtp_header.timestamp); @@ -134,6 +154,46 @@ class RtpSenderTest : public ::testing::Test { } }; +class RtpSenderVideoTest : public RtpSenderTest { + protected: + virtual void SetUp() override { + RtpSenderTest::SetUp(); + rtp_sender_video_.reset( + new RTPSenderVideo(&fake_clock_, rtp_sender_.get())); + } + rtc::scoped_ptr rtp_sender_video_; + + void VerifyCVOPacket(uint8_t* data, + size_t len, + bool expect_cvo, + RtpHeaderExtensionMap* map, + uint16_t seq_num, + VideoRotation rotation) { + webrtc::RtpUtility::RtpHeaderParser rtp_parser(data, len); + + webrtc::RTPHeader rtp_header; + size_t length = static_cast(rtp_sender_->BuildRTPheader( + packet_, kPayload, expect_cvo /* marker_bit */, kTimestamp, 0)); + if (expect_cvo) { + ASSERT_EQ(kRtpHeaderSize + rtp_sender_->RtpHeaderExtensionTotalLength(), + length); + } else { + ASSERT_EQ(kRtpHeaderSize, length); + } + ASSERT_TRUE(rtp_parser.Parse(rtp_header, map)); + ASSERT_FALSE(rtp_parser.RTCP()); + EXPECT_EQ(expect_cvo, rtp_header.markerBit); + EXPECT_EQ(payload_, rtp_header.payloadType); + EXPECT_EQ(seq_num, rtp_header.sequenceNumber); + EXPECT_EQ(kTimestamp, rtp_header.timestamp); + EXPECT_EQ(rtp_sender_->SSRC(), rtp_header.ssrc); + EXPECT_EQ(0, rtp_header.numCSRCs); + EXPECT_EQ(0U, rtp_header.paddingLength); + EXPECT_EQ(RTPSender::ConvertToCVOByte(rotation), + rtp_header.extension.videoRotation); + } +}; + TEST_F(RtpSenderTest, RegisterRtpTransmissionTimeOffsetHeaderExtension) { EXPECT_EQ(0u, rtp_sender_->RtpHeaderExtensionTotalLength()); EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( @@ -182,16 +242,40 @@ TEST_F(RtpSenderTest, RegisterRtpHeaderExtensions) { EXPECT_EQ(kRtpOneByteHeaderLength + kTransmissionTimeOffsetLength + kAbsoluteSendTimeLength + kAudioLevelLength, rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + EXPECT_EQ(kRtpOneByteHeaderLength + kTransmissionTimeOffsetLength + + kAbsoluteSendTimeLength + kAudioLevelLength + + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); + + // Deregister starts. EXPECT_EQ(0, rtp_sender_->DeregisterRtpHeaderExtension( kRtpExtensionTransmissionTimeOffset)); EXPECT_EQ(kRtpOneByteHeaderLength + kAbsoluteSendTimeLength + - kAudioLevelLength, rtp_sender_->RtpHeaderExtensionTotalLength()); + kAudioLevelLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); EXPECT_EQ(0, rtp_sender_->DeregisterRtpHeaderExtension( kRtpExtensionAbsoluteSendTime)); - EXPECT_EQ(kRtpOneByteHeaderLength + kAudioLevelLength, - rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ(kRtpOneByteHeaderLength + kAudioLevelLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); EXPECT_EQ(0, rtp_sender_->DeregisterRtpHeaderExtension( kRtpExtensionAudioLevel)); + EXPECT_EQ(kRtpOneByteHeaderLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ( + 0, rtp_sender_->DeregisterRtpHeaderExtension(kRtpExtensionVideoRotation)); + EXPECT_EQ(0u, rtp_sender_->RtpHeaderExtensionTotalLength()); +} + +TEST_F(RtpSenderTest, RegisterRtpVideoRotationHeaderExtension) { + EXPECT_EQ(0u, rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + EXPECT_EQ(kRtpOneByteHeaderLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); + EXPECT_EQ( + 0, rtp_sender_->DeregisterRtpHeaderExtension(kRtpExtensionVideoRotation)); EXPECT_EQ(0u, rtp_sender_->RtpHeaderExtensionTotalLength()); } @@ -216,6 +300,7 @@ TEST_F(RtpSenderTest, BuildRTPPacket) { EXPECT_EQ(0, rtp_header.extension.transmissionTimeOffset); EXPECT_EQ(0u, rtp_header.extension.absoluteSendTime); EXPECT_EQ(0u, rtp_header.extension.audioLevel); + EXPECT_EQ(0u, rtp_header.extension.videoRotation); } TEST_F(RtpSenderTest, BuildRTPPacketWithTransmissionOffsetExtension) { @@ -319,6 +404,57 @@ TEST_F(RtpSenderTest, BuildRTPPacketWithAbsoluteSendTimeExtension) { EXPECT_EQ(0u, rtp_header2.extension.absoluteSendTime); } +// Test CVO header extension is only set when marker bit is true. +TEST_F(RtpSenderTest, BuildRTPPacketWithVideoRotation_MarkerBit) { + rtp_sender_->SetVideoRotation(kRotation); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + + RtpHeaderExtensionMap map; + map.Register(kRtpExtensionVideoRotation, kVideoRotationExtensionId); + + size_t length = static_cast( + rtp_sender_->BuildRTPheader(packet_, kPayload, true, kTimestamp, 0)); + ASSERT_EQ(kRtpHeaderSize + rtp_sender_->RtpHeaderExtensionTotalLength(), + length); + + // Verify + webrtc::RtpUtility::RtpHeaderParser rtp_parser(packet_, length); + webrtc::RTPHeader rtp_header; + + ASSERT_TRUE(rtp_parser.Parse(rtp_header, &map)); + ASSERT_FALSE(rtp_parser.RTCP()); + VerifyRTPHeaderCommon(rtp_header); + EXPECT_EQ(length, rtp_header.headerLength); + EXPECT_TRUE(rtp_header.extension.hasVideoRotation); + EXPECT_EQ(RTPSender::ConvertToCVOByte(kRotation), + rtp_header.extension.videoRotation); +} + +// Test CVO header extension is not set when marker bit is false. +TEST_F(RtpSenderTest, BuildRTPPacketWithVideoRotation_NoMarkerBit) { + rtp_sender_->SetVideoRotation(kRotation); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + + RtpHeaderExtensionMap map; + map.Register(kRtpExtensionVideoRotation, kVideoRotationExtensionId); + + size_t length = static_cast( + rtp_sender_->BuildRTPheader(packet_, kPayload, false, kTimestamp, 0)); + ASSERT_EQ(kRtpHeaderSize, length); + + // Verify + webrtc::RtpUtility::RtpHeaderParser rtp_parser(packet_, length); + webrtc::RTPHeader rtp_header; + + ASSERT_TRUE(rtp_parser.Parse(rtp_header, &map)); + ASSERT_FALSE(rtp_parser.RTCP()); + VerifyRTPHeaderCommon(rtp_header, false); + EXPECT_EQ(length, rtp_header.headerLength); + EXPECT_FALSE(rtp_header.extension.hasVideoRotation); +} + TEST_F(RtpSenderTest, BuildRTPPacketWithAudioLevelExtension) { EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( kRtpExtensionAudioLevel, kAudioLevelExtensionId)); @@ -1168,4 +1304,34 @@ TEST_F(RtpSenderTest, BytesReportedCorrectly) { rtp_stats.transmitted.TotalBytes() + rtx_stats.transmitted.TotalBytes()); } + +// Verify that only the last packet of a frame has CVO byte set. +TEST_F(RtpSenderVideoTest, SendVideoWithCVO) { + RTPVideoHeader hdr = {0}; + hdr.rotation = kVideoRotation_90; + + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoRotation, kVideoRotationExtensionId)); + EXPECT_EQ(kRtpOneByteHeaderLength + kVideoRotationLength, + rtp_sender_->RtpHeaderExtensionTotalLength()); + + rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, kPayload, + kTimestamp, 0, packet_, sizeof(packet_), NULL, + NULL, &hdr); + + RtpHeaderExtensionMap map; + map.Register(kRtpExtensionVideoRotation, kVideoRotationExtensionId); + + // Verify that this packet doesn't have CVO byte. + VerifyCVOPacket( + reinterpret_cast(transport_.sent_packets_[0]->data()), + transport_.sent_packets_[0]->length(), false, &map, kSeqNum, + kVideoRotation_0); + + // Verify that this packet doesn't have CVO byte. + VerifyCVOPacket( + reinterpret_cast(transport_.sent_packets_[1]->data()), + transport_.sent_packets_[1]->length(), true, &map, kSeqNum + 1, + hdr.rotation); +} } // namespace webrtc diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc index bfcb5e29cf..c802bd5742 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -259,7 +259,7 @@ int32_t RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType, const size_t payloadSize, const RTPFragmentationHeader* fragmentation, VideoCodecInformation* codecInfo, - const RTPVideoTypeHeader* rtpTypeHdr) { + const RTPVideoHeader* rtpHdr) { if (payloadSize == 0) { return -1; } @@ -274,15 +274,8 @@ int32_t RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType, // Will be extracted in SendVP8 for VP8 codec; other codecs use 0 _numberFirstPartition = 0; - return Send(videoType, - frameType, - payloadType, - captureTimeStamp, - capture_time_ms, - payloadData, - payloadSize, - fragmentation, - rtpTypeHdr) + return Send(videoType, frameType, payloadType, captureTimeStamp, + capture_time_ms, payloadData, payloadSize, fragmentation, rtpHdr) ? 0 : -1; } @@ -307,14 +300,14 @@ bool RTPSenderVideo::Send(const RtpVideoCodecTypes videoType, const uint8_t* payloadData, const size_t payloadSize, const RTPFragmentationHeader* fragmentation, - const RTPVideoTypeHeader* rtpTypeHdr) { + const RTPVideoHeader* rtpHdr) { uint16_t rtp_header_length = _rtpSender.RTPHeaderLength(); size_t payload_bytes_to_send = payloadSize; const uint8_t* data = payloadData; size_t max_payload_length = _rtpSender.MaxDataPayloadLength(); rtc::scoped_ptr packetizer(RtpPacketizer::Create( - videoType, max_payload_length, rtpTypeHdr, frameType)); + videoType, max_payload_length, &(rtpHdr->codecHeader), frameType)); // TODO(changbin): we currently don't support to configure the codec to // output multiple partitions for VP8. Should remove below check after the @@ -337,6 +330,31 @@ bool RTPSenderVideo::Send(const RtpVideoCodecTypes videoType, // Set marker bit true if this is the last packet in frame. _rtpSender.BuildRTPheader( dataBuffer, payloadType, last, captureTimeStamp, capture_time_ms); + + // According to + // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ + // ts_126114v120700p.pdf Section 7.4.5: + // The MTSI client shall add the payload bytes as defined in this clause + // onto the last RTP packet in each group of packets which make up a key + // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265 + // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP + // packet in each group of packets which make up another type of frame + // (e.g. a P-Frame) only if the current value is different from the previous + // value sent. + // Here we are adding it to the last packet of every frame at this point. + if (!rtpHdr) { + assert(!_rtpSender.IsRtpHeaderExtensionRegistered( + kRtpExtensionVideoRotation)); + } else if (last) { + // Checking whether CVO header extension is registered will require taking + // a lock. It'll be a no-op if it's not registered. + size_t packetSize = payloadSize + rtp_header_length; + RtpUtility::RtpHeaderParser rtp_parser(dataBuffer, packetSize); + RTPHeader rtp_header; + rtp_parser.Parse(rtp_header); + _rtpSender.UpdateVideoRotation(dataBuffer, packetSize, rtp_header, + rtpHdr->rotation); + } if (SendVideoPacket(dataBuffer, payload_bytes_in_packet, rtp_header_length, diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h index 7338875c12..92c312f5e9 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h @@ -51,7 +51,7 @@ class RTPSenderVideo { const size_t payloadSize, const RTPFragmentationHeader* fragmentation, VideoCodecInformation* codecInfo, - const RTPVideoTypeHeader* rtpTypeHdr); + const RTPVideoHeader* rtpHdr); int32_t SendRTPIntraRequest(); @@ -101,7 +101,7 @@ class RTPSenderVideo { const uint8_t* payloadData, const size_t payloadSize, const RTPFragmentationHeader* fragmentation, - const RTPVideoTypeHeader* rtpTypeHdr); + const RTPVideoHeader* rtpHdr); private: RTPSenderInterface& _rtpSender; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_utility.cc b/webrtc/modules/rtp_rtcp/source/rtp_utility.cc index b12839e16e..d26ceda3cf 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_utility.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_utility.cc @@ -373,6 +373,10 @@ bool RtpHeaderParser::Parse(RTPHeader& header, header.extension.hasAudioLevel = false; header.extension.audioLevel = 0; + // May not be present in packet. + header.extension.hasVideoRotation = false; + header.extension.videoRotation = 0; + if (X) { /* RTP header extension, RFC 3550. 0 1 2 3 @@ -511,6 +515,21 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( header.extension.hasAbsoluteSendTime = true; break; } + case kRtpExtensionVideoRotation: { + if (len != 0) { + LOG(LS_WARNING) + << "Incorrect coordination of video coordination len: " << len; + return; + } + // 0 1 2 3 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | ID | len=0 |V|0 0 0 0 C F R R| 0x00 | 0x00 | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + header.extension.hasVideoRotation = true; + header.extension.videoRotation = ptr[0]; + break; + } default: { LOG(LS_WARNING) << "Extension type not implemented: " << type; return;