From e0c8b230e7ede712a29802c51a8e0590224bfed1 Mon Sep 17 00:00:00 2001 From: Johnny Lee Date: Tue, 11 Sep 2018 16:50:49 -0400 Subject: [PATCH] Frame marking RTP header extension (PART 1: implement extension) Bug: webrtc:7765 Change-Id: I23896d121afd6be4bce5ff4deaf736149efebcdb Reviewed-on: https://webrtc-review.googlesource.com/85200 Commit-Queue: Danil Chapovalov Reviewed-by: Stefan Holmer Reviewed-by: Sergey Silkin Reviewed-by: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#24695} --- AUTHORS | 1 + api/rtp_headers.cc | 4 +- api/rtp_headers.h | 4 + api/rtpparameters.cc | 7 +- api/rtpparameters.h | 4 + api/video/BUILD.gn | 1 + api/video/video_frame_marking.h | 29 +++++++ logging/rtc_event_log/rtc_event_log2text.cc | 2 + .../rtc_event_log/rtc_event_log_parser_new.cc | 2 + media/engine/webrtcvideoengine.cc | 3 + modules/rtp_rtcp/include/rtp_rtcp_defines.h | 1 + .../source/rtp_header_extension_map.cc | 1 + .../rtp_rtcp/source/rtp_header_extensions.cc | 80 +++++++++++++++++++ .../rtp_rtcp/source/rtp_header_extensions.h | 16 ++++ .../rtp_rtcp/source/rtp_packet_received.cc | 2 + modules/rtp_rtcp/source/rtp_receiver_video.cc | 6 ++ modules/rtp_rtcp/source/rtp_rtcp_impl.cc | 2 + modules/rtp_rtcp/source/rtp_utility.cc | 13 +++ modules/rtp_rtcp/source/rtp_video_header.h | 2 + modules/video_coding/frame_object.cc | 8 ++ modules/video_coding/frame_object.h | 1 + test/fuzzers/rtp_packet_fuzzer.cc | 4 + 22 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 api/video/video_frame_marking.h diff --git a/AUTHORS b/AUTHORS index 05b6bbeebb..f03daca644 100644 --- a/AUTHORS +++ b/AUTHORS @@ -90,6 +90,7 @@ Temasys Communications <*@temasys.io> The Chromium Authors <*@chromium.org> The WebRTC Authors <*@webrtc.org> Videxio AS <*@videxio.com> +Vidyo, Inc. <*@vidyo.com> Vonage Holdings Corp. <*@vonage.com> Wire Swiss GmbH <*@wire.com> Miguel Paris diff --git a/api/rtp_headers.cc b/api/rtp_headers.cc index a0b1a15cbb..da7f1ea9fd 100644 --- a/api/rtp_headers.cc +++ b/api/rtp_headers.cc @@ -34,7 +34,9 @@ RTPHeaderExtension::RTPHeaderExtension() videoRotation(kVideoRotation_0), hasVideoContentType(false), videoContentType(VideoContentType::UNSPECIFIED), - has_video_timing(false) {} + has_video_timing(false), + has_frame_marking(false), + frame_marking({false, false, false, false, false, 0xFF, 0, 0}) {} RTPHeaderExtension::RTPHeaderExtension(const RTPHeaderExtension& other) = default; diff --git a/api/rtp_headers.h b/api/rtp_headers.h index c762534ab3..41f39888b8 100644 --- a/api/rtp_headers.h +++ b/api/rtp_headers.h @@ -19,6 +19,7 @@ #include "absl/types/optional.h" #include "api/array_view.h" #include "api/video/video_content_type.h" +#include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" @@ -116,6 +117,9 @@ struct RTPHeaderExtension { bool has_video_timing; VideoSendTiming video_timing; + bool has_frame_marking; + FrameMarking frame_marking; + PlayoutDelay playout_delay = {-1, -1}; // For identification of a stream when ssrc is not signaled. See diff --git a/api/rtpparameters.cc b/api/rtpparameters.cc index 62ca3fc025..9beae82d7a 100644 --- a/api/rtpparameters.cc +++ b/api/rtpparameters.cc @@ -129,6 +129,10 @@ const int RtpExtension::kVideoTimingDefaultId = 8; const char RtpExtension::kMidUri[] = "urn:ietf:params:rtp-hdrext:sdes:mid"; const int RtpExtension::kMidDefaultId = 9; +const char RtpExtension::kFrameMarkingUri[] = + "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07"; +const int RtpExtension::kFrameMarkingDefaultId = 10; + const char RtpExtension::kEncryptHeaderExtensionsUri[] = "urn:ietf:params:rtp-hdrext:encrypt"; @@ -149,7 +153,8 @@ bool RtpExtension::IsSupportedForVideo(const std::string& uri) { uri == webrtc::RtpExtension::kPlayoutDelayUri || uri == webrtc::RtpExtension::kVideoContentTypeUri || uri == webrtc::RtpExtension::kVideoTimingUri || - uri == webrtc::RtpExtension::kMidUri; + uri == webrtc::RtpExtension::kMidUri || + uri == webrtc::RtpExtension::kFrameMarkingUri; } bool RtpExtension::IsEncryptionSupported(const std::string& uri) { diff --git a/api/rtpparameters.h b/api/rtpparameters.h index 9a29c08d1c..aa9b09e867 100644 --- a/api/rtpparameters.h +++ b/api/rtpparameters.h @@ -282,6 +282,10 @@ struct RtpExtension { static const char kVideoTimingUri[]; static const int kVideoTimingDefaultId; + // Header extension for video frame marking. + static const char kFrameMarkingUri[]; + static const int kFrameMarkingDefaultId; + // Header extension for transport sequence number, see url for details: // http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions static const char kTransportSequenceNumberUri[]; diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index dfc4f27540..d04126021d 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -19,6 +19,7 @@ rtc_source_set("video_frame") { "video_frame.h", "video_frame_buffer.cc", "video_frame_buffer.h", + "video_frame_marking.h", "video_rotation.h", "video_sink_interface.h", "video_source_interface.cc", diff --git a/api/video/video_frame_marking.h b/api/video/video_frame_marking.h new file mode 100644 index 0000000000..2a34852f1d --- /dev/null +++ b/api/video/video_frame_marking.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_FRAME_MARKING_H_ +#define API_VIDEO_VIDEO_FRAME_MARKING_H_ + +namespace webrtc { + +struct FrameMarking { + bool start_of_frame; + bool end_of_frame; + bool independent_frame; + bool discardable_frame; + bool base_layer_sync; + uint8_t temporal_id; + uint8_t layer_id; + uint8_t tl0_pic_idx; +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_FRAME_MARKING_H_ diff --git a/logging/rtc_event_log/rtc_event_log2text.cc b/logging/rtc_event_log/rtc_event_log2text.cc index f08e6ece5e..67cb6ecdf4 100644 --- a/logging/rtc_event_log/rtc_event_log2text.cc +++ b/logging/rtc_event_log/rtc_event_log2text.cc @@ -157,6 +157,8 @@ webrtc::RtpHeaderExtensionMap GetDefaultHeaderExtensionMap() { webrtc::RtpExtension::kVideoContentTypeDefaultId); default_map.Register( webrtc::RtpExtension::kVideoTimingDefaultId); + default_map.Register( + webrtc::RtpExtension::kFrameMarkingDefaultId); default_map.Register( webrtc::RtpExtension::kTransportSequenceNumberDefaultId); default_map.Register( diff --git a/logging/rtc_event_log/rtc_event_log_parser_new.cc b/logging/rtc_event_log/rtc_event_log_parser_new.cc index f905a04f4e..bb5b1b2cdf 100644 --- a/logging/rtc_event_log/rtc_event_log_parser_new.cc +++ b/logging/rtc_event_log/rtc_event_log_parser_new.cc @@ -228,6 +228,8 @@ webrtc::RtpHeaderExtensionMap GetDefaultHeaderExtensionMap() { webrtc::RtpExtension::kVideoContentTypeDefaultId); default_map.Register( webrtc::RtpExtension::kVideoTimingDefaultId); + default_map.Register( + webrtc::RtpExtension::kFrameMarkingDefaultId); default_map.Register( webrtc::RtpExtension::kTransportSequenceNumberDefaultId); default_map.Register( diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc index 69173842cd..18919bc30a 100644 --- a/media/engine/webrtcvideoengine.cc +++ b/media/engine/webrtcvideoengine.cc @@ -509,6 +509,9 @@ RtpCapabilities WebRtcVideoEngine::GetCapabilities() const { capabilities.header_extensions.push_back( webrtc::RtpExtension(webrtc::RtpExtension::kVideoTimingUri, webrtc::RtpExtension::kVideoTimingDefaultId)); + capabilities.header_extensions.push_back( + webrtc::RtpExtension(webrtc::RtpExtension::kFrameMarkingUri, + webrtc::RtpExtension::kFrameMarkingDefaultId)); // TODO(bugs.webrtc.org/4050): Add MID header extension as capability once MID // demuxing is completed. // capabilities.header_extensions.push_back(webrtc::RtpExtension( diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 34c7428790..51e6337583 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -103,6 +103,7 @@ enum RTPExtensionType { kRtpExtensionPlayoutDelay, kRtpExtensionVideoContentType, kRtpExtensionVideoTiming, + kRtpExtensionFrameMarking, kRtpExtensionRtpStreamId, kRtpExtensionRepairedRtpStreamId, kRtpExtensionMid, diff --git a/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/modules/rtp_rtcp/source/rtp_header_extension_map.cc index 9fa66c0819..23ccd0ef88 100644 --- a/modules/rtp_rtcp/source/rtp_header_extension_map.cc +++ b/modules/rtp_rtcp/source/rtp_header_extension_map.cc @@ -38,6 +38,7 @@ constexpr ExtensionInfo kExtensions[] = { CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), + CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.cc b/modules/rtp_rtcp/source/rtp_header_extensions.cc index 2dba4d7cfa..e9e4d508de 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.cc +++ b/modules/rtp_rtcp/source/rtp_header_extensions.cc @@ -350,6 +350,86 @@ bool VideoTimingExtension::Write(rtc::ArrayView data, return true; } +// Frame Marking. +// +// Meta-information about an RTP stream outside the encrypted media payload, +// useful for an RTP switch to do codec-agnostic selective forwarding +// without decrypting the payload. +// +// For non-scalable streams: +// 0 1 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | L = 0 |S|E|I|D|0 0 0 0| +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// For scalable streams: +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | L = 2 |S|E|I|D|B| TID | LID | TL0PICIDX | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +constexpr RTPExtensionType FrameMarkingExtension::kId; +constexpr const char FrameMarkingExtension::kUri[]; + +bool FrameMarkingExtension::IsScalable(uint8_t temporal_id, uint8_t layer_id) { + return temporal_id != kNoTemporalIdx || layer_id != kNoSpatialIdx; +} + +bool FrameMarkingExtension::Parse(rtc::ArrayView data, + FrameMarking* frame_marking) { + RTC_DCHECK(frame_marking); + + if (data.size() != 1 && data.size() != 3) + return false; + + frame_marking->start_of_frame = (data[0] & 0x80) != 0; + frame_marking->end_of_frame = (data[0] & 0x40) != 0; + frame_marking->independent_frame = (data[0] & 0x20) != 0; + frame_marking->discardable_frame = (data[0] & 0x10) != 0; + + if (data.size() == 3) { + frame_marking->base_layer_sync = (data[0] & 0x08) != 0; + frame_marking->temporal_id = data[0] & 0x7; + frame_marking->layer_id = data[1]; + frame_marking->tl0_pic_idx = data[2]; + } else { + // non-scalable + frame_marking->base_layer_sync = false; + frame_marking->temporal_id = kNoTemporalIdx; + frame_marking->layer_id = kNoSpatialIdx; + frame_marking->tl0_pic_idx = 0; + } + return true; +} + +size_t FrameMarkingExtension::ValueSize(const FrameMarking& frame_marking) { + if (IsScalable(frame_marking.temporal_id, frame_marking.layer_id)) + return 3; + else + return 1; +} + +bool FrameMarkingExtension::Write(rtc::ArrayView data, + const FrameMarking& frame_marking) { + RTC_DCHECK_GE(data.size(), 1); + RTC_CHECK_LE(frame_marking.temporal_id, 0x07); + data[0] = frame_marking.start_of_frame ? 0x80 : 0x00; + data[0] |= frame_marking.end_of_frame ? 0x40 : 0x00; + data[0] |= frame_marking.independent_frame ? 0x20 : 0x00; + data[0] |= frame_marking.discardable_frame ? 0x10 : 0x00; + + if (IsScalable(frame_marking.temporal_id, frame_marking.layer_id)) { + RTC_DCHECK_EQ(data.size(), 3); + data[0] |= frame_marking.base_layer_sync ? 0x08 : 0x00; + data[0] |= frame_marking.temporal_id & 0x07; + data[1] = frame_marking.layer_id; + data[2] = frame_marking.tl0_pic_idx; + } + return true; +} + bool BaseRtpStringExtension::Parse(rtc::ArrayView data, StringRtpHeaderExtension* str) { if (data.empty() || data[0] == 0) // Valid string extension can't be empty. diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h index 4e1afc1d97..84e9831479 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -153,6 +153,22 @@ class VideoTimingExtension { uint8_t idx); }; +class FrameMarkingExtension { + public: + static constexpr RTPExtensionType kId = kRtpExtensionFrameMarking; + static constexpr const char kUri[] = + "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07"; + + static bool Parse(rtc::ArrayView data, + FrameMarking* frame_marking); + static size_t ValueSize(const FrameMarking& frame_marking); + static bool Write(rtc::ArrayView data, + const FrameMarking& frame_marking); + + private: + static bool IsScalable(uint8_t temporal_id, uint8_t layer_id); +}; + // Base extension class for RTP header extensions which are strings. // Subclasses must defined kId and kUri static constexpr members. class BaseRtpStringExtension { diff --git a/modules/rtp_rtcp/source/rtp_packet_received.cc b/modules/rtp_rtcp/source/rtp_packet_received.cc index 6acc9b55d4..c8deb99d90 100644 --- a/modules/rtp_rtcp/source/rtp_packet_received.cc +++ b/modules/rtp_rtcp/source/rtp_packet_received.cc @@ -61,6 +61,8 @@ void RtpPacketReceived::GetHeader(RTPHeader* header) const { &header->extension.videoContentType); header->extension.has_video_timing = GetExtension(&header->extension.video_timing); + header->extension.has_frame_marking = + GetExtension(&header->extension.frame_marking); GetExtension(&header->extension.stream_id); GetExtension(&header->extension.repaired_stream_id); GetExtension(&header->extension.mid); diff --git a/modules/rtp_rtcp/source/rtp_receiver_video.cc b/modules/rtp_rtcp/source/rtp_receiver_video.cc index 1101becfb1..3c9b9e5581 100644 --- a/modules/rtp_rtcp/source/rtp_receiver_video.cc +++ b/modules/rtp_rtcp/source/rtp_receiver_video.cc @@ -74,6 +74,7 @@ int32_t RTPReceiverVideo::ParseRtpPacket(WebRtcRTPHeader* rtp_header, rtp_header->video_header().rotation = kVideoRotation_0; rtp_header->video_header().content_type = VideoContentType::UNSPECIFIED; rtp_header->video_header().video_timing.flags = VideoSendTiming::kInvalid; + rtp_header->video_header().frame_marking.temporal_id = kNoTemporalIdx; // Retrieve the video rotation information. if (rtp_header->header.extension.hasVideoRotation) { @@ -94,6 +95,11 @@ int32_t RTPReceiverVideo::ParseRtpPacket(WebRtcRTPHeader* rtp_header, rtp_header->video_header().playout_delay = rtp_header->header.extension.playout_delay; + if (rtp_header->header.extension.has_frame_marking) { + rtp_header->video_header().frame_marking = + rtp_header->header.extension.frame_marking; + } + return data_callback_->OnReceivedPayloadData(parsed_payload.payload, parsed_payload.payload_length, rtp_header) == 0 diff --git a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index fe8dbf385c..20ae519875 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -50,6 +50,8 @@ RTPExtensionType StringToRtpExtensionType(const std::string& extension) { return kRtpExtensionVideoContentType; if (extension == RtpExtension::kVideoTimingUri) return kRtpExtensionVideoTiming; + if (extension == RtpExtension::kFrameMarkingUri) + return kRtpExtensionFrameMarking; if (extension == RtpExtension::kMidUri) return kRtpExtensionMid; RTC_NOTREACHED() << "Looking up unsupported RTP extension."; diff --git a/modules/rtp_rtcp/source/rtp_utility.cc b/modules/rtp_rtcp/source/rtp_utility.cc index d150ce2eb3..5f5b3dc0f5 100644 --- a/modules/rtp_rtcp/source/rtp_utility.cc +++ b/modules/rtp_rtcp/source/rtp_utility.cc @@ -236,6 +236,10 @@ bool RtpHeaderParser::Parse( header->extension.has_video_timing = false; header->extension.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false}; + header->extension.has_frame_marking = false; + header->extension.frame_marking = {false, false, false, false, false, + kNoTemporalIdx, 0, 0}; + if (X) { /* RTP header extension, RFC 3550. 0 1 2 3 @@ -454,6 +458,15 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( &header->extension.video_timing); break; } + case kRtpExtensionFrameMarking: { + if (!FrameMarkingExtension::Parse(rtc::MakeArrayView(ptr, len + 1), + &header->extension.frame_marking)) { + RTC_LOG(LS_WARNING) << "Incorrect frame marking len: " << len; + return; + } + header->extension.has_frame_marking = true; + break; + } case kRtpExtensionRtpStreamId: { header->extension.stream_id.Set(rtc::MakeArrayView(ptr, len + 1)); break; diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h index 6c7a150b43..68b48847a2 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.h +++ b/modules/rtp_rtcp/source/rtp_video_header.h @@ -13,6 +13,7 @@ #include "absl/container/inlined_vector.h" #include "absl/types/variant.h" #include "api/video/video_content_type.h" +#include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "common_types.h" // NOLINT(build/include) @@ -56,6 +57,7 @@ struct RTPVideoHeader { PlayoutDelay playout_delay; VideoSendTiming video_timing; + FrameMarking frame_marking; RTPVideoTypeHeader video_type_header; }; diff --git a/modules/video_coding/frame_object.cc b/modules/video_coding/frame_object.cc index 0965d7a499..4f7724de3f 100644 --- a/modules/video_coding/frame_object.cc +++ b/modules/video_coding/frame_object.cc @@ -159,5 +159,13 @@ absl::optional RtpFrameObject::GetRtpVideoHeader() const { return packet->video_header; } +absl::optional RtpFrameObject::GetFrameMarking() const { + rtc::CritScope lock(&packet_buffer_->crit_); + VCMPacket* packet = packet_buffer_->GetPacket(first_seq_num_); + if (!packet) + return absl::nullopt; + return packet->video_header.frame_marking; +} + } // namespace video_coding } // namespace webrtc diff --git a/modules/video_coding/frame_object.h b/modules/video_coding/frame_object.h index 08c640432f..5fb5193080 100644 --- a/modules/video_coding/frame_object.h +++ b/modules/video_coding/frame_object.h @@ -41,6 +41,7 @@ class RtpFrameObject : public EncodedFrame { int64_t RenderTime() const override; bool delayed_by_retransmission() const override; absl::optional GetRtpVideoHeader() const; + absl::optional GetFrameMarking() const; private: rtc::scoped_refptr packet_buffer_; diff --git a/test/fuzzers/rtp_packet_fuzzer.cc b/test/fuzzers/rtp_packet_fuzzer.cc index 0e35c8ee41..ef9bf2608e 100644 --- a/test/fuzzers/rtp_packet_fuzzer.cc +++ b/test/fuzzers/rtp_packet_fuzzer.cc @@ -95,6 +95,10 @@ void FuzzOneInput(const uint8_t* data, size_t size) { VideoSendTiming timing; packet.GetExtension(&timing); break; + case kRtpExtensionFrameMarking: + FrameMarking frame_marking; + packet.GetExtension(&frame_marking); + break; case kRtpExtensionRtpStreamId: { std::string rsid; packet.GetExtension(&rsid);