From 9465978a3b627655b2c99389dc35bcd343866ee1 Mon Sep 17 00:00:00 2001 From: philipel Date: Mon, 15 Jun 2020 12:26:39 +0200 Subject: [PATCH] Remove framemarking RTP extension. BUG=webrtc:11637 Change-Id: I47f8e22473429c9762956444e27cfbafb201b208 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/176442 Commit-Queue: Philip Eliasson Reviewed-by: Tommi Reviewed-by: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#31522} --- api/rtp_headers.cc | 4 +- api/rtp_headers.h | 6 +- api/rtp_parameters.cc | 2 - api/rtp_parameters.h | 4 - api/video/BUILD.gn | 1 - api/video/video_frame_marking.h | 29 -- call/rtp_payload_params.cc | 17 - call/rtp_payload_params_unittest.cc | 48 --- media/engine/webrtc_video_engine.cc | 1 - media/engine/webrtc_video_engine_unittest.cc | 4 - modules/rtp_rtcp/include/rtp_rtcp_defines.h | 1 - .../source/rtp_header_extension_map.cc | 1 - .../rtp_rtcp/source/rtp_header_extensions.cc | 80 ---- .../rtp_rtcp/source/rtp_header_extensions.h | 18 - modules/rtp_rtcp/source/rtp_packet.cc | 1 - .../rtp_rtcp/source/rtp_packet_received.cc | 2 - modules/rtp_rtcp/source/rtp_sender.cc | 1 - modules/rtp_rtcp/source/rtp_sender_video.cc | 15 +- .../source/rtp_sender_video_unittest.cc | 49 --- modules/rtp_rtcp/source/rtp_utility.cc | 14 - modules/rtp_rtcp/source/rtp_video_header.h | 2 - modules/video_coding/encoded_frame.cc | 14 - modules/video_coding/frame_object.cc | 4 - modules/video_coding/frame_object.h | 1 - modules/video_coding/packet_buffer.cc | 13 +- .../rtp_frame_reference_finder.cc | 126 ------ .../rtp_frame_reference_finder_unittest.cc | 381 ++---------------- modules/video_coding/session_info.cc | 6 - .../rtp_frame_reference_finder_fuzzer.cc | 1 - test/fuzzers/rtp_packet_fuzzer.cc | 4 - video/rtp_video_stream_receiver.cc | 2 - video/rtp_video_stream_receiver2.cc | 2 - 32 files changed, 49 insertions(+), 805 deletions(-) delete mode 100644 api/video/video_frame_marking.h diff --git a/api/rtp_headers.cc b/api/rtp_headers.cc index bf973b6fe5..e0ad9eb26e 100644 --- a/api/rtp_headers.cc +++ b/api/rtp_headers.cc @@ -26,9 +26,7 @@ RTPHeaderExtension::RTPHeaderExtension() videoRotation(kVideoRotation_0), hasVideoContentType(false), videoContentType(VideoContentType::UNSPECIFIED), - has_video_timing(false), - has_frame_marking(false), - frame_marking({false, false, false, false, false, 0xFF, 0, 0}) {} + has_video_timing(false) {} RTPHeaderExtension::RTPHeaderExtension(const RTPHeaderExtension& other) = default; diff --git a/api/rtp_headers.h b/api/rtp_headers.h index 163347f675..454149ca6e 100644 --- a/api/rtp_headers.h +++ b/api/rtp_headers.h @@ -21,10 +21,9 @@ #include "api/units/timestamp.h" #include "api/video/color_space.h" #include "api/video/video_content_type.h" -#include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" -#include "common_types.h" // NOLINT(build/include) +#include "common_types.h" // NOLINT (build/include) namespace webrtc { @@ -143,9 +142,6 @@ struct RTPHeaderExtension { bool has_video_timing; VideoSendTiming video_timing; - bool has_frame_marking; - FrameMarking frame_marking; - PlayoutDelay playout_delay = {-1, -1}; // For identification of a stream when ssrc is not signaled. See diff --git a/api/rtp_parameters.cc b/api/rtp_parameters.cc index c44e4c4b4a..28acb68be6 100644 --- a/api/rtp_parameters.cc +++ b/api/rtp_parameters.cc @@ -119,7 +119,6 @@ constexpr char RtpExtension::kAbsoluteCaptureTimeUri[]; constexpr char RtpExtension::kVideoRotationUri[]; constexpr char RtpExtension::kVideoContentTypeUri[]; constexpr char RtpExtension::kVideoTimingUri[]; -constexpr char RtpExtension::kFrameMarkingUri[]; constexpr char RtpExtension::kGenericFrameDescriptorUri00[]; constexpr char RtpExtension::kDependencyDescriptorUri[]; constexpr char RtpExtension::kTransportSequenceNumberUri[]; @@ -158,7 +157,6 @@ bool RtpExtension::IsSupportedForVideo(absl::string_view uri) { uri == webrtc::RtpExtension::kVideoContentTypeUri || uri == webrtc::RtpExtension::kVideoTimingUri || uri == webrtc::RtpExtension::kMidUri || - uri == webrtc::RtpExtension::kFrameMarkingUri || uri == webrtc::RtpExtension::kGenericFrameDescriptorUri00 || uri == webrtc::RtpExtension::kDependencyDescriptorUri || uri == webrtc::RtpExtension::kColorSpaceUri || diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h index f831e51f68..addd406385 100644 --- a/api/rtp_parameters.h +++ b/api/rtp_parameters.h @@ -310,10 +310,6 @@ struct RTC_EXPORT RtpExtension { static constexpr char kVideoTimingUri[] = "http://www.webrtc.org/experiments/rtp-hdrext/video-timing"; - // Header extension for video frame marking. - static constexpr char kFrameMarkingUri[] = - "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07"; - // Experimental codec agnostic frame descriptor. static constexpr char kGenericFrameDescriptorUri00[] = "http://www.webrtc.org/experiments/rtp-hdrext/" diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 1f145b7571..ee7e51d4c4 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -21,7 +21,6 @@ rtc_library("video_rtp_headers") { "hdr_metadata.h", "video_content_type.cc", "video_content_type.h", - "video_frame_marking.h", "video_rotation.h", "video_timing.cc", "video_timing.h", diff --git a/api/video/video_frame_marking.h b/api/video/video_frame_marking.h deleted file mode 100644 index 2a34852f1d..0000000000 --- a/api/video/video_frame_marking.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef API_VIDEO_VIDEO_FRAME_MARKING_H_ -#define API_VIDEO_VIDEO_FRAME_MARKING_H_ - -namespace webrtc { - -struct FrameMarking { - bool start_of_frame; - bool end_of_frame; - bool independent_frame; - bool discardable_frame; - bool base_layer_sync; - uint8_t temporal_id; - uint8_t layer_id; - uint8_t tl0_pic_idx; -}; - -} // namespace webrtc - -#endif // API_VIDEO_VIDEO_FRAME_MARKING_H_ diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index 8cff22ad21..110db2e9fa 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -93,15 +93,6 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, h264_header.packetization_mode = info.codecSpecific.H264.packetization_mode; rtp->simulcastIdx = spatial_index.value_or(0); - rtp->frame_marking.temporal_id = kNoTemporalIdx; - if (info.codecSpecific.H264.temporal_idx != kNoTemporalIdx) { - rtp->frame_marking.temporal_id = info.codecSpecific.H264.temporal_idx; - rtp->frame_marking.layer_id = 0; - rtp->frame_marking.independent_frame = - info.codecSpecific.H264.idr_frame; - rtp->frame_marking.base_layer_sync = - info.codecSpecific.H264.base_layer_sync; - } return; } case kVideoCodecMultiplex: @@ -233,14 +224,6 @@ void RtpPayloadParams::SetCodecSpecific(RTPVideoHeader* rtp_video_header, vp9_header.tl0_pic_idx = state_.tl0_pic_idx; } } - if (rtp_video_header->codec == kVideoCodecH264) { - if (rtp_video_header->frame_marking.temporal_id != kNoTemporalIdx) { - if (rtp_video_header->frame_marking.temporal_id == 0) { - ++state_.tl0_pic_idx; - } - rtp_video_header->frame_marking.tl0_pic_idx = state_.tl0_pic_idx; - } - } if (generic_picture_id_experiment_ && rtp_video_header->codec == kVideoCodecGeneric) { rtp_video_header->video_type_header.emplace() diff --git a/call/rtp_payload_params_unittest.cc b/call/rtp_payload_params_unittest.cc index 98c1e524d9..a5510b0240 100644 --- a/call/rtp_payload_params_unittest.cc +++ b/call/rtp_payload_params_unittest.cc @@ -148,54 +148,6 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) { codec_info.codecSpecific.VP9.end_of_picture); } -TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_H264) { - RtpPayloadState state; - state.picture_id = kPictureId; - state.tl0_pic_idx = kInitialTl0PicIdx1; - RtpPayloadParams params(kSsrc1, &state, FieldTrialBasedConfig()); - - EncodedImage encoded_image; - CodecSpecificInfo codec_info; - CodecSpecificInfoH264* h264info = &codec_info.codecSpecific.H264; - codec_info.codecType = kVideoCodecH264; - h264info->packetization_mode = H264PacketizationMode::SingleNalUnit; - h264info->temporal_idx = kNoTemporalIdx; - - RTPVideoHeader header = - params.GetRtpVideoHeader(encoded_image, &codec_info, 10); - - EXPECT_EQ(0, header.simulcastIdx); - EXPECT_EQ(kVideoCodecH264, header.codec); - const auto& h264 = absl::get(header.video_type_header); - EXPECT_EQ(H264PacketizationMode::SingleNalUnit, h264.packetization_mode); - - // test temporal param 1 - h264info->temporal_idx = 1; - h264info->base_layer_sync = true; - h264info->idr_frame = false; - - header = params.GetRtpVideoHeader(encoded_image, &codec_info, 20); - - EXPECT_EQ(kVideoCodecH264, header.codec); - EXPECT_EQ(header.frame_marking.tl0_pic_idx, kInitialTl0PicIdx1); - EXPECT_EQ(header.frame_marking.temporal_id, h264info->temporal_idx); - EXPECT_EQ(header.frame_marking.base_layer_sync, h264info->base_layer_sync); - EXPECT_EQ(header.frame_marking.independent_frame, h264info->idr_frame); - - // test temporal param 2 - h264info->temporal_idx = 0; - h264info->base_layer_sync = false; - h264info->idr_frame = true; - - header = params.GetRtpVideoHeader(encoded_image, &codec_info, 30); - - EXPECT_EQ(kVideoCodecH264, header.codec); - EXPECT_EQ(header.frame_marking.tl0_pic_idx, kInitialTl0PicIdx1 + 1); - EXPECT_EQ(header.frame_marking.temporal_id, h264info->temporal_idx); - EXPECT_EQ(header.frame_marking.base_layer_sync, h264info->base_layer_sync); - EXPECT_EQ(header.frame_marking.independent_frame, h264info->idr_frame); -} - TEST(RtpPayloadParamsTest, PictureIdIsSetForVp8) { RtpPayloadState state; state.picture_id = kInitialPictureId1; diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 3976a6a1c5..1581c97b8d 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -609,7 +609,6 @@ WebRtcVideoEngine::GetRtpHeaderExtensions() const { webrtc::RtpExtension::kPlayoutDelayUri, webrtc::RtpExtension::kVideoContentTypeUri, webrtc::RtpExtension::kVideoTimingUri, - webrtc::RtpExtension::kFrameMarkingUri, webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri, webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) { result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv); diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index 3260ace2d7..673f05625d 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -347,10 +347,6 @@ TEST_F(WebRtcVideoEngineTest, SupportsVideoTimingHeaderExtension) { ExpectRtpCapabilitySupport(RtpExtension::kVideoTimingUri, true); } -TEST_F(WebRtcVideoEngineTest, SupportsFrameMarkingHeaderExtension) { - ExpectRtpCapabilitySupport(RtpExtension::kFrameMarkingUri, true); -} - TEST_F(WebRtcVideoEngineTest, SupportsColorSpaceHeaderExtension) { ExpectRtpCapabilitySupport(RtpExtension::kColorSpaceUri, true); } diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 5f63229184..1b72236bbb 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -65,7 +65,6 @@ enum RTPExtensionType : int { kRtpExtensionPlayoutDelay, kRtpExtensionVideoContentType, kRtpExtensionVideoTiming, - kRtpExtensionFrameMarking, kRtpExtensionRtpStreamId, kRtpExtensionRepairedRtpStreamId, kRtpExtensionMid, diff --git a/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/modules/rtp_rtcp/source/rtp_header_extension_map.cc index f59f9c4ebb..63562c5b89 100644 --- a/modules/rtp_rtcp/source/rtp_header_extension_map.cc +++ b/modules/rtp_rtcp/source/rtp_header_extension_map.cc @@ -41,7 +41,6 @@ constexpr ExtensionInfo kExtensions[] = { CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), - CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.cc b/modules/rtp_rtcp/source/rtp_header_extensions.cc index fefe6c618f..527874d785 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.cc +++ b/modules/rtp_rtcp/source/rtp_header_extensions.cc @@ -525,86 +525,6 @@ bool VideoTimingExtension::Write(rtc::ArrayView data, return true; } -// Frame Marking. -// -// Meta-information about an RTP stream outside the encrypted media payload, -// useful for an RTP switch to do codec-agnostic selective forwarding -// without decrypting the payload. -// -// For non-scalable streams: -// 0 1 -// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | ID | L = 0 |S|E|I|D|0 0 0 0| -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// -// For scalable streams: -// 0 1 2 3 -// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | ID | L = 2 |S|E|I|D|B| TID | LID | TL0PICIDX | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -constexpr RTPExtensionType FrameMarkingExtension::kId; -constexpr const char FrameMarkingExtension::kUri[]; - -bool FrameMarkingExtension::IsScalable(uint8_t temporal_id, uint8_t layer_id) { - return temporal_id != kNoTemporalIdx || layer_id != kNoSpatialIdx; -} - -bool FrameMarkingExtension::Parse(rtc::ArrayView data, - FrameMarking* frame_marking) { - RTC_DCHECK(frame_marking); - - if (data.size() != 1 && data.size() != 3) - return false; - - frame_marking->start_of_frame = (data[0] & 0x80) != 0; - frame_marking->end_of_frame = (data[0] & 0x40) != 0; - frame_marking->independent_frame = (data[0] & 0x20) != 0; - frame_marking->discardable_frame = (data[0] & 0x10) != 0; - - if (data.size() == 3) { - frame_marking->base_layer_sync = (data[0] & 0x08) != 0; - frame_marking->temporal_id = data[0] & 0x7; - frame_marking->layer_id = data[1]; - frame_marking->tl0_pic_idx = data[2]; - } else { - // non-scalable - frame_marking->base_layer_sync = false; - frame_marking->temporal_id = kNoTemporalIdx; - frame_marking->layer_id = kNoSpatialIdx; - frame_marking->tl0_pic_idx = 0; - } - return true; -} - -size_t FrameMarkingExtension::ValueSize(const FrameMarking& frame_marking) { - if (IsScalable(frame_marking.temporal_id, frame_marking.layer_id)) - return 3; - else - return 1; -} - -bool FrameMarkingExtension::Write(rtc::ArrayView data, - const FrameMarking& frame_marking) { - RTC_DCHECK_GE(data.size(), 1); - RTC_CHECK_LE(frame_marking.temporal_id, 0x07); - data[0] = frame_marking.start_of_frame ? 0x80 : 0x00; - data[0] |= frame_marking.end_of_frame ? 0x40 : 0x00; - data[0] |= frame_marking.independent_frame ? 0x20 : 0x00; - data[0] |= frame_marking.discardable_frame ? 0x10 : 0x00; - - if (IsScalable(frame_marking.temporal_id, frame_marking.layer_id)) { - RTC_DCHECK_EQ(data.size(), 3); - data[0] |= frame_marking.base_layer_sync ? 0x08 : 0x00; - data[0] |= frame_marking.temporal_id & 0x07; - data[1] = frame_marking.layer_id; - data[2] = frame_marking.tl0_pic_idx; - } - return true; -} - // Color space including HDR metadata as an optional field. // // RTP header extension to carry color space information and optionally HDR diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h index f4517bb513..8a81280f7b 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -19,7 +19,6 @@ #include "api/rtp_headers.h" #include "api/video/color_space.h" #include "api/video/video_content_type.h" -#include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" @@ -217,23 +216,6 @@ class VideoTimingExtension { uint8_t offset); }; -class FrameMarkingExtension { - public: - using value_type = FrameMarking; - static constexpr RTPExtensionType kId = kRtpExtensionFrameMarking; - static constexpr const char kUri[] = - "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07"; - - static bool Parse(rtc::ArrayView data, - FrameMarking* frame_marking); - static size_t ValueSize(const FrameMarking& frame_marking); - static bool Write(rtc::ArrayView data, - const FrameMarking& frame_marking); - - private: - static bool IsScalable(uint8_t temporal_id, uint8_t layer_id); -}; - class ColorSpaceExtension { public: using value_type = ColorSpace; diff --git a/modules/rtp_rtcp/source/rtp_packet.cc b/modules/rtp_rtcp/source/rtp_packet.cc index e054bb8306..767c9a0688 100644 --- a/modules/rtp_rtcp/source/rtp_packet.cc +++ b/modules/rtp_rtcp/source/rtp_packet.cc @@ -188,7 +188,6 @@ void RtpPacket::ZeroMutableExtensions() { case RTPExtensionType::kRtpExtensionAudioLevel: case RTPExtensionType::kRtpExtensionAbsoluteCaptureTime: case RTPExtensionType::kRtpExtensionColorSpace: - case RTPExtensionType::kRtpExtensionFrameMarking: case RTPExtensionType::kRtpExtensionGenericFrameDescriptor00: case RTPExtensionType::kRtpExtensionGenericFrameDescriptor02: case RTPExtensionType::kRtpExtensionMid: diff --git a/modules/rtp_rtcp/source/rtp_packet_received.cc b/modules/rtp_rtcp/source/rtp_packet_received.cc index 56aea8eb5e..feadee1db1 100644 --- a/modules/rtp_rtcp/source/rtp_packet_received.cc +++ b/modules/rtp_rtcp/source/rtp_packet_received.cc @@ -69,8 +69,6 @@ void RtpPacketReceived::GetHeader(RTPHeader* header) const { &header->extension.videoContentType); header->extension.has_video_timing = GetExtension(&header->extension.video_timing); - header->extension.has_frame_marking = - GetExtension(&header->extension.frame_marking); GetExtension(&header->extension.stream_id); GetExtension(&header->extension.repaired_stream_id); GetExtension(&header->extension.mid); diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc index c0960c2df8..af2752d6e3 100644 --- a/modules/rtp_rtcp/source/rtp_sender.cc +++ b/modules/rtp_rtcp/source/rtp_sender.cc @@ -108,7 +108,6 @@ bool IsNonVolatile(RTPExtensionType type) { case kRtpExtensionAbsoluteSendTime: case kRtpExtensionTransportSequenceNumber: case kRtpExtensionTransportSequenceNumber02: - case kRtpExtensionFrameMarking: case kRtpExtensionRtpStreamId: case kRtpExtensionMid: case kRtpExtensionGenericFrameDescriptor00: diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 721bc8bc07..58a8699688 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -312,14 +312,6 @@ void RTPSenderVideo::AddRtpHeaderExtensions( packet->SetExtension(*absolute_capture_time); } - if (video_header.codec == kVideoCodecH264 && - video_header.frame_marking.temporal_id != kNoTemporalIdx) { - FrameMarking frame_marking = video_header.frame_marking; - frame_marking.start_of_frame = first_packet; - frame_marking.end_of_frame = last_packet; - packet->SetExtension(frame_marking); - } - if (video_header.generic) { bool extension_is_set = false; if (video_structure_ != nullptr) { @@ -736,12 +728,7 @@ uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) { } uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; } }; - switch (header.codec) { - case kVideoCodecH264: - return header.frame_marking.temporal_id; - default: - return absl::visit(TemporalIdGetter(), header.video_type_header); - } + return absl::visit(TemporalIdGetter(), header.video_type_header); } bool RTPSenderVideo::UpdateConditionalRetransmit( diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index 29e48626ab..32e138f840 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -59,7 +59,6 @@ using ::testing::WithArgs; enum : int { // The first valid value is 1. kAbsoluteSendTimeExtensionId = 1, - kFrameMarkingExtensionId, kGenericDescriptorId, kDependencyDescriptorId, kTransmissionTimeOffsetExtensionId, @@ -95,8 +94,6 @@ class LoopbackTransportTest : public webrtc::Transport { kGenericDescriptorId); receivers_extensions_.Register( kDependencyDescriptorId); - receivers_extensions_.Register( - kFrameMarkingExtensionId); receivers_extensions_.Register( kAbsoluteCaptureTimeExtensionId); receivers_extensions_.Register( @@ -291,43 +288,6 @@ TEST_P(RtpSenderVideoTest, DeltaFrameHasCVOWhenNonZero) { EXPECT_EQ(kVideoRotation_90, rotation); } -TEST_P(RtpSenderVideoTest, CheckH264FrameMarking) { - uint8_t kFrame[kMaxPacketLength]; - rtp_module_->RegisterRtpHeaderExtension(FrameMarkingExtension::kUri, - kFrameMarkingExtensionId); - - RTPFragmentationHeader frag; - frag.VerifyAndAllocateFragmentationHeader(1); - frag.fragmentationOffset[0] = 0; - frag.fragmentationLength[0] = sizeof(kFrame); - - RTPVideoHeader hdr; - hdr.video_type_header.emplace().packetization_mode = - H264PacketizationMode::NonInterleaved; - hdr.codec = kVideoCodecH264; - hdr.frame_marking.temporal_id = kNoTemporalIdx; - hdr.frame_marking.tl0_pic_idx = 99; - hdr.frame_marking.base_layer_sync = true; - hdr.frame_type = VideoFrameType::kVideoFrameDelta; - rtp_sender_video_.SendVideo(kPayload, kType, kTimestamp, 0, kFrame, &frag, - hdr, kDefaultExpectedRetransmissionTimeMs); - - FrameMarking fm; - EXPECT_FALSE( - transport_.last_sent_packet().GetExtension(&fm)); - - hdr.frame_marking.temporal_id = 0; - hdr.frame_type = VideoFrameType::kVideoFrameDelta; - rtp_sender_video_.SendVideo(kPayload, kType, kTimestamp + 1, 0, kFrame, &frag, - hdr, kDefaultExpectedRetransmissionTimeMs); - - EXPECT_TRUE( - transport_.last_sent_packet().GetExtension(&fm)); - EXPECT_EQ(hdr.frame_marking.temporal_id, fm.temporal_id); - EXPECT_EQ(hdr.frame_marking.tl0_pic_idx, fm.tl0_pic_idx); - EXPECT_EQ(hdr.frame_marking.base_layer_sync, fm.base_layer_sync); -} - // Make sure rotation is parsed correctly when the Camera (C) and Flip (F) bits // are set in the CVO byte. TEST_P(RtpSenderVideoTest, SendVideoWithCameraAndFlipCVO) { @@ -369,7 +329,6 @@ TEST_P(RtpSenderVideoTest, RetransmissionTypesH264) { header.video_type_header.emplace().packetization_mode = H264PacketizationMode::NonInterleaved; header.codec = kVideoCodecH264; - header.frame_marking.temporal_id = kNoTemporalIdx; EXPECT_FALSE(rtp_sender_video_.AllowRetransmission( header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs)); @@ -380,14 +339,6 @@ TEST_P(RtpSenderVideoTest, RetransmissionTypesH264) { EXPECT_TRUE(rtp_sender_video_.AllowRetransmission( header, kConditionallyRetransmitHigherLayers, kDefaultExpectedRetransmissionTimeMs)); - - // Test higher level retransmit. - for (int tid = 0; tid <= kMaxTemporalStreams; ++tid) { - header.frame_marking.temporal_id = tid; - EXPECT_TRUE(rtp_sender_video_.AllowRetransmission( - header, kRetransmitHigherLayers | kRetransmitBaseLayer, - kDefaultExpectedRetransmissionTimeMs)); - } } TEST_P(RtpSenderVideoTest, RetransmissionTypesVP8BaseLayer) { diff --git a/modules/rtp_rtcp/source/rtp_utility.cc b/modules/rtp_rtcp/source/rtp_utility.cc index f76d7d0f0b..c25fd96fa5 100644 --- a/modules/rtp_rtcp/source/rtp_utility.cc +++ b/modules/rtp_rtcp/source/rtp_utility.cc @@ -17,7 +17,6 @@ #include "api/array_view.h" #include "api/video/video_content_type.h" -#include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "modules/rtp_rtcp/include/rtp_cvo.h" @@ -245,10 +244,6 @@ bool RtpHeaderParser::Parse(RTPHeader* header, header->extension.has_video_timing = false; header->extension.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false}; - header->extension.has_frame_marking = false; - header->extension.frame_marking = {false, false, false, false, - false, kNoTemporalIdx, 0, 0}; - if (X) { /* RTP header extension, RFC 3550. 0 1 2 3 @@ -497,15 +492,6 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( &header->extension.video_timing); break; } - case kRtpExtensionFrameMarking: { - if (!FrameMarkingExtension::Parse(rtc::MakeArrayView(ptr, len + 1), - &header->extension.frame_marking)) { - RTC_LOG(LS_WARNING) << "Incorrect frame marking len: " << len; - return; - } - header->extension.has_frame_marking = true; - break; - } case kRtpExtensionRtpStreamId: { std::string name(reinterpret_cast(ptr), len + 1); if (IsLegalRsidName(name)) { diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h index 5785ea9a54..514340add6 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.h +++ b/modules/rtp_rtcp/source/rtp_video_header.h @@ -19,7 +19,6 @@ #include "api/video/color_space.h" #include "api/video/video_codec_type.h" #include "api/video/video_content_type.h" -#include "api/video/video_frame_marking.h" #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" @@ -75,7 +74,6 @@ struct RTPVideoHeader { PlayoutDelay playout_delay = {-1, -1}; VideoSendTiming video_timing; - FrameMarking frame_marking = {false, false, false, false, false, 0xFF, 0, 0}; absl::optional color_space; RTPVideoTypeHeader video_type_header; }; diff --git a/modules/video_coding/encoded_frame.cc b/modules/video_coding/encoded_frame.cc index 1e9e374c64..3de62da9f5 100644 --- a/modules/video_coding/encoded_frame.cc +++ b/modules/video_coding/encoded_frame.cc @@ -135,20 +135,6 @@ void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) { } case kVideoCodecH264: { _codecSpecificInfo.codecType = kVideoCodecH264; - - // The following H264 codec specific data are not used elsewhere. - // Instead they are read directly from the frame marking extension. - // These codec specific data structures should be removed - // when frame marking is used. - _codecSpecificInfo.codecSpecific.H264.temporal_idx = kNoTemporalIdx; - if (header->frame_marking.temporal_id != kNoTemporalIdx) { - _codecSpecificInfo.codecSpecific.H264.temporal_idx = - header->frame_marking.temporal_id; - _codecSpecificInfo.codecSpecific.H264.base_layer_sync = - header->frame_marking.base_layer_sync; - _codecSpecificInfo.codecSpecific.H264.idr_frame = - header->frame_marking.independent_frame; - } break; } default: { diff --git a/modules/video_coding/frame_object.cc b/modules/video_coding/frame_object.cc index cb83999c94..2399e8f060 100644 --- a/modules/video_coding/frame_object.cc +++ b/modules/video_coding/frame_object.cc @@ -129,9 +129,5 @@ const RTPVideoHeader& RtpFrameObject::GetRtpVideoHeader() const { return rtp_video_header_; } -const FrameMarking& RtpFrameObject::GetFrameMarking() const { - return rtp_video_header_.frame_marking; -} - } // namespace video_coding } // namespace webrtc diff --git a/modules/video_coding/frame_object.h b/modules/video_coding/frame_object.h index f7988763d3..831b444df4 100644 --- a/modules/video_coding/frame_object.h +++ b/modules/video_coding/frame_object.h @@ -47,7 +47,6 @@ class RtpFrameObject : public EncodedFrame { int64_t RenderTime() const override; bool delayed_by_retransmission() const override; const RTPVideoHeader& GetRtpVideoHeader() const; - const FrameMarking& GetFrameMarking() const; private: RTPVideoHeader rtp_video_header_; diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index ad82211720..7da8a1c301 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -363,15 +363,10 @@ std::vector> PacketBuffer::FindFrames( VideoFrameType::kVideoFrameDelta; } - // With IPPP, if this is not a keyframe, make sure there are no gaps - // in the packet sequence numbers up until this point. - const uint8_t h264tid = - buffer_[start_index] != nullptr - ? buffer_[start_index]->video_header.frame_marking.temporal_id - : kNoTemporalIdx; - if (h264tid == kNoTemporalIdx && !is_h264_keyframe && - missing_packets_.upper_bound(start_seq_num) != - missing_packets_.begin()) { + // If this is not a keyframe, make sure there are no gaps in the packet + // sequence numbers up until this point. + if (!is_h264_keyframe && missing_packets_.upper_bound(start_seq_num) != + missing_packets_.begin()) { return found_frames; } } diff --git a/modules/video_coding/rtp_frame_reference_finder.cc b/modules/video_coding/rtp_frame_reference_finder.cc index bdef991b8b..2a43c275d6 100644 --- a/modules/video_coding/rtp_frame_reference_finder.cc +++ b/modules/video_coding/rtp_frame_reference_finder.cc @@ -108,8 +108,6 @@ RtpFrameReferenceFinder::ManageFrameInternal(RtpFrameObject* frame) { return ManageFrameVp8(frame); case kVideoCodecVP9: return ManageFrameVp9(frame); - case kVideoCodecH264: - return ManageFrameH264(frame); case kVideoCodecGeneric: if (auto* generic_header = absl::get_if( &frame->GetRtpVideoHeader().video_type_header)) { @@ -715,130 +713,6 @@ void RtpFrameReferenceFinder::UnwrapPictureIds(RtpFrameObject* frame) { frame->id.picture_id = unwrapper_.Unwrap(frame->id.picture_id); } -RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameH264( - RtpFrameObject* frame) { - const FrameMarking& rtp_frame_marking = frame->GetFrameMarking(); - - uint8_t tid = rtp_frame_marking.temporal_id; - bool blSync = rtp_frame_marking.base_layer_sync; - - if (tid == kNoTemporalIdx) - return ManageFramePidOrSeqNum(std::move(frame), kNoPictureId); - - // Protect against corrupted packets with arbitrary large temporal idx. - if (tid >= kMaxTemporalLayers) - return kDrop; - - frame->id.picture_id = frame->last_seq_num(); - - if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { - // For H264, use last_seq_num_gop_ to simply store last picture id - // as a pair of unpadded and padded sequence numbers. - if (last_seq_num_gop_.empty()) { - last_seq_num_gop_.insert(std::make_pair( - 0, std::make_pair(frame->id.picture_id, frame->id.picture_id))); - } - } - - // Stash if we have no keyframe yet. - if (last_seq_num_gop_.empty()) - return kStash; - - // Check for gap in sequence numbers. Store in |not_yet_received_seq_num_|. - if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) { - uint16_t last_pic_id_padded = last_seq_num_gop_.begin()->second.second; - if (AheadOf(frame->id.picture_id, last_pic_id_padded)) { - do { - last_pic_id_padded = last_pic_id_padded + 1; - not_yet_received_seq_num_.insert(last_pic_id_padded); - } while (last_pic_id_padded != frame->id.picture_id); - } - } - - int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(rtp_frame_marking.tl0_pic_idx); - - // Clean up info for base layers that are too old. - int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo; - auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx); - layer_info_.erase(layer_info_.begin(), clean_layer_info_to); - - // Clean up info about not yet received frames that are too old. - uint16_t old_picture_id = frame->id.picture_id - kMaxNotYetReceivedFrames * 2; - auto clean_frames_to = not_yet_received_seq_num_.lower_bound(old_picture_id); - not_yet_received_seq_num_.erase(not_yet_received_seq_num_.begin(), - clean_frames_to); - - if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { - frame->num_references = 0; - layer_info_[unwrapped_tl0].fill(-1); - UpdateDataH264(frame, unwrapped_tl0, tid); - return kHandOff; - } - - auto layer_info_it = - layer_info_.find(tid == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0); - - // Stash if we have no base layer frame yet. - if (layer_info_it == layer_info_.end()) - return kStash; - - // Base layer frame. Copy layer info from previous base layer frame. - if (tid == 0) { - layer_info_it = - layer_info_.insert(std::make_pair(unwrapped_tl0, layer_info_it->second)) - .first; - frame->num_references = 1; - frame->references[0] = layer_info_it->second[0]; - UpdateDataH264(frame, unwrapped_tl0, tid); - return kHandOff; - } - - // This frame only references its base layer frame. - if (blSync) { - frame->num_references = 1; - frame->references[0] = layer_info_it->second[0]; - UpdateDataH264(frame, unwrapped_tl0, tid); - return kHandOff; - } - - // Find all references for general frame. - frame->num_references = 0; - for (uint8_t layer = 0; layer <= tid; ++layer) { - // Stash if we have not yet received frames on this temporal layer. - if (layer_info_it->second[layer] == -1) - return kStash; - - // Drop if the last frame on this layer is ahead of this frame. A layer sync - // frame was received after this frame for the same base layer frame. - uint16_t last_frame_in_layer = layer_info_it->second[layer]; - if (AheadOf(last_frame_in_layer, frame->id.picture_id)) - return kDrop; - - // Stash and wait for missing frame between this frame and the reference - auto not_received_seq_num_it = - not_yet_received_seq_num_.upper_bound(last_frame_in_layer); - if (not_received_seq_num_it != not_yet_received_seq_num_.end() && - AheadOf(frame->id.picture_id, *not_received_seq_num_it)) { - return kStash; - } - - if (!(AheadOf(frame->id.picture_id, last_frame_in_layer))) { - RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id - << " and packet range [" << frame->first_seq_num() - << ", " << frame->last_seq_num() - << "] already received, " - " dropping frame."; - return kDrop; - } - - ++frame->num_references; - frame->references[layer] = last_frame_in_layer; - } - - UpdateDataH264(frame, unwrapped_tl0, tid); - return kHandOff; -} - void RtpFrameReferenceFinder::UpdateLastPictureIdWithPaddingH264() { auto seq_num_it = last_seq_num_gop_.begin(); diff --git a/modules/video_coding/rtp_frame_reference_finder_unittest.cc b/modules/video_coding/rtp_frame_reference_finder_unittest.cc index 9ded6bcb96..0c08ddd302 100644 --- a/modules/video_coding/rtp_frame_reference_finder_unittest.cc +++ b/modules/video_coding/rtp_frame_reference_finder_unittest.cc @@ -32,13 +32,11 @@ std::unique_ptr CreateFrame( uint16_t seq_num_end, bool keyframe, VideoCodecType codec, - const RTPVideoTypeHeader& video_type_header, - const FrameMarking& frame_markings) { + const RTPVideoTypeHeader& video_type_header) { RTPVideoHeader video_header; video_header.frame_type = keyframe ? VideoFrameType::kVideoFrameKey : VideoFrameType::kVideoFrameDelta; video_header.video_type_header = video_type_header; - video_header.frame_marking = frame_markings; // clang-format off return std::make_unique( @@ -92,7 +90,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, bool keyframe) { std::unique_ptr frame = CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecGeneric, - RTPVideoTypeHeader(), FrameMarking()); + RTPVideoTypeHeader()); reference_finder_->ManageFrame(std::move(frame)); } @@ -110,9 +108,8 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, vp8_header.tl0PicIdx = tl0; vp8_header.layerSync = sync; - std::unique_ptr frame = - CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecVP8, - vp8_header, FrameMarking()); + std::unique_ptr frame = CreateFrame( + seq_num_start, seq_num_end, keyframe, kVideoCodecVP8, vp8_header); reference_finder_->ManageFrame(std::move(frame)); } @@ -140,9 +137,8 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, vp9_header.gof = *ss; } - std::unique_ptr frame = - CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecVP9, - vp9_header, FrameMarking()); + std::unique_ptr frame = CreateFrame( + seq_num_start, seq_num_end, keyframe, kVideoCodecVP9, vp9_header); reference_finder_->ManageFrame(std::move(frame)); } @@ -166,26 +162,15 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, for (size_t i = 0; i < refs.size(); ++i) vp9_header.pid_diff[i] = refs[i]; - std::unique_ptr frame = - CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecVP9, - vp9_header, FrameMarking()); + std::unique_ptr frame = CreateFrame( + seq_num_start, seq_num_end, keyframe, kVideoCodecVP9, vp9_header); reference_finder_->ManageFrame(std::move(frame)); } - void InsertH264(uint16_t seq_num_start, - uint16_t seq_num_end, - bool keyframe, - uint8_t tid = kNoTemporalIdx, - int32_t tl0 = kNoTl0PicIdx, - bool sync = false) { - FrameMarking frame_marking{}; - frame_marking.temporal_id = tid; - frame_marking.tl0_pic_idx = tl0; - frame_marking.base_layer_sync = sync; - + void InsertH264(uint16_t seq_num_start, uint16_t seq_num_end, bool keyframe) { std::unique_ptr frame = CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecH264, - RTPVideoTypeHeader(), frame_marking); + RTPVideoTypeHeader()); reference_finder_->ManageFrame(std::move(frame)); } @@ -1440,14 +1425,28 @@ TEST_F(TestRtpFrameReferenceFinder, H264KeyFrameReferences) { CheckReferencesH264(sn); } -// Test with 1 temporal layer. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayers_0) { +TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrap) { + uint16_t sn = 0xFFFF; + + InsertH264(sn - 1, sn - 1, true); + InsertH264(sn, sn, false); + InsertH264(sn + 1, sn + 1, false); + InsertH264(sn + 2, sn + 2, false); + + ASSERT_EQ(4UL, frames_from_callback_.size()); + CheckReferencesH264(sn - 1); + CheckReferencesH264(sn, sn - 1); + CheckReferencesH264(sn + 1, sn); + CheckReferencesH264(sn + 2, sn + 1); +} + +TEST_F(TestRtpFrameReferenceFinder, H264Frames) { uint16_t sn = Rand(); - InsertH264(sn, sn, true, 0, 1); - InsertH264(sn + 1, sn + 1, false, 0, 2); - InsertH264(sn + 2, sn + 2, false, 0, 3); - InsertH264(sn + 3, sn + 3, false, 0, 4); + InsertH264(sn, sn, true); + InsertH264(sn + 1, sn + 1, false); + InsertH264(sn + 2, sn + 2, false); + InsertH264(sn + 3, sn + 3, false); ASSERT_EQ(4UL, frames_from_callback_.size()); CheckReferencesH264(sn); @@ -1456,37 +1455,16 @@ TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayers_0) { CheckReferencesH264(sn + 3, sn + 2); } -TEST_F(TestRtpFrameReferenceFinder, H264DuplicateTl1Frames) { +TEST_F(TestRtpFrameReferenceFinder, H264Reordering) { uint16_t sn = Rand(); - InsertH264(sn, sn, true, 0, 0); - InsertH264(sn + 1, sn + 1, false, 1, 0, true); - InsertH264(sn + 2, sn + 2, false, 0, 1); - InsertH264(sn + 3, sn + 3, false, 1, 1); - InsertH264(sn + 3, sn + 3, false, 1, 1); - InsertH264(sn + 4, sn + 4, false, 0, 2); - InsertH264(sn + 5, sn + 5, false, 1, 2); - - ASSERT_EQ(6UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn + 1, sn + 2); - CheckReferencesH264(sn + 4, sn + 2); - CheckReferencesH264(sn + 5, sn + 3, sn + 4); -} - -// Test with 1 temporal layer. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersReordering_0) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 1); - InsertH264(sn + 1, sn + 1, false, 0, 2); - InsertH264(sn + 3, sn + 3, false, 0, 4); - InsertH264(sn + 2, sn + 2, false, 0, 3); - InsertH264(sn + 5, sn + 5, false, 0, 6); - InsertH264(sn + 6, sn + 6, false, 0, 7); - InsertH264(sn + 4, sn + 4, false, 0, 5); + InsertH264(sn, sn, true); + InsertH264(sn + 1, sn + 1, false); + InsertH264(sn + 3, sn + 3, false); + InsertH264(sn + 2, sn + 2, false); + InsertH264(sn + 5, sn + 5, false); + InsertH264(sn + 6, sn + 6, false); + InsertH264(sn + 4, sn + 4, false); ASSERT_EQ(7UL, frames_from_callback_.size()); CheckReferencesH264(sn); @@ -1498,258 +1476,13 @@ TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersReordering_0) { CheckReferencesH264(sn + 6, sn + 5); } -// Test with 2 temporal layers in a 01 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayers_01) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 255); - InsertH264(sn + 1, sn + 1, false, 1, 255, true); - InsertH264(sn + 2, sn + 2, false, 0, 0); - InsertH264(sn + 3, sn + 3, false, 1, 0); - - ASSERT_EQ(4UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn + 1, sn + 2); -} - -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersMultiSn_01) { - uint16_t sn = Rand(); - - InsertH264(sn, sn + 3, true, 0, 255); - InsertH264(sn + 4, sn + 5, false, 1, 255, true); - InsertH264(sn + 6, sn + 8, false, 0, 0); - InsertH264(sn + 9, sn + 9, false, 1, 0); - - ASSERT_EQ(4UL, frames_from_callback_.size()); - CheckReferencesH264(sn + 3); - CheckReferencesH264(sn + 5, sn + 3); - CheckReferencesH264(sn + 8, sn + 3); - CheckReferencesH264(sn + 9, sn + 5, sn + 8); -} - -// Test with 2 temporal layers in a 01 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersReordering_01) { - uint16_t sn = Rand(); - - InsertH264(sn + 1, sn + 1, false, 1, 255, true); - InsertH264(sn, sn, true, 0, 255); - InsertH264(sn + 3, sn + 3, false, 1, 0); - InsertH264(sn + 5, sn + 5, false, 1, 1); - InsertH264(sn + 2, sn + 2, false, 0, 0); - InsertH264(sn + 4, sn + 4, false, 0, 1); - InsertH264(sn + 6, sn + 6, false, 0, 2); - InsertH264(sn + 7, sn + 7, false, 1, 2); - - ASSERT_EQ(8UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn + 1, sn + 2); - CheckReferencesH264(sn + 4, sn + 2); - CheckReferencesH264(sn + 5, sn + 3, sn + 4); - CheckReferencesH264(sn + 6, sn + 4); - CheckReferencesH264(sn + 7, sn + 5, sn + 6); -} - -// Test with 3 temporal layers in a 0212 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayers_0212) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 55); - InsertH264(sn + 1, sn + 1, false, 2, 55, true); - InsertH264(sn + 2, sn + 2, false, 1, 55, true); - InsertH264(sn + 3, sn + 3, false, 2, 55); - InsertH264(sn + 4, sn + 4, false, 0, 56); - InsertH264(sn + 5, sn + 5, false, 2, 56, true); - InsertH264(sn + 6, sn + 6, false, 1, 56, true); - InsertH264(sn + 7, sn + 7, false, 2, 56); - InsertH264(sn + 8, sn + 8, false, 0, 57); - InsertH264(sn + 9, sn + 9, false, 2, 57, true); - InsertH264(sn + 10, sn + 10, false, 1, 57, true); - InsertH264(sn + 11, sn + 11, false, 2, 57); - - ASSERT_EQ(12UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn, sn + 1, sn + 2); - CheckReferencesH264(sn + 4, sn); - CheckReferencesH264(sn + 5, sn + 4); - CheckReferencesH264(sn + 6, sn + 4); - CheckReferencesH264(sn + 7, sn + 4, sn + 5, sn + 6); - CheckReferencesH264(sn + 8, sn + 4); - CheckReferencesH264(sn + 9, sn + 8); - CheckReferencesH264(sn + 10, sn + 8); - CheckReferencesH264(sn + 11, sn + 8, sn + 9, sn + 10); -} - -// Test with 3 temporal layers in a 0212 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersMissingFrame_0212) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 55, false); - InsertH264(sn + 2, sn + 2, false, 1, 55, true); - InsertH264(sn + 3, sn + 3, false, 2, 55, false); - - ASSERT_EQ(2UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 2, sn); -} - -// Test with 3 temporal layers in a 0212 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersReordering_0212) { - uint16_t sn = Rand(); - - InsertH264(sn + 1, sn + 1, false, 2, 55, true); - InsertH264(sn, sn, true, 0, 55, false); - InsertH264(sn + 2, sn + 2, false, 1, 55, true); - InsertH264(sn + 4, sn + 4, false, 0, 56, false); - InsertH264(sn + 5, sn + 5, false, 2, 56, false); - InsertH264(sn + 3, sn + 3, false, 2, 55, false); - InsertH264(sn + 7, sn + 7, false, 2, 56, false); - InsertH264(sn + 9, sn + 9, false, 2, 57, true); - InsertH264(sn + 6, sn + 6, false, 1, 56, false); - InsertH264(sn + 8, sn + 8, false, 0, 57, false); - InsertH264(sn + 11, sn + 11, false, 2, 57, false); - InsertH264(sn + 10, sn + 10, false, 1, 57, true); - - ASSERT_EQ(12UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn, sn + 1, sn + 2); - CheckReferencesH264(sn + 4, sn); - CheckReferencesH264(sn + 5, sn + 2, sn + 3, sn + 4); - CheckReferencesH264(sn + 6, sn + 2, sn + 4); - CheckReferencesH264(sn + 7, sn + 4, sn + 5, sn + 6); - CheckReferencesH264(sn + 8, sn + 4); - CheckReferencesH264(sn + 9, sn + 8); - CheckReferencesH264(sn + 10, sn + 8); - CheckReferencesH264(sn + 11, sn + 8, sn + 9, sn + 10); -} - -TEST_F(TestRtpFrameReferenceFinder, H264InsertManyFrames_0212) { - uint16_t sn = Rand(); - - const int keyframes_to_insert = 50; - const int frames_per_keyframe = 120; // Should be a multiple of 4. - uint8_t tl0 = 128; - - for (int k = 0; k < keyframes_to_insert; ++k) { - InsertH264(sn, sn, true, 0, tl0, false); - InsertH264(sn + 1, sn + 1, false, 2, tl0, true); - InsertH264(sn + 2, sn + 2, false, 1, tl0, true); - InsertH264(sn + 3, sn + 3, false, 2, tl0, false); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn, sn + 1, sn + 2); - frames_from_callback_.clear(); - ++tl0; - - for (int f = 4; f < frames_per_keyframe; f += 4) { - uint16_t sf = sn + f; - - InsertH264(sf, sf, false, 0, tl0, false); - InsertH264(sf + 1, sf + 1, false, 2, tl0, false); - InsertH264(sf + 2, sf + 2, false, 1, tl0, false); - InsertH264(sf + 3, sf + 3, false, 2, tl0, false); - CheckReferencesH264(sf, sf - 4); - CheckReferencesH264(sf + 1, sf, sf - 1, sf - 2); - CheckReferencesH264(sf + 2, sf, sf - 2); - CheckReferencesH264(sf + 3, sf, sf + 1, sf + 2); - frames_from_callback_.clear(); - ++tl0; - } - - sn += frames_per_keyframe; - } -} - -TEST_F(TestRtpFrameReferenceFinder, H264LayerSync) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 0, false); - InsertH264(sn + 1, sn + 1, false, 1, 0, true); - InsertH264(sn + 2, sn + 2, false, 0, 1, false); - ASSERT_EQ(3UL, frames_from_callback_.size()); - - InsertH264(sn + 4, sn + 4, false, 0, 2, false); - InsertH264(sn + 5, sn + 5, false, 1, 2, true); - InsertH264(sn + 6, sn + 6, false, 0, 3, false); - InsertH264(sn + 7, sn + 7, false, 1, 3, false); - - ASSERT_EQ(7UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 4, sn + 2); - CheckReferencesH264(sn + 5, sn + 4); - CheckReferencesH264(sn + 6, sn + 4); - CheckReferencesH264(sn + 7, sn + 6, sn + 5); -} - -TEST_F(TestRtpFrameReferenceFinder, H264Tl1SyncFrameAfterTl1Frame) { - InsertH264(1000, 1000, true, 0, 247, true); - InsertH264(1001, 1001, false, 0, 248, false); - InsertH264(1002, 1002, false, 1, 248, false); // Will be dropped - InsertH264(1003, 1003, false, 1, 248, true); // due to this frame. - - ASSERT_EQ(3UL, frames_from_callback_.size()); - CheckReferencesH264(1000); - CheckReferencesH264(1001, 1000); - CheckReferencesH264(1003, 1001); -} - -TEST_F(TestRtpFrameReferenceFinder, H264DetectMissingFrame_0212) { - InsertH264(1, 1, true, 0, 1, false); - InsertH264(2, 2, false, 2, 1, true); - InsertH264(3, 3, false, 1, 1, true); - InsertH264(4, 4, false, 2, 1, false); - - InsertH264(6, 6, false, 2, 2, false); - InsertH264(7, 7, false, 1, 2, false); - InsertH264(8, 8, false, 2, 2, false); - ASSERT_EQ(4UL, frames_from_callback_.size()); - - InsertH264(5, 5, false, 0, 2, false); - ASSERT_EQ(8UL, frames_from_callback_.size()); - - CheckReferencesH264(1); - CheckReferencesH264(2, 1); - CheckReferencesH264(3, 1); - CheckReferencesH264(4, 3, 2, 1); - - CheckReferencesH264(5, 1); - CheckReferencesH264(6, 5, 4, 3); - CheckReferencesH264(7, 5, 3); - CheckReferencesH264(8, 7, 6, 5); -} - -TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrap) { - uint16_t sn = 0xFFFF; - - InsertH264(sn - 1, sn - 1, true, 0, 1); - InsertH264(sn, sn, false, 0, 2); - InsertH264(sn + 1, sn + 1, false, 0, 3); - InsertH264(sn + 2, sn + 2, false, 0, 4); - - ASSERT_EQ(4UL, frames_from_callback_.size()); - CheckReferencesH264(sn - 1); - CheckReferencesH264(sn, sn - 1); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn + 1); -} - TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrapMulti) { uint16_t sn = 0xFFFF; - InsertH264(sn - 3, sn - 2, true, 0, 1); - InsertH264(sn - 1, sn + 1, false, 0, 2); - InsertH264(sn + 2, sn + 3, false, 0, 3); - InsertH264(sn + 4, sn + 7, false, 0, 4); + InsertH264(sn - 3, sn - 2, true); + InsertH264(sn - 1, sn + 1, false); + InsertH264(sn + 2, sn + 3, false); + InsertH264(sn + 4, sn + 7, false); ASSERT_EQ(4UL, frames_from_callback_.size()); CheckReferencesH264(sn - 2); @@ -1758,35 +1491,5 @@ TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrapMulti) { CheckReferencesH264(sn + 7, sn + 3); } -TEST_F(TestRtpFrameReferenceFinder, H264Tl0PicIdxWrap) { - int numTl0Wraps = 1000; - int64_t sn = Rand(); - - for (int i = 0; i < numTl0Wraps; i++) { - for (int tl0 = 0; tl0 < 256; tl0 += 16, sn += 16) { - InsertH264(sn, sn, true, 0, tl0); - reference_finder_->ClearTo(sn); // Too many stashed frames cause errors. - - for (int k = 1; k < 8; k++) { - InsertH264(sn + k, sn + k, false, 0, tl0 + k); - } - - // Skip a TL0 index. - for (int k = 9; k < 16; k++) { - InsertH264(sn + k, sn + k, false, 0, tl0 + k); - } - - ASSERT_EQ(8UL, frames_from_callback_.size()); - - CheckReferencesH264(sn); - for (int k = 1; k < 8; k++) { - CheckReferencesH264(sn + k, sn + k - 1); - } - - frames_from_callback_.clear(); - } - } -} - } // namespace video_coding } // namespace webrtc diff --git a/modules/video_coding/session_info.cc b/modules/video_coding/session_info.cc index e51d293607..07b9a9d6b5 100644 --- a/modules/video_coding/session_info.cc +++ b/modules/video_coding/session_info.cc @@ -95,8 +95,6 @@ int VCMSessionInfo::TemporalId() const { return absl::get( packets_.front().video_header.video_type_header) .temporal_idx; - } else if (packets_.front().video_header.codec == kVideoCodecH264) { - return packets_.front().video_header.frame_marking.temporal_id; } else { return kNoTemporalIdx; } @@ -113,8 +111,6 @@ bool VCMSessionInfo::LayerSync() const { return absl::get( packets_.front().video_header.video_type_header) .temporal_up_switch; - } else if (packets_.front().video_header.codec == kVideoCodecH264) { - return packets_.front().video_header.frame_marking.base_layer_sync; } else { return false; } @@ -131,8 +127,6 @@ int VCMSessionInfo::Tl0PicId() const { return absl::get( packets_.front().video_header.video_type_header) .tl0_pic_idx; - } else if (packets_.front().video_header.codec == kVideoCodecH264) { - return packets_.front().video_header.frame_marking.tl0_pic_idx; } else { return kNoTl0PicIdx; } diff --git a/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc b/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc index a9f33feed6..8b19a088de 100644 --- a/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc +++ b/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc @@ -132,7 +132,6 @@ void FuzzOneInput(const uint8_t* data, size_t size) { break; } - reader.CopyTo(&video_header.frame_marking); video_header.generic = GenerateGenericFrameDependencies(&reader); // clang-format off diff --git a/test/fuzzers/rtp_packet_fuzzer.cc b/test/fuzzers/rtp_packet_fuzzer.cc index 6a4f5e7893..a22c643a44 100644 --- a/test/fuzzers/rtp_packet_fuzzer.cc +++ b/test/fuzzers/rtp_packet_fuzzer.cc @@ -112,10 +112,6 @@ void FuzzOneInput(const uint8_t* data, size_t size) { VideoSendTiming timing; packet.GetExtension(&timing); break; - case kRtpExtensionFrameMarking: - FrameMarking frame_marking; - packet.GetExtension(&frame_marking); - break; case kRtpExtensionRtpStreamId: { std::string rsid; packet.GetExtension(&rsid); diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 4544a12db0..8bbb5866a0 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -517,7 +517,6 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData( video_header.content_type = VideoContentType::UNSPECIFIED; video_header.video_timing.flags = VideoSendTiming::kInvalid; video_header.is_last_packet_in_frame |= rtp_packet.Marker(); - video_header.frame_marking.temporal_id = kNoTemporalIdx; if (const auto* vp9_header = absl::get_if(&video_header.video_type_header)) { @@ -535,7 +534,6 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData( } else { rtp_packet.GetExtension(&video_header.playout_delay); } - rtp_packet.GetExtension(&video_header.frame_marking); ParseGenericDependenciesResult generic_descriptor_state = ParseGenericDependenciesExtension(rtp_packet, &video_header); diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index 481b83e241..3f11bb77c4 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -485,7 +485,6 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData( video_header.content_type = VideoContentType::UNSPECIFIED; video_header.video_timing.flags = VideoSendTiming::kInvalid; video_header.is_last_packet_in_frame |= rtp_packet.Marker(); - video_header.frame_marking.temporal_id = kNoTemporalIdx; if (const auto* vp9_header = absl::get_if(&video_header.video_type_header)) { @@ -503,7 +502,6 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData( } else { rtp_packet.GetExtension(&video_header.playout_delay); } - rtp_packet.GetExtension(&video_header.frame_marking); ParseGenericDependenciesResult generic_descriptor_state = ParseGenericDependenciesExtension(rtp_packet, &video_header);