From 8aba8fe8516b91af99c46039957a17115ff0bcb4 Mon Sep 17 00:00:00 2001 From: philipel Date: Thu, 13 Jun 2019 15:13:16 +0200 Subject: [PATCH] Reland "Populate the GFD-00 for H264 and generic codecs." This is a reland of d3c6f9ccffe88749fde8bc1320baa1fe2db15b6b Original change's description: > Populate the GFD-00 for H264 and generic codecs. > > Bug: none > Change-Id: I368eb38740314280db87aaf8e179e9bd0fc20c3c > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/103502 > Commit-Queue: Philip Eliasson > Reviewed-by: Niels Moller > Cr-Commit-Position: refs/heads/master@{#28272} Bug: none Change-Id: Ic02590e5328783969d5480a8f413986ef7055f8e Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/142168 Reviewed-by: Philip Eliasson Reviewed-by: Niels Moller Commit-Queue: Philip Eliasson Cr-Commit-Position: refs/heads/master@{#28289} --- call/rtp_payload_params.cc | 81 +++++++++++++++++- call/rtp_payload_params.h | 9 ++ call/rtp_payload_params_unittest.cc | 127 ++++++++++++++++++++++++++++ 3 files changed, 215 insertions(+), 2 deletions(-) diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index 84ca65f395..599012314a 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -256,7 +256,7 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info, RTPVideoHeader* rtp_video_header) { switch (rtp_video_header->codec) { case VideoCodecType::kVideoCodecGeneric: - // TODO(philipel): Implement generic codec to new generic descriptor. + GenericToGeneric(frame_id, is_keyframe, rtp_video_header); return; case VideoCodecType::kVideoCodecVP8: if (codec_specific_info) { @@ -268,13 +268,90 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info, // TODO(philipel): Implement VP9 to new generic descriptor. return; case VideoCodecType::kVideoCodecH264: - // TODO(philipel): Implement H264 to new generic descriptor. + if (codec_specific_info) { + H264ToGeneric(codec_specific_info->codecSpecific.H264, frame_id, + is_keyframe, rtp_video_header); + } + return; case VideoCodecType::kVideoCodecMultiplex: return; } RTC_NOTREACHED() << "Unsupported codec."; } +void RtpPayloadParams::GenericToGeneric(int64_t shared_frame_id, + bool is_keyframe, + RTPVideoHeader* rtp_video_header) { + RTPVideoHeader::GenericDescriptorInfo& generic = + rtp_video_header->generic.emplace(); + + generic.frame_id = shared_frame_id; + + if (is_keyframe) { + last_shared_frame_id_[0].fill(-1); + } else { + int64_t frame_id = last_shared_frame_id_[0][0]; + RTC_DCHECK_NE(frame_id, -1); + RTC_DCHECK_LT(frame_id, shared_frame_id); + generic.dependencies.push_back(frame_id); + } + + last_shared_frame_id_[0][0] = shared_frame_id; +} + +void RtpPayloadParams::H264ToGeneric(const CodecSpecificInfoH264& h264_info, + int64_t shared_frame_id, + bool is_keyframe, + RTPVideoHeader* rtp_video_header) { + const int temporal_index = + h264_info.temporal_idx != kNoTemporalIdx ? h264_info.temporal_idx : 0; + + if (temporal_index >= RtpGenericFrameDescriptor::kMaxTemporalLayers) { + RTC_LOG(LS_WARNING) << "Temporal and/or spatial index is too high to be " + "used with generic frame descriptor."; + return; + } + + RTPVideoHeader::GenericDescriptorInfo& generic = + rtp_video_header->generic.emplace(); + + generic.frame_id = shared_frame_id; + generic.temporal_index = temporal_index; + + if (is_keyframe) { + RTC_DCHECK_EQ(temporal_index, 0); + last_shared_frame_id_[/*spatial index*/ 0].fill(-1); + last_shared_frame_id_[/*spatial index*/ 0][temporal_index] = + shared_frame_id; + return; + } + + if (h264_info.base_layer_sync) { + int64_t tl0_frame_id = last_shared_frame_id_[/*spatial index*/ 0][0]; + + for (int i = 1; i < RtpGenericFrameDescriptor::kMaxTemporalLayers; ++i) { + if (last_shared_frame_id_[/*spatial index*/ 0][i] < tl0_frame_id) { + last_shared_frame_id_[/*spatial index*/ 0][i] = -1; + } + } + + RTC_DCHECK_GE(tl0_frame_id, 0); + RTC_DCHECK_LT(tl0_frame_id, shared_frame_id); + generic.dependencies.push_back(tl0_frame_id); + } else { + for (int i = 0; i <= temporal_index; ++i) { + int64_t frame_id = last_shared_frame_id_[/*spatial index*/ 0][i]; + + if (frame_id != -1) { + RTC_DCHECK_LT(frame_id, shared_frame_id); + generic.dependencies.push_back(frame_id); + } + } + } + + last_shared_frame_id_[/*spatial_index*/ 0][temporal_index] = shared_frame_id; +} + void RtpPayloadParams::Vp8ToGeneric(const CodecSpecificInfoVP8& vp8_info, int64_t shared_frame_id, bool is_keyframe, diff --git a/call/rtp_payload_params.h b/call/rtp_payload_params.h index 3cbbb1ba69..b012398518 100644 --- a/call/rtp_payload_params.h +++ b/call/rtp_payload_params.h @@ -53,6 +53,15 @@ class RtpPayloadParams final { bool is_keyframe, RTPVideoHeader* rtp_video_header); + void H264ToGeneric(const CodecSpecificInfoH264& h264_info, + int64_t shared_frame_id, + bool is_keyframe, + RTPVideoHeader* rtp_video_header); + + void GenericToGeneric(int64_t shared_frame_id, + bool is_keyframe, + RTPVideoHeader* rtp_video_header); + // TODO(bugs.webrtc.org/10242): Delete SetDependenciesVp8Deprecated() and move // the logic in SetDependenciesVp8New() into Vp8ToGeneric() once all hardware // wrappers have been updated. diff --git a/call/rtp_payload_params_unittest.cc b/call/rtp_payload_params_unittest.cc index 6216034b8a..00003484a9 100644 --- a/call/rtp_payload_params_unittest.cc +++ b/call/rtp_payload_params_unittest.cc @@ -24,8 +24,12 @@ #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/include/video_codec_interface.h" #include "test/field_trial.h" +#include "test/gmock.h" #include "test/gtest.h" +using ::testing::ElementsAre; +using ::testing::IsEmpty; + namespace webrtc { namespace { const uint32_t kSsrc1 = 12345; @@ -335,6 +339,32 @@ TEST(RtpPayloadParamsTest, PictureIdForOldGenericFormat) { EXPECT_EQ(1, header.generic->frame_id); } +TEST(RtpPayloadParamsTest, GenericDescriptorForGenericCodec) { + test::ScopedFieldTrials generic_picture_id( + "WebRTC-GenericDescriptor/Enabled/"); + RtpPayloadState state{}; + + EncodedImage encoded_image; + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + CodecSpecificInfo codec_info; + codec_info.codecType = kVideoCodecGeneric; + + RtpPayloadParams params(kSsrc1, &state); + RTPVideoHeader header = + params.GetRtpVideoHeader(encoded_image, &codec_info, 0); + + EXPECT_EQ(kVideoCodecGeneric, header.codec); + ASSERT_TRUE(header.generic); + EXPECT_EQ(0, header.generic->frame_id); + EXPECT_THAT(header.generic->dependencies, IsEmpty()); + + encoded_image._frameType = VideoFrameType::kVideoFrameDelta; + header = params.GetRtpVideoHeader(encoded_image, &codec_info, 1); + ASSERT_TRUE(header.generic); + EXPECT_EQ(1, header.generic->frame_id); + EXPECT_THAT(header.generic->dependencies, ElementsAre(0)); +} + class RtpPayloadParamsVp8ToGenericTest : public ::testing::Test { public: enum LayerSync { kNoSync, kSync }; @@ -432,4 +462,101 @@ TEST_F(RtpPayloadParamsVp8ToGenericTest, FrameIdGaps) { ConvertAndCheck(1, 20, VideoFrameType::kVideoFrameDelta, kNoSync, {10, 15}); } +class RtpPayloadParamsH264ToGenericTest : public ::testing::Test { + public: + enum LayerSync { kNoSync, kSync }; + + RtpPayloadParamsH264ToGenericTest() + : generic_descriptor_field_trial_("WebRTC-GenericDescriptor/Enabled/"), + state_(), + params_(123, &state_) {} + + void ConvertAndCheck(int temporal_index, + int64_t shared_frame_id, + VideoFrameType frame_type, + LayerSync layer_sync, + const std::set& expected_deps, + uint16_t width = 0, + uint16_t height = 0) { + EncodedImage encoded_image; + encoded_image._frameType = frame_type; + encoded_image._encodedWidth = width; + encoded_image._encodedHeight = height; + + CodecSpecificInfo codec_info; + codec_info.codecType = kVideoCodecH264; + codec_info.codecSpecific.H264.temporal_idx = temporal_index; + codec_info.codecSpecific.H264.base_layer_sync = layer_sync == kSync; + + RTPVideoHeader header = + params_.GetRtpVideoHeader(encoded_image, &codec_info, shared_frame_id); + + ASSERT_TRUE(header.generic); + EXPECT_TRUE(header.generic->higher_spatial_layers.empty()); + EXPECT_EQ(header.generic->spatial_index, 0); + + EXPECT_EQ(header.generic->frame_id, shared_frame_id); + EXPECT_EQ(header.generic->temporal_index, temporal_index); + std::set actual_deps(header.generic->dependencies.begin(), + header.generic->dependencies.end()); + EXPECT_EQ(expected_deps, actual_deps); + + EXPECT_EQ(header.width, width); + EXPECT_EQ(header.height, height); + } + + protected: + test::ScopedFieldTrials generic_descriptor_field_trial_; + RtpPayloadState state_; + RtpPayloadParams params_; +}; + +TEST_F(RtpPayloadParamsH264ToGenericTest, Keyframe) { + ConvertAndCheck(0, 0, VideoFrameType::kVideoFrameKey, kNoSync, {}, 480, 360); + ConvertAndCheck(0, 1, VideoFrameType::kVideoFrameDelta, kNoSync, {0}); + ConvertAndCheck(0, 2, VideoFrameType::kVideoFrameKey, kNoSync, {}, 480, 360); +} + +TEST_F(RtpPayloadParamsH264ToGenericTest, TooHighTemporalIndex) { + ConvertAndCheck(0, 0, VideoFrameType::kVideoFrameKey, kNoSync, {}, 480, 360); + + EncodedImage encoded_image; + encoded_image._frameType = VideoFrameType::kVideoFrameDelta; + CodecSpecificInfo codec_info; + codec_info.codecType = kVideoCodecH264; + codec_info.codecSpecific.H264.temporal_idx = + RtpGenericFrameDescriptor::kMaxTemporalLayers; + codec_info.codecSpecific.H264.base_layer_sync = false; + + RTPVideoHeader header = + params_.GetRtpVideoHeader(encoded_image, &codec_info, 1); + EXPECT_FALSE(header.generic); +} + +TEST_F(RtpPayloadParamsH264ToGenericTest, LayerSync) { + // 02120212 pattern + ConvertAndCheck(0, 0, VideoFrameType::kVideoFrameKey, kNoSync, {}, 480, 360); + ConvertAndCheck(2, 1, VideoFrameType::kVideoFrameDelta, kNoSync, {0}); + ConvertAndCheck(1, 2, VideoFrameType::kVideoFrameDelta, kNoSync, {0}); + ConvertAndCheck(2, 3, VideoFrameType::kVideoFrameDelta, kNoSync, {0, 1, 2}); + + ConvertAndCheck(0, 4, VideoFrameType::kVideoFrameDelta, kNoSync, {0}); + ConvertAndCheck(2, 5, VideoFrameType::kVideoFrameDelta, kNoSync, {2, 3, 4}); + ConvertAndCheck(1, 6, VideoFrameType::kVideoFrameDelta, kSync, + {4}); // layer sync + ConvertAndCheck(2, 7, VideoFrameType::kVideoFrameDelta, kNoSync, {4, 5, 6}); +} + +TEST_F(RtpPayloadParamsH264ToGenericTest, FrameIdGaps) { + // 0101 pattern + ConvertAndCheck(0, 0, VideoFrameType::kVideoFrameKey, kNoSync, {}, 480, 360); + ConvertAndCheck(1, 1, VideoFrameType::kVideoFrameDelta, kNoSync, {0}); + + ConvertAndCheck(0, 5, VideoFrameType::kVideoFrameDelta, kNoSync, {0}); + ConvertAndCheck(1, 10, VideoFrameType::kVideoFrameDelta, kNoSync, {1, 5}); + + ConvertAndCheck(0, 15, VideoFrameType::kVideoFrameDelta, kNoSync, {5}); + ConvertAndCheck(1, 20, VideoFrameType::kVideoFrameDelta, kNoSync, {10, 15}); +} + } // namespace webrtc