From 6cf46b949755e113e21b49928e8929efa62da3c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Wed, 11 Jan 2023 15:55:53 +0100 Subject: [PATCH] Add RTPVideoHeader::SetFromMetadata() and FromMetadata(). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is now ready for plumbing to Chromium layers. Once it's exposed in JavaScript (behind flag!) we can evaluate whether all of this information is really needed or if the information is superflous (e.g. already contained in the raw bytes). Bug: webrtc:14709 Change-Id: I3837ef86046704a300ec8a108c8c9477bd91b9ce Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/290884 Reviewed-by: Harald Alvestrand Commit-Queue: Henrik Boström Cr-Commit-Position: refs/heads/main@{#39102} --- modules/rtp_rtcp/source/rtp_video_header.cc | 49 +++++ modules/rtp_rtcp/source/rtp_video_header.h | 3 + .../source/rtp_video_header_unittest.cc | 193 +++++++++++++++--- 3 files changed, 221 insertions(+), 24 deletions(-) diff --git a/modules/rtp_rtcp/source/rtp_video_header.cc b/modules/rtp_rtcp/source/rtp_video_header.cc index 8351c8c2ab..b07a7beec4 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.cc +++ b/modules/rtp_rtcp/source/rtp_video_header.cc @@ -17,6 +17,14 @@ RTPVideoHeader::GenericDescriptorInfo::GenericDescriptorInfo( const GenericDescriptorInfo& other) = default; RTPVideoHeader::GenericDescriptorInfo::~GenericDescriptorInfo() = default; +// static +RTPVideoHeader RTPVideoHeader::FromMetadata( + const VideoFrameMetadata& metadata) { + RTPVideoHeader rtp_video_header; + rtp_video_header.SetFromMetadata(metadata); + return rtp_video_header; +} + RTPVideoHeader::RTPVideoHeader() : video_timing() {} RTPVideoHeader::RTPVideoHeader(const RTPVideoHeader& other) = default; RTPVideoHeader::~RTPVideoHeader() = default; @@ -58,4 +66,45 @@ VideoFrameMetadata RTPVideoHeader::GetAsMetadata() const { return metadata; } +void RTPVideoHeader::SetFromMetadata(const VideoFrameMetadata& metadata) { + frame_type = metadata.GetFrameType(); + width = metadata.GetWidth(); + height = metadata.GetHeight(); + rotation = metadata.GetRotation(); + content_type = metadata.GetContentType(); + if (!metadata.GetFrameId().has_value()) { + generic = absl::nullopt; + } else { + generic.emplace(); + generic->frame_id = metadata.GetFrameId().value(); + generic->spatial_index = metadata.GetSpatialIndex(); + generic->temporal_index = metadata.GetTemporalIndex(); + generic->dependencies.assign(metadata.GetFrameDependencies().begin(), + metadata.GetFrameDependencies().end()); + generic->decode_target_indications.assign( + metadata.GetDecodeTargetIndications().begin(), + metadata.GetDecodeTargetIndications().end()); + } + is_last_frame_in_picture = metadata.GetIsLastFrameInPicture(); + simulcastIdx = metadata.GetSimulcastIdx(); + codec = metadata.GetCodec(); + switch (codec) { + case VideoCodecType::kVideoCodecVP8: + video_type_header = absl::get( + metadata.GetRTPVideoHeaderCodecSpecifics()); + break; + case VideoCodecType::kVideoCodecVP9: + video_type_header = absl::get( + metadata.GetRTPVideoHeaderCodecSpecifics()); + break; + case VideoCodecType::kVideoCodecH264: + video_type_header = absl::get( + metadata.GetRTPVideoHeaderCodecSpecifics()); + break; + default: + // Codec-specifics are not supported for this codec. + break; + } +} + } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h index c65bde9fab..d185c20c83 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.h +++ b/modules/rtp_rtcp/source/rtp_video_header.h @@ -58,6 +58,8 @@ struct RTPVideoHeader { std::bitset<32> active_decode_targets = ~uint32_t{0}; }; + static RTPVideoHeader FromMetadata(const VideoFrameMetadata& metadata); + RTPVideoHeader(); RTPVideoHeader(const RTPVideoHeader& other); @@ -65,6 +67,7 @@ struct RTPVideoHeader { // The subset of RTPVideoHeader that is exposed in the Insertable Streams API. VideoFrameMetadata GetAsMetadata() const; + void SetFromMetadata(const VideoFrameMetadata& metadata); absl::optional generic; diff --git a/modules/rtp_rtcp/source/rtp_video_header_unittest.cc b/modules/rtp_rtcp/source/rtp_video_header_unittest.cc index e6e5b48cc5..335fa1a8a0 100644 --- a/modules/rtp_rtcp/source/rtp_video_header_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_video_header_unittest.cc @@ -21,42 +21,77 @@ namespace { using ::testing::ElementsAre; using ::testing::IsEmpty; -TEST(RTPVideoHeaderTest, GetAsMetadataGetFrameType) { +TEST(RTPVideoHeaderTest, FrameType_GetAsMetadata) { RTPVideoHeader video_header; video_header.frame_type = VideoFrameType::kVideoFrameKey; VideoFrameMetadata metadata = video_header.GetAsMetadata(); EXPECT_EQ(metadata.GetFrameType(), VideoFrameType::kVideoFrameKey); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetWidth) { +TEST(RTPVideoHeaderTest, FrameType_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetFrameType(VideoFrameType::kVideoFrameKey); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ(video_header.frame_type, VideoFrameType::kVideoFrameKey); +} + +TEST(RTPVideoHeaderTest, Width_GetAsMetadata) { RTPVideoHeader video_header; video_header.width = 1280u; VideoFrameMetadata metadata = video_header.GetAsMetadata(); - EXPECT_EQ(metadata.GetWidth(), video_header.width); + EXPECT_EQ(metadata.GetWidth(), 1280u); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetHeight) { +TEST(RTPVideoHeaderTest, Width_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetWidth(1280u); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ(video_header.width, 1280u); +} + +TEST(RTPVideoHeaderTest, Height_GetAsMetadata) { RTPVideoHeader video_header; video_header.height = 720u; VideoFrameMetadata metadata = video_header.GetAsMetadata(); - EXPECT_EQ(metadata.GetHeight(), video_header.height); + EXPECT_EQ(metadata.GetHeight(), 720u); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetRotation) { +TEST(RTPVideoHeaderTest, Height_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetHeight(720u); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ(video_header.height, 720u); +} + +TEST(RTPVideoHeaderTest, Rotation_GetAsMetadata) { RTPVideoHeader video_header; video_header.rotation = VideoRotation::kVideoRotation_90; VideoFrameMetadata metadata = video_header.GetAsMetadata(); EXPECT_EQ(metadata.GetRotation(), VideoRotation::kVideoRotation_90); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetContentType) { +TEST(RTPVideoHeaderTest, Rotation_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetRotation(VideoRotation::kVideoRotation_90); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ(video_header.rotation, VideoRotation::kVideoRotation_90); +} + +TEST(RTPVideoHeaderTest, ContentType_GetAsMetadata) { RTPVideoHeader video_header; video_header.content_type = VideoContentType::SCREENSHARE; VideoFrameMetadata metadata = video_header.GetAsMetadata(); EXPECT_EQ(metadata.GetContentType(), VideoContentType::SCREENSHARE); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetFrameId) { +TEST(RTPVideoHeaderTest, ContentType_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetContentType(VideoContentType::SCREENSHARE); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ(video_header.content_type, VideoContentType::SCREENSHARE); +} + +TEST(RTPVideoHeaderTest, FrameId_GetAsMetadata) { RTPVideoHeader video_header; RTPVideoHeader::GenericDescriptorInfo& generic = video_header.generic.emplace(); @@ -65,14 +100,29 @@ TEST(RTPVideoHeaderTest, GetAsMetadataGetFrameId) { EXPECT_EQ(metadata.GetFrameId().value(), 10); } -TEST(RTPVideoHeaderTest, GetAsMetadataHasNoFrameIdForHeaderWithoutGeneric) { +TEST(RTPVideoHeaderTest, FrameId_GetAsMetadataWhenGenericIsMissing) { RTPVideoHeader video_header; VideoFrameMetadata metadata = video_header.GetAsMetadata(); ASSERT_FALSE(video_header.generic); EXPECT_FALSE(metadata.GetFrameId().has_value()); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetSpatialIndex) { +TEST(RTPVideoHeaderTest, FrameId_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetFrameId(10); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_TRUE(video_header.generic.has_value()); + EXPECT_EQ(video_header.generic->frame_id, 10); +} + +TEST(RTPVideoHeaderTest, FrameId_FromMetadataWhenFrameIdIsMissing) { + VideoFrameMetadata metadata; + metadata.SetFrameId(absl::nullopt); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_FALSE(video_header.generic.has_value()); +} + +TEST(RTPVideoHeaderTest, SpatialIndex_GetAsMetadata) { RTPVideoHeader video_header; RTPVideoHeader::GenericDescriptorInfo& generic = video_header.generic.emplace(); @@ -81,15 +131,23 @@ TEST(RTPVideoHeaderTest, GetAsMetadataGetSpatialIndex) { EXPECT_EQ(metadata.GetSpatialIndex(), 2); } -TEST(RTPVideoHeaderTest, - GetAsMetadataSpatialIndexIsZeroForHeaderWithoutGeneric) { +TEST(RTPVideoHeaderTest, SpatialIndex_GetAsMetadataWhenGenericIsMissing) { RTPVideoHeader video_header; VideoFrameMetadata metadata = video_header.GetAsMetadata(); ASSERT_FALSE(video_header.generic); EXPECT_EQ(metadata.GetSpatialIndex(), 0); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetTemporalIndex) { +TEST(RTPVideoHeaderTest, SpatialIndex_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetFrameId(123); // Must have a frame ID for related properties. + metadata.SetSpatialIndex(2); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_TRUE(video_header.generic.has_value()); + EXPECT_EQ(video_header.generic->spatial_index, 2); +} + +TEST(RTPVideoHeaderTest, TemporalIndex_GetAsMetadata) { RTPVideoHeader video_header; RTPVideoHeader::GenericDescriptorInfo& generic = video_header.generic.emplace(); @@ -98,15 +156,23 @@ TEST(RTPVideoHeaderTest, GetAsMetadataGetTemporalIndex) { EXPECT_EQ(metadata.GetTemporalIndex(), 3); } -TEST(RTPVideoHeaderTest, - GetAsMetadataTemporalIndexIsZeroForHeaderWithoutGeneric) { +TEST(RTPVideoHeaderTest, TemporalIndex_GetAsMetadataWhenGenericIsMissing) { RTPVideoHeader video_header; VideoFrameMetadata metadata = video_header.GetAsMetadata(); ASSERT_FALSE(video_header.generic); EXPECT_EQ(metadata.GetTemporalIndex(), 0); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetFrameDependencies) { +TEST(RTPVideoHeaderTest, TemporalIndex_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetFrameId(123); // Must have a frame ID for related properties. + metadata.SetTemporalIndex(3); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_TRUE(video_header.generic.has_value()); + EXPECT_EQ(video_header.generic->temporal_index, 3); +} + +TEST(RTPVideoHeaderTest, FrameDependencies_GetAsMetadata) { RTPVideoHeader video_header; RTPVideoHeader::GenericDescriptorInfo& generic = video_header.generic.emplace(); @@ -115,15 +181,24 @@ TEST(RTPVideoHeaderTest, GetAsMetadataGetFrameDependencies) { EXPECT_THAT(metadata.GetFrameDependencies(), ElementsAre(5, 6, 7)); } -TEST(RTPVideoHeaderTest, - GetAsMetadataFrameDependencyIsEmptyForHeaderWithoutGeneric) { +TEST(RTPVideoHeaderTest, FrameDependency_GetAsMetadataWhenGenericIsMissing) { RTPVideoHeader video_header; VideoFrameMetadata metadata = video_header.GetAsMetadata(); ASSERT_FALSE(video_header.generic); EXPECT_THAT(metadata.GetFrameDependencies(), IsEmpty()); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetDecodeTargetIndications) { +TEST(RTPVideoHeaderTest, FrameDependencies_FromMetadata) { + VideoFrameMetadata metadata; + absl::InlinedVector dependencies = {5, 6, 7}; + metadata.SetFrameId(123); // Must have a frame ID for related properties. + metadata.SetFrameDependencies(dependencies); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_TRUE(video_header.generic.has_value()); + EXPECT_THAT(video_header.generic->dependencies, ElementsAre(5, 6, 7)); +} + +TEST(RTPVideoHeaderTest, DecodeTargetIndications_GetAsMetadata) { RTPVideoHeader video_header; RTPVideoHeader::GenericDescriptorInfo& generic = video_header.generic.emplace(); @@ -134,28 +209,54 @@ TEST(RTPVideoHeaderTest, GetAsMetadataGetDecodeTargetIndications) { } TEST(RTPVideoHeaderTest, - GetAsMetadataGetDecodeTargetIndicationsIsEmptyForHeaderWithoutGeneric) { + DecodeTargetIndications_GetAsMetadataWhenGenericIsMissing) { RTPVideoHeader video_header; VideoFrameMetadata metadata = video_header.GetAsMetadata(); ASSERT_FALSE(video_header.generic); EXPECT_THAT(metadata.GetDecodeTargetIndications(), IsEmpty()); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetIsLastFrameInPicture) { +TEST(RTPVideoHeaderTest, DecodeTargetIndications_FromMetadata) { + VideoFrameMetadata metadata; + absl::InlinedVector decode_target_indications = { + DecodeTargetIndication::kSwitch}; + metadata.SetFrameId(123); // Must have a frame ID for related properties. + metadata.SetDecodeTargetIndications(decode_target_indications); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_TRUE(video_header.generic.has_value()); + EXPECT_THAT(video_header.generic->decode_target_indications, + ElementsAre(DecodeTargetIndication::kSwitch)); +} + +TEST(RTPVideoHeaderTest, IsLastFrameInPicture_GetAsMetadata) { RTPVideoHeader video_header; video_header.is_last_frame_in_picture = false; VideoFrameMetadata metadata = video_header.GetAsMetadata(); EXPECT_FALSE(metadata.GetIsLastFrameInPicture()); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetSimulcastIdx) { +TEST(RTPVideoHeaderTest, IsLastFrameInPicture_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetIsLastFrameInPicture(false); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_FALSE(video_header.is_last_frame_in_picture); +} + +TEST(RTPVideoHeaderTest, SimulcastIdx_GetAsMetadata) { RTPVideoHeader video_header; video_header.simulcastIdx = 123; VideoFrameMetadata metadata = video_header.GetAsMetadata(); EXPECT_EQ(metadata.GetSimulcastIdx(), 123); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetCodec) { +TEST(RTPVideoHeaderTest, SimulcastIdx_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetSimulcastIdx(123); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ(video_header.simulcastIdx, 123); +} + +TEST(RTPVideoHeaderTest, Codec_GetAsMetadata) { RTPVideoHeader video_header; video_header.codec = VideoCodecType::kVideoCodecVP9; video_header.video_type_header = RTPVideoHeaderVP9(); @@ -163,7 +264,15 @@ TEST(RTPVideoHeaderTest, GetAsMetadataGetCodec) { EXPECT_EQ(metadata.GetCodec(), VideoCodecType::kVideoCodecVP9); } -TEST(RTPVideoHeaderTest, GetAsMetadataGetRTPVideoHeaderCodecSpecifics) { +TEST(RTPVideoHeaderTest, Codec_FromMetadata) { + VideoFrameMetadata metadata; + metadata.SetCodec(VideoCodecType::kVideoCodecVP9); + metadata.SetRTPVideoHeaderCodecSpecifics(RTPVideoHeaderVP9()); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ(video_header.codec, VideoCodecType::kVideoCodecVP9); +} + +TEST(RTPVideoHeaderTest, RTPVideoHeaderCodecSpecifics_GetAsMetadata) { RTPVideoHeader video_header; { video_header.codec = VideoCodecType::kVideoCodecVP8; @@ -202,5 +311,41 @@ TEST(RTPVideoHeaderTest, GetAsMetadataGetRTPVideoHeaderCodecSpecifics) { } } +TEST(RTPVideoHeaderTest, RTPVideoHeaderCodecSpecifics_FromMetadata) { + VideoFrameMetadata metadata; + { + metadata.SetCodec(VideoCodecType::kVideoCodecVP8); + RTPVideoHeaderVP8 vp8_specifics; + vp8_specifics.InitRTPVideoHeaderVP8(); + vp8_specifics.pictureId = 42; + metadata.SetRTPVideoHeaderCodecSpecifics(vp8_specifics); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ( + absl::get(video_header.video_type_header).pictureId, + 42); + } + { + metadata.SetCodec(VideoCodecType::kVideoCodecVP9); + RTPVideoHeaderVP9 vp9_specifics; + vp9_specifics.InitRTPVideoHeaderVP9(); + vp9_specifics.max_picture_id = 42; + metadata.SetRTPVideoHeaderCodecSpecifics(vp9_specifics); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ(absl::get(video_header.video_type_header) + .max_picture_id, + 42); + } + { + metadata.SetCodec(VideoCodecType::kVideoCodecH264); + RTPVideoHeaderH264 h264_specifics; + h264_specifics.nalu_type = 42; + metadata.SetRTPVideoHeaderCodecSpecifics(h264_specifics); + RTPVideoHeader video_header = RTPVideoHeader::FromMetadata(metadata); + EXPECT_EQ( + absl::get(video_header.video_type_header).nalu_type, + 42); + } +} + } // namespace } // namespace webrtc