From ad1d9f0d7825acc2b86aacf875449d6cd2449045 Mon Sep 17 00:00:00 2001 From: Johannes Kron Date: Fri, 9 Nov 2018 11:12:36 +0100 Subject: [PATCH] Add RTP header extension for HDR metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:8651 Change-Id: I1c956eaac1532ac0d3820084edb4054a4cc9c68d Reviewed-on: https://webrtc-review.googlesource.com/c/109924 Commit-Queue: Johannes Kron Reviewed-by: Karl Wiberg Reviewed-by: Åsa Persson Reviewed-by: Alex Loiko Cr-Commit-Position: refs/heads/master@{#25578} --- api/rtp_headers.h | 4 + modules/rtp_rtcp/include/rtp_rtcp_defines.h | 1 + .../source/rtp_header_extension_map.cc | 1 + .../rtp_rtcp/source/rtp_header_extensions.cc | 130 ++++++++++++++++++ .../rtp_rtcp/source/rtp_header_extensions.h | 27 ++++ .../rtp_rtcp/source/rtp_packet_received.cc | 1 + .../rtp_rtcp/source/rtp_packet_unittest.cc | 35 +++++ modules/rtp_rtcp/source/rtp_utility.cc | 4 + test/fuzzers/rtp_packet_fuzzer.cc | 5 + 9 files changed, 208 insertions(+) diff --git a/api/rtp_headers.h b/api/rtp_headers.h index eff6223686..3e51f43591 100644 --- a/api/rtp_headers.h +++ b/api/rtp_headers.h @@ -15,7 +15,9 @@ #include #include +#include "absl/types/optional.h" #include "api/array_view.h" +#include "api/video/hdr_metadata.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" @@ -126,6 +128,8 @@ struct RTPHeaderExtension { // For identifying the media section used to interpret this RTP packet. See // https://tools.ietf.org/html/draft-ietf-mmusic-sdp-bundle-negotiation-38 Mid mid; + + absl::optional hdr_metadata; }; struct RTPHeader { diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 5b9a051525..138ed0a661 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -112,6 +112,7 @@ enum RTPExtensionType : int { kRtpExtensionRepairedRtpStreamId, kRtpExtensionMid, kRtpExtensionGenericFrameDescriptor, + kRtpExtensionHdrMetadata, kRtpExtensionNumberOfExtensions // Must be the last entity in the enum. }; diff --git a/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/modules/rtp_rtcp/source/rtp_header_extension_map.cc index dde25e324e..49857a0338 100644 --- a/modules/rtp_rtcp/source/rtp_header_extension_map.cc +++ b/modules/rtp_rtcp/source/rtp_header_extension_map.cc @@ -43,6 +43,7 @@ constexpr ExtensionInfo kExtensions[] = { CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), + CreateExtensionInfo(), }; // Because of kRtpExtensionNone, NumberOfExtension is 1 bigger than the actual diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.cc b/modules/rtp_rtcp/source/rtp_header_extensions.cc index 082e0e0d61..fe327b3011 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.cc +++ b/modules/rtp_rtcp/source/rtp_header_extensions.cc @@ -11,6 +11,7 @@ #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include +#include #include "modules/rtp_rtcp/include/rtp_cvo.h" #include "modules/rtp_rtcp/source/byte_io.h" @@ -433,6 +434,135 @@ bool FrameMarkingExtension::Write(rtc::ArrayView data, return true; } +// HDR Metadata. +// +// RTP header extension to carry HDR metadata. +// Float values are upscaled by a static factor and transmitted as integers. +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | length | luminance_max | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | | luminance_min | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | mastering_metadata.primary_r.x and .y | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | mastering_metadata.primary_g.x and .y | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | mastering_metadata.primary_b.x and .y | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | mastering_metadata.white.x and .y | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | max_content_light_level | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | max_frame_average_light_level | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +constexpr RTPExtensionType HdrMetadataExtension::kId; +constexpr uint8_t HdrMetadataExtension::kValueSizeBytes; +constexpr const char HdrMetadataExtension::kUri[]; + +bool HdrMetadataExtension::Parse(rtc::ArrayView data, + HdrMetadata* hdr_metadata) { + RTC_DCHECK(hdr_metadata); + if (data.size() != kValueSizeBytes) + return false; + + size_t offset = 0; + offset += ParseLuminance(data.data() + offset, + &hdr_metadata->mastering_metadata.luminance_max, + kLuminanceMaxDenominator); + offset += ParseLuminance(data.data() + offset, + &hdr_metadata->mastering_metadata.luminance_min, + kLuminanceMinDenominator); + offset += ParseChromaticity(data.data() + offset, + &hdr_metadata->mastering_metadata.primary_r); + offset += ParseChromaticity(data.data() + offset, + &hdr_metadata->mastering_metadata.primary_g); + offset += ParseChromaticity(data.data() + offset, + &hdr_metadata->mastering_metadata.primary_b); + offset += ParseChromaticity(data.data() + offset, + &hdr_metadata->mastering_metadata.white_point); + // TODO(kron): Do we need 32 bit here or is it enough with 16 bits? + // Also, what resolution is needed? + hdr_metadata->max_content_light_level = + ByteReader::ReadBigEndian(data.data() + offset); + offset += 4; + hdr_metadata->max_frame_average_light_level = + ByteReader::ReadBigEndian(data.data() + offset); + RTC_DCHECK_EQ(kValueSizeBytes, offset + 4); + return true; +} + +bool HdrMetadataExtension::Write(rtc::ArrayView data, + const HdrMetadata& hdr_metadata) { + RTC_DCHECK_EQ(data.size(), kValueSizeBytes); + size_t offset = 0; + offset += WriteLuminance(data.data() + offset, + hdr_metadata.mastering_metadata.luminance_max, + kLuminanceMaxDenominator); + offset += WriteLuminance(data.data() + offset, + hdr_metadata.mastering_metadata.luminance_min, + kLuminanceMinDenominator); + offset += WriteChromaticity(data.data() + offset, + hdr_metadata.mastering_metadata.primary_r); + offset += WriteChromaticity(data.data() + offset, + hdr_metadata.mastering_metadata.primary_g); + offset += WriteChromaticity(data.data() + offset, + hdr_metadata.mastering_metadata.primary_b); + offset += WriteChromaticity(data.data() + offset, + hdr_metadata.mastering_metadata.white_point); + + // TODO(kron): Do we need 32 bit here or is it enough with 16 bits? + // Also, what resolution is needed? + ByteWriter::WriteBigEndian(data.data() + offset, + hdr_metadata.max_content_light_level); + offset += 4; + ByteWriter::WriteBigEndian( + data.data() + offset, hdr_metadata.max_frame_average_light_level); + RTC_DCHECK_EQ(kValueSizeBytes, offset + 4); + return true; +} + +size_t HdrMetadataExtension::ParseChromaticity( + const uint8_t* data, + HdrMasteringMetadata::Chromaticity* p) { + uint16_t chromaticity_x_scaled = ByteReader::ReadBigEndian(data); + uint16_t chromaticity_y_scaled = + ByteReader::ReadBigEndian(data + 2); + p->x = static_cast(chromaticity_x_scaled) / kChromaticityDenominator; + p->y = static_cast(chromaticity_y_scaled) / kChromaticityDenominator; + return 4; // Return number of bytes read. +} + +size_t HdrMetadataExtension::ParseLuminance(const uint8_t* data, + float* f, + int denominator) { + uint32_t luminance_scaled = ByteReader::ReadBigEndian(data); + *f = static_cast(luminance_scaled) / denominator; + return 3; // Return number of bytes read. +} + +size_t HdrMetadataExtension::WriteChromaticity( + uint8_t* data, + const HdrMasteringMetadata::Chromaticity& p) { + RTC_DCHECK_GE(p.x, 0.0f); + RTC_DCHECK_GE(p.y, 0.0f); + ByteWriter::WriteBigEndian( + data, std::round(p.x * kChromaticityDenominator)); + ByteWriter::WriteBigEndian( + data + 2, std::round(p.y * kChromaticityDenominator)); + return 4; // Return number of bytes written. +} + +size_t HdrMetadataExtension::WriteLuminance(uint8_t* data, + float f, + int denominator) { + RTC_DCHECK_GE(f, 0.0f); + ByteWriter::WriteBigEndian(data, std::round(f * denominator)); + return 3; // Return number of bytes written. +} + bool BaseRtpStringExtension::Parse(rtc::ArrayView data, StringRtpHeaderExtension* str) { if (data.empty() || data[0] == 0) // Valid string extension can't be empty. diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h index 808356a5b8..ba43415fba 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -16,6 +16,7 @@ #include "api/array_view.h" #include "api/rtp_headers.h" +#include "api/video/hdr_metadata.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" @@ -181,6 +182,32 @@ class FrameMarkingExtension { static bool IsScalable(uint8_t temporal_id, uint8_t layer_id); }; +class HdrMetadataExtension { + public: + using value_type = HdrMetadata; + static constexpr RTPExtensionType kId = kRtpExtensionHdrMetadata; + static constexpr uint8_t kValueSizeBytes = 30; + // TODO(webrtc:8651): Change to a valid uri. + static constexpr const char kUri[] = "rtp-hdr-metadata-uri-placeholder"; + + static bool Parse(rtc::ArrayView data, + HdrMetadata* hdr_metadata); + static size_t ValueSize(const HdrMetadata&) { return kValueSizeBytes; } + static bool Write(rtc::ArrayView data, + const HdrMetadata& hdr_metadata); + + private: + static constexpr int kChromaticityDenominator = 10000; // 0.0001 resolution. + static constexpr int kLuminanceMaxDenominator = 100; // 0.01 resolution. + static constexpr int kLuminanceMinDenominator = 10000; // 0.0001 resolution. + static size_t ParseChromaticity(const uint8_t* data, + HdrMasteringMetadata::Chromaticity* p); + static size_t ParseLuminance(const uint8_t* data, float* f, int denominator); + static size_t WriteChromaticity(uint8_t* data, + const HdrMasteringMetadata::Chromaticity& p); + static size_t WriteLuminance(uint8_t* data, float f, int denominator); +}; + // Base extension class for RTP header extensions which are strings. // Subclasses must defined kId and kUri static constexpr members. class BaseRtpStringExtension { diff --git a/modules/rtp_rtcp/source/rtp_packet_received.cc b/modules/rtp_rtcp/source/rtp_packet_received.cc index 93c0a1e530..ff7b4e8719 100644 --- a/modules/rtp_rtcp/source/rtp_packet_received.cc +++ b/modules/rtp_rtcp/source/rtp_packet_received.cc @@ -69,6 +69,7 @@ void RtpPacketReceived::GetHeader(RTPHeader* header) const { GetExtension(&header->extension.repaired_stream_id); GetExtension(&header->extension.mid); GetExtension(&header->extension.playout_delay); + header->extension.hdr_metadata = GetExtension(); } } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_packet_unittest.cc b/modules/rtp_rtcp/source/rtp_packet_unittest.cc index b485df62d6..37a9a531de 100644 --- a/modules/rtp_rtcp/source/rtp_packet_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_packet_unittest.cc @@ -185,6 +185,24 @@ constexpr uint8_t kPacketWithLegacyTimingExtension[] = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; // clang-format on + +HdrMetadata CreateTestHdrMetadata() { + // Random but reasonable HDR metadata. + HdrMetadata hdr_metadata; + hdr_metadata.mastering_metadata.luminance_max = 2000.0; + hdr_metadata.mastering_metadata.luminance_min = 2.0001; + hdr_metadata.mastering_metadata.primary_r.x = 0.3003; + hdr_metadata.mastering_metadata.primary_r.y = 0.4004; + hdr_metadata.mastering_metadata.primary_g.x = 0.3201; + hdr_metadata.mastering_metadata.primary_g.y = 0.4604; + hdr_metadata.mastering_metadata.primary_b.x = 0.3409; + hdr_metadata.mastering_metadata.primary_b.y = 0.4907; + hdr_metadata.mastering_metadata.white_point.x = 0.4103; + hdr_metadata.mastering_metadata.white_point.y = 0.4806; + hdr_metadata.max_content_light_level = 2345; + hdr_metadata.max_frame_average_light_level = 1789; + return hdr_metadata; +} } // namespace TEST(RtpPacketTest, CreateMinimum) { @@ -801,4 +819,21 @@ TEST(RtpPacketTest, ParseLegacyTimingFrameExtension) { EXPECT_EQ(receivied_timing.flags, 0); } +TEST(RtpPacketTest, CreateAndParseHdrMetadataExtension) { + // Create packet with extension. + RtpPacket::ExtensionManager extensions(/*extmap-allow-mixed=*/true); + extensions.Register(1); + RtpPacket packet(&extensions); + const HdrMetadata kHdrMetadata = CreateTestHdrMetadata(); + EXPECT_TRUE(packet.SetExtension(kHdrMetadata)); + packet.SetPayloadSize(42); + + // Read packet with the extension. + RtpPacketReceived parsed(&extensions); + EXPECT_TRUE(parsed.Parse(packet.Buffer())); + HdrMetadata parsed_hdr_metadata; + EXPECT_TRUE(parsed.GetExtension(&parsed_hdr_metadata)); + EXPECT_EQ(kHdrMetadata, parsed_hdr_metadata); +} + } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_utility.cc b/modules/rtp_rtcp/source/rtp_utility.cc index 53a006d4f9..80f0eab682 100644 --- a/modules/rtp_rtcp/source/rtp_utility.cc +++ b/modules/rtp_rtcp/source/rtp_utility.cc @@ -507,6 +507,10 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( RTC_LOG(WARNING) << "RtpGenericFrameDescriptor unsupported by rtp header parser."; break; + case kRtpExtensionHdrMetadata: + RTC_LOG(WARNING) + << "RtpExtensionHdrMetadata unsupported by rtp header parser."; + break; case kRtpExtensionNone: case kRtpExtensionNumberOfExtensions: { RTC_NOTREACHED() << "Invalid extension type: " << type; diff --git a/test/fuzzers/rtp_packet_fuzzer.cc b/test/fuzzers/rtp_packet_fuzzer.cc index 8bf8e74aba..469fb364d2 100644 --- a/test/fuzzers/rtp_packet_fuzzer.cc +++ b/test/fuzzers/rtp_packet_fuzzer.cc @@ -121,6 +121,11 @@ void FuzzOneInput(const uint8_t* data, size_t size) { packet.GetExtension(&descriptor); break; } + case kRtpExtensionHdrMetadata: { + HdrMetadata hdr_metadata; + packet.GetExtension(&hdr_metadata); + break; + } } } }