From b3488d08db64659365ffa38f8ea1fac6928f235f Mon Sep 17 00:00:00 2001 From: Qiu Jianlin Date: Thu, 7 Dec 2023 08:12:12 +0800 Subject: [PATCH] Add SDP negotiation support for HEVC. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds neccessary checks for SDP negotiation with HEVC. Test: Manually apply the CL on Chromium and enable HEVC HW encoder, and add HEVC profiles in rtc video decoder/encoder factory, H265 is negotiated in SDP with correct FMTP lines added. Bug: webrtc:13485 Change-Id: I5557b20b646cc96c5acb578521204fe10df0dcf0 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/330202 Reviewed-by: Henrik Boström Reviewed-by: Harald Alvestrand Commit-Queue: Jianlin Qiu Cr-Commit-Position: refs/heads/main@{#41357} --- api/video_codecs/h265_profile_tier_level.h | 5 +- media/base/codec.cc | 27 +++++++ media/base/codec.h | 7 +- media/base/codec_unittest.cc | 61 +++++++++++++++ media/base/sdp_video_format_utils.cc | 59 +++++++++++++++ media/base/sdp_video_format_utils.h | 12 +++ media/base/sdp_video_format_utils_unittest.cc | 31 ++++++++ .../internal_decoder_factory_unittest.cc | 10 +++ .../internal_encoder_factory_unittest.cc | 13 ++++ pc/media_session.cc | 17 +++++ pc/media_session_unittest.cc | 74 +++++++++++++++++++ 11 files changed, 313 insertions(+), 3 deletions(-) diff --git a/api/video_codecs/h265_profile_tier_level.h b/api/video_codecs/h265_profile_tier_level.h index 3056d2b623..c757e54af1 100644 --- a/api/video_codecs/h265_profile_tier_level.h +++ b/api/video_codecs/h265_profile_tier_level.h @@ -101,8 +101,9 @@ RTC_EXPORT absl::optional ParseSdpForH265ProfileTierLevel( // Returns true if the parameters have the same H265 profile or neither contains // an H265 profile, otherwise false. -bool H265IsSameProfileTierLevel(const SdpVideoFormat::Parameters& params1, - const SdpVideoFormat::Parameters& params2); +RTC_EXPORT bool H265IsSameProfileTierLevel( + const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2); } // namespace webrtc diff --git a/media/base/codec.cc b/media/base/codec.cc index c4e1c6f1f3..05bdb7fe58 100644 --- a/media/base/codec.cc +++ b/media/base/codec.cc @@ -15,6 +15,9 @@ #include "api/audio_codecs/audio_format.h" #include "api/video_codecs/av1_profile.h" #include "api/video_codecs/h264_profile_level_id.h" +#ifdef RTC_ENABLE_H265 +#include "api/video_codecs/h265_profile_tier_level.h" +#endif #include "api/video_codecs/vp9_profile.h" #include "media/base/media_constants.h" #include "rtc_base/checks.h" @@ -41,6 +44,24 @@ bool IsSameH264PacketizationMode(const CodecParameterMap& left, GetH264PacketizationModeOrDefault(right); } +#ifdef RTC_ENABLE_H265 +std::string GetH265TxModeOrDefault(const CodecParameterMap& params) { + auto it = params.find(kH265FmtpTxMode); + if (it != params.end()) { + return it->second; + } + // If TxMode is not present, a value of "SRST" must be inferred. + // https://tools.ietf.org/html/rfc7798@section-7.1 + return "SRST"; +} + +bool IsSameH265TxMode(const CodecParameterMap& left, + const CodecParameterMap& right) { + return absl::EqualsIgnoreCase(GetH265TxModeOrDefault(left), + GetH265TxModeOrDefault(right)); +} +#endif + // Some (video) codecs are actually families of codecs and rely on parameters // to distinguish different incompatible family members. bool IsSameCodecSpecific(const std::string& name1, @@ -59,6 +80,12 @@ bool IsSameCodecSpecific(const std::string& name1, return webrtc::VP9IsSameProfile(params1, params2); if (either_name_matches(kAv1CodecName)) return webrtc::AV1IsSameProfile(params1, params2); +#ifdef RTC_ENABLE_H265 + if (either_name_matches(kH265CodecName)) { + return webrtc::H265IsSameProfileTierLevel(params1, params2) && + IsSameH265TxMode(params1, params2); + } +#endif return true; } diff --git a/media/base/codec.h b/media/base/codec.h index bd4239b251..bc8b841fb9 100644 --- a/media/base/codec.h +++ b/media/base/codec.h @@ -98,6 +98,9 @@ struct RTC_EXPORT Codec { absl::InlinedVector scalability_modes; + // H.265 only + absl::optional tx_mode; + // Non key-value parameters such as the telephone-event "0‐15" are // represented using an empty string as key, i.e. {"": "0-15"}. CodecParameterMap params; @@ -110,7 +113,9 @@ struct RTC_EXPORT Codec { // Indicates if this codec is compatible with the specified codec by // checking the assigned id and profile values for the relevant video codecs. - // H264 levels are not compared. + // For H.264, packetization modes will be compared; If H.265 is enabled, + // TxModes will be compared. + // H.264(and H.265, if enabled) levels are not compared. bool Matches(const Codec& codec) const; bool MatchesRtpCodec(const webrtc::RtpCodec& capability) const; diff --git a/media/base/codec_unittest.cc b/media/base/codec_unittest.cc index eb34530c38..4dc3b18c21 100644 --- a/media/base/codec_unittest.cc +++ b/media/base/codec_unittest.cc @@ -342,6 +342,67 @@ TEST(CodecTest, TestH264CodecMatches) { } } +#ifdef RTC_ENABLE_H265 +// Matching H.265 codecs should have matching profile/tier/level and tx-mode. +TEST(CodecTest, TestH265CodecMatches) { + constexpr char kProfile1[] = "1"; + constexpr char kTier1[] = "1"; + constexpr char kLevel3_1[] = "93"; + constexpr char kLevel4[] = "120"; + constexpr char kTxMrst[] = "MRST"; + + VideoCodec c_ptl_blank = + cricket::CreateVideoCodec(95, cricket::kH265CodecName); + + { + VideoCodec c_profile_1 = + cricket::CreateVideoCodec(95, cricket::kH265CodecName); + c_profile_1.params[cricket::kH265FmtpProfileId] = kProfile1; + + // Matches since profile-id unspecified defaults to "1". + EXPECT_TRUE(c_ptl_blank.Matches(c_profile_1)); + } + + { + VideoCodec c_tier_flag_1 = + cricket::CreateVideoCodec(95, cricket::kH265CodecName); + c_tier_flag_1.params[cricket::kH265FmtpTierFlag] = kTier1; + + // Does not match since profile-space unspecified defaults to "0". + EXPECT_FALSE(c_ptl_blank.Matches(c_tier_flag_1)); + } + + { + VideoCodec c_level_id_3_1 = + cricket::CreateVideoCodec(95, cricket::kH265CodecName); + c_level_id_3_1.params[cricket::kH265FmtpLevelId] = kLevel3_1; + + // Matches since level-id unspecified defautls to "93". + EXPECT_TRUE(c_ptl_blank.Matches(c_level_id_3_1)); + } + + { + VideoCodec c_level_id_4 = + cricket::CreateVideoCodec(95, cricket::kH265CodecName); + c_level_id_4.params[cricket::kH265FmtpLevelId] = kLevel4; + + // Does not match since different level-ids are specified. + EXPECT_FALSE(c_ptl_blank.Matches(c_level_id_4)); + } + + { + VideoCodec c_tx_mode_mrst = + cricket::CreateVideoCodec(95, cricket::kH265CodecName); + c_tx_mode_mrst.params[cricket::kH265FmtpTxMode] = kTxMrst; + + // Does not match since tx-mode implies to "SRST" and must be not specified + // when it is the only mode supported: + // https://datatracker.ietf.org/doc/html/draft-ietf-avtcore-hevc-webrtc + EXPECT_FALSE(c_ptl_blank.Matches(c_tx_mode_mrst)); + } +} +#endif + TEST(CodecTest, TestSetParamGetParamAndRemoveParam) { AudioCodec codec = cricket::CreateAudioCodec(0, "foo", 22222, 2); codec.SetParam("a", "1"); diff --git a/media/base/sdp_video_format_utils.cc b/media/base/sdp_video_format_utils.cc index a156afdc02..88d6d58504 100644 --- a/media/base/sdp_video_format_utils.cc +++ b/media/base/sdp_video_format_utils.cc @@ -15,6 +15,9 @@ #include #include "api/video_codecs/h264_profile_level_id.h" +#ifdef RTC_ENABLE_H265 +#include "api/video_codecs/h265_profile_tier_level.h" +#endif #include "rtc_base/checks.h" #include "rtc_base/string_to_number.h" @@ -27,6 +30,11 @@ const char kVPxFmtpMaxFrameRate[] = "max-fr"; // Max frame size for VP8 and VP9 video. const char kVPxFmtpMaxFrameSize[] = "max-fs"; const int kVPxFmtpFrameSizeSubBlockPixels = 256; +#ifdef RTC_ENABLE_H265 +constexpr char kH265ProfileId[] = "profile-id"; +constexpr char kH265TierFlag[] = "tier-flag"; +constexpr char kH265LevelId[] = "level-id"; +#endif bool IsH264LevelAsymmetryAllowed(const SdpVideoFormat::Parameters& params) { const auto it = params.find(kH264LevelAsymmetryAllowed); @@ -60,8 +68,59 @@ absl::optional ParsePositiveNumberFromParams( return i; } +#ifdef RTC_ENABLE_H265 +// Compares two H265Level and return the smaller. +H265Level H265LevelMin(H265Level a, H265Level b) { + return a <= b ? a : b; +} + +// Returns true if none of profile-id/tier-flag/level-id is specified +// explicitly in the param. +bool IsDefaultH265PTL(const SdpVideoFormat::Parameters& params) { + return !params.count(kH265ProfileId) && !params.count(kH265TierFlag) && + !params.count(kH265LevelId); +} +#endif + } // namespace +#ifdef RTC_ENABLE_H265 +// Set level according to https://tools.ietf.org/html/rfc7798#section-7.1 +void H265GenerateProfileTierLevelForAnswer( + const SdpVideoFormat::Parameters& local_supported_params, + const SdpVideoFormat::Parameters& remote_offered_params, + SdpVideoFormat::Parameters* answer_params) { + // If local and remote haven't set profile-id/tier-flag/level-id, they + // are both using the default PTL In this case, don't set PTL in answer + // either. + if (IsDefaultH265PTL(local_supported_params) && + IsDefaultH265PTL(remote_offered_params)) { + return; + } + + // Parse profile-tier-level. + const absl::optional local_profile_tier_level = + ParseSdpForH265ProfileTierLevel(local_supported_params); + const absl::optional remote_profile_tier_level = + ParseSdpForH265ProfileTierLevel(remote_offered_params); + // Profile and tier for local and remote codec must be valid and equal. + RTC_DCHECK(local_profile_tier_level); + RTC_DCHECK(remote_profile_tier_level); + RTC_DCHECK_EQ(local_profile_tier_level->profile, + remote_profile_tier_level->profile); + RTC_DCHECK_EQ(local_profile_tier_level->tier, + remote_profile_tier_level->tier); + + const H265Level answer_level = H265LevelMin(local_profile_tier_level->level, + remote_profile_tier_level->level); + + // Level-id in answer is changable as long as the highest level indicated by + // the answer is not higher than that indicated by the offer. See + // https://tools.ietf.org/html/rfc7798#section-7.2.2, sub-clause 2. + (*answer_params)[kH265LevelId] = H265LevelToString(answer_level); +} +#endif + // Set level according to https://tools.ietf.org/html/rfc6184#section-8.2.2. void H264GenerateProfileLevelIdForAnswer( const SdpVideoFormat::Parameters& local_supported_params, diff --git a/media/base/sdp_video_format_utils.h b/media/base/sdp_video_format_utils.h index 80c1e4d501..421cab885a 100644 --- a/media/base/sdp_video_format_utils.h +++ b/media/base/sdp_video_format_utils.h @@ -36,6 +36,18 @@ void H264GenerateProfileLevelIdForAnswer( const SdpVideoFormat::Parameters& remote_offered_params, SdpVideoFormat::Parameters* answer_params); +#ifdef RTC_ENABLE_H265 +// Works similarly as H264GenerateProfileLevelIdForAnswer, but generates codec +// parameters that will be used as answer for H.265. +// Media configuration parameters, except level-id, must be used symmetrically. +// For level-id, the highest level indicated by the answer must not be higher +// than that indicated by the offer. +void H265GenerateProfileTierLevelForAnswer( + const SdpVideoFormat::Parameters& local_supported_params, + const SdpVideoFormat::Parameters& remote_offered_params, + SdpVideoFormat::Parameters* answer_params); +#endif + // Parse max frame rate from SDP FMTP line. absl::nullopt is returned if the // field is missing or not a number. absl::optional ParseSdpForVPxMaxFrameRate( diff --git a/media/base/sdp_video_format_utils_unittest.cc b/media/base/sdp_video_format_utils_unittest.cc index d8ef9ab827..3ae2b3bf29 100644 --- a/media/base/sdp_video_format_utils_unittest.cc +++ b/media/base/sdp_video_format_utils_unittest.cc @@ -72,6 +72,37 @@ TEST(SdpVideoFormatUtilsTest, EXPECT_EQ("42e01f", answer_params["profile-level-id"]); } +#ifdef RTC_ENABLE_H265 +// Answer should not include explicit PTL info if neither local nor remote set +// any of them. +TEST(SdpVideoFormatUtilsTest, H265GenerateProfileTierLevelEmpty) { + SdpVideoFormat::Parameters answer_params; + H265GenerateProfileTierLevelForAnswer(SdpVideoFormat::Parameters(), + SdpVideoFormat::Parameters(), + &answer_params); + EXPECT_TRUE(answer_params.empty()); +} + +// Answer must use the minimum level as supported by both local and remote. +TEST(SdpVideoFormatUtilsTest, H265GenerateProfileTierLevelNoEmpty) { + constexpr char kLocallySupportedLevelId[] = "93"; + constexpr char kRemoteOfferedLevelId[] = "120"; + + SdpVideoFormat::Parameters local_params; + local_params["profile-id"] = "1"; + local_params["tier-flag"] = "0"; + local_params["level-id"] = kLocallySupportedLevelId; + SdpVideoFormat::Parameters remote_params; + remote_params["profile-id"] = "1"; + remote_params["tier-flag"] = "0"; + remote_params["level-id"] = kRemoteOfferedLevelId; + SdpVideoFormat::Parameters answer_params; + H265GenerateProfileTierLevelForAnswer(local_params, remote_params, + &answer_params); + EXPECT_EQ(kLocallySupportedLevelId, answer_params["level-id"]); +} +#endif + TEST(SdpVideoFormatUtilsTest, MaxFrameRateIsMissingOrInvalid) { SdpVideoFormat::Parameters params; absl::optional empty = ParseSdpForVPxMaxFrameRate(params); diff --git a/media/engine/internal_decoder_factory_unittest.cc b/media/engine/internal_decoder_factory_unittest.cc index bb2e24d5d8..51d6a94dd6 100644 --- a/media/engine/internal_decoder_factory_unittest.cc +++ b/media/engine/internal_decoder_factory_unittest.cc @@ -43,6 +43,8 @@ constexpr bool kDav1dIsIncluded = true; #else constexpr bool kDav1dIsIncluded = false; #endif +constexpr bool kH265Enabled = false; + constexpr VideoDecoderFactory::CodecSupport kSupported = { /*is_supported=*/true, /*is_power_efficient=*/false}; constexpr VideoDecoderFactory::CodecSupport kUnsupported = { @@ -99,6 +101,14 @@ TEST(InternalDecoderFactoryTest, Av1Profile0) { } } +// At current stage since internal H.265 decoder is not implemented, +TEST(InternalDecoderFactoryTest, H265IsNotEnabled) { + InternalDecoderFactory factory; + std::unique_ptr decoder = + factory.CreateVideoDecoder(SdpVideoFormat(cricket::kH265CodecName)); + EXPECT_EQ(static_cast(decoder), kH265Enabled); +} + #if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY) TEST(InternalDecoderFactoryTest, Av1) { InternalDecoderFactory factory; diff --git a/media/engine/internal_encoder_factory_unittest.cc b/media/engine/internal_encoder_factory_unittest.cc index a1c90b8cf4..b9ca6d88c2 100644 --- a/media/engine/internal_encoder_factory_unittest.cc +++ b/media/engine/internal_encoder_factory_unittest.cc @@ -33,6 +33,8 @@ constexpr bool kH264Enabled = true; #else constexpr bool kH264Enabled = false; #endif +constexpr bool kH265Enabled = false; + constexpr VideoEncoderFactory::CodecSupport kSupported = { /*is_supported=*/true, /*is_power_efficient=*/false}; constexpr VideoEncoderFactory::CodecSupport kUnsupported = { @@ -78,6 +80,17 @@ TEST(InternalEncoderFactoryTest, H264) { } } +// At current stage H.265 is not supported by internal encoder factory. +TEST(InternalEncoderFactoryTest, H265IsNotEnabled) { + InternalEncoderFactory factory; + std::unique_ptr encoder = + factory.CreateVideoEncoder(SdpVideoFormat(cricket::kH265CodecName)); + EXPECT_EQ(static_cast(encoder), kH265Enabled); + EXPECT_THAT( + factory.GetSupportedFormats(), + Not(Contains(Field(&SdpVideoFormat::name, cricket::kH265CodecName)))); +} + TEST(InternalEncoderFactoryTest, QueryCodecSupportWithScalabilityMode) { InternalEncoderFactory factory; // VP8 and VP9 supported for singles spatial layers. diff --git a/pc/media_session.cc b/pc/media_session.cc index 573e35225e..e3197c4076 100644 --- a/pc/media_session.cc +++ b/pc/media_session.cc @@ -728,6 +728,16 @@ void NegotiatePacketization(const Codec& local_codec, : absl::nullopt; } +#ifdef RTC_ENABLE_H265 +void NegotiateTxMode(const Codec& local_codec, + const Codec& remote_codec, + Codec* negotiated_codec) { + negotiated_codec->tx_mode = (local_codec.tx_mode == remote_codec.tx_mode) + ? local_codec.tx_mode + : absl::nullopt; +} +#endif + // Finds a codec in `codecs2` that matches `codec_to_match`, which is // a member of `codecs1`. If `codec_to_match` is an RED or RTX codec, both // the codecs themselves and their associated codecs must match. @@ -849,6 +859,13 @@ void NegotiateCodecs(const std::vector& local_codecs, webrtc::H264GenerateProfileLevelIdForAnswer(ours.params, theirs->params, &negotiated.params); } +#ifdef RTC_ENABLE_H265 + if (absl::EqualsIgnoreCase(ours.name, kH265CodecName)) { + webrtc::H265GenerateProfileTierLevelForAnswer( + ours.params, theirs->params, &negotiated.params); + NegotiateTxMode(ours, *theirs, &negotiated); + } +#endif negotiated.id = theirs->id; negotiated.name = theirs->name; negotiated_codecs->push_back(std::move(negotiated)); diff --git a/pc/media_session_unittest.cc b/pc/media_session_unittest.cc index 641f638e72..f4fd09cba0 100644 --- a/pc/media_session_unittest.cc +++ b/pc/media_session_unittest.cc @@ -4323,6 +4323,80 @@ TEST_F(MediaSessionDescriptionFactoryTest, EXPECT_EQ(vcd1->codecs()[0].id, vcd2->codecs()[0].id); } +#ifdef RTC_ENABLE_H265 +// Test verifying that negotiating codecs with the same tx-mode retains the +// tx-mode value. +TEST_F(MediaSessionDescriptionFactoryTest, H265TxModeIsEqualRetainIt) { + std::vector f1_codecs = {CreateVideoCodec(96, "H265")}; + f1_codecs.back().tx_mode = "mrst"; + f1_.set_video_codecs(f1_codecs, f1_codecs); + + std::vector f2_codecs = {CreateVideoCodec(96, "H265")}; + f2_codecs.back().tx_mode = "mrst"; + f2_.set_video_codecs(f2_codecs, f2_codecs); + + MediaSessionOptions opts; + AddMediaDescriptionOptions(MEDIA_TYPE_VIDEO, "video1", + RtpTransceiverDirection::kSendRecv, kActive, + &opts); + + // Create an offer with two video sections using same codecs. + std::unique_ptr offer = + f1_.CreateOfferOrError(opts, nullptr).MoveValue(); + ASSERT_TRUE(offer); + ASSERT_EQ(1u, offer->contents().size()); + const MediaContentDescription* vcd1 = + offer->contents()[0].media_description(); + ASSERT_EQ(1u, vcd1->codecs().size()); + EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst"); + + // Create answer and negotiate the codecs. + std::unique_ptr answer = + f2_.CreateAnswerOrError(offer.get(), opts, nullptr).MoveValue(); + ASSERT_TRUE(answer); + ASSERT_EQ(1u, answer->contents().size()); + vcd1 = answer->contents()[0].media_description(); + ASSERT_EQ(1u, vcd1->codecs().size()); + EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst"); +} + +// Test verifying that negotiating codecs with different tx_mode removes +// the tx_mode value. +TEST_F(MediaSessionDescriptionFactoryTest, H265TxModeIsDifferentDropCodecs) { + std::vector f1_codecs = {CreateVideoCodec(96, "H265")}; + f1_codecs.back().tx_mode = "mrst"; + f1_.set_video_codecs(f1_codecs, f1_codecs); + + std::vector f2_codecs = {CreateVideoCodec(96, "H265")}; + f2_codecs.back().tx_mode = "mrmt"; + f2_.set_video_codecs(f2_codecs, f2_codecs); + + MediaSessionOptions opts; + AddMediaDescriptionOptions(MEDIA_TYPE_VIDEO, "video1", + RtpTransceiverDirection::kSendRecv, kActive, + &opts); + + // Create an offer with two video sections using same codecs. + std::unique_ptr offer = + f1_.CreateOfferOrError(opts, nullptr).MoveValue(); + ASSERT_TRUE(offer); + ASSERT_EQ(1u, offer->contents().size()); + const VideoContentDescription* vcd1 = + offer->contents()[0].media_description()->as_video(); + ASSERT_EQ(1u, vcd1->codecs().size()); + EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst"); + + // Create answer and negotiate the codecs. + std::unique_ptr answer = + f2_.CreateAnswerOrError(offer.get(), opts, nullptr).MoveValue(); + ASSERT_TRUE(answer); + ASSERT_EQ(1u, answer->contents().size()); + vcd1 = answer->contents()[0].media_description()->as_video(); + ASSERT_EQ(1u, vcd1->codecs().size()); + EXPECT_EQ(vcd1->codecs()[0].tx_mode, absl::nullopt); +} +#endif + // Test verifying that negotiating codecs with the same packetization retains // the packetization value. TEST_F(MediaSessionDescriptionFactoryTest, PacketizationIsEqual) {