Add SDP negotiation support for HEVC.

This adds neccessary checks for SDP negotiation with HEVC.

Test: Manually apply the CL on Chromium and enable HEVC HW encoder,
and add HEVC profiles in rtc video decoder/encoder factory, H265 is
negotiated in SDP with correct FMTP lines added.

Bug: webrtc:13485
Change-Id: I5557b20b646cc96c5acb578521204fe10df0dcf0
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/330202
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Jianlin Qiu <jianlin.qiu@intel.com>
Cr-Commit-Position: refs/heads/main@{#41357}

api/video_codecs/h265_profile_tier_level.h

@@ -101,8 +101,9 @@ RTC_EXPORT absl::optional<H265ProfileTierLevel> ParseSdpForH265ProfileTierLevel(
 
 // Returns true if the parameters have the same H265 profile or neither contains
 // an H265 profile, otherwise false.
-bool H265IsSameProfileTierLevel(const SdpVideoFormat::Parameters& params1,
-                                const SdpVideoFormat::Parameters& params2);
+RTC_EXPORT bool H265IsSameProfileTierLevel(
+    const SdpVideoFormat::Parameters& params1,
+    const SdpVideoFormat::Parameters& params2);
 
 }  // namespace webrtc
 

media/base/codec.cc

@@ -15,6 +15,9 @@
 #include "api/audio_codecs/audio_format.h"
 #include "api/video_codecs/av1_profile.h"
 #include "api/video_codecs/h264_profile_level_id.h"
+#ifdef RTC_ENABLE_H265
+#include "api/video_codecs/h265_profile_tier_level.h"
+#endif
 #include "api/video_codecs/vp9_profile.h"
 #include "media/base/media_constants.h"
 #include "rtc_base/checks.h"
@@ -41,6 +44,24 @@ bool IsSameH264PacketizationMode(const CodecParameterMap& left,
          GetH264PacketizationModeOrDefault(right);
 }
 
+#ifdef RTC_ENABLE_H265
+std::string GetH265TxModeOrDefault(const CodecParameterMap& params) {
+  auto it = params.find(kH265FmtpTxMode);
+  if (it != params.end()) {
+    return it->second;
+  }
+  // If TxMode is not present, a value of "SRST" must be inferred.
+  // https://tools.ietf.org/html/rfc7798@section-7.1
+  return "SRST";
+}
+
+bool IsSameH265TxMode(const CodecParameterMap& left,
+                      const CodecParameterMap& right) {
+  return absl::EqualsIgnoreCase(GetH265TxModeOrDefault(left),
+                                GetH265TxModeOrDefault(right));
+}
+#endif
+
 // Some (video) codecs are actually families of codecs and rely on parameters
 // to distinguish different incompatible family members.
 bool IsSameCodecSpecific(const std::string& name1,
@@ -59,6 +80,12 @@ bool IsSameCodecSpecific(const std::string& name1,
     return webrtc::VP9IsSameProfile(params1, params2);
   if (either_name_matches(kAv1CodecName))
     return webrtc::AV1IsSameProfile(params1, params2);
+#ifdef RTC_ENABLE_H265
+  if (either_name_matches(kH265CodecName)) {
+    return webrtc::H265IsSameProfileTierLevel(params1, params2) &&
+           IsSameH265TxMode(params1, params2);
+  }
+#endif
   return true;
 }
 

media/base/codec.h

@@ -98,6 +98,9 @@ struct RTC_EXPORT Codec {
   absl::InlinedVector<webrtc::ScalabilityMode, webrtc::kScalabilityModeCount>
       scalability_modes;
 
+  // H.265 only
+  absl::optional<std::string> tx_mode;
+
   // Non key-value parameters such as the telephone-event "0‐15" are
   // represented using an empty string as key, i.e. {"": "0-15"}.
   CodecParameterMap params;
@@ -110,7 +113,9 @@ struct RTC_EXPORT Codec {
 
   // Indicates if this codec is compatible with the specified codec by
   // checking the assigned id and profile values for the relevant video codecs.
-  // H264 levels are not compared.
+  // For H.264, packetization modes will be compared; If H.265 is enabled,
+  // TxModes will be compared.
+  // H.264(and H.265, if enabled) levels are not compared.
   bool Matches(const Codec& codec) const;
   bool MatchesRtpCodec(const webrtc::RtpCodec& capability) const;
 

media/base/codec_unittest.cc

@@ -342,6 +342,67 @@ TEST(CodecTest, TestH264CodecMatches) {
   }
 }
 
+#ifdef RTC_ENABLE_H265
+// Matching H.265 codecs should have matching profile/tier/level and tx-mode.
+TEST(CodecTest, TestH265CodecMatches) {
+  constexpr char kProfile1[] = "1";
+  constexpr char kTier1[] = "1";
+  constexpr char kLevel3_1[] = "93";
+  constexpr char kLevel4[] = "120";
+  constexpr char kTxMrst[] = "MRST";
+
+  VideoCodec c_ptl_blank =
+      cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+
+  {
+    VideoCodec c_profile_1 =
+        cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+    c_profile_1.params[cricket::kH265FmtpProfileId] = kProfile1;
+
+    // Matches since profile-id unspecified defaults to "1".
+    EXPECT_TRUE(c_ptl_blank.Matches(c_profile_1));
+  }
+
+  {
+    VideoCodec c_tier_flag_1 =
+        cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+    c_tier_flag_1.params[cricket::kH265FmtpTierFlag] = kTier1;
+
+    // Does not match since profile-space unspecified defaults to "0".
+    EXPECT_FALSE(c_ptl_blank.Matches(c_tier_flag_1));
+  }
+
+  {
+    VideoCodec c_level_id_3_1 =
+        cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+    c_level_id_3_1.params[cricket::kH265FmtpLevelId] = kLevel3_1;
+
+    // Matches since level-id unspecified defautls to "93".
+    EXPECT_TRUE(c_ptl_blank.Matches(c_level_id_3_1));
+  }
+
+  {
+    VideoCodec c_level_id_4 =
+        cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+    c_level_id_4.params[cricket::kH265FmtpLevelId] = kLevel4;
+
+    // Does not match since different level-ids are specified.
+    EXPECT_FALSE(c_ptl_blank.Matches(c_level_id_4));
+  }
+
+  {
+    VideoCodec c_tx_mode_mrst =
+        cricket::CreateVideoCodec(95, cricket::kH265CodecName);
+    c_tx_mode_mrst.params[cricket::kH265FmtpTxMode] = kTxMrst;
+
+    // Does not match since tx-mode implies to "SRST" and must be not specified
+    // when it is the only mode supported:
+    // https://datatracker.ietf.org/doc/html/draft-ietf-avtcore-hevc-webrtc
+    EXPECT_FALSE(c_ptl_blank.Matches(c_tx_mode_mrst));
+  }
+}
+#endif
+
 TEST(CodecTest, TestSetParamGetParamAndRemoveParam) {
   AudioCodec codec = cricket::CreateAudioCodec(0, "foo", 22222, 2);
   codec.SetParam("a", "1");

media/base/sdp_video_format_utils.cc

@@ -15,6 +15,9 @@
 #include <utility>
 
 #include "api/video_codecs/h264_profile_level_id.h"
+#ifdef RTC_ENABLE_H265
+#include "api/video_codecs/h265_profile_tier_level.h"
+#endif
 #include "rtc_base/checks.h"
 #include "rtc_base/string_to_number.h"
 
@@ -27,6 +30,11 @@ const char kVPxFmtpMaxFrameRate[] = "max-fr";
 // Max frame size for VP8 and VP9 video.
 const char kVPxFmtpMaxFrameSize[] = "max-fs";
 const int kVPxFmtpFrameSizeSubBlockPixels = 256;
+#ifdef RTC_ENABLE_H265
+constexpr char kH265ProfileId[] = "profile-id";
+constexpr char kH265TierFlag[] = "tier-flag";
+constexpr char kH265LevelId[] = "level-id";
+#endif
 
 bool IsH264LevelAsymmetryAllowed(const SdpVideoFormat::Parameters& params) {
   const auto it = params.find(kH264LevelAsymmetryAllowed);
@@ -60,8 +68,59 @@ absl::optional<int> ParsePositiveNumberFromParams(
   return i;
 }
 
+#ifdef RTC_ENABLE_H265
+// Compares two H265Level and return the smaller.
+H265Level H265LevelMin(H265Level a, H265Level b) {
+  return a <= b ? a : b;
+}
+
+// Returns true if none of profile-id/tier-flag/level-id is specified
+// explicitly in the param.
+bool IsDefaultH265PTL(const SdpVideoFormat::Parameters& params) {
+  return !params.count(kH265ProfileId) && !params.count(kH265TierFlag) &&
+         !params.count(kH265LevelId);
+}
+#endif
+
 }  // namespace
 
+#ifdef RTC_ENABLE_H265
+// Set level according to https://tools.ietf.org/html/rfc7798#section-7.1
+void H265GenerateProfileTierLevelForAnswer(
+    const SdpVideoFormat::Parameters& local_supported_params,
+    const SdpVideoFormat::Parameters& remote_offered_params,
+    SdpVideoFormat::Parameters* answer_params) {
+  // If local and remote haven't set profile-id/tier-flag/level-id, they
+  // are both using the default PTL In this case, don't set PTL in answer
+  // either.
+  if (IsDefaultH265PTL(local_supported_params) &&
+      IsDefaultH265PTL(remote_offered_params)) {
+    return;
+  }
+
+  // Parse profile-tier-level.
+  const absl::optional<H265ProfileTierLevel> local_profile_tier_level =
+      ParseSdpForH265ProfileTierLevel(local_supported_params);
+  const absl::optional<H265ProfileTierLevel> remote_profile_tier_level =
+      ParseSdpForH265ProfileTierLevel(remote_offered_params);
+  // Profile and tier for local and remote codec must be valid and equal.
+  RTC_DCHECK(local_profile_tier_level);
+  RTC_DCHECK(remote_profile_tier_level);
+  RTC_DCHECK_EQ(local_profile_tier_level->profile,
+                remote_profile_tier_level->profile);
+  RTC_DCHECK_EQ(local_profile_tier_level->tier,
+                remote_profile_tier_level->tier);
+
+  const H265Level answer_level = H265LevelMin(local_profile_tier_level->level,
+                                              remote_profile_tier_level->level);
+
+  // Level-id in answer is changable as long as the highest level indicated by
+  // the answer is not higher than that indicated by the offer. See
+  // https://tools.ietf.org/html/rfc7798#section-7.2.2, sub-clause 2.
+  (*answer_params)[kH265LevelId] = H265LevelToString(answer_level);
+}
+#endif
+
 // Set level according to https://tools.ietf.org/html/rfc6184#section-8.2.2.
 void H264GenerateProfileLevelIdForAnswer(
     const SdpVideoFormat::Parameters& local_supported_params,

media/base/sdp_video_format_utils.h

@@ -36,6 +36,18 @@ void H264GenerateProfileLevelIdForAnswer(
     const SdpVideoFormat::Parameters& remote_offered_params,
     SdpVideoFormat::Parameters* answer_params);
 
+#ifdef RTC_ENABLE_H265
+// Works similarly as H264GenerateProfileLevelIdForAnswer, but generates codec
+// parameters that will be used as answer for H.265.
+// Media configuration parameters, except level-id, must be used symmetrically.
+// For level-id, the highest level indicated by the answer must not be higher
+// than that indicated by the offer.
+void H265GenerateProfileTierLevelForAnswer(
+    const SdpVideoFormat::Parameters& local_supported_params,
+    const SdpVideoFormat::Parameters& remote_offered_params,
+    SdpVideoFormat::Parameters* answer_params);
+#endif
+
 // Parse max frame rate from SDP FMTP line. absl::nullopt is returned if the
 // field is missing or not a number.
 absl::optional<int> ParseSdpForVPxMaxFrameRate(

media/base/sdp_video_format_utils_unittest.cc

@@ -72,6 +72,37 @@ TEST(SdpVideoFormatUtilsTest,
   EXPECT_EQ("42e01f", answer_params["profile-level-id"]);
 }
 
+#ifdef RTC_ENABLE_H265
+// Answer should not include explicit PTL info if neither local nor remote set
+// any of them.
+TEST(SdpVideoFormatUtilsTest, H265GenerateProfileTierLevelEmpty) {
+  SdpVideoFormat::Parameters answer_params;
+  H265GenerateProfileTierLevelForAnswer(SdpVideoFormat::Parameters(),
+                                        SdpVideoFormat::Parameters(),
+                                        &answer_params);
+  EXPECT_TRUE(answer_params.empty());
+}
+
+// Answer must use the minimum level as supported by both local and remote.
+TEST(SdpVideoFormatUtilsTest, H265GenerateProfileTierLevelNoEmpty) {
+  constexpr char kLocallySupportedLevelId[] = "93";
+  constexpr char kRemoteOfferedLevelId[] = "120";
+
+  SdpVideoFormat::Parameters local_params;
+  local_params["profile-id"] = "1";
+  local_params["tier-flag"] = "0";
+  local_params["level-id"] = kLocallySupportedLevelId;
+  SdpVideoFormat::Parameters remote_params;
+  remote_params["profile-id"] = "1";
+  remote_params["tier-flag"] = "0";
+  remote_params["level-id"] = kRemoteOfferedLevelId;
+  SdpVideoFormat::Parameters answer_params;
+  H265GenerateProfileTierLevelForAnswer(local_params, remote_params,
+                                        &answer_params);
+  EXPECT_EQ(kLocallySupportedLevelId, answer_params["level-id"]);
+}
+#endif
+
 TEST(SdpVideoFormatUtilsTest, MaxFrameRateIsMissingOrInvalid) {
   SdpVideoFormat::Parameters params;
   absl::optional<int> empty = ParseSdpForVPxMaxFrameRate(params);

media/engine/internal_decoder_factory_unittest.cc

@@ -43,6 +43,8 @@ constexpr bool kDav1dIsIncluded = true;
 #else
 constexpr bool kDav1dIsIncluded = false;
 #endif
+constexpr bool kH265Enabled = false;
+
 constexpr VideoDecoderFactory::CodecSupport kSupported = {
     /*is_supported=*/true, /*is_power_efficient=*/false};
 constexpr VideoDecoderFactory::CodecSupport kUnsupported = {
@@ -99,6 +101,14 @@ TEST(InternalDecoderFactoryTest, Av1Profile0) {
   }
 }
 
+// At current stage since internal H.265 decoder is not implemented,
+TEST(InternalDecoderFactoryTest, H265IsNotEnabled) {
+  InternalDecoderFactory factory;
+  std::unique_ptr<VideoDecoder> decoder =
+      factory.CreateVideoDecoder(SdpVideoFormat(cricket::kH265CodecName));
+  EXPECT_EQ(static_cast<bool>(decoder), kH265Enabled);
+}
+
 #if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY)
 TEST(InternalDecoderFactoryTest, Av1) {
   InternalDecoderFactory factory;

media/engine/internal_encoder_factory_unittest.cc

@@ -33,6 +33,8 @@ constexpr bool kH264Enabled = true;
 #else
 constexpr bool kH264Enabled = false;
 #endif
+constexpr bool kH265Enabled = false;
+
 constexpr VideoEncoderFactory::CodecSupport kSupported = {
     /*is_supported=*/true, /*is_power_efficient=*/false};
 constexpr VideoEncoderFactory::CodecSupport kUnsupported = {
@@ -78,6 +80,17 @@ TEST(InternalEncoderFactoryTest, H264) {
   }
 }
 
+// At current stage H.265 is not supported by internal encoder factory.
+TEST(InternalEncoderFactoryTest, H265IsNotEnabled) {
+  InternalEncoderFactory factory;
+  std::unique_ptr<VideoEncoder> encoder =
+      factory.CreateVideoEncoder(SdpVideoFormat(cricket::kH265CodecName));
+  EXPECT_EQ(static_cast<bool>(encoder), kH265Enabled);
+  EXPECT_THAT(
+      factory.GetSupportedFormats(),
+      Not(Contains(Field(&SdpVideoFormat::name, cricket::kH265CodecName))));
+}
+
 TEST(InternalEncoderFactoryTest, QueryCodecSupportWithScalabilityMode) {
   InternalEncoderFactory factory;
   // VP8 and VP9 supported for singles spatial layers.

pc/media_session.cc

@@ -728,6 +728,16 @@ void NegotiatePacketization(const Codec& local_codec,
           : absl::nullopt;
 }
 
+#ifdef RTC_ENABLE_H265
+void NegotiateTxMode(const Codec& local_codec,
+                     const Codec& remote_codec,
+                     Codec* negotiated_codec) {
+  negotiated_codec->tx_mode = (local_codec.tx_mode == remote_codec.tx_mode)
+                                  ? local_codec.tx_mode
+                                  : absl::nullopt;
+}
+#endif
+
 // Finds a codec in `codecs2` that matches `codec_to_match`, which is
 // a member of `codecs1`. If `codec_to_match` is an RED or RTX codec, both
 // the codecs themselves and their associated codecs must match.
@@ -849,6 +859,13 @@ void NegotiateCodecs(const std::vector<Codec>& local_codecs,
         webrtc::H264GenerateProfileLevelIdForAnswer(ours.params, theirs->params,
                                                     &negotiated.params);
       }
+#ifdef RTC_ENABLE_H265
+      if (absl::EqualsIgnoreCase(ours.name, kH265CodecName)) {
+        webrtc::H265GenerateProfileTierLevelForAnswer(
+            ours.params, theirs->params, &negotiated.params);
+        NegotiateTxMode(ours, *theirs, &negotiated);
+      }
+#endif
       negotiated.id = theirs->id;
       negotiated.name = theirs->name;
       negotiated_codecs->push_back(std::move(negotiated));

pc/media_session_unittest.cc

@@ -4323,6 +4323,80 @@ TEST_F(MediaSessionDescriptionFactoryTest,
   EXPECT_EQ(vcd1->codecs()[0].id, vcd2->codecs()[0].id);
 }
 
+#ifdef RTC_ENABLE_H265
+// Test verifying that negotiating codecs with the same tx-mode retains the
+// tx-mode value.
+TEST_F(MediaSessionDescriptionFactoryTest, H265TxModeIsEqualRetainIt) {
+  std::vector f1_codecs = {CreateVideoCodec(96, "H265")};
+  f1_codecs.back().tx_mode = "mrst";
+  f1_.set_video_codecs(f1_codecs, f1_codecs);
+
+  std::vector f2_codecs = {CreateVideoCodec(96, "H265")};
+  f2_codecs.back().tx_mode = "mrst";
+  f2_.set_video_codecs(f2_codecs, f2_codecs);
+
+  MediaSessionOptions opts;
+  AddMediaDescriptionOptions(MEDIA_TYPE_VIDEO, "video1",
+                             RtpTransceiverDirection::kSendRecv, kActive,
+                             &opts);
+
+  // Create an offer with two video sections using same codecs.
+  std::unique_ptr<SessionDescription> offer =
+      f1_.CreateOfferOrError(opts, nullptr).MoveValue();
+  ASSERT_TRUE(offer);
+  ASSERT_EQ(1u, offer->contents().size());
+  const MediaContentDescription* vcd1 =
+      offer->contents()[0].media_description();
+  ASSERT_EQ(1u, vcd1->codecs().size());
+  EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst");
+
+  // Create answer and negotiate the codecs.
+  std::unique_ptr<SessionDescription> answer =
+      f2_.CreateAnswerOrError(offer.get(), opts, nullptr).MoveValue();
+  ASSERT_TRUE(answer);
+  ASSERT_EQ(1u, answer->contents().size());
+  vcd1 = answer->contents()[0].media_description();
+  ASSERT_EQ(1u, vcd1->codecs().size());
+  EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst");
+}
+
+// Test verifying that negotiating codecs with different tx_mode removes
+// the tx_mode value.
+TEST_F(MediaSessionDescriptionFactoryTest, H265TxModeIsDifferentDropCodecs) {
+  std::vector f1_codecs = {CreateVideoCodec(96, "H265")};
+  f1_codecs.back().tx_mode = "mrst";
+  f1_.set_video_codecs(f1_codecs, f1_codecs);
+
+  std::vector f2_codecs = {CreateVideoCodec(96, "H265")};
+  f2_codecs.back().tx_mode = "mrmt";
+  f2_.set_video_codecs(f2_codecs, f2_codecs);
+
+  MediaSessionOptions opts;
+  AddMediaDescriptionOptions(MEDIA_TYPE_VIDEO, "video1",
+                             RtpTransceiverDirection::kSendRecv, kActive,
+                             &opts);
+
+  // Create an offer with two video sections using same codecs.
+  std::unique_ptr<SessionDescription> offer =
+      f1_.CreateOfferOrError(opts, nullptr).MoveValue();
+  ASSERT_TRUE(offer);
+  ASSERT_EQ(1u, offer->contents().size());
+  const VideoContentDescription* vcd1 =
+      offer->contents()[0].media_description()->as_video();
+  ASSERT_EQ(1u, vcd1->codecs().size());
+  EXPECT_EQ(vcd1->codecs()[0].tx_mode, "mrst");
+
+  // Create answer and negotiate the codecs.
+  std::unique_ptr<SessionDescription> answer =
+      f2_.CreateAnswerOrError(offer.get(), opts, nullptr).MoveValue();
+  ASSERT_TRUE(answer);
+  ASSERT_EQ(1u, answer->contents().size());
+  vcd1 = answer->contents()[0].media_description()->as_video();
+  ASSERT_EQ(1u, vcd1->codecs().size());
+  EXPECT_EQ(vcd1->codecs()[0].tx_mode, absl::nullopt);
+}
+#endif
+
 // Test verifying that negotiating codecs with the same packetization retains
 // the packetization value.
 TEST_F(MediaSessionDescriptionFactoryTest, PacketizationIsEqual) {