From 79e5e721b54b3821f33be96bda3f1be69429cd53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Wed, 22 Jan 2025 15:13:37 +0100 Subject: [PATCH] Add unidirectional codec support ("offer to send" use case). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL implements allowing sendonly codecs in setCodecPreferences(), i.e. this spec PR: https://github.com/w3c/webrtc-pc/pull/3018. It also makes the setCodecPreferences() ignore level IDs in the filtering algorithm (but not in the sCP method call) as per this spec PR: https://github.com/w3c/webrtc-pc/pull/3023. In short, before this CL, setCodecPreferences() threw an exception if a codec was preferred that is not present in receiver codec capabilities. After this CL, setCodecPreferences() allows you to prefer codecs that are *either* in the sender capabilities *or* the receiver capabilities. - This allows you to "offer to send", i.e. prefer sendonly codecs on a sendonly transceiver. - The filtering on direction is handled by RtpTransceiver::filtered_codec_preferences() which is called during SDP offer/answer (sdp_offer_answer.cc). Also as per spec changes, if this filtering results in not having any codecs to offer or answer then this results in not having any codec preferences as opposed to throwing an exception (old behavior). - Two old peer_connection_media_unittest.cc tests are updated to reflect the API failing less. This CL adds both unit tests (rtp_transceiver_unittest.cc) and full stack integration tests (peer_connection_encodings_integrationtest.cc). It also makes us pass the following Web Platform Tests in Chrome: https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/web_tests/external/wpt/webrtc/protocol/h265-level-id.https.html Bug: chromium:381407888 Change-Id: I98a5ad1acccb56db0538e4d47975b8a725102c33 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/374520 Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Henrik Boström Reviewed-by: Harald Alvestrand Reviewed-by: Evan Shrubsole Cr-Commit-Position: refs/heads/main@{#43788} --- ...er_connection_encodings_integrationtest.cc | 196 ++++++++++++- pc/peer_connection_media_unittest.cc | 13 +- pc/rtp_transceiver.cc | 163 +++++++---- pc/rtp_transceiver.h | 16 +- pc/rtp_transceiver_unittest.cc | 261 ++++++++++++++++++ pc/sdp_offer_answer.cc | 2 +- 6 files changed, 592 insertions(+), 59 deletions(-) diff --git a/pc/peer_connection_encodings_integrationtest.cc b/pc/peer_connection_encodings_integrationtest.cc index 1b6eeaf52e..de66c11d2e 100644 --- a/pc/peer_connection_encodings_integrationtest.cc +++ b/pc/peer_connection_encodings_integrationtest.cc @@ -63,6 +63,7 @@ using ::testing::Each; using ::testing::Eq; using ::testing::Field; using ::testing::Gt; +using ::testing::HasSubstr; using ::testing::IsSupersetOf; using ::testing::IsTrue; using ::testing::Key; @@ -2995,13 +2996,13 @@ INSTANTIATE_TEST_SUITE_P(StandardPath, "AV1"), StringParamToString()); -#ifdef RTC_ENABLE_H265 // These tests use fake encoders and decoders, allowing testing of codec // preferences, SDP negotiation and get/setParamaters(). But because the codecs // implementations are fake, these tests do not encode or decode any frames. class PeerConnectionEncodingsFakeCodecsIntegrationTest : public PeerConnectionEncodingsIntegrationTest { public: +#ifdef RTC_ENABLE_H265 scoped_refptr CreatePcWithFakeH265( std::unique_ptr field_trials = nullptr) { std::unique_ptr @@ -3026,8 +3027,62 @@ class PeerConnectionEncodingsFakeCodecsIntegrationTest std::move(video_decoder_factory), std::move(field_trials)); return pc_wrapper; } +#endif // RTC_ENABLE_H265 + + // Creates a PC where we have H264 with one sendonly, one recvonly and one + // sendrecv "profile-level-id". The sendrecv one is constrained baseline. + scoped_refptr CreatePcWithUnidirectionalH264( + std::unique_ptr field_trials = nullptr) { + std::unique_ptr + video_encoder_factory = + std::make_unique(); + SdpVideoFormat h264_constrained_baseline = + SdpVideoFormat("H264", + {{"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + {"profile-level-id", "42f00b"}}, // sendrecv + {ScalabilityMode::kL1T1}); + video_encoder_factory->AddSupportedVideoCodec(h264_constrained_baseline); + video_encoder_factory->AddSupportedVideoCodec( + SdpVideoFormat("H264", + {{"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + {"profile-level-id", "640034"}}, // sendonly + {ScalabilityMode::kL1T1})); + std::unique_ptr + video_decoder_factory = + std::make_unique(); + video_decoder_factory->AddSupportedVideoCodec(h264_constrained_baseline); + video_decoder_factory->AddSupportedVideoCodec( + SdpVideoFormat("H264", + {{"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + {"profile-level-id", "f4001f"}}, // recvonly + {ScalabilityMode::kL1T1})); + auto pc_wrapper = make_ref_counted( + "pc", &pss_, background_thread_.get(), background_thread_.get()); + pc_wrapper->CreatePc( + {}, CreateBuiltinAudioEncoderFactory(), + CreateBuiltinAudioDecoderFactory(), std::move(video_encoder_factory), + std::move(video_decoder_factory), std::move(field_trials)); + return pc_wrapper; + } + + std::string LocalDescriptionStr(PeerConnectionTestWrapper* pc_wrapper) { + const SessionDescriptionInterface* local_description = + pc_wrapper->pc()->local_description(); + if (!local_description) { + return ""; + } + std::string str; + if (!local_description->ToString(&str)) { + return ""; + } + return str; + } }; +#ifdef RTC_ENABLE_H265 TEST_F(PeerConnectionEncodingsFakeCodecsIntegrationTest, H265Singlecast) { rtc::scoped_refptr local_pc_wrapper = CreatePcWithFakeH265(); @@ -3160,4 +3215,143 @@ TEST_F(PeerConnectionEncodingsFakeCodecsIntegrationTest, } #endif // RTC_ENABLE_H265 +TEST_F(PeerConnectionEncodingsFakeCodecsIntegrationTest, + H264UnidirectionalNegotiation) { + rtc::scoped_refptr local_pc_wrapper = + CreatePcWithUnidirectionalH264(); + rtc::scoped_refptr remote_pc_wrapper = + CreatePcWithUnidirectionalH264(); + ExchangeIceCandidates(local_pc_wrapper, remote_pc_wrapper); + + rtc::scoped_refptr transceiver = + local_pc_wrapper->pc() + ->AddTransceiver(cricket::MEDIA_TYPE_VIDEO) + .MoveValue(); + + // Filter on codec name and assert that sender capabilities have codecs for + // {sendrecv, sendonly} and the receiver capabilities have codecs for + // {sendrecv, recvonly}. + std::vector send_codecs = + local_pc_wrapper->pc_factory() + ->GetRtpSenderCapabilities(cricket::MEDIA_TYPE_VIDEO) + .codecs; + send_codecs.erase(std::remove_if(send_codecs.begin(), send_codecs.end(), + [](const RtpCodecCapability& codec) { + return codec.name != "H264"; + }), + send_codecs.end()); + std::vector recv_codecs = + local_pc_wrapper->pc_factory() + ->GetRtpReceiverCapabilities(cricket::MEDIA_TYPE_VIDEO) + .codecs; + recv_codecs.erase(std::remove_if(recv_codecs.begin(), recv_codecs.end(), + [](const RtpCodecCapability& codec) { + RTC_LOG(LS_ERROR) << codec.name; + return codec.name != "H264"; + }), + recv_codecs.end()); + ASSERT_THAT(send_codecs, SizeIs(2u)); + ASSERT_THAT(recv_codecs, SizeIs(2u)); + EXPECT_EQ(send_codecs[0], recv_codecs[0]); + EXPECT_NE(send_codecs[1], recv_codecs[1]); + RtpCodecCapability& sendrecv_codec = send_codecs[0]; + RtpCodecCapability& sendonly_codec = send_codecs[1]; + RtpCodecCapability& recvonly_codec = recv_codecs[1]; + + // Preferring sendonly + recvonly on a sendrecv transceiver is the same as + // not having any preferences, meaning the sendrecv codec (not listed) is the + // one being negotiated. + std::vector preferred_codecs = {sendonly_codec, + recvonly_codec}; + EXPECT_THAT(transceiver->SetCodecPreferences(preferred_codecs), IsRtcOk()); + EXPECT_THAT( + transceiver->SetDirectionWithError(RtpTransceiverDirection::kSendRecv), + IsRtcOk()); + Negotiate(local_pc_wrapper, remote_pc_wrapper); + std::string local_sdp = LocalDescriptionStr(local_pc_wrapper.get()); + EXPECT_THAT(local_sdp, + HasSubstr(sendrecv_codec.parameters["profile-level-id"])); + EXPECT_THAT(local_sdp, + Not(HasSubstr(sendonly_codec.parameters["profile-level-id"]))); + EXPECT_THAT(local_sdp, + Not(HasSubstr(recvonly_codec.parameters["profile-level-id"]))); + + // Prefer all codecs and expect that the SDP offer contains the relevant + // codecs after filtering. Complete O/A each time. + preferred_codecs = {sendrecv_codec, sendonly_codec, recvonly_codec}; + EXPECT_THAT(transceiver->SetCodecPreferences(preferred_codecs), IsRtcOk()); + // Transceiver direction: sendrecv. + EXPECT_THAT( + transceiver->SetDirectionWithError(RtpTransceiverDirection::kSendRecv), + IsRtcOk()); + Negotiate(local_pc_wrapper, remote_pc_wrapper); + local_sdp = LocalDescriptionStr(local_pc_wrapper.get()); + EXPECT_THAT(local_sdp, + HasSubstr(sendrecv_codec.parameters["profile-level-id"])); + EXPECT_THAT(local_sdp, + Not(HasSubstr(sendonly_codec.parameters["profile-level-id"]))); + EXPECT_THAT(local_sdp, + Not(HasSubstr(recvonly_codec.parameters["profile-level-id"]))); + // Transceiver direction: sendonly. + EXPECT_THAT( + transceiver->SetDirectionWithError(RtpTransceiverDirection::kSendOnly), + IsRtcOk()); + Negotiate(local_pc_wrapper, remote_pc_wrapper); + local_sdp = LocalDescriptionStr(local_pc_wrapper.get()); + EXPECT_THAT(local_sdp, + HasSubstr(sendrecv_codec.parameters["profile-level-id"])); + EXPECT_THAT(local_sdp, + HasSubstr(sendonly_codec.parameters["profile-level-id"])); + EXPECT_THAT(local_sdp, + Not(HasSubstr(recvonly_codec.parameters["profile-level-id"]))); + // Transceiver direction: recvonly. + EXPECT_THAT( + transceiver->SetDirectionWithError(RtpTransceiverDirection::kRecvOnly), + IsRtcOk()); + Negotiate(local_pc_wrapper, remote_pc_wrapper); + local_sdp = LocalDescriptionStr(local_pc_wrapper.get()); + EXPECT_THAT(local_sdp, + HasSubstr(sendrecv_codec.parameters["profile-level-id"])); + EXPECT_THAT(local_sdp, + Not(HasSubstr(sendonly_codec.parameters["profile-level-id"]))); + EXPECT_THAT(local_sdp, + HasSubstr(recvonly_codec.parameters["profile-level-id"])); + + // Test that offering a sendonly codec on a sendonly transceiver is possible. + // - Note that we don't complete the negotiation this time because we're not + // capable of receiving the codec. + preferred_codecs = {sendonly_codec}; + EXPECT_THAT(transceiver->SetCodecPreferences(preferred_codecs), IsRtcOk()); + EXPECT_THAT( + transceiver->SetDirectionWithError(RtpTransceiverDirection::kSendOnly), + IsRtcOk()); + std::unique_ptr offer = + CreateOffer(local_pc_wrapper); + EXPECT_TRUE(Await({SetLocalDescription(local_pc_wrapper, offer.get())})); + local_sdp = LocalDescriptionStr(local_pc_wrapper.get()); + EXPECT_THAT(local_sdp, + Not(HasSubstr(sendrecv_codec.parameters["profile-level-id"]))); + EXPECT_THAT(local_sdp, + HasSubstr(sendonly_codec.parameters["profile-level-id"])); + EXPECT_THAT(local_sdp, + Not(HasSubstr(recvonly_codec.parameters["profile-level-id"]))); + // Test that offering recvonly codec on a recvonly transceiver is possible. + // - Note that we don't complete the negotiation this time because we're not + // capable of sending the codec. + preferred_codecs = {recvonly_codec}; + EXPECT_THAT(transceiver->SetCodecPreferences(preferred_codecs), IsRtcOk()); + EXPECT_THAT( + transceiver->SetDirectionWithError(RtpTransceiverDirection::kRecvOnly), + IsRtcOk()); + offer = CreateOffer(local_pc_wrapper); + EXPECT_TRUE(Await({SetLocalDescription(local_pc_wrapper, offer.get())})); + local_sdp = LocalDescriptionStr(local_pc_wrapper.get()); + EXPECT_THAT(local_sdp, + Not(HasSubstr(sendrecv_codec.parameters["profile-level-id"]))); + EXPECT_THAT(local_sdp, + Not(HasSubstr(sendonly_codec.parameters["profile-level-id"]))); + EXPECT_THAT(local_sdp, + HasSubstr(recvonly_codec.parameters["profile-level-id"])); +} + } // namespace webrtc diff --git a/pc/peer_connection_media_unittest.cc b/pc/peer_connection_media_unittest.cc index 27bf1435d6..cdc9d0e22e 100644 --- a/pc/peer_connection_media_unittest.cc +++ b/pc/peer_connection_media_unittest.cc @@ -60,6 +60,7 @@ #ifdef WEBRTC_ANDROID #include "pc/test/android_test_initializer.h" #endif +#include "api/test/rtc_error_matchers.h" #include "rtc_base/virtual_socket_server.h" #include "test/gmock.h" @@ -1512,8 +1513,9 @@ TEST_F(PeerConnectionMediaTestUnifiedPlan, return codec.name.find("_only_") != std::string::npos; }); - auto result = transceiver->SetCodecPreferences(codecs); - EXPECT_EQ(RTCErrorType::INVALID_MODIFICATION, result.type()); + // This is OK, however because the codec is send-only and the transciever is + // not send-only, it would get filtered out during negotiation. + EXPECT_THAT(transceiver->SetCodecPreferences(codecs), IsRtcOk()); } TEST_F(PeerConnectionMediaTestUnifiedPlan, @@ -2044,7 +2046,7 @@ TEST_F(PeerConnectionMediaTestUnifiedPlan, } TEST_F(PeerConnectionMediaTestUnifiedPlan, - SetCodecPreferencesReceiveOnlyWithSendOnlyTransceiverStops) { + SetCodecPreferencesRecvOnlyCodecOnSendOnlyTransceiver) { auto fake_engine = std::make_unique(); std::vector audio_codecs; @@ -2065,7 +2067,10 @@ TEST_F(PeerConnectionMediaTestUnifiedPlan, EXPECT_TRUE(audio_transceiver->SetCodecPreferences(capabilities.codecs).ok()); RTCOfferAnswerOptions options; EXPECT_TRUE(caller->SetLocalDescription(caller->CreateOffer(options))); - EXPECT_EQ(audio_transceiver->direction(), RtpTransceiverDirection::kStopped); + // The transceiver is still sendonly (not stopped) because preferring a codec + // that is not applicable to the sendonly use case is the same as not having + // any codec preferences. + EXPECT_EQ(audio_transceiver->direction(), RtpTransceiverDirection::kSendOnly); } TEST_F(PeerConnectionMediaTestUnifiedPlan, SetCodecPreferencesVideoNoRtx) { diff --git a/pc/rtp_transceiver.cc b/pc/rtp_transceiver.cc index 788b49145e..dee1df89bd 100644 --- a/pc/rtp_transceiver.cc +++ b/pc/rtp_transceiver.cc @@ -44,6 +44,7 @@ #include "api/video/video_bitrate_allocator_factory.h" #include "api/video_codecs/scalability_mode.h" #include "media/base/codec.h" +#include "media/base/codec_comparators.h" #include "media/base/media_channel.h" #include "media/base/media_config.h" #include "media/base/media_engine.h" @@ -64,56 +65,45 @@ namespace webrtc { namespace { +bool HasAnyMediaCodec(const std::vector& codecs) { + return absl::c_any_of(codecs, [](const RtpCodecCapability& codec) { + return codec.IsMediaCodec(); + }); +} + RTCError VerifyCodecPreferences( - const std::vector& unfiltered_codecs, - const std::vector& recv_codecs, - const FieldTrialsView& field_trials) { - // If the intersection between codecs and - // RTCRtpReceiver.getCapabilities(kind).codecs only contains RTX, RED, FEC - // codecs or Comfort Noise codecs or is an empty set, throw + const std::vector& codecs, + const std::vector& send_codecs, + const std::vector& recv_codecs) { + // `codec_capabilities` is the union of `send_codecs` and `recv_codecs`. + std::vector codec_capabilities; + codec_capabilities.reserve(send_codecs.size() + recv_codecs.size()); + codec_capabilities.insert(codec_capabilities.end(), send_codecs.begin(), + send_codecs.end()); + codec_capabilities.insert(codec_capabilities.end(), recv_codecs.begin(), + recv_codecs.end()); + // If a media codec is not recognized from `codec_capabilities`, throw // InvalidModificationError. - // This ensures that we always have something to offer, regardless of - // transceiver.direction. - // TODO(fippo): clean up the filtering killswitch - std::vector codecs = unfiltered_codecs; - if (!absl::c_any_of(codecs, [&recv_codecs](const RtpCodecCapability& codec) { - return codec.IsMediaCodec() && - absl::c_any_of(recv_codecs, - [&codec](const cricket::Codec& recv_codec) { - return recv_codec.MatchesRtpCodec(codec); + if (!absl::c_all_of(codecs, [&codec_capabilities]( + const RtpCodecCapability& codec) { + return !codec.IsMediaCodec() || + absl::c_any_of(codec_capabilities, + [&codec](const cricket::Codec& codec_capability) { + return IsSameRtpCodec(codec_capability, codec); }); })) { LOG_AND_RETURN_ERROR(RTCErrorType::INVALID_MODIFICATION, - "Invalid codec preferences: Missing codec from recv " - "codec capabilities."); + "Invalid codec preferences: Missing codec from codec " + "capabilities."); } - - // Let codecCapabilities RTCRtpReceiver.getCapabilities(kind).codecs. - // For each codec in codecs, If - // codec is not in codecCapabilities, throw InvalidModificationError. - for (const auto& codec_preference : codecs) { - bool is_recv_codec = absl::c_any_of( - recv_codecs, [&codec_preference](const cricket::Codec& codec) { - return codec.MatchesRtpCodec(codec_preference); - }); - if (!is_recv_codec) { - LOG_AND_RETURN_ERROR( - RTCErrorType::INVALID_MODIFICATION, - std::string("Invalid codec preferences: invalid codec with name \"") + - codec_preference.name + "\"."); - } - } - - // Check we have a real codec (not just rtx, red, fec or CN) - if (absl::c_all_of(codecs, [](const RtpCodecCapability& codec) { - return !codec.IsMediaCodec(); - })) { + // If `codecs` only contains entries for RTX, RED, FEC or Comfort Noise, throw + // InvalidModificationError. + if (!HasAnyMediaCodec(codecs)) { LOG_AND_RETURN_ERROR( RTCErrorType::INVALID_MODIFICATION, "Invalid codec preferences: codec list must have a non " - "RTX, RED or FEC entry."); + "RTX, RED, FEC or Comfort Noise entry."); } - return RTCError::OK(); } @@ -668,32 +658,105 @@ RTCError RtpTransceiver::SetCodecPreferences( // to codecs and abort these steps. if (codec_capabilities.empty()) { codec_preferences_.clear(); + sendrecv_codec_preferences_.clear(); + sendonly_codec_preferences_.clear(); + recvonly_codec_preferences_.clear(); return RTCError::OK(); } - // 4. Remove any duplicate values in codecs. std::vector codecs; absl::c_remove_copy_if(codec_capabilities, std::back_inserter(codecs), [&codecs](const RtpCodecCapability& codec) { return absl::c_linear_search(codecs, codec); }); + // TODO(https://crbug.com/webrtc/391530822): Move logic in + // MediaSessionDescriptionFactory to this level. + return UpdateCodecPreferencesCaches(codecs); +} - // 6. to 8. - RTCError result; - std::vector recv_codecs; +RTCError RtpTransceiver::UpdateCodecPreferencesCaches( + const std::vector& codecs) { + // Get codec capabilities from media engine. + std::vector send_codecs, recv_codecs; if (media_type_ == cricket::MEDIA_TYPE_AUDIO) { + send_codecs = media_engine()->voice().send_codecs(); recv_codecs = media_engine()->voice().recv_codecs(); } else if (media_type_ == cricket::MEDIA_TYPE_VIDEO) { + send_codecs = media_engine()->video().send_codecs(); recv_codecs = media_engine()->video().recv_codecs(context()->use_rtx()); } - result = VerifyCodecPreferences(codecs, recv_codecs, - context()->env().field_trials()); - - if (result.ok()) { - codec_preferences_ = codecs; + RTCError error = VerifyCodecPreferences(codecs, send_codecs, recv_codecs); + if (!error.ok()) { + return error; } + codec_preferences_ = codecs; + // Update the filtered views of `codec_preferences_` so that we don't have + // to query codec capabilities when calling filtered_codec_preferences() or + // every time the direction changes. + sendrecv_codec_preferences_.clear(); + sendonly_codec_preferences_.clear(); + recvonly_codec_preferences_.clear(); + for (const RtpCodecCapability& codec : codec_preferences_) { + if (!codec.IsMediaCodec()) { + // Non-media codecs don't need to be filtered at this level. + sendrecv_codec_preferences_.push_back(codec); + sendonly_codec_preferences_.push_back(codec); + recvonly_codec_preferences_.push_back(codec); + continue; + } + // Is this a send codec, receive codec or both? + bool is_send_codec = + absl::c_any_of(send_codecs, [&codec](const cricket::Codec& send_codec) { + return IsSameRtpCodecIgnoringLevel(send_codec, codec); + }); + bool is_recv_codec = + absl::c_any_of(recv_codecs, [&codec](const cricket::Codec& recv_codec) { + return IsSameRtpCodecIgnoringLevel(recv_codec, codec); + }); + // The codec being neither for sending or receving is not possible because + // of prior validation by VerifyCodecPreferences(). + RTC_CHECK(is_send_codec || is_recv_codec); + if (is_send_codec && is_recv_codec) { + sendrecv_codec_preferences_.push_back(codec); + } + if (is_send_codec) { + sendonly_codec_preferences_.push_back(codec); + } + if (is_recv_codec) { + recvonly_codec_preferences_.push_back(codec); + } + } + // If filtering results in an empty list this is the same as not having any + // preferences. + if (!HasAnyMediaCodec(sendrecv_codec_preferences_)) { + sendrecv_codec_preferences_.clear(); + } + if (!HasAnyMediaCodec(sendonly_codec_preferences_)) { + sendonly_codec_preferences_.clear(); + } + if (!HasAnyMediaCodec(recvonly_codec_preferences_)) { + recvonly_codec_preferences_.clear(); + } + return RTCError::OK(); +} - return result; +std::vector RtpTransceiver::codec_preferences() const { + return codec_preferences_; +} + +std::vector RtpTransceiver::filtered_codec_preferences() + const { + switch (direction_) { + case RtpTransceiverDirection::kSendRecv: + case RtpTransceiverDirection::kInactive: + case RtpTransceiverDirection::kStopped: + return sendrecv_codec_preferences_; + case RtpTransceiverDirection::kSendOnly: + return sendonly_codec_preferences_; + case RtpTransceiverDirection::kRecvOnly: + return recvonly_codec_preferences_; + } + return codec_preferences_; } std::vector diff --git a/pc/rtp_transceiver.h b/pc/rtp_transceiver.h index e544cece34..5908415bef 100644 --- a/pc/rtp_transceiver.h +++ b/pc/rtp_transceiver.h @@ -275,9 +275,13 @@ class RtpTransceiver : public RtpTransceiverInterface { void StopInternal() override; RTCError SetCodecPreferences( rtc::ArrayView codecs) override; - std::vector codec_preferences() const override { - return codec_preferences_; - } + // TODO(https://crbug.com/webrtc/391275081): Delete codec_preferences() in + // favor of filtered_codec_preferences() because it's not used anywhere. + std::vector codec_preferences() const override; + // A direction()-filtered view of codec_preferences(). If this filtering + // results in not having any media codecs, an empty list is returned to mean + // "no preferences". + std::vector filtered_codec_preferences() const; std::vector GetHeaderExtensionsToNegotiate() const override; std::vector GetNegotiatedHeaderExtensions() @@ -311,6 +315,9 @@ class RtpTransceiver : public RtpTransceiverInterface { // are updated before deleting it. void DeleteChannel(); + RTCError UpdateCodecPreferencesCaches( + const std::vector& codecs); + // Enforce that this object is created, used and destroyed on one thread. TaskQueueBase* const thread_; const bool unified_plan_; @@ -340,6 +347,9 @@ class RtpTransceiver : public RtpTransceiverInterface { std::unique_ptr channel_ = nullptr; ConnectionContext* const context_; std::vector codec_preferences_; + std::vector sendrecv_codec_preferences_; + std::vector sendonly_codec_preferences_; + std::vector recvonly_codec_preferences_; std::vector header_extensions_to_negotiate_; // `negotiated_header_extensions_` is read and written to on the signaling diff --git a/pc/rtp_transceiver_unittest.cc b/pc/rtp_transceiver_unittest.cc index dc23c818b2..bc01313887 100644 --- a/pc/rtp_transceiver_unittest.cc +++ b/pc/rtp_transceiver_unittest.cc @@ -20,7 +20,9 @@ #include "api/environment/environment_factory.h" #include "api/peer_connection_interface.h" #include "api/rtp_parameters.h" +#include "api/test/rtc_error_matchers.h" #include "media/base/fake_media_engine.h" +#include "pc/rtp_parameters_conversion.h" #include "pc/test/enable_fake_media.h" #include "pc/test/mock_channel_interface.h" #include "pc/test/mock_rtp_receiver_internal.h" @@ -37,6 +39,7 @@ using ::testing::Optional; using ::testing::Property; using ::testing::Return; using ::testing::ReturnRef; +using ::testing::SizeIs; namespace webrtc { @@ -168,6 +171,7 @@ class RtpTransceiverUnifiedPlanTest : public RtpTransceiverTest { /* on_negotiation_needed= */ [] {}); } + protected: rtc::AutoThread main_thread_; }; @@ -197,6 +201,263 @@ TEST_F(RtpTransceiverUnifiedPlanTest, StopSetsDirection) { *transceiver->current_direction()); } +class RtpTransceiverFilteredCodecPreferencesTest + : public RtpTransceiverUnifiedPlanTest { + public: + RtpTransceiverFilteredCodecPreferencesTest() + : transceiver_(CreateTransceiver( + MockSender(cricket::MediaType::MEDIA_TYPE_VIDEO), + MockReceiver(cricket::MediaType::MEDIA_TYPE_VIDEO))) {} + + struct H264CodecCapabilities { + cricket::Codec cricket_sendrecv_codec; + RtpCodecCapability sendrecv_codec; + cricket::Codec cricket_sendonly_codec; + RtpCodecCapability sendonly_codec; + cricket::Codec cricket_recvonly_codec; + RtpCodecCapability recvonly_codec; + cricket::Codec cricket_rtx_codec; + RtpCodecCapability rtx_codec; + }; + + // For H264, the profile and level IDs are entangled and not ignored by + // IsSameRtpCodecIgnoringLevel(). + H264CodecCapabilities ConfigureH264CodecCapabilities() { + cricket::Codec cricket_sendrecv_codec = cricket::CreateVideoCodec( + SdpVideoFormat("H264", + {{"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + {"profile-level-id", "42f00b"}}, + {ScalabilityMode::kL1T1})); + cricket::Codec cricket_sendonly_codec = cricket::CreateVideoCodec( + SdpVideoFormat("H264", + {{"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + {"profile-level-id", "640034"}}, + {ScalabilityMode::kL1T1})); + cricket::Codec cricket_recvonly_codec = cricket::CreateVideoCodec( + SdpVideoFormat("H264", + {{"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + {"profile-level-id", "f4001f"}}, + {ScalabilityMode::kL1T1})); + cricket::Codec cricket_rtx_codec = cricket::CreateVideoRtxCodec( + cricket::Codec::kIdNotSet, cricket::Codec::kIdNotSet); + media_engine()->SetVideoSendCodecs( + {cricket_sendrecv_codec, cricket_sendonly_codec, cricket_rtx_codec}); + media_engine()->SetVideoRecvCodecs( + {cricket_sendrecv_codec, cricket_recvonly_codec, cricket_rtx_codec}); + return { + .cricket_sendrecv_codec = cricket_sendrecv_codec, + .sendrecv_codec = ToRtpCodecCapability(cricket_sendrecv_codec), + .cricket_sendonly_codec = cricket_sendonly_codec, + .sendonly_codec = ToRtpCodecCapability(cricket_sendonly_codec), + .cricket_recvonly_codec = cricket_recvonly_codec, + .recvonly_codec = ToRtpCodecCapability(cricket_recvonly_codec), + .cricket_rtx_codec = cricket_rtx_codec, + .rtx_codec = ToRtpCodecCapability(cricket_rtx_codec), + }; + } + +#ifdef RTC_ENABLE_H265 + struct H265CodecCapabilities { + // The level-id from sender getCapabilities() or receiver getCapabilities(). + static constexpr const char* kSendOnlyLevel = "180"; + static constexpr const char* kRecvOnlyLevel = "156"; + // A valid H265 level-id, but one not present in either getCapabilities(). + static constexpr const char* kLevelNotInCapabilities = "135"; + + cricket::Codec cricket_sendonly_codec; + RtpCodecCapability sendonly_codec; + cricket::Codec cricket_recvonly_codec; + RtpCodecCapability recvonly_codec; + }; + + // For H265, the profile and level IDs are separate and are ignored by + // IsSameRtpCodecIgnoringLevel(). + H265CodecCapabilities ConfigureH265CodecCapabilities() { + cricket::Codec cricket_sendonly_codec = cricket::CreateVideoCodec( + SdpVideoFormat("H265", + {{"profile-id", "1"}, + {"tier-flag", "0"}, + {"level-id", H265CodecCapabilities::kSendOnlyLevel}, + {"tx-mode", "SRST"}}, + {ScalabilityMode::kL1T1})); + cricket::Codec cricket_recvonly_codec = cricket::CreateVideoCodec( + SdpVideoFormat("H265", + {{"profile-id", "1"}, + {"tier-flag", "0"}, + {"level-id", H265CodecCapabilities::kRecvOnlyLevel}, + {"tx-mode", "SRST"}}, + {ScalabilityMode::kL1T1})); + media_engine()->SetVideoSendCodecs({cricket_sendonly_codec}); + media_engine()->SetVideoRecvCodecs({cricket_recvonly_codec}); + return { + .cricket_sendonly_codec = cricket_sendonly_codec, + .sendonly_codec = ToRtpCodecCapability(cricket_sendonly_codec), + .cricket_recvonly_codec = cricket_recvonly_codec, + .recvonly_codec = ToRtpCodecCapability(cricket_recvonly_codec), + }; + } +#endif // RTC_ENABLE_H265 + + protected: + rtc::scoped_refptr transceiver_; +}; + +TEST_F(RtpTransceiverFilteredCodecPreferencesTest, EmptyByDefault) { + ConfigureH264CodecCapabilities(); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kSendRecv), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), SizeIs(0)); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kSendOnly), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), SizeIs(0)); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kRecvOnly), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), SizeIs(0)); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kInactive), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), SizeIs(0)); +} + +TEST_F(RtpTransceiverFilteredCodecPreferencesTest, OrderIsMaintained) { + const auto codecs = ConfigureH264CodecCapabilities(); + std::vector codec_capabilities = {codecs.sendrecv_codec, + codecs.rtx_codec}; + EXPECT_THAT(transceiver_->SetCodecPreferences(codec_capabilities), IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codec_capabilities[0], codec_capabilities[1])); + // Reverse order. + codec_capabilities = {codecs.rtx_codec, codecs.sendrecv_codec}; + EXPECT_THAT(transceiver_->SetCodecPreferences(codec_capabilities), IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codec_capabilities[0], codec_capabilities[1])); +} + +TEST_F(RtpTransceiverFilteredCodecPreferencesTest, + FiltersCodecsBasedOnDirection) { + const auto codecs = ConfigureH264CodecCapabilities(); + std::vector codec_capabilities = { + codecs.sendonly_codec, codecs.sendrecv_codec, codecs.recvonly_codec}; + EXPECT_THAT(transceiver_->SetCodecPreferences(codec_capabilities), IsRtcOk()); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kSendRecv), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codecs.sendrecv_codec)); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kSendOnly), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codecs.sendonly_codec, codecs.sendrecv_codec)); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kRecvOnly), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codecs.sendrecv_codec, codecs.recvonly_codec)); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kInactive), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codecs.sendrecv_codec)); +} + +TEST_F(RtpTransceiverFilteredCodecPreferencesTest, + RtxIsIncludedAfterFiltering) { + const auto codecs = ConfigureH264CodecCapabilities(); + std::vector codec_capabilities = {codecs.recvonly_codec, + codecs.rtx_codec}; + EXPECT_THAT(transceiver_->SetCodecPreferences(codec_capabilities), IsRtcOk()); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kRecvOnly), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codecs.recvonly_codec, codecs.rtx_codec)); +} + +TEST_F(RtpTransceiverFilteredCodecPreferencesTest, + NoMediaIsTheSameAsNoPreference) { + const auto codecs = ConfigureH264CodecCapabilities(); + std::vector codec_capabilities = {codecs.recvonly_codec, + codecs.rtx_codec}; + EXPECT_THAT(transceiver_->SetCodecPreferences(codec_capabilities), IsRtcOk()); + + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kSendOnly), + IsRtcOk()); + // After filtering the only codec that remains is RTX which is not a media + // codec, this is the same as not having any preferences. + EXPECT_THAT(transceiver_->filtered_codec_preferences(), SizeIs(0)); + + // But the preferences are remembered in case the direction changes such that + // we do have a media codec. + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kRecvOnly), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codecs.recvonly_codec, codecs.rtx_codec)); +} + +#ifdef RTC_ENABLE_H265 +TEST_F(RtpTransceiverFilteredCodecPreferencesTest, + H265LevelIdIsIgnoredByFilter) { + const auto codecs = ConfigureH265CodecCapabilities(); + std::vector codec_capabilities = {codecs.sendonly_codec, + codecs.recvonly_codec}; + EXPECT_THAT(transceiver_->SetCodecPreferences(codec_capabilities), IsRtcOk()); + // Regardless of direction, both codecs are preferred due to ignoring levels. + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kSendOnly), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codec_capabilities[0], codec_capabilities[1])); + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kRecvOnly), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codec_capabilities[0], codec_capabilities[1])); + EXPECT_THAT( + transceiver_->SetDirectionWithError(RtpTransceiverDirection::kSendRecv), + IsRtcOk()); + EXPECT_THAT(transceiver_->filtered_codec_preferences(), + ElementsAre(codec_capabilities[0], codec_capabilities[1])); +} + +TEST_F(RtpTransceiverFilteredCodecPreferencesTest, + H265LevelIdHasToFromSenderOrReceiverCapabilities) { + ConfigureH265CodecCapabilities(); + cricket::Codec cricket_codec = cricket::CreateVideoCodec(SdpVideoFormat( + "H265", + {{"profile-id", "1"}, + {"tier-flag", "0"}, + {"level-id", H265CodecCapabilities::kLevelNotInCapabilities}, + {"tx-mode", "SRST"}}, + {ScalabilityMode::kL1T1})); + + std::vector codec_capabilities = { + ToRtpCodecCapability(cricket_codec)}; + EXPECT_THAT(transceiver_->SetCodecPreferences(codec_capabilities), + IsRtcErrorWithTypeAndMessage( + RTCErrorType::INVALID_MODIFICATION, + "Invalid codec preferences: Missing codec from codec " + "capabilities.")); +} +#endif // RTC_ENABLE_H265 + class RtpTransceiverTestForHeaderExtensions : public RtpTransceiverUnifiedPlanTest { public: diff --git a/pc/sdp_offer_answer.cc b/pc/sdp_offer_answer.cc index 7ff0aaeabd..22c6e7f38f 100644 --- a/pc/sdp_offer_answer.cc +++ b/pc/sdp_offer_answer.cc @@ -776,7 +776,7 @@ cricket::MediaDescriptionOptions GetMediaDescriptionOptionsForTransceiver( cricket::MediaDescriptionOptions media_description_options( transceiver->media_type(), mid, transceiver->direction(), stopped); media_description_options.codec_preferences = - transceiver->codec_preferences(); + transceiver->filtered_codec_preferences(); media_description_options.header_extensions = transceiver->GetHeaderExtensionsToNegotiate(); // This behavior is specified in JSEP. The gist is that: