From da72666d94b7045e32879f98be4c38f6c5bd5f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20Bostr=C3=B6m?= Date: Wed, 21 Aug 2024 10:15:46 +0200 Subject: [PATCH] Support standard simulcast with `requested_resolution`. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to spec, if you ask for three encodings you get three encodings (duh). But according to legacy code, if you ask for three encodings AND codec is VP9, then surely you meant a single encoding that is kSVC where the other encodings influence the scalability mode of the first encoding. Standard simulcast support in VP9 was shipped as an opt-in feature where you have to specify `scalability_mode` and `scale_resolution_down_by` in order to let WebRTC know that you want to disable the legacy path. But `scale_resolution_down_by` is not the only way to configure resolution, there is also the `requested_resolution` code path. This CL adds standard simulcast support for this code path as well. Prior to this change, our parameterized test would have passed in VP8 but failed in VP9. With this change the test passes for all codecs. Bug: webrtc:361124448 Change-Id: Ic5a7136de8abf430813fd01342862775fca145fb Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/360100 Commit-Queue: Henrik Boström Reviewed-by: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/main@{#42822} --- media/engine/webrtc_video_engine.cc | 3 +- ...er_connection_encodings_integrationtest.cc | 74 ++++++++++++++++++- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index ac7d0f884a..fcd7b23938 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2145,7 +2145,8 @@ WebRtcVideoSendChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig( for (const webrtc::RtpEncodingParameters& encoding : rtp_parameters_.encodings) { if (encoding.scalability_mode.has_value() && - encoding.scale_resolution_down_by.has_value()) { + (encoding.scale_resolution_down_by.has_value() || + encoding.requested_resolution.has_value())) { legacy_scalability_mode = false; break; } diff --git a/pc/peer_connection_encodings_integrationtest.cc b/pc/peer_connection_encodings_integrationtest.cc index a497acb61c..0400b92182 100644 --- a/pc/peer_connection_encodings_integrationtest.cc +++ b/pc/peer_connection_encodings_integrationtest.cc @@ -2041,7 +2041,8 @@ TEST_F(PeerConnectionEncodingsIntegrationTest, } // Tests that use the standard path (specifying both `scalability_mode` and -// `scale_resolution_down_by`) should pass for all codecs. +// `scale_resolution_down_by` or `requested_resolution`) should pass for all +// codecs. class PeerConnectionEncodingsIntegrationParameterizedTest : public PeerConnectionEncodingsIntegrationTest, public ::testing::WithParamInterface { @@ -2111,6 +2112,7 @@ TEST_P(PeerConnectionEncodingsIntegrationParameterizedTest, AllLayersInactive) { EXPECT_EQ(*outbound_rtps[2]->bytes_sent, 0u); } +// Configure 4:2:1 using `scale_resolution_down_by`. TEST_P(PeerConnectionEncodingsIntegrationParameterizedTest, Simulcast) { rtc::scoped_refptr local_pc_wrapper = CreatePc(); if (SkipTestDueToAv1Missing(local_pc_wrapper)) { @@ -2120,7 +2122,7 @@ TEST_P(PeerConnectionEncodingsIntegrationParameterizedTest, Simulcast) { ExchangeIceCandidates(local_pc_wrapper, remote_pc_wrapper); std::vector layers = - CreateLayers({"f", "h", "q"}, /*active=*/true); + CreateLayers({"q", "h", "f"}, /*active=*/true); rtc::scoped_refptr transceiver = AddTransceiverWithSimulcastLayers(local_pc_wrapper, remote_pc_wrapper, layers); @@ -2158,7 +2160,73 @@ TEST_P(PeerConnectionEncodingsIntegrationParameterizedTest, Simulcast) { ASSERT_TRUE_WAIT(HasOutboundRtpBytesSent(local_pc_wrapper, 3u), kLongTimeoutForRampingUp.ms()); EXPECT_TRUE(OutboundRtpResolutionsAreLessThanOrEqualToExpectations( - local_pc_wrapper, {{"f", 320, 180}, {"h", 640, 360}, {"q", 1280, 720}})); + local_pc_wrapper, {{"q", 320, 180}, {"h", 640, 360}, {"f", 1280, 720}})); + // Verify codec and scalability mode. + rtc::scoped_refptr report = GetStats(local_pc_wrapper); + std::vector outbound_rtps = + report->GetStatsOfType(); + ASSERT_THAT(outbound_rtps, SizeIs(3u)); + EXPECT_THAT(GetCurrentCodecMimeType(report, *outbound_rtps[0]), + StrCaseEq(mime_type_)); + EXPECT_THAT(GetCurrentCodecMimeType(report, *outbound_rtps[1]), + StrCaseEq(mime_type_)); + EXPECT_THAT(GetCurrentCodecMimeType(report, *outbound_rtps[2]), + StrCaseEq(mime_type_)); + EXPECT_THAT(*outbound_rtps[0]->scalability_mode, StrEq("L1T3")); + EXPECT_THAT(*outbound_rtps[1]->scalability_mode, StrEq("L1T3")); + EXPECT_THAT(*outbound_rtps[2]->scalability_mode, StrEq("L1T3")); +} + +// Configure 4:2:1 using `requested_resolution`. +TEST_P(PeerConnectionEncodingsIntegrationParameterizedTest, + SimulcastWithRequestedResolution) { + rtc::scoped_refptr local_pc_wrapper = CreatePc(); + if (SkipTestDueToAv1Missing(local_pc_wrapper)) { + return; + } + rtc::scoped_refptr remote_pc_wrapper = CreatePc(); + ExchangeIceCandidates(local_pc_wrapper, remote_pc_wrapper); + + std::vector layers = + CreateLayers({"q", "h", "f"}, /*active=*/true); + rtc::scoped_refptr transceiver = + AddTransceiverWithSimulcastLayers(local_pc_wrapper, remote_pc_wrapper, + layers); + std::vector codecs = + GetCapabilitiesAndRestrictToCodec(remote_pc_wrapper, codec_name_); + transceiver->SetCodecPreferences(codecs); + + rtc::scoped_refptr sender = transceiver->sender(); + RtpParameters parameters = sender->GetParameters(); + ASSERT_THAT(parameters.encodings, SizeIs(3)); + parameters.encodings[0].scalability_mode = "L1T3"; + parameters.encodings[0].requested_resolution = {.width = 320, .height = 180}; + parameters.encodings[1].scalability_mode = "L1T3"; + parameters.encodings[1].requested_resolution = {.width = 640, .height = 360}; + parameters.encodings[2].scalability_mode = "L1T3"; + parameters.encodings[2].requested_resolution = {.width = 1280, .height = 720}; + sender->SetParameters(parameters); + + NegotiateWithSimulcastTweaks(local_pc_wrapper, remote_pc_wrapper); + local_pc_wrapper->WaitForConnection(); + remote_pc_wrapper->WaitForConnection(); + + // GetParameters() does not report any fallback. + parameters = sender->GetParameters(); + ASSERT_THAT(parameters.encodings, SizeIs(3)); + EXPECT_THAT(parameters.encodings[0].scalability_mode, + Optional(std::string("L1T3"))); + EXPECT_THAT(parameters.encodings[1].scalability_mode, + Optional(std::string("L1T3"))); + EXPECT_THAT(parameters.encodings[2].scalability_mode, + Optional(std::string("L1T3"))); + + // Wait until media is flowing on all three layers. + // Ramp up time is needed before all three layers are sending. + ASSERT_TRUE_WAIT(HasOutboundRtpBytesSent(local_pc_wrapper, 3u), + kLongTimeoutForRampingUp.ms()); + EXPECT_TRUE(OutboundRtpResolutionsAreLessThanOrEqualToExpectations( + local_pc_wrapper, {{"q", 320, 180}, {"h", 640, 360}, {"f", 1280, 720}})); // Verify codec and scalability mode. rtc::scoped_refptr report = GetStats(local_pc_wrapper); std::vector outbound_rtps =