From fb1959625d61b9d10e2b63fee8096e486191516b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Mon, 16 Aug 2021 16:41:56 +0200 Subject: [PATCH] Allow setting different number of temporal layers per simulcast layer. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting different number of temporal layers is supported by SimulcastEncodeAdapter and LibvpxVp8Encoder will fallback to SimulcastEncoderAdapter if InitEncode fails. Bug: none Change-Id: I8a09ee1e6c70a0006317957c0802d019a0d28ca2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228642 Reviewed-by: Erik Språng Reviewed-by: Harald Alvestrand Commit-Queue: Åsa Persson Cr-Commit-Position: refs/heads/master@{#34785} --- api/rtp_parameters.h | 2 - media/base/media_engine.cc | 9 - media/engine/webrtc_video_engine_unittest.cc | 49 ++--- pc/rtp_sender_receiver_unittest.cc | 24 +++ video/video_send_stream_tests.cc | 209 +++++++++++++++++++ 5 files changed, 246 insertions(+), 47 deletions(-) diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h index 71ae9843d1..2664ae734a 100644 --- a/api/rtp_parameters.h +++ b/api/rtp_parameters.h @@ -489,8 +489,6 @@ struct RTC_EXPORT RtpEncodingParameters { // Specifies the number of temporal layers for video (if the feature is // supported by the codec implementation). - // TODO(asapersson): Different number of temporal layers are not supported - // per simulcast layer. // Screencast support is experimental. absl::optional num_temporal_layers; diff --git a/media/base/media_engine.cc b/media/base/media_engine.cc index 36a9694cfb..21c3787382 100644 --- a/media/base/media_engine.cc +++ b/media/base/media_engine.cc @@ -106,15 +106,6 @@ webrtc::RTCError CheckRtpParametersValues( "num_temporal_layers to an invalid number."); } } - if (i > 0 && (rtp_parameters.encodings[i].num_temporal_layers != - rtp_parameters.encodings[i - 1].num_temporal_layers)) { - LOG_AND_RETURN_ERROR( - RTCErrorType::INVALID_MODIFICATION, - "Attempted to set RtpParameters num_temporal_layers " - "at encoding layer i: " + - rtc::ToString(i) + - " to a different value than other encoding layers."); - } } return webrtc::RTCError::OK(); diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index 7c1bf6e448..025a553040 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -7706,28 +7706,6 @@ TEST_F(WebRtcVideoChannelTest, channel_->SetRtpSendParameters(last_ssrc_, parameters).type()); } -TEST_F(WebRtcVideoChannelTest, - SetRtpSendParametersNumTemporalLayersFailsForInvalidModification) { - const size_t kNumSimulcastStreams = 3; - SetUpSimulcast(true, false); - - // Get and set the rtp encoding parameters. - webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); - EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); - - // No/all layers should be set. - parameters.encodings[0].num_temporal_layers = 1; - EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION, - channel_->SetRtpSendParameters(last_ssrc_, parameters).type()); - - // Different values not supported. - parameters.encodings[0].num_temporal_layers = 1; - parameters.encodings[1].num_temporal_layers = 2; - parameters.encodings[2].num_temporal_layers = 2; - EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION, - channel_->SetRtpSendParameters(last_ssrc_, parameters).type()); -} - TEST_F(WebRtcVideoChannelTest, GetAndSetRtpSendParametersNumTemporalLayers) { const size_t kNumSimulcastStreams = 3; SetUpSimulcast(true, false); @@ -7767,9 +7745,9 @@ TEST_F(WebRtcVideoChannelTest, NumTemporalLayersPropagatedToEncoder) { // Change the value and set it on the VideoChannel. webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); - parameters.encodings[0].num_temporal_layers = 2; + parameters.encodings[0].num_temporal_layers = 3; parameters.encodings[1].num_temporal_layers = 2; - parameters.encodings[2].num_temporal_layers = 2; + parameters.encodings[2].num_temporal_layers = 1; EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); // Verify that the new value is propagated down to the encoder. @@ -7778,16 +7756,16 @@ TEST_F(WebRtcVideoChannelTest, NumTemporalLayersPropagatedToEncoder) { webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams); EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size()); - EXPECT_EQ(2UL, encoder_config.simulcast_layers[0].num_temporal_layers); + EXPECT_EQ(3UL, encoder_config.simulcast_layers[0].num_temporal_layers); EXPECT_EQ(2UL, encoder_config.simulcast_layers[1].num_temporal_layers); - EXPECT_EQ(2UL, encoder_config.simulcast_layers[2].num_temporal_layers); + EXPECT_EQ(1UL, encoder_config.simulcast_layers[2].num_temporal_layers); // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of // VideoStreams are created appropriately for the simulcast case. EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); - EXPECT_EQ(2UL, stream->GetVideoStreams()[0].num_temporal_layers); + EXPECT_EQ(3UL, stream->GetVideoStreams()[0].num_temporal_layers); EXPECT_EQ(2UL, stream->GetVideoStreams()[1].num_temporal_layers); - EXPECT_EQ(2UL, stream->GetVideoStreams()[2].num_temporal_layers); + EXPECT_EQ(1UL, stream->GetVideoStreams()[2].num_temporal_layers); // No parameter changed, encoder should not be reconfigured. EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); @@ -7809,29 +7787,28 @@ TEST_F(WebRtcVideoChannelTest, channel_->SetSend(true); frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame()); - // Change rtp encoding parameters, num_temporal_layers not changed. + // Change rtp encoding parameters. webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_); EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); - parameters.encodings[0].min_bitrate_bps = 33000; + parameters.encodings[0].num_temporal_layers = 2; + parameters.encodings[2].num_temporal_layers = 1; EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok()); // Verify that no value is propagated down to the encoder. webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy(); EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams); EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size()); - EXPECT_FALSE(encoder_config.simulcast_layers[0].num_temporal_layers); + EXPECT_EQ(2UL, encoder_config.simulcast_layers[0].num_temporal_layers); EXPECT_FALSE(encoder_config.simulcast_layers[1].num_temporal_layers); - EXPECT_FALSE(encoder_config.simulcast_layers[2].num_temporal_layers); + EXPECT_EQ(1UL, encoder_config.simulcast_layers[2].num_temporal_layers); // FakeVideoSendStream calls CreateEncoderStreams, test that the vector of // VideoStreams are created appropriately for the simulcast case. EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size()); - EXPECT_EQ(kDefaultNumTemporalLayers, - stream->GetVideoStreams()[0].num_temporal_layers); + EXPECT_EQ(2UL, stream->GetVideoStreams()[0].num_temporal_layers); EXPECT_EQ(kDefaultNumTemporalLayers, stream->GetVideoStreams()[1].num_temporal_layers); - EXPECT_EQ(kDefaultNumTemporalLayers, - stream->GetVideoStreams()[2].num_temporal_layers); + EXPECT_EQ(1UL, stream->GetVideoStreams()[2].num_temporal_layers); EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); } diff --git a/pc/rtp_sender_receiver_unittest.cc b/pc/rtp_sender_receiver_unittest.cc index a8140e82f2..f36183366a 100644 --- a/pc/rtp_sender_receiver_unittest.cc +++ b/pc/rtp_sender_receiver_unittest.cc @@ -1260,6 +1260,30 @@ TEST_F(RtpSenderReceiverTest, VideoSenderDetectInvalidScaleResolutionDownBy) { DestroyVideoRtpSender(); } +TEST_F(RtpSenderReceiverTest, VideoSenderCanSetNumTemporalLayers) { + CreateVideoRtpSender(); + + RtpParameters params = video_rtp_sender_->GetParameters(); + params.encodings[0].num_temporal_layers = 2; + + EXPECT_TRUE(video_rtp_sender_->SetParameters(params).ok()); + params = video_rtp_sender_->GetParameters(); + EXPECT_EQ(2, params.encodings[0].num_temporal_layers); + + DestroyVideoRtpSender(); +} + +TEST_F(RtpSenderReceiverTest, VideoSenderDetectInvalidNumTemporalLayers) { + CreateVideoRtpSender(); + + RtpParameters params = video_rtp_sender_->GetParameters(); + params.encodings[0].num_temporal_layers = webrtc::kMaxTemporalStreams + 1; + RTCError result = video_rtp_sender_->SetParameters(params); + EXPECT_EQ(RTCErrorType::INVALID_RANGE, result.type()); + + DestroyVideoRtpSender(); +} + TEST_F(RtpSenderReceiverTest, VideoSenderCanSetMaxFramerate) { CreateVideoRtpSender(); diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index 8cb43136e5..6d6ab050a7 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -25,13 +25,18 @@ #include "call/rtp_transport_controller_send.h" #include "call/simulated_network.h" #include "call/video_send_stream.h" +#include "media/engine/internal_encoder_factory.h" +#include "media/engine/simulcast_encoder_adapter.h" +#include "media/engine/webrtc_video_engine.h" #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h" #include "modules/rtp_rtcp/source/rtcp_sender.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet.h" #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "modules/rtp_rtcp/source/rtp_util.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h" +#include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" #include "rtc_base/checks.h" @@ -122,6 +127,10 @@ class VideoSendStreamTest : public test::CallTest { uint8_t num_spatial_layers); void TestRequestSourceRotateVideo(bool support_orientation_ext); + + void TestTemporalLayers(VideoEncoderFactory* encoder_factory, + const std::string& payload_name, + const std::vector& num_temporal_layers); }; TEST_F(VideoSendStreamTest, CanStartStartedStream) { @@ -3971,4 +3980,204 @@ TEST_F(VideoSendStreamTest, SwitchesToScreenshareAndBack) { RunBaseTest(&test); } +void VideoSendStreamTest::TestTemporalLayers( + VideoEncoderFactory* encoder_factory, + const std::string& payload_name, + const std::vector& num_temporal_layers) { + static constexpr int kMaxBitrateBps = 1000000; + static constexpr int kMinFramesToObservePerStream = 8; + + class TemporalLayerObserver + : public test::EndToEndTest, + public test::FrameGeneratorCapturer::SinkWantsObserver { + public: + TemporalLayerObserver(VideoEncoderFactory* encoder_factory, + const std::string& payload_name, + const std::vector& num_temporal_layers) + : EndToEndTest(kDefaultTimeoutMs), + encoder_factory_(encoder_factory), + payload_name_(payload_name), + num_temporal_layers_(num_temporal_layers), + depacketizer_(CreateVideoRtpDepacketizer( + PayloadStringToCodecType(payload_name))) {} + + private: + void OnFrameGeneratorCapturerCreated( + test::FrameGeneratorCapturer* frame_generator_capturer) override { + frame_generator_capturer->ChangeResolution(640, 360); + } + + void OnSinkWantsChanged(rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) override {} + + void ModifySenderBitrateConfig( + BitrateConstraints* bitrate_config) override { + bitrate_config->start_bitrate_bps = kMaxBitrateBps / 2; + } + + size_t GetNumVideoStreams() const override { + return num_temporal_layers_.size(); + } + + void ModifyVideoConfigs( + VideoSendStream::Config* send_config, + std::vector* receive_configs, + VideoEncoderConfig* encoder_config) override { + send_config->encoder_settings.encoder_factory = encoder_factory_; + send_config->rtp.payload_name = payload_name_; + send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType; + encoder_config->video_format.name = payload_name_; + encoder_config->codec_type = PayloadStringToCodecType(payload_name_); + encoder_config->video_stream_factory = + rtc::make_ref_counted( + payload_name_, /*max_qp=*/56, /*is_screenshare=*/false, + /*conference_mode=*/false); + encoder_config->max_bitrate_bps = kMaxBitrateBps; + + for (size_t i = 0; i < num_temporal_layers_.size(); ++i) { + VideoStream& stream = encoder_config->simulcast_layers[i]; + stream.num_temporal_layers = num_temporal_layers_[i]; + configured_num_temporal_layers_[send_config->rtp.ssrcs[i]] = + num_temporal_layers_[i]; + } + } + + struct ParsedPacket { + uint32_t timestamp; + uint32_t ssrc; + int temporal_idx; + }; + + bool ParsePayload(const uint8_t* packet, + size_t length, + ParsedPacket& parsed) const { + RtpPacket rtp_packet; + EXPECT_TRUE(rtp_packet.Parse(packet, length)); + + if (rtp_packet.payload_size() == 0) { + return false; // Padding packet. + } + parsed.timestamp = rtp_packet.Timestamp(); + parsed.ssrc = rtp_packet.Ssrc(); + + absl::optional parsed_payload = + depacketizer_->Parse(rtp_packet.PayloadBuffer()); + EXPECT_TRUE(parsed_payload); + + if (const auto* vp8_header = absl::get_if( + &parsed_payload->video_header.video_type_header)) { + parsed.temporal_idx = vp8_header->temporalIdx; + } else { + RTC_NOTREACHED(); + } + return true; + } + + Action OnSendRtp(const uint8_t* packet, size_t length) override { + ParsedPacket parsed; + if (!ParsePayload(packet, length, parsed)) + return SEND_PACKET; + + uint32_t ssrc = parsed.ssrc; + int temporal_idx = + parsed.temporal_idx == kNoTemporalIdx ? 0 : parsed.temporal_idx; + max_observed_tl_idxs_[ssrc] = + std::max(temporal_idx, max_observed_tl_idxs_[ssrc]); + + if (last_observed_packet_.count(ssrc) == 0 || + parsed.timestamp != last_observed_packet_[ssrc].timestamp) { + num_observed_frames_[ssrc]++; + } + last_observed_packet_[ssrc] = parsed; + + if (HighestTemporalLayerSentPerStream()) + observation_complete_.Set(); + + return SEND_PACKET; + } + + bool HighestTemporalLayerSentPerStream() const { + if (num_observed_frames_.size() != + configured_num_temporal_layers_.size()) { + return false; + } + for (const auto& num_frames : num_observed_frames_) { + if (num_frames.second < kMinFramesToObservePerStream) { + return false; + } + } + if (max_observed_tl_idxs_.size() != + configured_num_temporal_layers_.size()) { + return false; + } + for (const auto& max_tl_idx : max_observed_tl_idxs_) { + uint32_t ssrc = max_tl_idx.first; + int configured_num_tls = + configured_num_temporal_layers_.find(ssrc)->second; + if (max_tl_idx.second != configured_num_tls - 1) + return false; + } + return true; + } + + void PerformTest() override { EXPECT_TRUE(Wait()); } + + VideoEncoderFactory* const encoder_factory_; + const std::string payload_name_; + const std::vector num_temporal_layers_; + const std::unique_ptr depacketizer_; + // Mapped by SSRC. + std::map configured_num_temporal_layers_; + std::map max_observed_tl_idxs_; + std::map num_observed_frames_; + std::map last_observed_packet_; + } test(encoder_factory, payload_name, num_temporal_layers); + + RunBaseTest(&test); +} + +TEST_F(VideoSendStreamTest, TestTemporalLayersVp8) { + InternalEncoderFactory internal_encoder_factory; + test::FunctionVideoEncoderFactory encoder_factory( + [&internal_encoder_factory]() { + return std::make_unique( + &internal_encoder_factory, SdpVideoFormat("VP8")); + }); + + TestTemporalLayers(&encoder_factory, "VP8", + /*num_temporal_layers=*/{2}); +} + +TEST_F(VideoSendStreamTest, TestTemporalLayersVp8Simulcast) { + InternalEncoderFactory internal_encoder_factory; + test::FunctionVideoEncoderFactory encoder_factory( + [&internal_encoder_factory]() { + return std::make_unique( + &internal_encoder_factory, SdpVideoFormat("VP8")); + }); + + TestTemporalLayers(&encoder_factory, "VP8", + /*num_temporal_layers=*/{2, 2}); +} + +TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithDifferentNumTls) { + InternalEncoderFactory internal_encoder_factory; + test::FunctionVideoEncoderFactory encoder_factory( + [&internal_encoder_factory]() { + return std::make_unique( + &internal_encoder_factory, SdpVideoFormat("VP8")); + }); + + TestTemporalLayers(&encoder_factory, "VP8", + /*num_temporal_layers=*/{3, 1}); +} + +TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithoutSimAdapter) { + test::FunctionVideoEncoderFactory encoder_factory( + []() { return VP8Encoder::Create(); }); + + TestTemporalLayers(&encoder_factory, "VP8", + /*num_temporal_layers=*/{2, 2}); +} + } // namespace webrtc