From 8234ead6d9671ba01861c0e11042a02c7335c8dd Mon Sep 17 00:00:00 2001 From: Seth Hampson Date: Fri, 2 Feb 2018 15:16:24 -0800 Subject: [PATCH] Allows the application to set active simulcast streams. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently all simulcast streams are set as active by default. This update takes the values of the rtp encoding parameters and wires those values down to the VideoSendStream and VideoStreamEncoder, so that the appropriate simulcast streams can be turned off and on. This includes adding more application specific controls in the EncoderStreamFactory. Bug: webrtc:8653 Change-Id: Iaa7da3209cea0f0db72543981a319e319705cb00 Reviewed-on: https://webrtc-review.googlesource.com/47245 Reviewed-by: Erik Språng Reviewed-by: Peter Thatcher Commit-Queue: Seth Hampson Cr-Commit-Position: refs/heads/master@{#21920} --- media/engine/webrtcvideoengine.cc | 84 +++++++++++++++------- media/engine/webrtcvideoengine_unittest.cc | 79 +++++++++++++++++++- video/video_quality_test.cc | 4 ++ 3 files changed, 139 insertions(+), 28 deletions(-) diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc index 6c10ca8152..dbf862a86b 100644 --- a/media/engine/webrtcvideoengine.cc +++ b/media/engine/webrtcvideoengine.cc @@ -1805,14 +1805,25 @@ webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters( rtp_parameters_.encodings[0].max_bitrate_bps) || (new_parameters.encodings[0].bitrate_priority != rtp_parameters_.encodings[0].bitrate_priority); + // TODO(bugs.webrtc.org/8807): The active field as well should not require + // a full encoder reconfiguration, but it needs to update both the bitrate + // allocator and the video bitrate allocator. + bool new_send_state = false; + for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) { + if (new_parameters.encodings[i].active != + rtp_parameters_.encodings[i].active) { + new_send_state = true; + } + } rtp_parameters_ = new_parameters; // Codecs are currently handled at the WebRtcVideoChannel level. rtp_parameters_.codecs.clear(); - if (reconfigure_encoder) { + if (reconfigure_encoder || new_send_state) { ReconfigureEncoder(); } - // Encoding may have been activated/deactivated. - UpdateSendState(); + if (new_send_state) { + UpdateSendState(); + } return webrtc::RTCError::OK(); } @@ -1846,12 +1857,15 @@ WebRtcVideoChannel::WebRtcVideoSendStream::ValidateRtpParameters( void WebRtcVideoChannel::WebRtcVideoSendStream::UpdateSendState() { RTC_DCHECK_RUN_ON(&thread_checker_); - // TODO(bugs.webrtc.org/8653): Handle multiple encodings by creating a - // vector of bools corresponding to the appropriate active streams, and call - // stream_->UpdateActiveSimulcastLayers(). - if (sending_ && rtp_parameters_.encodings[0].active) { + if (sending_) { RTC_DCHECK(stream_ != nullptr); - stream_->Start(); + std::vector active_layers(rtp_parameters_.encodings.size()); + for (size_t i = 0; i < active_layers.size(); ++i) { + active_layers[i] = rtp_parameters_.encodings[i].active; + } + // This updates what simulcast layers are sending, and possibly starts + // or stops the VideoSendStream. + stream_->UpdateActiveSimulcastLayers(active_layers); } else { if (stream_ != nullptr) { stream_->Stop(); @@ -1907,6 +1921,18 @@ WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig( encoder_config.bitrate_priority = rtp_parameters_.encodings[0].bitrate_priority; + // Application-controlled state is held in the encoder_config's + // simulcast_layers. Currently this is used to control which simulcast layers + // are active. + RTC_DCHECK_GE(rtp_parameters_.encodings.size(), + encoder_config.number_of_streams); + RTC_DCHECK_GT(encoder_config.number_of_streams, 0); + encoder_config.simulcast_layers.resize(encoder_config.number_of_streams); + for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) { + encoder_config.simulcast_layers[i].active = + rtp_parameters_.encodings[i].active; + } + int max_qp = kDefaultQpMax; codec.GetParam(kCodecParamMaxQuantization, &max_qp); encoder_config.video_stream_factory = @@ -2607,13 +2633,22 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( (!conference_mode_ || !cricket::UseSimulcastScreenshare())) { RTC_DCHECK_EQ(1, encoder_config.number_of_streams); } + RTC_DCHECK_EQ(encoder_config.simulcast_layers.size(), + encoder_config.number_of_streams); + std::vector layers; + if (encoder_config.number_of_streams > 1 || (CodecNamesEq(codec_name_, kVp8CodecName) && is_screencast_ && conference_mode_)) { - return GetSimulcastConfig(encoder_config.number_of_streams, width, height, - encoder_config.max_bitrate_bps, - encoder_config.bitrate_priority, max_qp_, - max_framerate_, is_screencast_); + layers = GetSimulcastConfig(encoder_config.number_of_streams, width, height, + encoder_config.max_bitrate_bps, + encoder_config.bitrate_priority, max_qp_, + max_framerate_, is_screencast_); + // Update the active simulcast layers. + for (size_t i = 0; i < layers.size(); ++i) { + layers[i].active = encoder_config.simulcast_layers[i].active; + } + return layers; } // For unset max bitrates set default bitrate for non-simulcast. @@ -2622,23 +2657,22 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( ? encoder_config.max_bitrate_bps : GetMaxDefaultVideoBitrateKbps(width, height) * 1000; - webrtc::VideoStream stream; - stream.width = width; - stream.height = height; - stream.max_framerate = max_framerate_; - stream.min_bitrate_bps = GetMinVideoBitrateBps(); - stream.target_bitrate_bps = stream.max_bitrate_bps = max_bitrate_bps; - stream.max_qp = max_qp_; - stream.bitrate_priority = encoder_config.bitrate_priority; + webrtc::VideoStream layer; + layer.width = width; + layer.height = height; + layer.max_framerate = max_framerate_; + layer.min_bitrate_bps = GetMinVideoBitrateBps(); + layer.target_bitrate_bps = layer.max_bitrate_bps = max_bitrate_bps; + layer.max_qp = max_qp_; + layer.bitrate_priority = encoder_config.bitrate_priority; if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screencast_) { - stream.temporal_layer_thresholds_bps.resize(GetDefaultVp9TemporalLayers() - - 1); + layer.temporal_layer_thresholds_bps.resize(GetDefaultVp9TemporalLayers() - + 1); } - std::vector streams; - streams.push_back(stream); - return streams; + layers.push_back(layer); + return layers; } } // namespace cricket diff --git a/media/engine/webrtcvideoengine_unittest.cc b/media/engine/webrtcvideoengine_unittest.cc index 9accef0321..4fceab88e0 100644 --- a/media/engine/webrtcvideoengine_unittest.cc +++ b/media/engine/webrtcvideoengine_unittest.cc @@ -4516,9 +4516,7 @@ TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersPrioritySimulcastStreams) { // Test that a stream will not be sending if its encoding is made inactive // through SetRtpSendParameters. -// TODO(bugs.webrtc.org/8653): Update this test when we support active/inactive -// for individual encodings. -TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersEncodingsActive) { +TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersOneEncodingActive) { FakeVideoSendStream* stream = AddSendStream(); EXPECT_TRUE(channel_->SetSend(true)); EXPECT_TRUE(stream->IsSending()); @@ -4537,6 +4535,81 @@ TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersEncodingsActive) { EXPECT_TRUE(stream->IsSending()); } +// Tests that when active is updated for any simulcast layer then the send +// stream's sending state will be updated and it will be reconfigured with the +// new appropriate active simulcast streams. +TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersMultipleEncodingsActive) { + // Create the stream params with multiple ssrcs for simulcast. + const int kNumSimulcastStreams = 3; + std::vector ssrcs = MAKE_VECTOR(kSsrcs3); + StreamParams stream_params = CreateSimStreamParams("cname", ssrcs); + FakeVideoSendStream* fake_video_send_stream = AddSendStream(stream_params); + uint32_t primary_ssrc = stream_params.first_ssrc(); + + // Using the FakeVideoCapturer, we manually send a full size frame. This + // allows us to test that ReconfigureEncoder is called appropriately. + cricket::FakeVideoCapturer capturer; + VideoOptions options; + EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, true, &options, &capturer)); + EXPECT_EQ(cricket::CS_RUNNING, + capturer.Start(cricket::VideoFormat( + 1920, 1080, cricket::VideoFormat::FpsToInterval(30), + cricket::FOURCC_I420))); + channel_->SetSend(true); + EXPECT_TRUE(capturer.CaptureFrame()); + + // Check that all encodings are initially active. + webrtc::RtpParameters parameters = + channel_->GetRtpSendParameters(primary_ssrc); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_TRUE(parameters.encodings[0].active); + EXPECT_TRUE(parameters.encodings[1].active); + EXPECT_TRUE(parameters.encodings[2].active); + EXPECT_TRUE(fake_video_send_stream->IsSending()); + + // Only turn on only the middle stream. + parameters.encodings[0].active = false; + parameters.encodings[1].active = true; + parameters.encodings[2].active = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(primary_ssrc, parameters).ok()); + // Verify that the active fields are set on the VideoChannel. + parameters = channel_->GetRtpSendParameters(primary_ssrc); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_FALSE(parameters.encodings[0].active); + EXPECT_TRUE(parameters.encodings[1].active); + EXPECT_FALSE(parameters.encodings[2].active); + // Check that the VideoSendStream is updated appropriately. This means its + // send state was updated and it was reconfigured. + EXPECT_TRUE(fake_video_send_stream->IsSending()); + std::vector simulcast_streams = + fake_video_send_stream->GetVideoStreams(); + EXPECT_EQ(kNumSimulcastStreams, simulcast_streams.size()); + EXPECT_FALSE(simulcast_streams[0].active); + EXPECT_TRUE(simulcast_streams[1].active); + EXPECT_FALSE(simulcast_streams[2].active); + + // Turn off all streams. + parameters.encodings[0].active = false; + parameters.encodings[1].active = false; + parameters.encodings[2].active = false; + EXPECT_TRUE(channel_->SetRtpSendParameters(primary_ssrc, parameters).ok()); + // Verify that the active fields are set on the VideoChannel. + parameters = channel_->GetRtpSendParameters(primary_ssrc); + EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size()); + EXPECT_FALSE(parameters.encodings[0].active); + EXPECT_FALSE(parameters.encodings[1].active); + EXPECT_FALSE(parameters.encodings[2].active); + // Check that the VideoSendStream is off. + EXPECT_FALSE(fake_video_send_stream->IsSending()); + simulcast_streams = fake_video_send_stream->GetVideoStreams(); + EXPECT_EQ(kNumSimulcastStreams, simulcast_streams.size()); + EXPECT_FALSE(simulcast_streams[0].active); + EXPECT_FALSE(simulcast_streams[1].active); + EXPECT_FALSE(simulcast_streams[2].active); + + EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, true, nullptr, nullptr)); +} + // Test that if a stream is reconfigured (due to a codec change or other // change) while its encoding is still inactive, it doesn't start sending. TEST_F(WebRtcVideoChannelTest, diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index a0e967199d..f31a03a313 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -1259,6 +1259,7 @@ void VideoQualityTest::FillScalabilitySettings( params->video[video_idx].min_transmit_bps; encoder_config.number_of_streams = num_streams; encoder_config.spatial_layers = params->ss[video_idx].spatial_layers; + encoder_config.simulcast_layers = std::vector(num_streams); encoder_config.video_stream_factory = new rtc::RefCountedObject( params->video[video_idx].codec, kDefaultMaxQp, @@ -1412,6 +1413,8 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, params_.ss[video_idx].streams[i].max_bitrate_bps; } if (params_.ss[video_idx].infer_streams) { + video_encoder_configs_[video_idx].simulcast_layers = + std::vector(params_.ss[video_idx].streams.size()); video_encoder_configs_[video_idx].video_stream_factory = new rtc::RefCountedObject( params_.video[video_idx].codec, @@ -1613,6 +1616,7 @@ void VideoQualityTest::SetupThumbnails(Transport* send_transport, thumbnail_encoder_config.video_stream_factory = new rtc::RefCountedObject(params_.ss[0].streams); } else { + thumbnail_encoder_config.simulcast_layers = std::vector(1); thumbnail_encoder_config.video_stream_factory = new rtc::RefCountedObject( params_.video[0].codec, params_.ss[0].streams[0].max_qp,