From 93be66cdaaaf98462e273f10621cea24645e8588 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Thu, 2 Apr 2020 14:10:27 +0200 Subject: [PATCH] Calculate video padding for vp9 in the same way as for vp8 Bug: webrtc:11476 Change-Id: I8d7b5aac91868e10061605cc5043226ee916cc09 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/172722 Reviewed-by: Evan Shrubsole Reviewed-by: Niels Moller Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#30982} --- api/video/video_stream_encoder_interface.h | 1 + video/video_send_stream_impl.cc | 30 +++--- video/video_send_stream_impl.h | 1 + video/video_send_stream_impl_unittest.cc | 117 ++++++++++++++++++++- video/video_stream_encoder.cc | 32 +++--- video/video_stream_encoder_unittest.cc | 1 + 6 files changed, 152 insertions(+), 30 deletions(-) diff --git a/api/video/video_stream_encoder_interface.h b/api/video/video_stream_encoder_interface.h index 253fb04306..8e1df0f858 100644 --- a/api/video/video_stream_encoder_interface.h +++ b/api/video/video_stream_encoder_interface.h @@ -44,6 +44,7 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { public: virtual void OnEncoderConfigurationChanged( std::vector streams, + bool is_svc, VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps) = 0; }; diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index 405de7cd3b..bfd6216892 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -58,12 +58,16 @@ bool TransportSeqNumExtensionConfigured(const VideoSendStream::Config& config) { // Calculate max padding bitrate for a multi layer codec. int CalculateMaxPadBitrateBps(const std::vector& streams, + bool is_svc, VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps, bool pad_to_min_bitrate, bool alr_probing) { int pad_up_to_bitrate_bps = 0; + RTC_DCHECK(!is_svc || streams.size() <= 1) << "Only one stream is allowed in " + "SVC mode."; + // Filter out only the active streams; std::vector active_streams; for (const VideoStream& stream : streams) { @@ -71,7 +75,13 @@ int CalculateMaxPadBitrateBps(const std::vector& streams, active_streams.emplace_back(stream); } - if (active_streams.size() > 1) { + if (active_streams.size() > 1 || (!active_streams.empty() && is_svc)) { + // Simulcast or SVC is used. + // if SVC is used, stream bitrates should already encode svc bitrates: + // min_bitrate = min bitrate of a lowest svc layer. + // target_bitrate = sum of target bitrates of lower layers + min bitrate + // of the last one (as used in the calculations below). + // max_bitrate = sum of all active layers' max_bitrate. if (alr_probing) { // With alr probing, just pad to the min bitrate of the lowest stream, // probing will handle the rest of the rampup. @@ -471,22 +481,23 @@ MediaStreamAllocationConfig VideoSendStreamImpl::GetAllocationConfig() const { void VideoSendStreamImpl::OnEncoderConfigurationChanged( std::vector streams, + bool is_svc, VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps) { if (!worker_queue_->IsCurrent()) { rtc::WeakPtr send_stream = weak_ptr_; - worker_queue_->PostTask([send_stream, streams, content_type, + worker_queue_->PostTask([send_stream, streams, is_svc, content_type, min_transmit_bitrate_bps]() mutable { if (send_stream) { send_stream->OnEncoderConfigurationChanged( - std::move(streams), content_type, min_transmit_bitrate_bps); + std::move(streams), is_svc, content_type, min_transmit_bitrate_bps); } }); return; } + RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size()); TRACE_EVENT0("webrtc", "VideoSendStream::OnEncoderConfigurationChanged"); - RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size()); RTC_DCHECK_RUN_ON(worker_queue_); const VideoCodecType codec_type = @@ -516,14 +527,9 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( encoder_max_bitrate_bps_); // TODO(bugs.webrtc.org/10266): Query the VideoBitrateAllocator instead. - if (codec_type == kVideoCodecVP9) { - max_padding_bitrate_ = has_alr_probing_ ? streams[0].min_bitrate_bps - : streams[0].target_bitrate_bps; - } else { - max_padding_bitrate_ = CalculateMaxPadBitrateBps( - streams, content_type, min_transmit_bitrate_bps, - config_->suspend_below_min_bitrate, has_alr_probing_); - } + max_padding_bitrate_ = CalculateMaxPadBitrateBps( + streams, is_svc, content_type, min_transmit_bitrate_bps, + config_->suspend_below_min_bitrate, has_alr_probing_); // Clear stats for disabled layers. for (size_t i = streams.size(); i < config_->rtp.ssrcs.size(); ++i) { diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h index 4195efcf82..d3f87e3bf3 100644 --- a/video/video_send_stream_impl.h +++ b/video/video_send_stream_impl.h @@ -116,6 +116,7 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, void OnEncoderConfigurationChanged( std::vector streams, + bool is_svc, VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps) override; diff --git a/video/video_send_stream_impl_unittest.cc b/video/video_send_stream_impl_unittest.cc index 2b5a90a5f0..24519b118f 100644 --- a/video/video_send_stream_impl_unittest.cc +++ b/video/video_send_stream_impl_unittest.cc @@ -241,7 +241,7 @@ TEST_F(VideoSendStreamImplTest, UpdatesObserverOnConfigurationChange) { static_cast(vss_impl.get()) ->OnEncoderConfigurationChanged( - std::vector{qvga_stream, vga_stream}, + std::vector{qvga_stream, vga_stream}, false, VideoEncoderConfig::ContentType::kRealtimeVideo, min_transmit_bitrate_bps); vss_impl->Stop(); @@ -309,7 +309,7 @@ TEST_F(VideoSendStreamImplTest, UpdatesObserverOnConfigurationChangeWithAlr) { static_cast(vss_impl.get()) ->OnEncoderConfigurationChanged( - std::vector{low_stream, high_stream}, + std::vector{low_stream, high_stream}, false, VideoEncoderConfig::ContentType::kScreen, min_transmit_bitrate_bps); vss_impl->Stop(); @@ -371,7 +371,7 @@ TEST_F(VideoSendStreamImplTest, static_cast(vss_impl.get()) ->OnEncoderConfigurationChanged( - std::vector{low_stream, high_stream}, + std::vector{low_stream, high_stream}, false, VideoEncoderConfig::ContentType::kRealtimeVideo, /*min_transmit_bitrate_bps=*/0); vss_impl->Stop(); @@ -690,7 +690,7 @@ TEST_F(VideoSendStreamImplTest, CallsVideoStreamEncoderOnBitrateUpdate) { static_cast(vss_impl.get()) ->OnEncoderConfigurationChanged( - std::vector{qvga_stream}, + std::vector{qvga_stream}, false, VideoEncoderConfig::ContentType::kRealtimeVideo, min_transmit_bitrate_bps); @@ -816,7 +816,7 @@ TEST_F(VideoSendStreamImplTest, DisablesPaddingOnPausedEncoder) { // Reconfigure e.g. due to a fake frame. static_cast(vss_impl.get()) ->OnEncoderConfigurationChanged( - std::vector{qvga_stream}, + std::vector{qvga_stream}, false, VideoEncoderConfig::ContentType::kRealtimeVideo, min_transmit_bitrate_bps); // Still no padding because no actual frames were passed, only @@ -893,5 +893,112 @@ TEST_F(VideoSendStreamImplTest, KeepAliveOnDroppedFrame) { ASSERT_TRUE(done.Wait(5000)); } +TEST_F(VideoSendStreamImplTest, ConfiguresBitratesForSvcWithAlr) { + test_queue_.SendTask( + [this] { + const bool kSuspend = false; + config_.suspend_below_min_bitrate = kSuspend; + config_.rtp.extensions.emplace_back( + RtpExtension::kTransportSequenceNumberUri, 1); + config_.periodic_alr_bandwidth_probing = true; + auto vss_impl = CreateVideoSendStreamImpl( + kDefaultInitialBitrateBps, kDefaultBitratePriority, + VideoEncoderConfig::ContentType::kScreen); + vss_impl->Start(); + + // Svc + VideoStream stream; + stream.width = 1920; + stream.height = 1080; + stream.max_framerate = 30; + stream.min_bitrate_bps = 60000; + stream.target_bitrate_bps = 6000000; + stream.max_bitrate_bps = 1250000; + stream.num_temporal_layers = 2; + stream.max_qp = 56; + stream.bitrate_priority = 1; + + int min_transmit_bitrate_bps = 400000; + + config_.rtp.ssrcs.emplace_back(1); + config_.rtp.ssrcs.emplace_back(2); + + EXPECT_CALL(bitrate_allocator_, AddObserver(vss_impl.get(), _)) + .WillRepeatedly(Invoke([&](BitrateAllocatorObserver*, + MediaStreamAllocationConfig config) { + EXPECT_EQ(config.min_bitrate_bps, + static_cast(stream.min_bitrate_bps)); + EXPECT_EQ(config.max_bitrate_bps, + static_cast(stream.max_bitrate_bps)); + if (config.pad_up_bitrate_bps != 0) { + EXPECT_EQ(config.pad_up_bitrate_bps, + static_cast(min_transmit_bitrate_bps)); + } + EXPECT_EQ(config.enforce_min_bitrate, !kSuspend); + })); + + static_cast(vss_impl.get()) + ->OnEncoderConfigurationChanged( + std::vector{stream}, true, + VideoEncoderConfig::ContentType::kScreen, + min_transmit_bitrate_bps); + vss_impl->Stop(); + }, + RTC_FROM_HERE); +} + +TEST_F(VideoSendStreamImplTest, ConfiguresBitratesForSvcNoAlr) { + test_queue_.SendTask( + [this] { + const bool kSuspend = false; + config_.suspend_below_min_bitrate = kSuspend; + config_.rtp.extensions.emplace_back( + RtpExtension::kTransportSequenceNumberUri, 1); + config_.periodic_alr_bandwidth_probing = false; + auto vss_impl = CreateVideoSendStreamImpl( + kDefaultInitialBitrateBps, kDefaultBitratePriority, + VideoEncoderConfig::ContentType::kScreen); + vss_impl->Start(); + + // Svc + VideoStream stream; + stream.width = 1920; + stream.height = 1080; + stream.max_framerate = 30; + stream.min_bitrate_bps = 60000; + stream.target_bitrate_bps = 6000000; + stream.max_bitrate_bps = 1250000; + stream.num_temporal_layers = 2; + stream.max_qp = 56; + stream.bitrate_priority = 1; + + int min_transmit_bitrate_bps = 400000; + + config_.rtp.ssrcs.emplace_back(1); + config_.rtp.ssrcs.emplace_back(2); + + EXPECT_CALL(bitrate_allocator_, AddObserver(vss_impl.get(), _)) + .WillRepeatedly(Invoke([&](BitrateAllocatorObserver*, + MediaStreamAllocationConfig config) { + EXPECT_EQ(config.min_bitrate_bps, + static_cast(stream.min_bitrate_bps)); + EXPECT_EQ(config.max_bitrate_bps, + static_cast(stream.max_bitrate_bps)); + if (config.pad_up_bitrate_bps != 0) { + EXPECT_EQ(config.pad_up_bitrate_bps, + static_cast(stream.target_bitrate_bps)); + } + EXPECT_EQ(config.enforce_min_bitrate, !kSuspend); + })); + + static_cast(vss_impl.get()) + ->OnEncoderConfigurationChanged( + std::vector{stream}, true, + VideoEncoderConfig::ContentType::kScreen, + min_transmit_bitrate_bps); + vss_impl->Stop(); + }, + RTC_FROM_HERE); +} } // namespace internal } // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 8252cb29ef..4ad9a52197 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -516,18 +516,6 @@ void VideoStreamEncoder::ReconfigureEncoder() { RTC_LOG(LS_ERROR) << "Failed to create encoder configuration."; } - // Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9. - if (encoder_config_.codec_type == kVideoCodecVP9) { - // Lower max bitrate to the level codec actually can produce. - streams[0].max_bitrate_bps = - std::min(streams[0].max_bitrate_bps, - SvcRateAllocator::GetMaxBitrate(codec).bps()); - streams[0].min_bitrate_bps = codec.spatialLayers[0].minBitrate * 1000; - // target_bitrate_bps specifies the maximum padding bitrate. - streams[0].target_bitrate_bps = - SvcRateAllocator::GetPaddingBitrate(codec).bps(); - } - char log_stream_buf[4 * 1024]; rtc::SimpleStringBuilder log_stream(log_stream_buf); log_stream << "ReconfigureEncoder:\n"; @@ -717,8 +705,26 @@ void VideoStreamEncoder::ReconfigureEncoder() { pending_encoder_reconfiguration_ = false; + bool is_svc = false; + // Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9 + // and leave only one stream containing all necessary information. + if (encoder_config_.codec_type == kVideoCodecVP9) { + // Lower max bitrate to the level codec actually can produce. + streams[0].max_bitrate_bps = + std::min(streams[0].max_bitrate_bps, + SvcRateAllocator::GetMaxBitrate(codec).bps()); + streams[0].min_bitrate_bps = codec.spatialLayers[0].minBitrate * 1000; + // target_bitrate_bps specifies the maximum padding bitrate. + streams[0].target_bitrate_bps = + SvcRateAllocator::GetPaddingBitrate(codec).bps(); + streams[0].width = streams.back().width; + streams[0].height = streams.back().height; + is_svc = codec.VP9()->numberOfSpatialLayers > 1; + streams.resize(1); + } + sink_->OnEncoderConfigurationChanged( - std::move(streams), encoder_config_.content_type, + std::move(streams), is_svc, encoder_config_.content_type, encoder_config_.min_transmit_bitrate_bps); resource_adaptation_processor_->ConfigureQualityScaler(info); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 934bf09fe5..bb85776500 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -1141,6 +1141,7 @@ class VideoStreamEncoderTest : public ::testing::Test { void OnEncoderConfigurationChanged( std::vector streams, + bool is_svc, VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps) override { rtc::CritScope lock(&crit_);