From c91c4233e30f747aed65ffdfd193247aa8f6372a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Mon, 1 Feb 2021 09:20:05 +0100 Subject: [PATCH] LibvpxVp9Encoder: add option to configure resolution_bitrate_limits. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: none Change-Id: Icdd7333296d652b1e0c159226df702084303475c Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/204701 Reviewed-by: Sergey Silkin Commit-Queue: Åsa Persson Cr-Commit-Position: refs/heads/master@{#33121} --- modules/video_coding/BUILD.gn | 1 + .../codecs/vp9/libvpx_vp9_encoder.cc | 4 + .../codecs/vp9/libvpx_vp9_encoder.h | 3 + .../codecs/vp9/test/vp9_impl_unittest.cc | 21 ++++ rtc_base/experiments/encoder_info_settings.cc | 3 + rtc_base/experiments/encoder_info_settings.h | 7 ++ video/adaptation/bitrate_constraint.cc | 20 +-- .../video_stream_encoder_resource_manager.cc | 16 +++ .../video_stream_encoder_resource_manager.h | 1 + video/video_stream_encoder.cc | 39 ++++++ video/video_stream_encoder_unittest.cc | 117 ++++++++++++++++++ 11 files changed, 214 insertions(+), 18 deletions(-) diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 2a71c76298..30fbaaaad0 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -586,6 +586,7 @@ rtc_library("webrtc_vp9") { "../../media:rtc_vp9_profile", "../../rtc_base", "../../rtc_base:checks", + "../../rtc_base/experiments:encoder_info_settings", "../../rtc_base/experiments:field_trial_parser", "../../rtc_base/experiments:rate_control_settings", "../../rtc_base/synchronization:mutex", diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc index 81223019fd..2bb4110b01 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc @@ -1718,6 +1718,10 @@ VideoEncoder::EncoderInfo LibvpxVp9Encoder::GetEncoderInfo() const { VideoFrameBuffer::Type::kNV12}; } } + if (!encoder_info_override_.resolution_bitrate_limits().empty()) { + info.resolution_bitrate_limits = + encoder_info_override_.resolution_bitrate_limits(); + } return info; } diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h index 037c760c17..4791584eeb 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h @@ -28,6 +28,7 @@ #include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h" #include "modules/video_coding/svc/scalable_video_controller.h" #include "modules/video_coding/utility/framerate_controller.h" +#include "rtc_base/experiments/encoder_info_settings.h" #include "vpx/vp8cx.h" namespace webrtc { @@ -230,6 +231,8 @@ class LibvpxVp9Encoder : public VP9Encoder { int num_steady_state_frames_; // Only set config when this flag is set. bool config_changed_; + + const LibvpxVp9EncoderInfoSettings encoder_info_override_; }; } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index 3d658838ed..b0e0e4504f 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -1636,6 +1636,27 @@ TEST_F(TestVp9Impl, Profile0PreferredPixelFormats) { VideoFrameBuffer::Type::kI420)); } +TEST_F(TestVp9Impl, EncoderInfoWithoutResolutionBitrateLimits) { + EXPECT_TRUE(encoder_->GetEncoderInfo().resolution_bitrate_limits.empty()); +} + +TEST_F(TestVp9Impl, EncoderInfoWithBitrateLimitsFromFieldTrial) { + test::ScopedFieldTrials field_trials( + "WebRTC-LibvpxVp9Encoder-GetEncoderInfoOverride/" + "frame_size_pixels:123|456|789," + "min_start_bitrate_bps:11000|22000|33000," + "min_bitrate_bps:44000|55000|66000," + "max_bitrate_bps:77000|88000|99000/"); + SetUp(); + + EXPECT_THAT( + encoder_->GetEncoderInfo().resolution_bitrate_limits, + ::testing::ElementsAre( + VideoEncoder::ResolutionBitrateLimits{123, 11000, 44000, 77000}, + VideoEncoder::ResolutionBitrateLimits{456, 22000, 55000, 88000}, + VideoEncoder::ResolutionBitrateLimits{789, 33000, 66000, 99000})); +} + TEST_F(TestVp9Impl, EncoderInfoFpsAllocation) { const uint8_t kNumSpatialLayers = 3; const uint8_t kNumTemporalLayers = 3; diff --git a/rtc_base/experiments/encoder_info_settings.cc b/rtc_base/experiments/encoder_info_settings.cc index fa6b843fb9..513c63240c 100644 --- a/rtc_base/experiments/encoder_info_settings.cc +++ b/rtc_base/experiments/encoder_info_settings.cc @@ -75,4 +75,7 @@ SimulcastEncoderAdapterEncoderInfoSettings:: : EncoderInfoSettings( "WebRTC-SimulcastEncoderAdapter-GetEncoderInfoOverride") {} +LibvpxVp9EncoderInfoSettings::LibvpxVp9EncoderInfoSettings() + : EncoderInfoSettings("WebRTC-LibvpxVp9Encoder-GetEncoderInfoOverride") {} + } // namespace webrtc diff --git a/rtc_base/experiments/encoder_info_settings.h b/rtc_base/experiments/encoder_info_settings.h index 9cacdbd2a5..19609fb377 100644 --- a/rtc_base/experiments/encoder_info_settings.h +++ b/rtc_base/experiments/encoder_info_settings.h @@ -57,6 +57,13 @@ class SimulcastEncoderAdapterEncoderInfoSettings : public EncoderInfoSettings { ~SimulcastEncoderAdapterEncoderInfoSettings() override {} }; +// EncoderInfo settings for LibvpxVp9Encoder. +class LibvpxVp9EncoderInfoSettings : public EncoderInfoSettings { + public: + LibvpxVp9EncoderInfoSettings(); + ~LibvpxVp9EncoderInfoSettings() override {} +}; + } // namespace webrtc #endif // RTC_BASE_EXPERIMENTS_ENCODER_INFO_SETTINGS_H_ diff --git a/video/adaptation/bitrate_constraint.cc b/video/adaptation/bitrate_constraint.cc index 28b5058747..c24bbb9853 100644 --- a/video/adaptation/bitrate_constraint.cc +++ b/video/adaptation/bitrate_constraint.cc @@ -19,23 +19,6 @@ namespace webrtc { -namespace { -bool IsSimulcast(const VideoEncoderConfig& encoder_config) { - const std::vector& simulcast_layers = - encoder_config.simulcast_layers; - - bool is_simulcast = simulcast_layers.size() > 1; - bool is_lowest_layer_active = simulcast_layers[0].active; - int num_active_layers = - std::count_if(simulcast_layers.begin(), simulcast_layers.end(), - [](const VideoStream& layer) { return layer.active; }); - - // We can't distinguish between simulcast and singlecast when only the - // lowest spatial layer is active. Treat this case as simulcast. - return is_simulcast && (num_active_layers > 1 || is_lowest_layer_active); -} -} // namespace - BitrateConstraint::BitrateConstraint() : encoder_settings_(absl::nullopt), encoder_target_bitrate_bps_(absl::nullopt) { @@ -70,7 +53,8 @@ bool BitrateConstraint::IsAdaptationUpAllowed( return true; } - if (IsSimulcast(encoder_settings_->encoder_config())) { + if (VideoStreamEncoderResourceManager::IsSimulcast( + encoder_settings_->encoder_config())) { // Resolution bitrate limits usage is restricted to singlecast. return true; } diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 96f888d4f6..22f0b56e35 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -706,4 +706,20 @@ VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels( return pixels; } +bool VideoStreamEncoderResourceManager::IsSimulcast( + const VideoEncoderConfig& encoder_config) { + const std::vector& simulcast_layers = + encoder_config.simulcast_layers; + + bool is_simulcast = simulcast_layers.size() > 1; + bool is_lowest_layer_active = simulcast_layers[0].active; + int num_active_layers = + std::count_if(simulcast_layers.begin(), simulcast_layers.end(), + [](const VideoStream& layer) { return layer.active; }); + + // We can't distinguish between simulcast and singlecast when only the + // lowest spatial layer is active. Treat this case as simulcast. + return is_simulcast && (num_active_layers > 1 || is_lowest_layer_active); +} + } // namespace webrtc diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index d6b9dd1910..e5cbeb0692 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -148,6 +148,7 @@ class VideoStreamEncoderResourceManager static absl::optional GetSingleActiveLayerPixels( const VideoCodec& codec); + static bool IsSimulcast(const VideoEncoderConfig& encoder_config); private: class InitialFrameDropper; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index c8d6021259..025481ae30 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -348,6 +348,41 @@ int NumActiveStreams(const std::vector& streams) { return num_active; } +void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info, + const VideoEncoderConfig& encoder_config, + VideoCodec* codec) { + if (codec->codecType != VideoCodecType::kVideoCodecVP9 || + VideoStreamEncoderResourceManager::IsSimulcast(encoder_config)) { + // Resolution bitrate limits usage is restricted to singlecast. + return; + } + + // Get bitrate limits for active stream. + absl::optional pixels = + VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels(*codec); + if (!pixels.has_value()) { + return; + } + absl::optional bitrate_limits = + encoder_info.GetEncoderBitrateLimitsForResolution(*pixels); + if (!bitrate_limits.has_value()) { + return; + } + + for (int i = 0; i < codec->VP9()->numberOfSpatialLayers; ++i) { + if (codec->spatialLayers[i].active) { + codec->spatialLayers[i].minBitrate = + bitrate_limits->min_bitrate_bps / 1000; + codec->spatialLayers[i].maxBitrate = + bitrate_limits->max_bitrate_bps / 1000; + codec->spatialLayers[i].targetBitrate = + std::min(codec->spatialLayers[i].targetBitrate, + codec->spatialLayers[i].maxBitrate); + break; + } + } +} + void ApplyEncoderBitrateLimitsIfSingleActiveStream( const VideoEncoder::EncoderInfo& encoder_info, const std::vector& encoder_config_layers, @@ -901,6 +936,10 @@ void VideoStreamEncoder::ReconfigureEncoder() { // thus some cropping might be needed. crop_width_ = last_frame_info_->width - codec.width; crop_height_ = last_frame_info_->height - codec.height; + if (encoder_bitrate_limits_) { + ApplyVp9BitrateLimits(encoder_->GetEncoderInfo(), encoder_config_, + &codec); + } } char log_stream_buf[4 * 1024]; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 8220daa465..b0cc6c021e 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -5356,6 +5356,123 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) { + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p( + 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p( + 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p( + 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000); + fake_encoder_.SetResolutionBitrateLimits( + {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p}); + + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1, + &video_encoder_config); + VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings(); + vp9_settings.numberOfSpatialLayers = 3; + // Since only one layer is active - automatic resize should be enabled. + vp9_settings.automaticResizeOn = true; + video_encoder_config.encoder_specific_settings = + new rtc::RefCountedObject( + vp9_settings); + video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps; + video_encoder_config.content_type = + VideoEncoderConfig::ContentType::kRealtimeVideo; + // Simulcast layers are used to indicate which spatial layers are active. + video_encoder_config.simulcast_layers.resize(3); + video_encoder_config.simulcast_layers[0].active = false; + video_encoder_config.simulcast_layers[1].active = true; + video_encoder_config.simulcast_layers[2].active = false; + + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + // The encoder bitrate limits for 360p should be used. + video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1); + EXPECT_EQ(fake_encoder_.video_codec().codecType, + VideoCodecType::kVideoCodecVP9); + EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2); + EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active); + EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width); + EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height); + EXPECT_EQ(static_cast(kEncoderLimits360p.min_bitrate_bps), + fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits360p.max_bitrate_bps), + fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000); + + // The encoder bitrate limits for 270p should be used. + video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1); + EXPECT_EQ(fake_encoder_.video_codec().codecType, + VideoCodecType::kVideoCodecVP9); + EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2); + EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active); + EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width); + EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height); + EXPECT_EQ(static_cast(kEncoderLimits270p.min_bitrate_bps), + fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000); + EXPECT_EQ(static_cast(kEncoderLimits270p.max_bitrate_bps), + fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) { + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p( + 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000); + const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p( + 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000); + fake_encoder_.SetResolutionBitrateLimits( + {kEncoderLimits180p, kEncoderLimits720p}); + + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1, + &video_encoder_config); + VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings(); + vp9_settings.numberOfSpatialLayers = 3; + // Since only one layer is active - automatic resize should be enabled. + vp9_settings.automaticResizeOn = true; + video_encoder_config.encoder_specific_settings = + new rtc::RefCountedObject( + vp9_settings); + video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps; + video_encoder_config.content_type = + VideoEncoderConfig::ContentType::kRealtimeVideo; + // Simulcast layers are used to indicate which spatial layers are active. + video_encoder_config.simulcast_layers.resize(3); + video_encoder_config.simulcast_layers[0].active = true; + video_encoder_config.simulcast_layers[1].active = false; + video_encoder_config.simulcast_layers[2].active = false; + + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + // Limits not applied on lowest stream, limits for 180p should not be used. + video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + EXPECT_FALSE(WaitForFrame(1000)); + EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1); + EXPECT_EQ(fake_encoder_.video_codec().codecType, + VideoCodecType::kVideoCodecVP9); + EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3); + EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active); + EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width); + EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height); + EXPECT_NE(static_cast(kEncoderLimits180p.min_bitrate_bps), + fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000); + EXPECT_NE(static_cast(kEncoderLimits180p.max_bitrate_bps), + fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenResolutionIncreases) { const int kWidth = 640;