diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 9329f22452..bc7d4da245 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -61,6 +61,9 @@ constexpr uint32_t kVp832ByteAlign = 32u; constexpr int kRtpTicksPerSecond = 90000; constexpr int kRtpTicksPerMs = kRtpTicksPerSecond / 1000; +constexpr double kLowRateFactor = 1.0; +constexpr double kHighRateFactor = 2.0; + // VP8 denoiser states. enum denoiserState : uint32_t { kDenoiserOff, @@ -72,8 +75,17 @@ enum denoiserState : uint32_t { kDenoiserOnAdaptive }; +// These settings correspond to the settings in vpx_codec_enc_cfg. +struct Vp8RateSettings { + uint32_t rc_undershoot_pct; + uint32_t rc_overshoot_pct; + uint32_t rc_buf_sz; + uint32_t rc_buf_optimal_sz; + uint32_t rc_dropframe_thresh; +}; + // Greatest common divisior -static int GCD(int a, int b) { +int GCD(int a, int b) { int c = a % b; while (c != 0) { a = b; @@ -83,6 +95,56 @@ static int GCD(int a, int b) { return b; } +uint32_t Interpolate(uint32_t low, + uint32_t high, + double bandwidth_headroom_factor) { + RTC_DCHECK_GE(bandwidth_headroom_factor, kLowRateFactor); + RTC_DCHECK_LE(bandwidth_headroom_factor, kHighRateFactor); + + // |factor| is between 0.0 and 1.0. + const double factor = bandwidth_headroom_factor - kLowRateFactor; + + return static_cast(((1.0 - factor) * low) + (factor * high) + 0.5); +} + +Vp8RateSettings GetRateSettings(double bandwidth_headroom_factor) { + static const Vp8RateSettings low_settings{1000u, 0u, 100u, 30u, 40u}; + static const Vp8RateSettings high_settings{100u, 15u, 1000u, 600u, 5u}; + + if (bandwidth_headroom_factor <= kLowRateFactor) { + return low_settings; + } else if (bandwidth_headroom_factor >= kHighRateFactor) { + return high_settings; + } + + Vp8RateSettings settings; + settings.rc_undershoot_pct = + Interpolate(low_settings.rc_undershoot_pct, + high_settings.rc_undershoot_pct, bandwidth_headroom_factor); + settings.rc_overshoot_pct = + Interpolate(low_settings.rc_overshoot_pct, high_settings.rc_overshoot_pct, + bandwidth_headroom_factor); + settings.rc_buf_sz = + Interpolate(low_settings.rc_buf_sz, high_settings.rc_buf_sz, + bandwidth_headroom_factor); + settings.rc_buf_optimal_sz = + Interpolate(low_settings.rc_buf_optimal_sz, + high_settings.rc_buf_optimal_sz, bandwidth_headroom_factor); + settings.rc_dropframe_thresh = + Interpolate(low_settings.rc_dropframe_thresh, + high_settings.rc_dropframe_thresh, bandwidth_headroom_factor); + return settings; +} + +void UpdateRateSettings(vpx_codec_enc_cfg_t* config, + const Vp8RateSettings& new_settings) { + config->rc_undershoot_pct = new_settings.rc_undershoot_pct; + config->rc_overshoot_pct = new_settings.rc_overshoot_pct; + config->rc_buf_sz = new_settings.rc_buf_sz; + config->rc_buf_optimal_sz = new_settings.rc_buf_optimal_sz; + config->rc_dropframe_thresh = new_settings.rc_dropframe_thresh; +} + static_assert(Vp8EncoderConfig::kMaxPeriodicity == VPX_TS_MAX_PERIODICITY, "Vp8EncoderConfig::kMaxPeriodicity must be kept in sync with the " "constant in libvpx."); @@ -317,6 +379,14 @@ void LibvpxVp8Encoder::SetRates(const RateControlParameters& parameters) { UpdateVpxConfiguration(stream_idx, frame_buffer_controller_.get(), &configurations_[i]); + if (rate_control_settings_.Vp8DynamicRateSettings()) { + // Tweak rate control settings based on available network headroom. + UpdateRateSettings( + &configurations_[i], + GetRateSettings(parameters.bandwidth_allocation.bps() / + parameters.bitrate.get_sum_bps())); + } + vpx_codec_err_t err = libvpx_->codec_enc_config_set(&encoders_[i], &configurations_[i]); if (err != VPX_CODEC_OK) { diff --git a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index d267b15f6b..a2597ef51a 100644 --- a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -23,12 +23,15 @@ #include "modules/video_coding/codecs/vp8/test/mock_libvpx_interface.h" #include "modules/video_coding/utility/vp8_header_parser.h" #include "rtc_base/time_utils.h" +#include "test/field_trial.h" #include "test/video_codec_settings.h" namespace webrtc { using ::testing::_; +using ::testing::AllOf; using ::testing::ElementsAreArray; +using ::testing::Field; using ::testing::Invoke; using ::testing::NiceMock; using ::testing::Return; @@ -119,16 +122,80 @@ TEST_F(TestVp8Impl, SetRates) { const uint32_t kBitrateBps = 300000; VideoBitrateAllocation bitrate_allocation; bitrate_allocation.SetBitrate(0, 0, kBitrateBps); - EXPECT_CALL(*vpx, codec_enc_config_set(_, _)) - .WillOnce( - Invoke([&](vpx_codec_ctx_t* ctx, const vpx_codec_enc_cfg_t* cfg) { - EXPECT_EQ(cfg->rc_target_bitrate, kBitrateBps / 1000); - return VPX_CODEC_OK; - })); + EXPECT_CALL( + *vpx, + codec_enc_config_set( + _, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate, + kBitrateBps / 1000), + Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 100u), + Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 15u), + Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 1000u), + Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 600u), + Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 30u)))) + .WillOnce(Return(VPX_CODEC_OK)); encoder.SetRates(VideoEncoder::RateControlParameters( bitrate_allocation, static_cast(codec_settings_.maxFramerate))); } +TEST_F(TestVp8Impl, DynamicSetRates) { + test::ScopedFieldTrials field_trials( + "WebRTC-VideoRateControl/vp8_dynamic_rate:true/"); + auto* const vpx = new NiceMock(); + LibvpxVp8Encoder encoder((std::unique_ptr(vpx))); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings_, 1, 1000)); + + const uint32_t kBitrateBps = 300000; + VideoEncoder::RateControlParameters rate_settings; + rate_settings.bitrate.SetBitrate(0, 0, kBitrateBps); + rate_settings.framerate_fps = + static_cast(codec_settings_.maxFramerate); + + // Set rates with no headroom. + rate_settings.bandwidth_allocation = DataRate::bps(kBitrateBps); + EXPECT_CALL( + *vpx, + codec_enc_config_set( + _, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate, + kBitrateBps / 1000), + Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 1000u), + Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 0u), + Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 100u), + Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 30u), + Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 40u)))) + .WillOnce(Return(VPX_CODEC_OK)); + encoder.SetRates(rate_settings); + + // Set rates with max headroom. + rate_settings.bandwidth_allocation = DataRate::bps(kBitrateBps * 2); + EXPECT_CALL( + *vpx, codec_enc_config_set( + _, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate, + kBitrateBps / 1000), + Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 100u), + Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 15u), + Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 1000u), + Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 600u), + Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 5u)))) + .WillOnce(Return(VPX_CODEC_OK)); + encoder.SetRates(rate_settings); + + // Set rates with headroom half way. + rate_settings.bandwidth_allocation = DataRate::bps((3 * kBitrateBps) / 2); + EXPECT_CALL( + *vpx, + codec_enc_config_set( + _, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate, + kBitrateBps / 1000), + Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 550u), + Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 8u), + Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 550u), + Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 315u), + Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 23u)))) + .WillOnce(Return(VPX_CODEC_OK)); + encoder.SetRates(rate_settings); +} + TEST_F(TestVp8Impl, EncodeFrameAndRelease) { EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index 0018403889..ad2a7eb8ef 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -50,6 +50,18 @@ int kMaxNumTiles4kVideo = 8; // Maximum allowed PID difference for differnet per-layer frame-rate case. const int kMaxAllowedPidDIff = 30; +constexpr double kLowRateFactor = 1.0; +constexpr double kHighRateFactor = 2.0; + +// These settings correspond to the settings in vpx_codec_enc_cfg. +struct Vp8RateSettings { + uint32_t rc_undershoot_pct; + uint32_t rc_overshoot_pct; + uint32_t rc_buf_sz; + uint32_t rc_buf_optimal_sz; + uint32_t rc_dropframe_thresh; +}; + // Only positive speeds, range for real-time coding currently is: 5 - 8. // Lower means slower/better quality, higher means fastest/lower quality. int GetCpuSpeed(int width, int height) { @@ -137,6 +149,56 @@ bool MoreLayersEnabled(const VideoBitrateAllocation& first, return false; } +uint32_t Interpolate(uint32_t low, + uint32_t high, + double bandwidth_headroom_factor) { + RTC_DCHECK_GE(bandwidth_headroom_factor, kLowRateFactor); + RTC_DCHECK_LE(bandwidth_headroom_factor, kHighRateFactor); + + // |factor| is between 0.0 and 1.0. + const double factor = bandwidth_headroom_factor - kLowRateFactor; + + return static_cast(((1.0 - factor) * low) + (factor * high) + 0.5); +} + +Vp8RateSettings GetRateSettings(double bandwidth_headroom_factor) { + static const Vp8RateSettings low_settings{1000u, 0u, 100u, 30u, 40u}; + static const Vp8RateSettings high_settings{100u, 15u, 1000u, 600u, 5u}; + + if (bandwidth_headroom_factor <= kLowRateFactor) { + return low_settings; + } else if (bandwidth_headroom_factor >= kHighRateFactor) { + return high_settings; + } + + Vp8RateSettings settings; + settings.rc_undershoot_pct = + Interpolate(low_settings.rc_undershoot_pct, + high_settings.rc_undershoot_pct, bandwidth_headroom_factor); + settings.rc_overshoot_pct = + Interpolate(low_settings.rc_overshoot_pct, high_settings.rc_overshoot_pct, + bandwidth_headroom_factor); + settings.rc_buf_sz = + Interpolate(low_settings.rc_buf_sz, high_settings.rc_buf_sz, + bandwidth_headroom_factor); + settings.rc_buf_optimal_sz = + Interpolate(low_settings.rc_buf_optimal_sz, + high_settings.rc_buf_optimal_sz, bandwidth_headroom_factor); + settings.rc_dropframe_thresh = + Interpolate(low_settings.rc_dropframe_thresh, + high_settings.rc_dropframe_thresh, bandwidth_headroom_factor); + return settings; +} + +void UpdateRateSettings(vpx_codec_enc_cfg_t* config, + const Vp8RateSettings& new_settings) { + config->rc_undershoot_pct = new_settings.rc_undershoot_pct; + config->rc_overshoot_pct = new_settings.rc_overshoot_pct; + config->rc_buf_sz = new_settings.rc_buf_sz; + config->rc_buf_optimal_sz = new_settings.rc_buf_optimal_sz; + config->rc_dropframe_thresh = new_settings.rc_dropframe_thresh; +} + } // namespace void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt, @@ -170,6 +232,8 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) external_ref_control_(false), // Set in InitEncode because of tests. trusted_rate_controller_(RateControlSettings::ParseFromFieldTrials() .LibvpxVp9TrustedRateController()), + dynamic_rate_settings_( + RateControlSettings::ParseFromFieldTrials().Vp9DynamicRateSettings()), full_superframe_drop_(true), first_frame_in_picture_(true), ss_info_needed_(false), @@ -333,7 +397,7 @@ void VP9EncoderImpl::SetRates(const RateControlParameters& parameters) { } codec_.maxFramerate = static_cast(parameters.framerate_fps + 0.5); - requested_bitrate_allocation_ = parameters.bitrate; + requested_rate_settings_ = parameters; return; } @@ -798,11 +862,20 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id); - if (requested_bitrate_allocation_) { + if (requested_rate_settings_) { + if (dynamic_rate_settings_) { + // Tweak rate control settings based on available network headroom. + UpdateRateSettings( + config_, + GetRateSettings( + requested_rate_settings_->bandwidth_allocation.bps() / + requested_rate_settings_->bitrate.get_sum_bps())); + } + bool more_layers_requested = MoreLayersEnabled( - *requested_bitrate_allocation_, current_bitrate_allocation_); + requested_rate_settings_->bitrate, current_bitrate_allocation_); bool less_layers_requested = MoreLayersEnabled( - current_bitrate_allocation_, *requested_bitrate_allocation_); + current_bitrate_allocation_, requested_rate_settings_->bitrate); // In SVC can enable new layers only if all lower layers are encoded and at // the base temporal layer. // This will delay rate allocation change until the next frame on the base @@ -813,8 +886,8 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, inter_layer_pred_ != InterLayerPredMode::kOn || (layer_id.spatial_layer_id == 0 && layer_id.temporal_layer_id == 0); if (!more_layers_requested || can_upswitch) { - current_bitrate_allocation_ = *requested_bitrate_allocation_; - requested_bitrate_allocation_ = absl::nullopt; + current_bitrate_allocation_ = requested_rate_settings_->bitrate; + requested_rate_settings_ = absl::nullopt; if (!SetSvcRates(current_bitrate_allocation_)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index fe7494274c..acc03bff5c 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -123,10 +123,11 @@ class VP9EncoderImpl : public VP9Encoder { InterLayerPredMode inter_layer_pred_; bool external_ref_control_; const bool trusted_rate_controller_; + const bool dynamic_rate_settings_; const bool full_superframe_drop_; bool first_frame_in_picture_; VideoBitrateAllocation current_bitrate_allocation_; - absl::optional requested_bitrate_allocation_; + absl::optional requested_rate_settings_; bool ss_info_needed_; std::vector framerate_controller_; diff --git a/rtc_base/experiments/rate_control_settings.cc b/rtc_base/experiments/rate_control_settings.cc index 7ef42dea94..b181432e88 100644 --- a/rtc_base/experiments/rate_control_settings.cc +++ b/rtc_base/experiments/rate_control_settings.cc @@ -83,12 +83,15 @@ RateControlSettings::RateControlSettings( kDefaultScreenshareHysteresisFactor)), probe_max_allocation_("probe_max_allocation", true), bitrate_adjuster_("bitrate_adjuster", false), - vp8_s0_boost_("vp8_s0_boost", true) { + vp8_s0_boost_("vp8_s0_boost", true), + vp8_dynamic_rate_("vp8_dynamic_rate", false), + vp9_dynamic_rate_("vp9_dynamic_rate", false) { ParseFieldTrial({&congestion_window_, &congestion_window_pushback_}, key_value_config->Lookup("WebRTC-CongestionWindow")); ParseFieldTrial({&pacing_factor_, &alr_probing_, &trust_vp8_, &trust_vp9_, &video_hysteresis_, &screenshare_hysteresis_, - &probe_max_allocation_, &bitrate_adjuster_, &vp8_s0_boost_}, + &probe_max_allocation_, &bitrate_adjuster_, &vp8_s0_boost_, + &vp8_dynamic_rate_, &vp9_dynamic_rate_}, key_value_config->Lookup("WebRTC-VideoRateControl")); } @@ -141,10 +144,18 @@ bool RateControlSettings::Vp8BoostBaseLayerQuality() const { return vp8_s0_boost_.Get(); } +bool RateControlSettings::Vp8DynamicRateSettings() const { + return vp8_dynamic_rate_.Get(); +} + bool RateControlSettings::LibvpxVp9TrustedRateController() const { return trust_vp9_.Get(); } +bool RateControlSettings::Vp9DynamicRateSettings() const { + return vp9_dynamic_rate_.Get(); +} + double RateControlSettings::GetSimulcastHysteresisFactor( VideoCodecMode mode) const { if (mode == VideoCodecMode::kScreensharing) { diff --git a/rtc_base/experiments/rate_control_settings.h b/rtc_base/experiments/rate_control_settings.h index 0898f9be42..c907b33a0b 100644 --- a/rtc_base/experiments/rate_control_settings.h +++ b/rtc_base/experiments/rate_control_settings.h @@ -42,7 +42,9 @@ class RateControlSettings final { bool LibvpxVp8TrustedRateController() const; bool Vp8BoostBaseLayerQuality() const; + bool Vp8DynamicRateSettings() const; bool LibvpxVp9TrustedRateController() const; + bool Vp9DynamicRateSettings() const; // TODO(bugs.webrtc.org/10272): Remove one of these when we have merged // VideoCodecMode and VideoEncoderConfig::ContentType. @@ -71,6 +73,8 @@ class RateControlSettings final { FieldTrialParameter probe_max_allocation_; FieldTrialParameter bitrate_adjuster_; FieldTrialParameter vp8_s0_boost_; + FieldTrialParameter vp8_dynamic_rate_; + FieldTrialParameter vp9_dynamic_rate_; }; } // namespace webrtc