From 7a82467d0db0d61f466a1da54b94f6a136726a3c Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Thu, 18 Jun 2020 19:16:53 +0200 Subject: [PATCH] Fix vp9 svc singlecast mode and enable quality scaler for vp9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1) Fix several typos and small mistakes which could lead to crashes 2) Adjust bitrates if leading layers are disabled 3) Wire up webrtc quality scaler Bug: webrtc:11319 Change-Id: I16e52bdb1c315d64906288e4f2be55fe698d5ceb Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/177525 Commit-Queue: Ilya Nikolaevskiy Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#31546} --- modules/video_coding/codecs/vp9/svc_config.cc | 13 ++++++ .../codecs/vp9/svc_rate_allocator.cc | 3 +- modules/video_coding/codecs/vp9/vp9_impl.cc | 45 +++++++++++++++++-- modules/video_coding/codecs/vp9/vp9_impl.h | 9 ++++ .../video_coding/video_codec_initializer.cc | 6 ++- video/quality_scaling_tests.cc | 3 +- video/video_stream_encoder.cc | 3 +- 7 files changed, 74 insertions(+), 8 deletions(-) diff --git a/modules/video_coding/codecs/vp9/svc_config.cc b/modules/video_coding/codecs/vp9/svc_config.cc index 118d8a77c1..cc7743ad25 100644 --- a/modules/video_coding/codecs/vp9/svc_config.cc +++ b/modules/video_coding/codecs/vp9/svc_config.cc @@ -121,6 +121,19 @@ std::vector ConfigureSvcNormalVideo(size_t input_width, spatial_layers.push_back(spatial_layer); } + // A workaround for sitiation when single HD layer is left with minBitrate + // about 500kbps. This would mean that there will always be at least 500kbps + // allocated to video regardless of how low is the actual BWE. + // Also, boost maxBitrate for the first layer to account for lost ability to + // predict from previous layers. + if (first_active_layer > 0) { + spatial_layers[0].minBitrate = kMinVp9SvcBitrateKbps; + // TODO(ilnik): tune this value or come up with a different formula to + // ensure that all singlecast configurations look good and not too much + // bitrate is added. + spatial_layers[0].maxBitrate *= 1.1; + } + return spatial_layers; } diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc index cc9a0d8997..25bca63c0e 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc @@ -140,7 +140,8 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec, } } upper_bound += DataRate::KilobitsPerSec( - codec.spatialLayers[num_active_layers - 1].minBitrate); + codec.spatialLayers[first_active_layer + num_active_layers - 1] + .minBitrate); // Do a binary search until upper and lower bound is the highest bitrate for // |num_active_layers| - 1 layers and lowest bitrate for |num_active_layers| diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index 25e0ac7e8d..d29c19dc8c 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -52,6 +52,15 @@ const int kMaxAllowedPidDiff = 30; constexpr double kLowRateFactor = 1.0; constexpr double kHighRateFactor = 2.0; +// TODO(ilink): Tune these thresholds further. +// Selected using ConverenceMotion_1280_720_50.yuv clip. +// No toggling observed on any link capacity from 100-2000kbps. +// HD was reached consistently when link capacity was 1500kbps. +// Set resolutions are a bit more conservative than svc_config.cc sets, e.g. +// for 300kbps resolution converged to 270p instead of 360p. +constexpr int kLowVp9QpThreshold = 149; +constexpr int kHighVp9QpThreshold = 205; + // These settings correspond to the settings in vpx_codec_enc_cfg. struct Vp9RateSettings { uint32_t rc_undershoot_pct; @@ -248,6 +257,8 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) "WebRTC-VP9VariableFramerateScreenshare")), variable_framerate_controller_( variable_framerate_experiment_.framerate_limit), + quality_scaler_experiment_( + ParseQualityScalerConfig("WebRTC-VP9QualityScaler")), num_steady_state_frames_(0), config_changed_(true) { codec_ = {}; @@ -398,7 +409,6 @@ bool VP9EncoderImpl::SetSvcRates( expect_no_more_active_layers = seen_active_layer; } } - RTC_DCHECK_GT(num_active_spatial_layers_, 0); if (higher_layers_enabled && !force_key_frame_) { // Prohibit drop of all layers for the next frame, so newly enabled @@ -566,7 +576,13 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, // put some key-frames at will even in VPX_KF_DISABLED kf_mode. config_->kf_max_dist = inst->VP9().keyFrameInterval; config_->kf_min_dist = config_->kf_max_dist; - config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0; + if (quality_scaler_experiment_.enabled) { + // In that experiment webrtc wide quality scaler is used instead of libvpx + // internal scaler. + config_->rc_resize_allowed = 0; + } else { + config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0; + } // Determine number of threads based on the image size and #cores. config_->g_threads = NumberOfThreads(config_->g_w, config_->g_h, settings.number_of_cores); @@ -1555,7 +1571,12 @@ VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const { EncoderInfo info; info.supports_native_handle = false; info.implementation_name = "libvpx"; - info.scaling_settings = VideoEncoder::ScalingSettings::kOff; + if (quality_scaler_experiment_.enabled) { + info.scaling_settings = VideoEncoder::ScalingSettings( + quality_scaler_experiment_.low_qp, quality_scaler_experiment_.high_qp); + } else { + info.scaling_settings = VideoEncoder::ScalingSettings::kOff; + } info.has_trusted_rate_controller = trusted_rate_controller_; info.is_hardware_accelerated = false; info.has_internal_source = false; @@ -1628,6 +1649,24 @@ VP9EncoderImpl::ParseVariableFramerateConfig(std::string group_name) { return config; } +// static +VP9EncoderImpl::QualityScalerExperiment +VP9EncoderImpl::ParseQualityScalerConfig(std::string group_name) { + FieldTrialFlag disabled = FieldTrialFlag("Disabled"); + FieldTrialParameter low_qp("low_qp", kLowVp9QpThreshold); + FieldTrialParameter high_qp("hihg_qp", kHighVp9QpThreshold); + ParseFieldTrial({&disabled, &low_qp, &high_qp}, + field_trial::FindFullName(group_name)); + QualityScalerExperiment config; + config.enabled = !disabled.Get(); + RTC_LOG(LS_INFO) << "Webrtc quality scaler for vp9 is " + << (config.enabled ? "enabled." : "disabled"); + config.low_qp = low_qp.Get(); + config.high_qp = high_qp.Get(); + + return config; +} + VP9DecoderImpl::VP9DecoderImpl() : decode_complete_callback_(nullptr), inited_(false), diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index 066ce20a6a..fae94c752b 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -175,6 +175,15 @@ class VP9EncoderImpl : public VP9Encoder { static VariableFramerateExperiment ParseVariableFramerateConfig( std::string group_name); FramerateController variable_framerate_controller_; + + const struct QualityScalerExperiment { + int low_qp; + int high_qp; + bool enabled; + } quality_scaler_experiment_; + static QualityScalerExperiment ParseQualityScalerConfig( + std::string group_name); + int num_steady_state_frames_; // Only set config when this flag is set. bool config_changed_; diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc index 8671df71df..7f36f99f89 100644 --- a/modules/video_coding/video_codec_initializer.cc +++ b/modules/video_coding/video_codec_initializer.cc @@ -75,7 +75,9 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( static_cast(streams.size()); video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; bool codec_active = false; - for (const VideoStream& stream : streams) { + // Active configuration might not be fully copied to |streams| for SVC yet. + // Therefore the |config| is checked here. + for (const VideoStream& stream : config.simulcast_layers) { if (stream.active) { codec_active = true; break; @@ -205,7 +207,7 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( spatial_layers.back().maxBitrate = video_codec.maxBitrate; } - for (size_t spatial_idx = 0; + for (size_t spatial_idx = first_active_layer; spatial_idx < config.simulcast_layers.size() && spatial_idx < spatial_layers.size(); ++spatial_idx) { diff --git a/video/quality_scaling_tests.cc b/video/quality_scaling_tests.cc index 19b9e8c36c..65a23dbbcc 100644 --- a/video/quality_scaling_tests.cc +++ b/video/quality_scaling_tests.cc @@ -233,7 +233,8 @@ TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrateWithScalingOff) { TEST_F(QualityScalingTest, NoAdaptDownForHighQp_Vp9) { // VP9 QP thresholds, low:1, high:1 -> high QP. - test::ScopedFieldTrials field_trials(kPrefix + "0,0,1,1,0,0" + kEnd); + test::ScopedFieldTrials field_trials(kPrefix + "0,0,1,1,0,0" + kEnd + + "WebRTC-VP9QualityScaler/Disabled/"); // QualityScaler always disabled. const bool kAutomaticResize = true; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index ef02cc2343..0ed73a3e63 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1730,7 +1730,8 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { bool simulcast_or_svc = (send_codec_.codecType == VideoCodecType::kVideoCodecVP9 && send_codec_.VP9().numberOfSpatialLayers > 1) || - send_codec_.numberOfSimulcastStreams > 1; + send_codec_.numberOfSimulcastStreams > 1 || + encoder_config_.simulcast_layers.size() > 1; if (simulcast_or_svc || !stream_resource_manager_.DropInitialFrames() || !encoder_target_bitrate_bps_.has_value()) {