diff --git a/modules/video_coding/utility/quality_scaler.cc b/modules/video_coding/utility/quality_scaler.cc index 1f1f192964..42c40c5c58 100644 --- a/modules/video_coding/utility/quality_scaler.cc +++ b/modules/video_coding/utility/quality_scaler.cc @@ -134,6 +134,11 @@ int64_t QualityScaler::GetSamplingPeriodMs() const { return sampling_period_ms_ * initial_scale_factor_; } +void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) { + RTC_DCHECK_RUN_ON(&task_checker_); + thresholds_ = thresholds; +} + void QualityScaler::ReportDroppedFrameByMediaOpt() { RTC_DCHECK_RUN_ON(&task_checker_); framedrop_percent_media_opt_.AddSample(100); diff --git a/modules/video_coding/utility/quality_scaler.h b/modules/video_coding/utility/quality_scaler.h index 7886fc0908..367db0e197 100644 --- a/modules/video_coding/utility/quality_scaler.h +++ b/modules/video_coding/utility/quality_scaler.h @@ -60,6 +60,8 @@ class QualityScaler { // Inform the QualityScaler of the last seen QP. void ReportQp(int qp, int64_t time_sent_us); + void SetQpThresholds(VideoEncoder::QpThresholds thresholds); + // The following members declared protected for testing purposes. protected: QualityScaler(rtc::TaskQueue* task_queue, @@ -80,7 +82,7 @@ class QualityScaler { AdaptationObserverInterface* const observer_ RTC_GUARDED_BY(&task_checker_); SequenceChecker task_checker_; - const VideoEncoder::QpThresholds thresholds_; + VideoEncoder::QpThresholds thresholds_ RTC_GUARDED_BY(&task_checker_); const int64_t sampling_period_ms_; bool fast_rampup_ RTC_GUARDED_BY(&task_checker_); rtc::MovingAverage average_qp_ RTC_GUARDED_BY(&task_checker_); diff --git a/rtc_base/experiments/BUILD.gn b/rtc_base/experiments/BUILD.gn index 3a78afe9a5..849e740ce6 100644 --- a/rtc_base/experiments/BUILD.gn +++ b/rtc_base/experiments/BUILD.gn @@ -107,7 +107,9 @@ rtc_static_library("balanced_degradation_settings") { deps = [ ":field_trial_parser", "../:rtc_base_approved", + "../../api/video_codecs:video_codecs_api", "../../system_wrappers:field_trial", + "//third_party/abseil-cpp/absl/types:optional", ] } diff --git a/rtc_base/experiments/balanced_degradation_settings.cc b/rtc_base/experiments/balanced_degradation_settings.cc index d6a33e11fd..a8d7d4d1ca 100644 --- a/rtc_base/experiments/balanced_degradation_settings.cc +++ b/rtc_base/experiments/balanced_degradation_settings.cc @@ -23,7 +23,23 @@ constexpr int kMinFps = 1; constexpr int kMaxFps = 100; std::vector DefaultConfigs() { - return {{320 * 240, 7}, {480 * 270, 10}, {640 * 480, 15}}; + return {{320 * 240, 7, {0, 0}, {0, 0}, {0, 0}, {0, 0}}, + {480 * 270, 10, {0, 0}, {0, 0}, {0, 0}, {0, 0}}, + {640 * 480, 15, {0, 0}, {0, 0}, {0, 0}, {0, 0}}}; +} + +bool IsValidThreshold( + const BalancedDegradationSettings::QpThreshold& threshold) { + if (threshold.GetLow().has_value() != threshold.GetHigh().has_value()) { + RTC_LOG(LS_WARNING) << "Neither or both values should be set."; + return false; + } + if (threshold.GetLow().has_value() && threshold.GetHigh().has_value() && + threshold.GetLow().value() >= threshold.GetHigh().value()) { + RTC_LOG(LS_WARNING) << "Invalid threshold value, low >= high threshold."; + return false; + } + return true; } bool IsValid(const std::vector& configs) { @@ -40,7 +56,24 @@ bool IsValid(const std::vector& configs) { for (size_t i = 1; i < configs.size(); ++i) { if (configs[i].pixels < configs[i - 1].pixels || configs[i].fps < configs[i - 1].fps) { - RTC_LOG(LS_WARNING) << "Invalid parameter value provided."; + RTC_LOG(LS_WARNING) << "Invalid fps/pixel value provided."; + return false; + } + if (((configs[i].vp8.low > 0) != (configs[i - 1].vp8.low > 0)) || + ((configs[i].vp9.low > 0) != (configs[i - 1].vp9.low > 0)) || + ((configs[i].h264.low > 0) != (configs[i - 1].h264.low > 0)) || + ((configs[i].generic.low > 0) != (configs[i - 1].generic.low > 0)) || + ((configs[i].vp8.high > 0) != (configs[i - 1].vp8.high > 0)) || + ((configs[i].vp9.high > 0) != (configs[i - 1].vp9.high > 0)) || + ((configs[i].h264.high > 0) != (configs[i - 1].h264.high > 0)) || + ((configs[i].generic.high > 0) != (configs[i - 1].generic.high > 0))) { + RTC_LOG(LS_WARNING) << "Invalid threshold value, all/none should be set."; + return false; + } + } + for (const auto& config : configs) { + if (!IsValidThreshold(config.vp8) || !IsValidThreshold(config.vp9) || + !IsValidThreshold(config.h264) || !IsValidThreshold(config.generic)) { return false; } } @@ -54,17 +87,86 @@ std::vector GetValidOrDefault( } return DefaultConfigs(); } + +absl::optional GetThresholds( + VideoCodecType type, + const BalancedDegradationSettings::Config& config) { + absl::optional low; + absl::optional high; + + switch (type) { + case kVideoCodecVP8: + low = config.vp8.GetLow(); + high = config.vp8.GetHigh(); + break; + case kVideoCodecVP9: + low = config.vp9.GetLow(); + high = config.vp9.GetHigh(); + break; + case kVideoCodecH264: + low = config.h264.GetLow(); + high = config.h264.GetHigh(); + break; + case kVideoCodecGeneric: + low = config.generic.GetLow(); + high = config.generic.GetHigh(); + break; + default: + break; + } + + if (low && high) { + RTC_LOG(LS_INFO) << "QP thresholds: low: " << *low << ", high: " << *high; + return absl::optional( + VideoEncoder::QpThresholds(*low, *high)); + } + return absl::nullopt; +} } // namespace +absl::optional BalancedDegradationSettings::QpThreshold::GetLow() const { + return (low > 0) ? absl::optional(low) : absl::nullopt; +} + +absl::optional BalancedDegradationSettings::QpThreshold::GetHigh() const { + return (high > 0) ? absl::optional(high) : absl::nullopt; +} + BalancedDegradationSettings::Config::Config() = default; -BalancedDegradationSettings::Config::Config(int pixels, int fps) - : pixels(pixels), fps(fps) {} +BalancedDegradationSettings::Config::Config(int pixels, + int fps, + QpThreshold vp8, + QpThreshold vp9, + QpThreshold h264, + QpThreshold generic) + : pixels(pixels), + fps(fps), + vp8(vp8), + vp9(vp9), + h264(h264), + generic(generic) {} BalancedDegradationSettings::BalancedDegradationSettings() { FieldTrialStructList configs( {FieldTrialStructMember("pixels", [](Config* c) { return &c->pixels; }), - FieldTrialStructMember("fps", [](Config* c) { return &c->fps; })}, + FieldTrialStructMember("fps", [](Config* c) { return &c->fps; }), + FieldTrialStructMember("vp8_qp_low", + [](Config* c) { return &c->vp8.low; }), + FieldTrialStructMember("vp8_qp_high", + [](Config* c) { return &c->vp8.high; }), + FieldTrialStructMember("vp9_qp_low", + [](Config* c) { return &c->vp9.low; }), + FieldTrialStructMember("vp9_qp_high", + [](Config* c) { return &c->vp9.high; }), + FieldTrialStructMember("h264_qp_low", + [](Config* c) { return &c->h264.low; }), + FieldTrialStructMember("h264_qp_high", + [](Config* c) { return &c->h264.high; }), + FieldTrialStructMember("generic_qp_low", + [](Config* c) { return &c->generic.low; }), + FieldTrialStructMember("generic_qp_high", + [](Config* c) { return &c->generic.high; })}, {}); ParseFieldTrial({&configs}, field_trial::FindFullName(kFieldTrial)); @@ -96,4 +198,19 @@ int BalancedDegradationSettings::MaxFps(int pixels) const { return std::numeric_limits::max(); } +absl::optional +BalancedDegradationSettings::GetQpThresholds(VideoCodecType type, + int pixels) const { + return GetThresholds(type, GetConfig(pixels)); +} + +BalancedDegradationSettings::Config BalancedDegradationSettings::GetConfig( + int pixels) const { + for (const auto& config : configs_) { + if (pixels <= config.pixels) + return config; + } + return configs_.back(); // Use last above highest pixels. +} + } // namespace webrtc diff --git a/rtc_base/experiments/balanced_degradation_settings.h b/rtc_base/experiments/balanced_degradation_settings.h index f006493595..ef4b5879df 100644 --- a/rtc_base/experiments/balanced_degradation_settings.h +++ b/rtc_base/experiments/balanced_degradation_settings.h @@ -13,6 +13,9 @@ #include +#include "absl/types/optional.h" +#include "api/video_codecs/video_encoder.h" + namespace webrtc { class BalancedDegradationSettings { @@ -20,17 +23,40 @@ class BalancedDegradationSettings { BalancedDegradationSettings(); ~BalancedDegradationSettings(); - struct Config { - Config(); - Config(int pixels, int fps); + struct QpThreshold { + QpThreshold() {} + QpThreshold(int low, int high) : low(low), high(high) {} - bool operator==(const Config& o) const { - return pixels == o.pixels && fps == o.fps; + bool operator==(const QpThreshold& o) const { + return low == o.low && high == o.high; } - int pixels = 0; // The video frame size. - int fps = 0; // The framerate to be used if the frame size is less than - // or equal to |pixels|. + absl::optional GetLow() const; + absl::optional GetHigh() const; + int low = 0; + int high = 0; + }; + + struct Config { + Config(); + Config(int pixels, + int fps, + QpThreshold vp8, + QpThreshold vp9, + QpThreshold h264, + QpThreshold generic); + + bool operator==(const Config& o) const { + return pixels == o.pixels && fps == o.fps && vp8 == o.vp8 && + vp9 == o.vp9 && h264 == o.h264 && generic == o.generic; + } + + int pixels = 0; // The video frame size. + int fps = 0; // The framerate and thresholds to be used if the frame + QpThreshold vp8; // size is less than or equal to |pixels|. + QpThreshold vp9; + QpThreshold h264; + QpThreshold generic; }; // Returns configurations from field trial on success (default on failure). @@ -40,7 +66,14 @@ class BalancedDegradationSettings { int MinFps(int pixels) const; int MaxFps(int pixels) const; + // Gets QpThresholds for the codec |type| based on |pixels|. + absl::optional GetQpThresholds( + VideoCodecType type, + int pixels) const; + private: + Config GetConfig(int pixels) const; + std::vector configs_; }; diff --git a/rtc_base/experiments/balanced_degradation_settings_unittest.cc b/rtc_base/experiments/balanced_degradation_settings_unittest.cc index 7a6eb6599d..8cbadacf90 100644 --- a/rtc_base/experiments/balanced_degradation_settings_unittest.cc +++ b/rtc_base/experiments/balanced_degradation_settings_unittest.cc @@ -22,9 +22,12 @@ namespace { void VerifyIsDefault( const std::vector& config) { EXPECT_THAT(config, ::testing::ElementsAre( - BalancedDegradationSettings::Config{320 * 240, 7}, - BalancedDegradationSettings::Config{480 * 270, 10}, - BalancedDegradationSettings::Config{640 * 480, 15})); + BalancedDegradationSettings::Config{ + 320 * 240, 7, {0, 0}, {0, 0}, {0, 0}, {0, 0}}, + BalancedDegradationSettings::Config{ + 480 * 270, 10, {0, 0}, {0, 0}, {0, 0}, {0, 0}}, + BalancedDegradationSettings::Config{ + 640 * 480, 15, {0, 0}, {0, 0}, {0, 0}, {0, 0}})); } } // namespace @@ -32,18 +35,26 @@ TEST(BalancedDegradationSettings, GetsDefaultConfigIfNoList) { webrtc::test::ScopedFieldTrials field_trials(""); BalancedDegradationSettings settings; VerifyIsDefault(settings.GetConfigs()); + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecVP8, 1)); + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecVP9, 1)); + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecH264, 1)); + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecGeneric, 1)); + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecMultiplex, 1)); } TEST(BalancedDegradationSettings, GetsConfig) { webrtc::test::ScopedFieldTrials field_trials( "WebRTC-Video-BalancedDegradationSettings/" - "pixels:1000|2000|3000,fps:5|15|25/"); + "pixels:11|22|33,fps:5|15|25,other:4|5|6/"); BalancedDegradationSettings settings; - EXPECT_THAT( - settings.GetConfigs(), - ::testing::ElementsAre(BalancedDegradationSettings::Config{1000, 5}, - BalancedDegradationSettings::Config{2000, 15}, - BalancedDegradationSettings::Config{3000, 25})); + EXPECT_THAT(settings.GetConfigs(), + ::testing::ElementsAre( + BalancedDegradationSettings::Config{ + 11, 5, {0, 0}, {0, 0}, {0, 0}, {0, 0}}, + BalancedDegradationSettings::Config{ + 22, 15, {0, 0}, {0, 0}, {0, 0}, {0, 0}}, + BalancedDegradationSettings::Config{ + 33, 25, {0, 0}, {0, 0}, {0, 0}, {0, 0}})); } TEST(BalancedDegradationSettings, GetsDefaultConfigForZeroFpsValue) { @@ -96,4 +107,128 @@ TEST(BalancedDegradationSettings, GetsMaxFps) { EXPECT_EQ(std::numeric_limits::max(), settings.MaxFps(2000 + 1)); } +TEST(BalancedDegradationSettings, QpThresholdsNotSetByDefault) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25/"); + BalancedDegradationSettings settings; + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecVP8, 1)); + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecVP9, 1)); + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecH264, 1)); + EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecGeneric, 1)); +} + +TEST(BalancedDegradationSettings, GetsConfigWithQpThresholds) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25,vp8_qp_low:89|90|88," + "vp8_qp_high:90|91|92,vp9_qp_low:27|28|29,vp9_qp_high:120|130|140," + "h264_qp_low:12|13|14,h264_qp_high:20|30|40,generic_qp_low:7|6|5," + "generic_qp_high:22|23|24/"); + BalancedDegradationSettings settings; + EXPECT_THAT(settings.GetConfigs(), + ::testing::ElementsAre( + BalancedDegradationSettings::Config{ + 1000, 5, {89, 90}, {27, 120}, {12, 20}, {7, 22}}, + BalancedDegradationSettings::Config{ + 2000, 15, {90, 91}, {28, 130}, {13, 30}, {6, 23}}, + BalancedDegradationSettings::Config{ + 3000, 25, {88, 92}, {29, 140}, {14, 40}, {5, 24}})); +} + +TEST(BalancedDegradationSettings, GetsDefaultConfigIfOnlyHasLowThreshold) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25,vp8_qp_low:89|90|88/"); + BalancedDegradationSettings settings; + VerifyIsDefault(settings.GetConfigs()); +} + +TEST(BalancedDegradationSettings, GetsDefaultConfigIfOnlyHasHighThreshold) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25,vp8_qp_high:90|91|92/"); + BalancedDegradationSettings settings; + VerifyIsDefault(settings.GetConfigs()); +} + +TEST(BalancedDegradationSettings, GetsDefaultConfigIfLowEqualsHigh) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25," + "vp8_qp_low:89|90|88,vp8_qp_high:90|91|88/"); + BalancedDegradationSettings settings; + VerifyIsDefault(settings.GetConfigs()); +} + +TEST(BalancedDegradationSettings, GetsDefaultConfigIfLowGreaterThanHigh) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25," + "vp8_qp_low:89|90|88,vp8_qp_high:90|91|87/"); + BalancedDegradationSettings settings; + VerifyIsDefault(settings.GetConfigs()); +} + +TEST(BalancedDegradationSettings, GetsDefaultConfigForZeroQpValue) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25," + "vp8_qp_low:89|0|88,vp8_qp_high:90|91|92/"); + BalancedDegradationSettings settings; + VerifyIsDefault(settings.GetConfigs()); +} + +TEST(BalancedDegradationSettings, GetsVp8QpThresholds) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25," + "vp8_qp_low:89|90|88,vp8_qp_high:90|91|92/"); + BalancedDegradationSettings settings; + EXPECT_EQ(89, settings.GetQpThresholds(kVideoCodecVP8, 1)->low); + EXPECT_EQ(90, settings.GetQpThresholds(kVideoCodecVP8, 1)->high); + EXPECT_EQ(90, settings.GetQpThresholds(kVideoCodecVP8, 1000)->high); + EXPECT_EQ(91, settings.GetQpThresholds(kVideoCodecVP8, 1001)->high); + EXPECT_EQ(91, settings.GetQpThresholds(kVideoCodecVP8, 2000)->high); + EXPECT_EQ(92, settings.GetQpThresholds(kVideoCodecVP8, 2001)->high); + EXPECT_EQ(92, settings.GetQpThresholds(kVideoCodecVP8, 3000)->high); + EXPECT_EQ(92, settings.GetQpThresholds(kVideoCodecVP8, 3001)->high); +} + +TEST(BalancedDegradationSettings, GetsVp9QpThresholds) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25," + "vp9_qp_low:55|56|57,vp9_qp_high:155|156|157/"); + BalancedDegradationSettings settings; + const auto thresholds = settings.GetQpThresholds(kVideoCodecVP9, 1000); + EXPECT_TRUE(thresholds); + EXPECT_EQ(55, thresholds->low); + EXPECT_EQ(155, thresholds->high); +} + +TEST(BalancedDegradationSettings, GetsH264QpThresholds) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25," + "h264_qp_low:21|22|23,h264_qp_high:41|43|42/"); + BalancedDegradationSettings settings; + const auto thresholds = settings.GetQpThresholds(kVideoCodecH264, 2000); + EXPECT_TRUE(thresholds); + EXPECT_EQ(22, thresholds->low); + EXPECT_EQ(43, thresholds->high); +} + +TEST(BalancedDegradationSettings, GetsGenericQpThresholds) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-BalancedDegradationSettings/" + "pixels:1000|2000|3000,fps:5|15|25," + "generic_qp_low:2|3|4,generic_qp_high:22|23|24/"); + BalancedDegradationSettings settings; + const auto thresholds = settings.GetQpThresholds(kVideoCodecGeneric, 3000); + EXPECT_TRUE(thresholds); + EXPECT_EQ(4, thresholds->low); + EXPECT_EQ(24, thresholds->high); +} + } // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index ebaaa449a8..a2bf3ef392 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -873,6 +873,16 @@ void VideoStreamEncoder::ConfigureQualityScaler( initial_framedrop_ = kMaxInitialFramedrop; } + if (degradation_preference_ == DegradationPreference::BALANCED && + quality_scaler_ && last_frame_info_) { + absl::optional thresholds = + balanced_settings_.GetQpThresholds(encoder_config_.codec_type, + last_frame_info_->pixel_count()); + if (thresholds) { + quality_scaler_->SetQpThresholds(*thresholds); + } + } + encoder_stats_observer_->OnAdaptationChanged( VideoStreamEncoderObserver::AdaptationReason::kNone, GetActiveCounts(kCpu), GetActiveCounts(kQuality)); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 8bc3dc4aff..3756e80401 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -282,7 +282,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // Set depending on degradation preferences. DegradationPreference degradation_preference_ RTC_GUARDED_BY(&encoder_queue_); - BalancedDegradationSettings balanced_settings_; + const BalancedDegradationSettings balanced_settings_; struct AdaptationRequest { // The pixel count produced by the source at the time of the adaptation.