diff --git a/api/video/video_bitrate_allocation.cc b/api/video/video_bitrate_allocation.cc index 32e72467f5..e189db1c19 100644 --- a/api/video/video_bitrate_allocation.cc +++ b/api/video/video_bitrate_allocation.cc @@ -18,7 +18,8 @@ namespace webrtc { -VideoBitrateAllocation::VideoBitrateAllocation() : sum_(0) {} +VideoBitrateAllocation::VideoBitrateAllocation() + : sum_(0), is_bw_limited_(false) {} bool VideoBitrateAllocation::SetBitrate(size_t spatial_index, size_t temporal_index, diff --git a/api/video/video_bitrate_allocation.h b/api/video/video_bitrate_allocation.h index da58a5b2bc..56c0f64da3 100644 --- a/api/video/video_bitrate_allocation.h +++ b/api/video/video_bitrate_allocation.h @@ -80,9 +80,15 @@ class RTC_EXPORT VideoBitrateAllocation { std::string ToString() const; + // Indicates if the allocation has some layers/streams disabled due to + // low available bandwidth. + void set_bw_limited(bool limited) { is_bw_limited_ = limited; } + bool is_bw_limited() const { return is_bw_limited_; } + private: uint32_t sum_; absl::optional bitrates_[kMaxSpatialLayers][kMaxTemporalStreams]; + bool is_bw_limited_; }; } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc index 86b677d6c8..7d5c724e30 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc @@ -211,7 +211,8 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( } const size_t first_active_layer = GetFirstActiveLayer(codec_); - size_t num_spatial_layers = GetNumActiveSpatialLayers(codec_); + const size_t num_active_layers = GetNumActiveSpatialLayers(codec_); + size_t num_spatial_layers = num_active_layers; if (num_spatial_layers == 0) { return VideoBitrateAllocation(); // All layers are deactivated. @@ -244,13 +245,16 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( } last_active_layer_count_ = num_spatial_layers; + VideoBitrateAllocation allocation; if (codec_.mode == VideoCodecMode::kRealtimeVideo) { - return GetAllocationNormalVideo(total_bitrate, first_active_layer, - num_spatial_layers); + allocation = GetAllocationNormalVideo(total_bitrate, first_active_layer, + num_spatial_layers); } else { - return GetAllocationScreenSharing(total_bitrate, first_active_layer, - num_spatial_layers); + allocation = GetAllocationScreenSharing(total_bitrate, first_active_layer, + num_spatial_layers); } + allocation.set_bw_limited(num_spatial_layers < num_active_layers); + return allocation; } VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc index 06240a32d8..6a677a2a6f 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc @@ -224,6 +224,24 @@ TEST(SvcRateAllocatorTest, DeactivateLowerLayers) { } } +TEST(SvcRateAllocatorTest, SignalsBwLimited) { + VideoCodec codec = Configure(1280, 720, 3, 1, false); + SvcRateAllocator allocator = SvcRateAllocator(codec); + + // Rough estimate calculated by hand. + uint32_t min_to_enable_all = 900000; + + EXPECT_TRUE( + allocator + .Allocate(VideoBitrateAllocationParameters(min_to_enable_all / 2, 30)) + .is_bw_limited()); + + EXPECT_FALSE( + allocator + .Allocate(VideoBitrateAllocationParameters(min_to_enable_all, 30)) + .is_bw_limited()); +} + TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) { VideoCodec codec = Configure(1280, 720, 3, 1, false); EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U); diff --git a/modules/video_coding/utility/simulcast_rate_allocator.cc b/modules/video_coding/utility/simulcast_rate_allocator.cc index 15b8e543a1..f074ee945c 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -166,6 +166,7 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers( min_bitrate = std::min(hysteresis_factor * min_bitrate, target_bitrate); } if (left_in_stable_allocation < min_bitrate) { + allocated_bitrates->set_bw_limited(true); break; } diff --git a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index eb01481646..e85ae3bc29 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -221,6 +221,27 @@ TEST_F(SimulcastRateAllocatorTest, SingleSimulcastBelowMin) { ExpectEqual(expected, GetAllocation(0)); } +TEST_F(SimulcastRateAllocatorTest, SignalsBwLimited) { + // Enough to enable all layers. + const int kVeryBigBitrate = 100000; + // With simulcast, use the min bitrate from the ss spec instead of the global. + SetupCodec3SL3TL({true, true, true}); + CreateAllocator(); + + EXPECT_TRUE( + GetAllocation(codec_.simulcastStream[0].minBitrate - 10).is_bw_limited()); + EXPECT_TRUE( + GetAllocation(codec_.simulcastStream[0].targetBitrate).is_bw_limited()); + EXPECT_TRUE(GetAllocation(codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].minBitrate) + .is_bw_limited()); + EXPECT_FALSE(GetAllocation(codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + codec_.simulcastStream[2].minBitrate) + .is_bw_limited()); + EXPECT_FALSE(GetAllocation(kVeryBigBitrate).is_bw_limited()); +} + TEST_F(SimulcastRateAllocatorTest, SingleSimulcastAboveMax) { codec_.numberOfSimulcastStreams = 1; codec_.simulcastStream[0].minBitrate = kMinBitrateKbps; @@ -655,6 +676,7 @@ TEST_P(ScreenshareRateAllocationTest, BitrateBelowTl0) { EXPECT_EQ(kLegacyScreenshareTargetBitrateKbps, allocation.get_sum_kbps()); EXPECT_EQ(kLegacyScreenshareTargetBitrateKbps, allocation.GetBitrate(0, 0) / 1000); + EXPECT_EQ(allocation.is_bw_limited(), GetParam()); } TEST_P(ScreenshareRateAllocationTest, BitrateAboveTl0) { @@ -674,6 +696,7 @@ TEST_P(ScreenshareRateAllocationTest, BitrateAboveTl0) { allocation.GetBitrate(0, 0) / 1000); EXPECT_EQ(target_bitrate_kbps - kLegacyScreenshareTargetBitrateKbps, allocation.GetBitrate(0, 1) / 1000); + EXPECT_EQ(allocation.is_bw_limited(), GetParam()); } TEST_F(ScreenshareRateAllocationTest, BitrateAboveTl1) { @@ -692,6 +715,7 @@ TEST_F(ScreenshareRateAllocationTest, BitrateAboveTl1) { EXPECT_EQ( kLegacyScreenshareMaxBitrateKbps - kLegacyScreenshareTargetBitrateKbps, allocation.GetBitrate(0, 1) / 1000); + EXPECT_FALSE(allocation.is_bw_limited()); } // This tests when the screenshare is inactive it should be allocated 0 bitrate diff --git a/video/send_statistics_proxy.cc b/video/send_statistics_proxy.cc index 5bf3427c25..58fb82e824 100644 --- a/video/send_statistics_proxy.cc +++ b/video/send_statistics_proxy.cc @@ -147,6 +147,7 @@ SendStatisticsProxy::SendStatisticsProxy( last_num_spatial_layers_(0), last_num_simulcast_streams_(0), last_spatial_layer_use_{}, + bw_limited_layers_(false), uma_container_( new UmaSamplesContainer(GetUmaPrefix(content_type_), stats_, clock)) { } @@ -1073,10 +1074,21 @@ void SendStatisticsProxy::OnAdaptationChanged( break; } - bool is_cpu_limited = cpu_counts.num_resolution_reductions > 0 || - cpu_counts.num_framerate_reductions > 0; - bool is_bandwidth_limited = quality_counts.num_resolution_reductions > 0 || - quality_counts.num_framerate_reductions > 0; + cpu_downscales_ = cpu_counts.num_resolution_reductions.value_or(-1); + quality_downscales_ = quality_counts.num_resolution_reductions.value_or(-1); + + cpu_counts_ = cpu_counts; + quality_counts_ = quality_counts; + + UpdateAdaptationStats(); +} + +void SendStatisticsProxy::UpdateAdaptationStats() { + bool is_cpu_limited = cpu_counts_.num_resolution_reductions > 0 || + cpu_counts_.num_framerate_reductions > 0; + bool is_bandwidth_limited = quality_counts_.num_resolution_reductions > 0 || + quality_counts_.num_framerate_reductions > 0 || + bw_limited_layers_; if (is_bandwidth_limited) { // We may be both CPU limited and bandwidth limited at the same time but // there is no way to express this in standardized stats. Heuristically, @@ -1092,21 +1104,27 @@ void SendStatisticsProxy::OnAdaptationChanged( QualityLimitationReason::kNone); } - UpdateAdaptationStats(cpu_counts, quality_counts); -} - -void SendStatisticsProxy::UpdateAdaptationStats( - const AdaptationSteps& cpu_counts, - const AdaptationSteps& quality_counts) { - cpu_downscales_ = cpu_counts.num_resolution_reductions.value_or(-1); - quality_downscales_ = quality_counts.num_resolution_reductions.value_or(-1); - - stats_.cpu_limited_resolution = cpu_counts.num_resolution_reductions > 0; - stats_.cpu_limited_framerate = cpu_counts.num_framerate_reductions > 0; - stats_.bw_limited_resolution = quality_counts.num_resolution_reductions > 0; - stats_.bw_limited_framerate = quality_counts.num_framerate_reductions > 0; + stats_.cpu_limited_resolution = cpu_counts_.num_resolution_reductions > 0; + stats_.cpu_limited_framerate = cpu_counts_.num_framerate_reductions > 0; + stats_.bw_limited_resolution = quality_counts_.num_resolution_reductions > 0; + stats_.bw_limited_framerate = quality_counts_.num_framerate_reductions > 0; + // If bitrate allocator has disabled some layers frame-rate or resolution are + // limited depending on the encoder configuration. + if (bw_limited_layers_) { + switch (content_type_) { + case VideoEncoderConfig::ContentType::kRealtimeVideo: { + stats_.bw_limited_resolution = true; + break; + } + case VideoEncoderConfig::ContentType::kScreen: { + stats_.bw_limited_framerate = true; + break; + } + } + } stats_.quality_limitation_reason = quality_limitation_reason_tracker_.current_reason(); + // |stats_.quality_limitation_durations_ms| depends on the current time // when it is polled; it is updated in SendStatisticsProxy::GetStats(). } @@ -1134,6 +1152,9 @@ void SendStatisticsProxy::OnBitrateAllocationUpdated( rtc::CritScope lock(&crit_); + bw_limited_layers_ = allocation.is_bw_limited(); + UpdateAdaptationStats(); + if (spatial_layers != last_spatial_layer_use_) { // If the number of spatial layers has changed, the resolution change is // not due to quality limitations, it is because the configuration diff --git a/video/send_statistics_proxy.h b/video/send_statistics_proxy.h index 6955ef6de9..e690803899 100644 --- a/video/send_statistics_proxy.h +++ b/video/send_statistics_proxy.h @@ -223,9 +223,7 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver, void SetAdaptTimer(const AdaptationSteps& counts, StatsTimer* timer) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); - void UpdateAdaptationStats(const AdaptationSteps& cpu_counts, - const AdaptationSteps& quality_counts) - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + void UpdateAdaptationStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); void TryUpdateInitialQualityResolutionAdaptUp( const AdaptationSteps& quality_counts) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); @@ -263,6 +261,11 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver, int last_num_simulcast_streams_ RTC_GUARDED_BY(crit_); std::array last_spatial_layer_use_ RTC_GUARDED_BY(crit_); + // Indicates if the latest bitrate allocation had layers disabled by low + // available bandwidth. + bool bw_limited_layers_ RTC_GUARDED_BY(crit_); + AdaptationSteps cpu_counts_ RTC_GUARDED_BY(crit_); + AdaptationSteps quality_counts_ RTC_GUARDED_BY(crit_); struct EncoderChangeEvent { std::string previous_encoder_implementation; diff --git a/video/send_statistics_proxy_unittest.cc b/video/send_statistics_proxy_unittest.cc index 47ce644ddf..4823e95c7a 100644 --- a/video/send_statistics_proxy_unittest.cc +++ b/video/send_statistics_proxy_unittest.cc @@ -1371,6 +1371,79 @@ TEST_F(SendStatisticsProxyTest, 0u, statistics_proxy_->GetStats().quality_limitation_resolution_changes); } +TEST_F(SendStatisticsProxyTest, + QualityLimitationReasonsAreCorrectForContentType) { + // Realtime case. + // Configure two streams. + VideoEncoderConfig config; + config.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; + config.number_of_streams = 2; + VideoStream stream1; + stream1.width = kWidth / 2; + stream1.height = kHeight / 2; + VideoStream stream2; + stream2.width = kWidth; + stream2.height = kHeight; + statistics_proxy_->OnEncoderReconfigured(config, {stream1, stream2}); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_framerate); + EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason, + QualityLimitationReason::kNone); + // Bw disabled one layer. + VideoCodec codec; + codec.numberOfSimulcastStreams = 2; + codec.simulcastStream[0].active = true; + codec.simulcastStream[1].active = true; + VideoBitrateAllocation allocation; + // Some positive bitrate only on the first stream. + allocation.SetBitrate(0, 0, 10000); + allocation.SetBitrate(1, 0, 0); + allocation.set_bw_limited(true); + statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation); + EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_framerate); + EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason, + QualityLimitationReason::kBandwidth); + // Bw enabled all layers. + allocation.SetBitrate(1, 0, 10000); + allocation.set_bw_limited(false); + statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_framerate); + EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason, + QualityLimitationReason::kNone); + + // Screencast case + // Configure two streams. + config.content_type = VideoEncoderConfig::ContentType::kScreen; + config.number_of_streams = 2; + stream1.width = kWidth; + stream1.height = kHeight; + statistics_proxy_->OnEncoderReconfigured(config, {stream1, stream2}); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_framerate); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); + EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason, + QualityLimitationReason::kNone); + // Bw disabled one layer. + // Some positive bitrate only on the second stream. + allocation.SetBitrate(0, 0, 10000); + allocation.SetBitrate(1, 0, 0); + allocation.set_bw_limited(true); + statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation); + EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_framerate); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); + EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason, + QualityLimitationReason::kBandwidth); + // Bw enabled all layers. + allocation.SetBitrate(1, 0, 10000); + allocation.set_bw_limited(false); + statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); + EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason, + QualityLimitationReason::kNone); +} + TEST_F(SendStatisticsProxyTest, SwitchContentTypeUpdatesHistograms) { for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) statistics_proxy_->OnIncomingFrame(kWidth, kHeight); @@ -1982,6 +2055,7 @@ TEST_F(SendStatisticsProxyTest, GetStatsReportsBandwidthLimitedResolution) { // Configure two streams. VideoEncoderConfig config; config.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; + config.number_of_streams = 2; VideoStream stream1; stream1.width = kWidth / 2; stream1.height = kHeight / 2; @@ -2044,6 +2118,26 @@ TEST_F(SendStatisticsProxyTest, GetStatsReportsBandwidthLimitedResolution) { quality_counts); statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution); + + // Adapt up. + quality_counts.num_resolution_reductions = 0; + statistics_proxy_->OnAdaptationChanged( + VideoStreamEncoderObserver::AdaptationReason::kQuality, cpu_counts, + quality_counts); + statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); + EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); + + // Bw disabled one layer. + VideoCodec codec; + codec.numberOfSimulcastStreams = 2; + codec.simulcastStream[0].active = true; + codec.simulcastStream[1].active = true; + VideoBitrateAllocation allocation; + // Some positive bitrate only on the second stream. + allocation.SetBitrate(1, 0, 10000); + allocation.set_bw_limited(true); + statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation); + EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution); } TEST_F(SendStatisticsProxyTest, GetStatsReportsTargetMediaBitrate) {