From 96c1b9c5ea035bfc37917c9a48a26953dbb943e3 Mon Sep 17 00:00:00 2001 From: Dan Tan Date: Tue, 30 Jul 2024 17:01:12 +0000 Subject: [PATCH] Add variables to lend unused audio bits to video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL only adds variables necessary for the feature, which will be implemented in later CLs. Bug: webrtc:350555527 Change-Id: I71e56666e629f56168d316bf693150c0df0e2ecf Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/356740 Reviewed-by: Erik Språng Commit-Queue: Dan Tan Reviewed-by: Jakob Ivarsson‎ Cr-Commit-Position: refs/heads/main@{#42698} --- audio/audio_send_stream.cc | 8 +++++++- audio/audio_send_stream.h | 1 + audio/audio_send_stream_unittest.cc | 14 +++++++++++++ call/bitrate_allocator.h | 12 ++++++++++++ call/bitrate_allocator_unittest.cc | 3 +++ video/video_send_stream_impl.cc | 11 ++++++++++- video/video_send_stream_impl.h | 1 + video/video_send_stream_impl_unittest.cc | 25 ++++++++++++++++++++++++ 8 files changed, 73 insertions(+), 2 deletions(-) diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc index 18f28c67eb..5b3f62412e 100644 --- a/audio/audio_send_stream.cc +++ b/audio/audio_send_stream.cc @@ -491,6 +491,11 @@ uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { return 0; } +absl::optional AudioSendStream::GetUsedRate() const { + // TODO(bugs.webrtc.org/35055527): Implement + return absl::nullopt; +} + void AudioSendStream::SetTransportOverhead( int transport_overhead_per_packet_bytes) { RTC_DCHECK_RUN_ON(&worker_thread_checker_); @@ -807,7 +812,8 @@ void AudioSendStream::ConfigureBitrateObserver() { constraints->min.bps(), constraints->max.bps(), 0, priority_bitrate.bps(), true, allocation_settings_.bitrate_priority.value_or( - config_.bitrate_priority)}); + config_.bitrate_priority), + TrackRateElasticity::kCanContributeUnusedRate}); registered_with_allocator_ = true; } diff --git a/audio/audio_send_stream.h b/audio/audio_send_stream.h index 5ba5d54fe8..3845e8dca0 100644 --- a/audio/audio_send_stream.h +++ b/audio/audio_send_stream.h @@ -97,6 +97,7 @@ class AudioSendStream final : public webrtc::AudioSendStream, // Implements BitrateAllocatorObserver. uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; + absl::optional GetUsedRate() const override; void SetTransportOverhead(int transport_overhead_per_packet_bytes); diff --git a/audio/audio_send_stream_unittest.cc b/audio/audio_send_stream_unittest.cc index 45f5391373..666d2635a6 100644 --- a/audio/audio_send_stream_unittest.cc +++ b/audio/audio_send_stream_unittest.cc @@ -928,6 +928,20 @@ TEST(AudioSendStreamTest, DefaultsHonorsPriorityBitrate) { send_stream->Stop(); } +TEST(AudioSendStreamTest, DefaultsToContributeUnusedBitrate) { + ConfigHelper helper(true, true, true); + auto send_stream = helper.CreateAudioSendStream(); + EXPECT_CALL( + *helper.bitrate_allocator(), + AddObserver(send_stream.get(), + Field(&MediaStreamAllocationConfig::rate_elasticity, + TrackRateElasticity::kCanContributeUnusedRate))); + EXPECT_CALL(*helper.channel_send(), StartSend()); + send_stream->Start(); + EXPECT_CALL(*helper.channel_send(), StopSend()); + send_stream->Stop(); +} + TEST(AudioSendStreamTest, OverridesPriorityBitrate) { ConfigHelper helper(true, true, true); ScopedKeyValueConfig field_trials(helper.field_trials, diff --git a/call/bitrate_allocator.h b/call/bitrate_allocator.h index 204fc6f94d..366ac3f82b 100644 --- a/call/bitrate_allocator.h +++ b/call/bitrate_allocator.h @@ -22,6 +22,7 @@ #include "api/call/bitrate_allocation.h" #include "api/sequence_checker.h" #include "api/transport/network_types.h" +#include "api/units/data_rate.h" #include "rtc_base/system/no_unique_address.h" namespace webrtc { @@ -37,6 +38,8 @@ class BitrateAllocatorObserver { // Returns the amount of protection used by the BitrateAllocatorObserver // implementation, as bitrate in bps. virtual uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) = 0; + // Returns the bitrate consumed (vs allocated) by BitrateAllocatorObserver + virtual absl::optional GetUsedRate() const = 0; protected: virtual ~BitrateAllocatorObserver() {} @@ -45,6 +48,12 @@ class BitrateAllocatorObserver { // Struct describing parameters for how a media stream should get bitrate // allocated to it. +enum class TrackRateElasticity { + kCanContributeUnusedRate, + kCanConsumeExtraRate, + kCanContributeAndConsume +}; + struct MediaStreamAllocationConfig { // Minimum bitrate supported by track. 0 equals no min bitrate. uint32_t min_bitrate_bps; @@ -61,6 +70,7 @@ struct MediaStreamAllocationConfig { // observers. If an observer has twice the bitrate_priority of other // observers, it should be allocated twice the bitrate above its min. double bitrate_priority; + absl::optional rate_elasticity; }; // Interface used for mocking @@ -82,10 +92,12 @@ struct AllocatableTrack { : observer(observer), config(allocation_config), allocated_bitrate_bps(-1), + last_used_bitrate_bps(-1), media_ratio(1.0) {} BitrateAllocatorObserver* observer; MediaStreamAllocationConfig config; int64_t allocated_bitrate_bps; + int64_t last_used_bitrate_bps; double media_ratio; // Part of the total bitrate used for media [0.0, 1.0]. uint32_t LastAllocatedBitrate() const; diff --git a/call/bitrate_allocator_unittest.cc b/call/bitrate_allocator_unittest.cc index 69bdd83397..0ceacd0a85 100644 --- a/call/bitrate_allocator_unittest.cc +++ b/call/bitrate_allocator_unittest.cc @@ -75,6 +75,9 @@ class TestBitrateObserver : public BitrateAllocatorObserver { last_probing_interval_ms_ = update.bwe_period.ms(); return update.target_bitrate.bps() * protection_ratio_; } + absl::optional GetUsedRate() const override { + return absl::nullopt; + } uint32_t last_bitrate_bps_; uint8_t last_fraction_loss_; int64_t last_rtt_ms_; diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index b42c577887..181f146540 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -780,7 +780,10 @@ MediaStreamAllocationConfig VideoSendStreamImpl::GetAllocationConfig() const { static_cast(disable_padding_ ? 0 : max_padding_bitrate_), encoder_av1_priority_bitrate_override_bps_, !config_.suspend_below_min_bitrate, - encoder_bitrate_priority_}; + encoder_bitrate_priority_, + (content_type_ == VideoEncoderConfig::ContentType::kRealtimeVideo) + ? absl::optional(TrackRateElasticity::kCanConsumeExtraRate) + : absl::nullopt}; } void VideoSendStreamImpl::OnEncoderConfigurationChanged( @@ -944,5 +947,11 @@ uint32_t VideoSendStreamImpl::OnBitrateUpdated(BitrateAllocationUpdate update) { return protection_bitrate_bps; } +absl::optional VideoSendStreamImpl::GetUsedRate() const { + // This value is for real-time video. Screenshare may have unused bandwidth + // that can be shared, and this needs to be changed to support that. + return absl::nullopt; +} + } // namespace internal } // namespace webrtc diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h index a2ad8ad8fc..ea405ff056 100644 --- a/video/video_send_stream_impl.h +++ b/video/video_send_stream_impl.h @@ -146,6 +146,7 @@ class VideoSendStreamImpl : public webrtc::VideoSendStream, absl::optional GetPacingFactorOverride() const; // Implements BitrateAllocatorObserver. uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; + absl::optional GetUsedRate() const override; // Implements VideoStreamEncoderInterface::EncoderSink void OnEncoderConfigurationChanged( diff --git a/video/video_send_stream_impl_unittest.cc b/video/video_send_stream_impl_unittest.cc index 29ac08e065..08ac956f51 100644 --- a/video/video_send_stream_impl_unittest.cc +++ b/video/video_send_stream_impl_unittest.cc @@ -1214,5 +1214,30 @@ TEST_F(VideoSendStreamImplTest, ConfiguresBitratesForSvc) { vss_impl->Stop(); } } + +TEST_F(VideoSendStreamImplTest, TestElasticityForRealtimeVideo) { + auto vss_impl = CreateVideoSendStreamImpl( + TestVideoEncoderConfig(VideoEncoderConfig::ContentType::kRealtimeVideo)); + EXPECT_CALL(bitrate_allocator_, + AddObserver(vss_impl.get(), + Field(&MediaStreamAllocationConfig::rate_elasticity, + TrackRateElasticity::kCanConsumeExtraRate))); + vss_impl->Start(); + EXPECT_CALL(bitrate_allocator_, RemoveObserver(vss_impl.get())); + vss_impl->Stop(); +} + +TEST_F(VideoSendStreamImplTest, TestElasticityForScreenshare) { + auto vss_impl = CreateVideoSendStreamImpl( + TestVideoEncoderConfig(VideoEncoderConfig::ContentType::kScreen)); + EXPECT_CALL(bitrate_allocator_, + AddObserver(vss_impl.get(), + Field(&MediaStreamAllocationConfig::rate_elasticity, + absl::nullopt))); + vss_impl->Start(); + EXPECT_CALL(bitrate_allocator_, RemoveObserver(vss_impl.get())); + vss_impl->Stop(); +} + } // namespace internal } // namespace webrtc