Add variables to lend unused audio bits to video

This CL only adds variables necessary for the feature, which will be
implemented in later CLs.

Bug: webrtc:350555527
Change-Id: I71e56666e629f56168d316bf693150c0df0e2ecf
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/356740
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Dan Tan <dwtan@google.com>
Reviewed-by: Jakob Ivarsson‎ <jakobi@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42698}

audio/audio_send_stream.cc

@@ -491,6 +491,11 @@ uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) {
   return 0;
 }
 
+absl::optional<DataRate> AudioSendStream::GetUsedRate() const {
+  // TODO(bugs.webrtc.org/35055527): Implement
+  return absl::nullopt;
+}
+
 void AudioSendStream::SetTransportOverhead(
     int transport_overhead_per_packet_bytes) {
   RTC_DCHECK_RUN_ON(&worker_thread_checker_);
@@ -807,7 +812,8 @@ void AudioSendStream::ConfigureBitrateObserver() {
           constraints->min.bps<uint32_t>(), constraints->max.bps<uint32_t>(), 0,
           priority_bitrate.bps(), true,
           allocation_settings_.bitrate_priority.value_or(
-              config_.bitrate_priority)});
+              config_.bitrate_priority),
+          TrackRateElasticity::kCanContributeUnusedRate});
 
   registered_with_allocator_ = true;
 }

audio/audio_send_stream.h

@@ -97,6 +97,7 @@ class AudioSendStream final : public webrtc::AudioSendStream,
 
   // Implements BitrateAllocatorObserver.
   uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override;
+  absl::optional<DataRate> GetUsedRate() const override;
 
   void SetTransportOverhead(int transport_overhead_per_packet_bytes);
 

audio/audio_send_stream_unittest.cc

@@ -928,6 +928,20 @@ TEST(AudioSendStreamTest, DefaultsHonorsPriorityBitrate) {
   send_stream->Stop();
 }
 
+TEST(AudioSendStreamTest, DefaultsToContributeUnusedBitrate) {
+  ConfigHelper helper(true, true, true);
+  auto send_stream = helper.CreateAudioSendStream();
+  EXPECT_CALL(
+      *helper.bitrate_allocator(),
+      AddObserver(send_stream.get(),
+                  Field(&MediaStreamAllocationConfig::rate_elasticity,
+                        TrackRateElasticity::kCanContributeUnusedRate)));
+  EXPECT_CALL(*helper.channel_send(), StartSend());
+  send_stream->Start();
+  EXPECT_CALL(*helper.channel_send(), StopSend());
+  send_stream->Stop();
+}
+
 TEST(AudioSendStreamTest, OverridesPriorityBitrate) {
   ConfigHelper helper(true, true, true);
   ScopedKeyValueConfig field_trials(helper.field_trials,

call/bitrate_allocator.h

@@ -22,6 +22,7 @@
 #include "api/call/bitrate_allocation.h"
 #include "api/sequence_checker.h"
 #include "api/transport/network_types.h"
+#include "api/units/data_rate.h"
 #include "rtc_base/system/no_unique_address.h"
 
 namespace webrtc {
@@ -37,6 +38,8 @@ class BitrateAllocatorObserver {
   // Returns the amount of protection used by the BitrateAllocatorObserver
   // implementation, as bitrate in bps.
   virtual uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) = 0;
+  // Returns the bitrate consumed (vs allocated) by BitrateAllocatorObserver
+  virtual absl::optional<DataRate> GetUsedRate() const = 0;
 
  protected:
   virtual ~BitrateAllocatorObserver() {}
@@ -45,6 +48,12 @@ class BitrateAllocatorObserver {
 // Struct describing parameters for how a media stream should get bitrate
 // allocated to it.
 
+enum class TrackRateElasticity {
+  kCanContributeUnusedRate,
+  kCanConsumeExtraRate,
+  kCanContributeAndConsume
+};
+
 struct MediaStreamAllocationConfig {
   // Minimum bitrate supported by track. 0 equals no min bitrate.
   uint32_t min_bitrate_bps;
@@ -61,6 +70,7 @@ struct MediaStreamAllocationConfig {
   // observers. If an observer has twice the bitrate_priority of other
   // observers, it should be allocated twice the bitrate above its min.
   double bitrate_priority;
+  absl::optional<TrackRateElasticity> rate_elasticity;
 };
 
 // Interface used for mocking
@@ -82,10 +92,12 @@ struct AllocatableTrack {
       : observer(observer),
         config(allocation_config),
         allocated_bitrate_bps(-1),
+        last_used_bitrate_bps(-1),
         media_ratio(1.0) {}
   BitrateAllocatorObserver* observer;
   MediaStreamAllocationConfig config;
   int64_t allocated_bitrate_bps;
+  int64_t last_used_bitrate_bps;
   double media_ratio;  // Part of the total bitrate used for media [0.0, 1.0].
 
   uint32_t LastAllocatedBitrate() const;

call/bitrate_allocator_unittest.cc

@@ -75,6 +75,9 @@ class TestBitrateObserver : public BitrateAllocatorObserver {
     last_probing_interval_ms_ = update.bwe_period.ms();
     return update.target_bitrate.bps() * protection_ratio_;
   }
+  absl::optional<DataRate> GetUsedRate() const override {
+    return absl::nullopt;
+  }
   uint32_t last_bitrate_bps_;
   uint8_t last_fraction_loss_;
   int64_t last_rtt_ms_;

video/video_send_stream_impl.cc

@@ -780,7 +780,10 @@ MediaStreamAllocationConfig VideoSendStreamImpl::GetAllocationConfig() const {
       static_cast<uint32_t>(disable_padding_ ? 0 : max_padding_bitrate_),
       encoder_av1_priority_bitrate_override_bps_,
       !config_.suspend_below_min_bitrate,
-      encoder_bitrate_priority_};
+      encoder_bitrate_priority_,
+      (content_type_ == VideoEncoderConfig::ContentType::kRealtimeVideo)
+          ? absl::optional(TrackRateElasticity::kCanConsumeExtraRate)
+          : absl::nullopt};
 }
 
 void VideoSendStreamImpl::OnEncoderConfigurationChanged(
@@ -944,5 +947,11 @@ uint32_t VideoSendStreamImpl::OnBitrateUpdated(BitrateAllocationUpdate update) {
   return protection_bitrate_bps;
 }
 
+absl::optional<DataRate> VideoSendStreamImpl::GetUsedRate() const {
+  // This value is for real-time video. Screenshare may have unused bandwidth
+  // that can be shared, and this needs to be changed to support that.
+  return absl::nullopt;
+}
+
 }  // namespace internal
 }  // namespace webrtc

video/video_send_stream_impl.h

@@ -146,6 +146,7 @@ class VideoSendStreamImpl : public webrtc::VideoSendStream,
   absl::optional<float> GetPacingFactorOverride() const;
   // Implements BitrateAllocatorObserver.
   uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override;
+  absl::optional<DataRate> GetUsedRate() const override;
 
   // Implements VideoStreamEncoderInterface::EncoderSink
   void OnEncoderConfigurationChanged(

video/video_send_stream_impl_unittest.cc

@@ -1214,5 +1214,30 @@ TEST_F(VideoSendStreamImplTest, ConfiguresBitratesForSvc) {
     vss_impl->Stop();
   }
 }
+
+TEST_F(VideoSendStreamImplTest, TestElasticityForRealtimeVideo) {
+  auto vss_impl = CreateVideoSendStreamImpl(
+      TestVideoEncoderConfig(VideoEncoderConfig::ContentType::kRealtimeVideo));
+  EXPECT_CALL(bitrate_allocator_,
+              AddObserver(vss_impl.get(),
+                          Field(&MediaStreamAllocationConfig::rate_elasticity,
+                                TrackRateElasticity::kCanConsumeExtraRate)));
+  vss_impl->Start();
+  EXPECT_CALL(bitrate_allocator_, RemoveObserver(vss_impl.get()));
+  vss_impl->Stop();
+}
+
+TEST_F(VideoSendStreamImplTest, TestElasticityForScreenshare) {
+  auto vss_impl = CreateVideoSendStreamImpl(
+      TestVideoEncoderConfig(VideoEncoderConfig::ContentType::kScreen));
+  EXPECT_CALL(bitrate_allocator_,
+              AddObserver(vss_impl.get(),
+                          Field(&MediaStreamAllocationConfig::rate_elasticity,
+                                absl::nullopt)));
+  vss_impl->Start();
+  EXPECT_CALL(bitrate_allocator_, RemoveObserver(vss_impl.get()));
+  vss_impl->Stop();
+}
+
 }  // namespace internal
 }  // namespace webrtc