Add accounting of actual audio bit usage

Part of a set of CL to allow video to borrow underused audio bitrate.

Bug: webrtc:35055527
Change-Id: Idb504cbbc5794c06b28bdc21b3d860c9da9df175
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/358202
Reviewed-by: Jakob Ivarsson‎ <jakobi@webrtc.org>
Commit-Queue: Dan Tan <dwtan@google.com>
Cr-Commit-Position: refs/heads/main@{#42733}

audio/audio_send_stream.cc

@@ -492,8 +492,7 @@ uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) {
 }
 
 absl::optional<DataRate> AudioSendStream::GetUsedRate() const {
-  // TODO(bugs.webrtc.org/35055527): Implement
+  return channel_send_->GetUsedRate();
-  return absl::nullopt;
 }
 
 void AudioSendStream::SetTransportOverhead(
@@ -518,6 +517,7 @@ void AudioSendStream::UpdateOverheadPerPacket() {
   if (registered_with_allocator_) {
     ConfigureBitrateObserver();
   }
+  channel_send_->RegisterPacketOverhead(overhead_per_packet_bytes);
 }
 
 size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const {

audio/audio_send_stream_unittest.cc

@@ -560,6 +560,7 @@ TEST(AudioSendStreamTest, AudioNetworkAdaptorReceivesOverhead) {
             }));
     EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
         .WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>()));
+    EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
 
     auto send_stream = helper.CreateAudioSendStream();
 
@@ -672,6 +673,7 @@ TEST(AudioSendStreamTest, SSBweWithOverhead) {
                                       "WebRTC-Audio-LegacyOverhead/Disabled/");
     EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
         .WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>()));
+    EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
     auto send_stream = helper.CreateAudioSendStream();
     const DataRate bitrate =
         DataRate::BitsPerSec(helper.config().max_bitrate_bps) +
@@ -694,6 +696,7 @@ TEST(AudioSendStreamTest, SSBweWithOverheadMinRespected) {
         "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/");
     EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
         .WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>()));
+    EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
     auto send_stream = helper.CreateAudioSendStream();
     const DataRate bitrate = DataRate::KilobitsPerSec(6) + kMinOverheadRate;
     EXPECT_CALL(*helper.channel_send(),
@@ -714,6 +717,7 @@ TEST(AudioSendStreamTest, SSBweWithOverheadMaxRespected) {
         "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/");
     EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
         .WillRepeatedly(Return(kOverheadPerPacket.bytes<size_t>()));
+    EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
     auto send_stream = helper.CreateAudioSendStream();
     const DataRate bitrate = DataRate::KilobitsPerSec(64) + kMaxOverheadRate;
     EXPECT_CALL(*helper.channel_send(),
@@ -796,6 +800,7 @@ TEST(AudioSendStreamTest, OnTransportOverheadChanged) {
 
     // CallEncoder will be called on overhead change.
     EXPECT_CALL(*helper.channel_send(), CallEncoder);
+    EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead);
 
     const size_t transport_overhead_per_packet_bytes = 333;
     send_stream->SetTransportOverhead(transport_overhead_per_packet_bytes);
@@ -811,6 +816,8 @@ TEST(AudioSendStreamTest, DoesntCallEncoderWhenOverheadUnchanged) {
     auto send_stream = helper.CreateAudioSendStream();
     auto new_config = helper.config();
 
+    EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead).Times(2);
+
     // CallEncoder will be called on overhead change.
     EXPECT_CALL(*helper.channel_send(), CallEncoder);
     const size_t transport_overhead_per_packet_bytes = 333;
@@ -832,6 +839,7 @@ TEST(AudioSendStreamTest, AudioOverheadChanged) {
     const size_t audio_overhead_per_packet_bytes = 555;
     EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
         .WillRepeatedly(Return(audio_overhead_per_packet_bytes));
+    EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead).Times(2);
     auto send_stream = helper.CreateAudioSendStream();
 
     BitrateAllocationUpdate update;
@@ -862,6 +870,7 @@ TEST(AudioSendStreamTest, OnAudioAndTransportOverheadChanged) {
     const size_t audio_overhead_per_packet_bytes = 555;
     EXPECT_CALL(*helper.rtp_rtcp(), ExpectedPerPacketOverhead)
         .WillRepeatedly(Return(audio_overhead_per_packet_bytes));
+    EXPECT_CALL(*helper.channel_send(), RegisterPacketOverhead).Times(2);
     auto send_stream = helper.CreateAudioSendStream();
     auto new_config = helper.config();
 

audio/channel_send.cc

@@ -57,6 +57,47 @@ constexpr int64_t kMinRetransmissionWindowMs = 30;
 class RtpPacketSenderProxy;
 class TransportSequenceNumberProxy;
 
+class AudioBitrateAccountant {
+ public:
+  void RegisterPacketOverhead(int packet_byte_overhead) {
+    packet_overhead_ = DataSize::Bytes(packet_byte_overhead);
+  }
+
+  void Reset() {
+    rate_last_frame_ = DataRate::BitsPerSec(0);
+    next_frame_duration_ = TimeDelta::Millis(0);
+    report_rate_ = absl::nullopt;
+  }
+
+  // A new frame is formed when bytesize is nonzero.
+  void UpdateBpsEstimate(DataSize payload_size, TimeDelta frame_duration) {
+    next_frame_duration_ += frame_duration;
+    // Do not have a full frame yet.
+    if (payload_size.bytes() == 0)
+      return;
+
+    // We report the larger of the rates computed using the last frame, and
+    // second last frame. Under DTX, frame sizes sometimes alternate, it is
+    // preferable to report the upper envelop.
+    DataRate rate_cur_frame =
+        (payload_size + packet_overhead_) / next_frame_duration_;
+
+    report_rate_ =
+        (rate_cur_frame > rate_last_frame_) ? rate_cur_frame : rate_last_frame_;
+
+    rate_last_frame_ = rate_cur_frame;
+    next_frame_duration_ = TimeDelta::Millis(0);
+  }
+
+  absl::optional<DataRate> GetUsedRate() const { return report_rate_; }
+
+ private:
+  TimeDelta next_frame_duration_ = TimeDelta::Millis(0);
+  DataSize packet_overhead_ = DataSize::Bytes(72);
+  DataRate rate_last_frame_ = DataRate::BitsPerSec(0);
+  absl::optional<DataRate> report_rate_;
+};
+
 class ChannelSend : public ChannelSendInterface,
                     public AudioPacketizationCallback,  // receive encoded
                                                         // packets from the ACM
@@ -152,6 +193,17 @@ class ChannelSend : public ChannelSendInterface,
   // ReportBlockDataObserver.
   void OnReportBlockDataUpdated(ReportBlockData report_block) override;
 
+  // Reports actual bitrate used (vs allocated).
+  absl::optional<DataRate> GetUsedRate() const override {
+    MutexLock lock(&bitrate_accountant_mutex_);
+    return bitrate_accountant_.GetUsedRate();
+  }
+
+  void RegisterPacketOverhead(int packet_byte_overhead) override {
+    MutexLock lock(&bitrate_accountant_mutex_);
+    bitrate_accountant_.RegisterPacketOverhead(packet_byte_overhead);
+  }
+
  private:
   // From AudioPacketizationCallback in the ACM
   int32_t SendData(AudioFrameType frameType,
@@ -240,6 +292,10 @@ class ChannelSend : public ChannelSendInterface,
   RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_checker_;
 
   SdpAudioFormat encoder_format_;
+
+  mutable Mutex bitrate_accountant_mutex_;
+  AudioBitrateAccountant bitrate_accountant_
+      RTC_GUARDED_BY(bitrate_accountant_mutex_);
 };
 
 const int kTelephoneEventAttenuationdB = 10;
@@ -800,10 +856,15 @@ void ChannelSend::ProcessAndEncodeAudio(
         // encoding is done and payload is ready for packetization and
         // transmission. Otherwise, it will return without invoking the
         // callback.
-        if (audio_coding_->Add10MsData(*audio_frame) < 0) {
+        int32_t encoded_bytes = audio_coding_->Add10MsData(*audio_frame);
+        MutexLock lock(&bitrate_accountant_mutex_);
+        if (encoded_bytes < 0) {
           RTC_DLOG(LS_ERROR) << "ACM::Add10MsData() failed.";
+          bitrate_accountant_.Reset();
           return;
         }
+        bitrate_accountant_.UpdateBpsEstimate(DataSize::Bytes(encoded_bytes),
+                                              TimeDelta::Millis(10));
       });
 }
 

audio/channel_send.h

@@ -110,6 +110,12 @@ class ChannelSendInterface {
   virtual void SetEncoderToPacketizerFrameTransformer(
       rtc::scoped_refptr<webrtc::FrameTransformerInterface>
           frame_transformer) = 0;
+
+  // Returns payload bitrate actually used.
+  virtual absl::optional<DataRate> GetUsedRate() const = 0;
+
+  // Registers per packet byte overhead.
+  virtual void RegisterPacketOverhead(int packet_byte_overhead) = 0;
 };
 
 std::unique_ptr<ChannelSendInterface> CreateChannelSend(

audio/channel_send_unittest.cc

@@ -306,6 +306,62 @@ TEST_F(ChannelSendTest, AudioLevelsAttachedToInsertedTransformedFrame) {
   EXPECT_TRUE_WAIT(sent_audio_level, 1000);
   EXPECT_EQ(*sent_audio_level, audio_level);
 }
+
+// Ensure that GetUsedRate returns null if no frames are coded.
+TEST_F(ChannelSendTest, NoUsedRateInitially) {
+  channel_->StartSend();
+  auto used_rate = channel_->GetUsedRate();
+  EXPECT_EQ(used_rate, absl::nullopt);
+}
+
+// Ensure that GetUsedRate returns value with one coded frame.
+TEST_F(ChannelSendTest, ValidUsedRateWithOneCodedFrame) {
+  channel_->StartSend();
+  EXPECT_CALL(transport_, SendRtp).Times(1);
+  ProcessNextFrame();
+  ProcessNextFrame();
+  auto used_rate = channel_->GetUsedRate();
+  EXPECT_GT(used_rate.value().bps(), 0);
+}
+
+// Ensure that GetUsedRate returns value with one coded frame.
+TEST_F(ChannelSendTest, UsedRateIsLargerofLastTwoFrames) {
+  channel_->StartSend();
+  channel_->CallEncoder(
+      [&](AudioEncoder* encoder) { encoder->OnReceivedOverhead(72); });
+  DataRate lowrate = DataRate::BitsPerSec(40000);
+  DataRate highrate = DataRate::BitsPerSec(80000);
+  BitrateAllocationUpdate update;
+  update.bwe_period = TimeDelta::Millis(100);
+
+  update.target_bitrate = lowrate;
+  channel_->OnBitrateAllocation(update);
+  EXPECT_CALL(transport_, SendRtp).Times(1);
+  ProcessNextFrame();
+  ProcessNextFrame();
+  // Last two frames have rates [32kbps, -], yielding 32kbps.
+  auto used_rate_1 = channel_->GetUsedRate();
+
+  update.target_bitrate = highrate;
+  channel_->OnBitrateAllocation(update);
+  EXPECT_CALL(transport_, SendRtp).Times(1);
+  ProcessNextFrame();
+  ProcessNextFrame();
+  // Last two frames have rates [54kbps, 32kbps], yielding 54kbps
+  auto used_rate_2 = channel_->GetUsedRate();
+
+  update.target_bitrate = lowrate;
+  channel_->OnBitrateAllocation(update);
+  EXPECT_CALL(transport_, SendRtp).Times(1);
+  ProcessNextFrame();
+  ProcessNextFrame();
+  // Last two frames have rates [32kbps 54kbps], yielding 54kbps
+  auto used_rate_3 = channel_->GetUsedRate();
+
+  EXPECT_GT(used_rate_2, used_rate_1);
+  EXPECT_EQ(used_rate_3, used_rate_2);
+}
+
 }  // namespace
 }  // namespace voe
 }  // namespace webrtc

audio/mock_voe_channel_proxy.h

@@ -182,6 +182,11 @@ class MockChannelSend : public voe::ChannelSendInterface {
       SetEncoderToPacketizerFrameTransformer,
       (rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer),
       (override));
+  MOCK_METHOD(absl::optional<DataRate>, GetUsedRate, (), (const, override));
+  MOCK_METHOD(void,
+              RegisterPacketOverhead,
+              (int packet_byte_overhead),
+              (override));
 };
 }  // namespace test
 }  // namespace webrtc