Send absolute capture time through audio coding module.

Bug: webrtc:10739
Change-Id: I44e0305be7c84b253172511c2977b1d700e40c1b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167067
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Chen Xing <chxg@google.com>
Commit-Queue: Minyue Li <minyue@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30363}

audio/channel_send.cc

@@ -146,17 +146,19 @@ class ChannelSend : public ChannelSendInterface,
   // From AudioPacketizationCallback in the ACM
   int32_t SendData(AudioFrameType frameType,
                    uint8_t payloadType,
-                   uint32_t timeStamp,
+                   uint32_t rtp_timestamp,
                    const uint8_t* payloadData,
-                   size_t payloadSize) override;
+                   size_t payloadSize,
+                   int64_t absolute_capture_timestamp_ms) override;
 
   void OnUplinkPacketLossRate(float packet_loss_rate);
   bool InputMute() const;
 
   int32_t SendRtpAudio(AudioFrameType frameType,
                        uint8_t payloadType,
-                       uint32_t timeStamp,
+                       uint32_t rtp_timestamp,
-                       rtc::ArrayView<const uint8_t> payload)
+                       rtc::ArrayView<const uint8_t> payload,
+                       int64_t absolute_capture_timestamp_ms)
       RTC_RUN_ON(encoder_queue_);
 
   void OnReceivedRtt(int64_t rtt_ms);
@@ -360,18 +362,21 @@ class VoERtcpObserver : public RtcpBandwidthObserver {
 
 int32_t ChannelSend::SendData(AudioFrameType frameType,
                               uint8_t payloadType,
-                              uint32_t timeStamp,
+                              uint32_t rtp_timestamp,
                               const uint8_t* payloadData,
-                              size_t payloadSize) {
+                              size_t payloadSize,
+                              int64_t absolute_capture_timestamp_ms) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   rtc::ArrayView<const uint8_t> payload(payloadData, payloadSize);
-  return SendRtpAudio(frameType, payloadType, timeStamp, payload);
+  return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload,
+                      absolute_capture_timestamp_ms);
 }
 
 int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType,
                                   uint8_t payloadType,
-                                  uint32_t timeStamp,
+                                  uint32_t rtp_timestamp,
-                                  rtc::ArrayView<const uint8_t> payload) {
+                                  rtc::ArrayView<const uint8_t> payload,
+                                  int64_t absolute_capture_timestamp_ms) {
   if (_includeAudioLevelIndication) {
     // Store current audio level in the RTP sender.
     // The level will be used in combination with voice-activity state
@@ -419,7 +424,7 @@ int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType,
 
   // Push data from ACM to RTP/RTCP-module to deliver audio frame for
   // packetization.
-  if (!_rtpRtcpModule->OnSendingRtpFrame(timeStamp,
+  if (!_rtpRtcpModule->OnSendingRtpFrame(rtp_timestamp,
                                          // Leaving the time when this frame was
                                          // received from the capture device as
                                          // undefined for voice for now.
@@ -433,10 +438,12 @@ int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType,
   // call.
   // TODO(nisse): Delete RTCPSender:timestamp_offset_, and see if we can confine
   // knowledge of the offset to a single place.
-  const uint32_t rtp_timestamp = timeStamp + _rtpRtcpModule->StartTimestamp();
+
   // This call will trigger Transport::SendPacket() from the RTP/RTCP module.
-  if (!rtp_sender_audio_->SendAudio(frameType, payloadType, rtp_timestamp,
+  if (!rtp_sender_audio_->SendAudio(
-                                    payload.data(), payload.size())) {
+          frameType, payloadType,
+          rtp_timestamp + _rtpRtcpModule->StartTimestamp(), payload.data(),
+          payload.size(), absolute_capture_timestamp_ms)) {
     RTC_DLOG(LS_ERROR)
         << "ChannelSend::SendData() failed to send data to RTP/RTCP module";
     return -1;

modules/audio_coding/acm2/acm_receiver_unittest.cc

@@ -107,7 +107,8 @@ class AcmReceiverTestOldApi : public AudioPacketizationCallback,
                uint8_t payload_type,
                uint32_t timestamp,
                const uint8_t* payload_data,
-               size_t payload_len_bytes) override {
+               size_t payload_len_bytes,
+               int64_t absolute_capture_timestamp_ms) override {
     if (frame_type == AudioFrameType::kEmptyFrame)
       return 0;
 

modules/audio_coding/acm2/acm_send_test.cc

@@ -126,7 +126,8 @@ int32_t AcmSendTestOldApi::SendData(AudioFrameType frame_type,
                                     uint8_t payload_type,
                                     uint32_t timestamp,
                                     const uint8_t* payload_data,
-                                    size_t payload_len_bytes) {
+                                    size_t payload_len_bytes,
+                                    int64_t absolute_capture_timestamp_ms) {
   // Store the packet locally.
   frame_type_ = frame_type;
   payload_type_ = payload_type;

modules/audio_coding/acm2/acm_send_test.h

@@ -54,7 +54,8 @@ class AcmSendTestOldApi : public AudioPacketizationCallback,
                    uint8_t payload_type,
                    uint32_t timestamp,
                    const uint8_t* payload_data,
-                   size_t payload_len_bytes) override;
+                   size_t payload_len_bytes,
+                   int64_t absolute_capture_timestamp_ms) override;
 
   AudioCodingModule* acm() { return acm_.get(); }
 

modules/audio_coding/acm2/audio_coding_module.cc

@@ -11,7 +11,6 @@
 #include "modules/audio_coding/include/audio_coding_module.h"
 
 #include <assert.h>
-
 #include <algorithm>
 #include <cstdint>
 
@@ -110,6 +109,7 @@ class AudioCodingModuleImpl final : public AudioCodingModule {
     // If a re-mix is required (up or down), this buffer will store a re-mixed
     // version of the input.
     std::vector<int16_t> buffer;
+    int64_t absolute_capture_timestamp_ms;
   };
 
   InputData input_data_ RTC_GUARDED_BY(acm_crit_sect_);
@@ -253,6 +253,7 @@ int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
                     int64_t{input_data.input_timestamp - last_timestamp_} *
                         encoder_stack_->RtpTimestampRateHz(),
                     int64_t{encoder_stack_->SampleRateHz()}));
+
   last_timestamp_ = input_data.input_timestamp;
   last_rtp_timestamp_ = rtp_timestamp;
   first_frame_ = false;
@@ -302,7 +303,8 @@ int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
     if (packetization_callback_) {
       packetization_callback_->SendData(
           frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
-          encode_buffer_.data(), encode_buffer_.size());
+          encode_buffer_.data(), encode_buffer_.size(),
+          input_data.absolute_capture_timestamp_ms);
     }
 
     if (vad_callback_) {
@@ -392,6 +394,9 @@ int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
   input_data->input_timestamp = ptr_frame->timestamp_;
   input_data->length_per_channel = ptr_frame->samples_per_channel_;
   input_data->audio_channel = current_num_channels;
+  // TODO(bugs.webrtc.org/10739): Assign from a corresponding field in
+  // audio_frame when it is added in AudioFrame.
+  input_data->absolute_capture_timestamp_ms = 0;
 
   if (!same_num_channels) {
     // Remixes the input frame to the output data and in the process resize the

modules/audio_coding/acm2/audio_coding_module_unittest.cc

@@ -111,7 +111,8 @@ class PacketizationCallbackStubOldApi : public AudioPacketizationCallback {
                    uint8_t payload_type,
                    uint32_t timestamp,
                    const uint8_t* payload_data,
-                   size_t payload_len_bytes) override {
+                   size_t payload_len_bytes,
+                   int64_t absolute_capture_timestamp_ms) override {
     rtc::CritScope lock(&crit_sect_);
     ++num_calls_;
     last_frame_type_ = frame_type;

modules/audio_coding/include/audio_coding_module.h

@@ -44,7 +44,21 @@ class AudioPacketizationCallback {
                            uint8_t payload_type,
                            uint32_t timestamp,
                            const uint8_t* payload_data,
-                           size_t payload_len_bytes) = 0;
+                           size_t payload_len_bytes,
+                           int64_t absolute_capture_timestamp_ms) {
+    // TODO(bugs.webrtc.org/10739): Deprecate the old SendData and make this one
+    // pure virtual.
+    RTC_NOTREACHED() << "This method must be overridden, or not used.";
+    return -1;
+  }
+  virtual int32_t SendData(AudioFrameType frame_type,
+                           uint8_t payload_type,
+                           uint32_t timestamp,
+                           const uint8_t* payload_data,
+                           size_t payload_len_bytes) {
+    return SendData(frame_type, payload_type, timestamp, payload_data,
+                    payload_len_bytes, 0);
+  }
 };
 
 // Callback class used for reporting VAD decision

modules/audio_coding/neteq/tools/rtp_encode.cc

@@ -112,7 +112,8 @@ class Packetizer : public AudioPacketizationCallback {
                    uint8_t payload_type,
                    uint32_t timestamp,
                    const uint8_t* payload_data,
-                   size_t payload_len_bytes) override {
+                   size_t payload_len_bytes,
+                   int64_t absolute_capture_timestamp_ms) override {
     if (payload_len_bytes == 0) {
       return 0;
     }

modules/audio_coding/test/Channel.cc

@@ -23,7 +23,8 @@ int32_t Channel::SendData(AudioFrameType frameType,
                           uint8_t payloadType,
                           uint32_t timeStamp,
                           const uint8_t* payloadData,
-                          size_t payloadSize) {
+                          size_t payloadSize,
+                          int64_t absolute_capture_timestamp_ms) {
   RTPHeader rtp_header;
   int32_t status;
   size_t payloadDataSize = payloadSize;

modules/audio_coding/test/Channel.h

@@ -51,7 +51,8 @@ class Channel : public AudioPacketizationCallback {
                    uint8_t payloadType,
                    uint32_t timeStamp,
                    const uint8_t* payloadData,
-                   size_t payloadSize) override;
+                   size_t payloadSize,
+                   int64_t absolute_capture_timestamp_ms) override;
 
   void RegisterReceiverACM(AudioCodingModule* acm);
 

modules/audio_coding/test/EncodeDecodeTest.cc

@@ -33,7 +33,8 @@ int32_t TestPacketization::SendData(const AudioFrameType /* frameType */,
                                     const uint8_t payloadType,
                                     const uint32_t timeStamp,
                                     const uint8_t* payloadData,
-                                    const size_t payloadSize) {
+                                    const size_t payloadSize,
+                                    int64_t absolute_capture_timestamp_ms) {
   _rtpStream->Write(payloadType, timeStamp, _seqNo++, payloadData, payloadSize,
                     _frequency);
   return 1;

modules/audio_coding/test/EncodeDecodeTest.h

@@ -32,7 +32,8 @@ class TestPacketization : public AudioPacketizationCallback {
                    const uint8_t payloadType,
                    const uint32_t timeStamp,
                    const uint8_t* payloadData,
-                   const size_t payloadSize) override;
+                   const size_t payloadSize,
+                   int64_t absolute_capture_timestamp_ms) override;
 
  private:
   static void MakeRTPheader(uint8_t* rtpHeader,

modules/audio_coding/test/TestAllCodecs.cc

@@ -64,7 +64,8 @@ int32_t TestPack::SendData(AudioFrameType frame_type,
                            uint8_t payload_type,
                            uint32_t timestamp,
                            const uint8_t* payload_data,
-                           size_t payload_size) {
+                           size_t payload_size,
+                           int64_t absolute_capture_timestamp_ms) {
   RTPHeader rtp_header;
   int32_t status;
 

modules/audio_coding/test/TestAllCodecs.h

@@ -29,7 +29,8 @@ class TestPack : public AudioPacketizationCallback {
                    uint8_t payload_type,
                    uint32_t timestamp,
                    const uint8_t* payload_data,
-                   size_t payload_size) override;
+                   size_t payload_size,
+                   int64_t absolute_capture_timestamp_ms) override;
 
   size_t payload_size();
   uint32_t timestamp_diff();

modules/audio_coding/test/TestStereo.cc

@@ -44,7 +44,8 @@ int32_t TestPackStereo::SendData(const AudioFrameType frame_type,
                                  const uint8_t payload_type,
                                  const uint32_t timestamp,
                                  const uint8_t* payload_data,
-                                 const size_t payload_size) {
+                                 const size_t payload_size,
+                                 int64_t absolute_capture_timestamp_ms) {
   RTPHeader rtp_header;
   int32_t status = 0;
 

modules/audio_coding/test/TestStereo.h

@@ -35,7 +35,8 @@ class TestPackStereo : public AudioPacketizationCallback {
                    const uint8_t payload_type,
                    const uint32_t timestamp,
                    const uint8_t* payload_data,
-                   const size_t payload_size) override;
+                   const size_t payload_size,
+                   int64_t absolute_capture_timestamp_ms) override;
 
   uint16_t payload_size();
   uint32_t timestamp_diff();

modules/audio_coding/test/opus_test.cc

@@ -337,7 +337,7 @@ void OpusTest::Run(TestPackStereo* channel,
 
         // Send data to the channel. "channel" will handle the loss simulation.
         channel->SendData(AudioFrameType::kAudioFrameSpeech, payload_type_,
-                          rtp_timestamp_, bitstream, bitstream_len_byte);
+                          rtp_timestamp_, bitstream, bitstream_len_byte, 0);
         if (first_packet) {
           first_packet = false;
           start_time_stamp = rtp_timestamp_;