From 99d6d8115b374cd57bfd9195d8eb28de26ffc134 Mon Sep 17 00:00:00 2001 From: Minyue Li Date: Wed, 29 Jan 2020 10:25:12 +0100 Subject: [PATCH] Adding absolute capture timestamp to AudioTrackSinkInterface. Bug: webrtc:10739 Change-Id: I8c134cbe82452ac71625cd0c810c783a73f17822 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167532 Commit-Queue: Minyue Li Reviewed-by: Chen Xing Reviewed-by: Karl Wiberg Cr-Commit-Position: refs/heads/master@{#30408} --- api/media_stream_interface.h | 20 +++++++++++++++++++- media/base/audio_source.h | 14 +++++++++----- media/base/fake_media_engine.cc | 3 ++- media/base/fake_media_engine.h | 3 ++- media/engine/webrtc_voice_engine.cc | 5 ++++- pc/remote_audio_source.cc | 5 ++++- pc/rtp_sender.cc | 14 ++++++++------ pc/rtp_sender.h | 14 +++++++++++++- 8 files changed, 61 insertions(+), 17 deletions(-) diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index dde6272fa5..79d463d788 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -202,7 +202,25 @@ class AudioTrackSinkInterface { int bits_per_sample, int sample_rate, size_t number_of_channels, - size_t number_of_frames) = 0; + size_t number_of_frames) { + RTC_NOTREACHED() << "This method must be overridden, or not used."; + } + + // In this method, |absolute_capture_timestamp_ms|, when available, is + // supposed to deliver the timestamp when this audio frame was originally + // captured. This timestamp MUST be based on the same clock as + // rtc::TimeMillis(). + virtual void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) { + // TODO(bugs.webrtc.org/10739): Deprecate the old OnData and make this one + // pure virtual. + return OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, + number_of_frames); + } protected: virtual ~AudioTrackSinkInterface() {} diff --git a/media/base/audio_source.h b/media/base/audio_source.h index d5cf629173..8a8796800b 100644 --- a/media/base/audio_source.h +++ b/media/base/audio_source.h @@ -13,6 +13,8 @@ #include +#include "absl/types/optional.h" + namespace cricket { // Abstract interface for providing the audio data. @@ -23,11 +25,13 @@ class AudioSource { class Sink { public: // Callback to receive data from the AudioSource. - virtual void OnData(const void* audio_data, - int bits_per_sample, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames) = 0; + virtual void OnData( + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) = 0; // Called when the AudioSource is going away. virtual void OnClose() = 0; diff --git a/media/base/fake_media_engine.cc b/media/base/fake_media_engine.cc index c31ef97786..8fc844fa78 100644 --- a/media/base/fake_media_engine.cc +++ b/media/base/fake_media_engine.cc @@ -39,7 +39,8 @@ void FakeVoiceMediaChannel::VoiceChannelAudioSink::OnData( int bits_per_sample, int sample_rate, size_t number_of_channels, - size_t number_of_frames) {} + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) {} void FakeVoiceMediaChannel::VoiceChannelAudioSink::OnClose() { source_ = nullptr; } diff --git a/media/base/fake_media_engine.h b/media/base/fake_media_engine.h index 3df8f85965..32ca11fe41 100644 --- a/media/base/fake_media_engine.h +++ b/media/base/fake_media_engine.h @@ -368,7 +368,8 @@ class FakeVoiceMediaChannel : public RtpHelper { int bits_per_sample, int sample_rate, size_t number_of_channels, - size_t number_of_frames) override; + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override; void OnClose() override; AudioSource* source() const; diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc index a36fc6e250..59ac1ee9cb 100644 --- a/media/engine/webrtc_voice_engine.cc +++ b/media/engine/webrtc_voice_engine.cc @@ -870,7 +870,8 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream int bits_per_sample, int sample_rate, size_t number_of_channels, - size_t number_of_frames) override { + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override { RTC_DCHECK_EQ(16, bits_per_sample); RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); RTC_DCHECK(stream_); @@ -879,6 +880,8 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream audio_frame->timestamp_, static_cast(audio_data), number_of_frames, sample_rate, audio_frame->speech_type_, audio_frame->vad_activity_, number_of_channels); + // TODO(bugs.webrtc.org/10739): pass absolute_capture_timestamp_ms to + // stream_. stream_->SendAudioData(std::move(audio_frame)); } diff --git a/pc/remote_audio_source.cc b/pc/remote_audio_source.cc index 05d0899f2a..da00402e41 100644 --- a/pc/remote_audio_source.cc +++ b/pc/remote_audio_source.cc @@ -144,8 +144,11 @@ void RemoteAudioSource::OnData(const AudioSinkInterface::Data& audio) { // Called on the externally-owned audio callback thread, via/from webrtc. rtc::CritScope lock(&sink_lock_); for (auto* sink : sinks_) { + // When peerconnection acts as an audio source, it should not provide + // absolute capture timestamp. sink->OnData(audio.data, 16, audio.sample_rate, audio.channels, - audio.samples_per_channel); + audio.samples_per_channel, + /*absolute_capture_timestamp_ms=*/absl::nullopt); } } diff --git a/pc/rtp_sender.cc b/pc/rtp_sender.cc index 402ad97920..73cfcd045a 100644 --- a/pc/rtp_sender.cc +++ b/pc/rtp_sender.cc @@ -372,15 +372,17 @@ LocalAudioSinkAdapter::~LocalAudioSinkAdapter() { sink_->OnClose(); } -void LocalAudioSinkAdapter::OnData(const void* audio_data, - int bits_per_sample, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames) { +void LocalAudioSinkAdapter::OnData( + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) { rtc::CritScope lock(&lock_); if (sink_) { sink_->OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, - number_of_frames); + number_of_frames, absolute_capture_timestamp_ms); } } diff --git a/pc/rtp_sender.h b/pc/rtp_sender.h index 82ef711a6c..fcf8448e4c 100644 --- a/pc/rtp_sender.h +++ b/pc/rtp_sender.h @@ -213,7 +213,19 @@ class LocalAudioSinkAdapter : public AudioTrackSinkInterface, int bits_per_sample, int sample_rate, size_t number_of_channels, - size_t number_of_frames) override; + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override; + + // AudioSinkInterface implementation. + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames) override { + OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, + number_of_frames, + /*absolute_capture_timestamp_ms=*/absl::nullopt); + } // cricket::AudioSource implementation. void SetSink(cricket::AudioSource::Sink* sink) override;