diff --git a/webrtc/audio/audio_receive_stream.cc b/webrtc/audio/audio_receive_stream.cc index aaf77a8a97..1f24b2ca27 100644 --- a/webrtc/audio/audio_receive_stream.cc +++ b/webrtc/audio/audio_receive_stream.cc @@ -21,6 +21,8 @@ #include "webrtc/base/logging.h" #include "webrtc/base/timeutils.h" #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h" +#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" #include "webrtc/voice_engine/channel_proxy.h" #include "webrtc/voice_engine/include/voe_base.h" #include "webrtc/voice_engine/include/voe_codec.h" @@ -81,6 +83,8 @@ AudioReceiveStream::AudioReceiveStream( RTC_DCHECK(remote_bitrate_estimator); RTC_DCHECK(rtp_header_parser_); + module_process_thread_checker_.DetachFromThread(); + VoiceEngineImpl* voe_impl = static_cast(voice_engine()); channel_proxy_ = voe_impl->GetChannelProxy(config_.voe_channel_id); channel_proxy_->SetRtcEventLog(event_log); @@ -125,7 +129,7 @@ AudioReceiveStream::AudioReceiveStream( } AudioReceiveStream::~AudioReceiveStream() { - RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); LOG(LS_INFO) << "~AudioReceiveStream: " << config_.ToString(); if (playing_) { Stop(); @@ -138,7 +142,7 @@ AudioReceiveStream::~AudioReceiveStream() { } void AudioReceiveStream::Start() { - RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (playing_) { return; } @@ -159,7 +163,7 @@ void AudioReceiveStream::Start() { } void AudioReceiveStream::Stop() { - RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (!playing_) { return; } @@ -170,7 +174,7 @@ void AudioReceiveStream::Stop() { } webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { - RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); webrtc::AudioReceiveStream::Stats stats; stats.remote_ssrc = config_.rtp.remote_ssrc; ScopedVoEInterface codec(voice_engine()); @@ -220,22 +224,78 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { } void AudioReceiveStream::SetSink(std::unique_ptr sink) { - RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_proxy_->SetSink(std::move(sink)); } void AudioReceiveStream::SetGain(float gain) { - RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); channel_proxy_->SetChannelOutputVolumeScaling(gain); } -const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const { - RTC_DCHECK_RUN_ON(&thread_checker_); - return config_; +AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo( + int sample_rate_hz, + AudioFrame* audio_frame) { + return channel_proxy_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame); +} + +int AudioReceiveStream::Ssrc() const { + return config_.rtp.remote_ssrc; +} + +int AudioReceiveStream::PreferredSampleRate() const { + return channel_proxy_->NeededFrequency(); +} + +int AudioReceiveStream::id() const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return config_.rtp.remote_ssrc; +} + +rtc::Optional AudioReceiveStream::GetInfo() const { + RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + Syncable::Info info; + + RtpRtcp* rtp_rtcp = nullptr; + RtpReceiver* rtp_receiver = nullptr; + channel_proxy_->GetRtpRtcp(&rtp_rtcp, &rtp_receiver); + RTC_DCHECK(rtp_rtcp); + RTC_DCHECK(rtp_receiver); + + if (!rtp_receiver->Timestamp(&info.latest_received_capture_timestamp)) { + return rtc::Optional(); + } + if (!rtp_receiver->LastReceivedTimeMs(&info.latest_receive_time_ms)) { + return rtc::Optional(); + } + if (rtp_rtcp->RemoteNTP(&info.capture_time_ntp_secs, + &info.capture_time_ntp_frac, + nullptr, + nullptr, + &info.capture_time_source_clock) != 0) { + return rtc::Optional(); + } + + int jitter_buffer_delay_ms = 0; + int playout_buffer_delay_ms = 0; + channel_proxy_->GetDelayEstimate(&jitter_buffer_delay_ms, + &playout_buffer_delay_ms); + info.current_delay_ms = jitter_buffer_delay_ms + playout_buffer_delay_ms; + return rtc::Optional(info); +} + +uint32_t AudioReceiveStream::GetPlayoutTimestamp() const { + // Called on video capture thread. + return channel_proxy_->GetPlayoutTimestamp(); +} + +void AudioReceiveStream::SetMinimumPlayoutDelay(int delay_ms) { + RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + return channel_proxy_->SetMinimumPlayoutDelay(delay_ms); } void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) { - RTC_DCHECK(thread_checker_.CalledOnValidThread()); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (send_stream) { VoiceEngineImpl* voe_impl = static_cast(voice_engine()); std::unique_ptr send_channel_proxy = @@ -247,7 +307,7 @@ void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) { } void AudioReceiveStream::SignalNetworkState(NetworkState state) { - RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_DCHECK_RUN_ON(&worker_thread_checker_); } bool AudioReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) { @@ -286,24 +346,9 @@ bool AudioReceiveStream::DeliverRtp(const uint8_t* packet, return channel_proxy_->ReceivedRTPPacket(packet, length, packet_time); } -AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo( - int sample_rate_hz, - AudioFrame* audio_frame) { - return channel_proxy_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame); -} - -int AudioReceiveStream::PreferredSampleRate() const { - return channel_proxy_->NeededFrequency(); -} - -int AudioReceiveStream::Ssrc() const { - return config_.rtp.remote_ssrc; -} - -internal::AudioState* AudioReceiveStream::audio_state() const { - auto* audio_state = static_cast(audio_state_.get()); - RTC_DCHECK(audio_state); - return audio_state; +const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return config_; } VoiceEngine* AudioReceiveStream::voice_engine() const { @@ -312,6 +357,12 @@ VoiceEngine* AudioReceiveStream::voice_engine() const { return voice_engine; } +internal::AudioState* AudioReceiveStream::audio_state() const { + auto* audio_state = static_cast(audio_state_.get()); + RTC_DCHECK(audio_state); + return audio_state; +} + int AudioReceiveStream::SetVoiceEnginePlayout(bool playout) { ScopedVoEInterface base(voice_engine()); if (playout) { @@ -320,6 +371,5 @@ int AudioReceiveStream::SetVoiceEnginePlayout(bool playout) { return base->StopPlayout(config_.voe_channel_id); } } - } // namespace internal } // namespace webrtc diff --git a/webrtc/audio/audio_receive_stream.h b/webrtc/audio/audio_receive_stream.h index 7dfc5d6bb8..6721c7ee65 100644 --- a/webrtc/audio/audio_receive_stream.h +++ b/webrtc/audio/audio_receive_stream.h @@ -18,13 +18,13 @@ #include "webrtc/base/constructormagic.h" #include "webrtc/base/thread_checker.h" #include "webrtc/call/audio_receive_stream.h" -#include "webrtc/call/audio_state.h" +#include "webrtc/call/syncable.h" #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h" namespace webrtc { +class PacketRouter; class RemoteBitrateEstimator; class RtcEventLog; -class PacketRouter; namespace voe { class ChannelProxy; @@ -34,7 +34,8 @@ namespace internal { class AudioSendStream; class AudioReceiveStream final : public webrtc::AudioReceiveStream, - public AudioMixer::Source { + public AudioMixer::Source, + public Syncable { public: AudioReceiveStream(PacketRouter* packet_router, RemoteBitrateEstimator* remote_bitrate_estimator, @@ -50,6 +51,18 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream, void SetSink(std::unique_ptr sink) override; void SetGain(float gain) override; + // AudioMixer::Source + AudioFrameInfo GetAudioFrameWithInfo(int sample_rate_hz, + AudioFrame* audio_frame) override; + int Ssrc() const override; + int PreferredSampleRate() const override; + + // Syncable + int id() const override; + rtc::Optional GetInfo() const override; + uint32_t GetPlayoutTimestamp() const override; + void SetMinimumPlayoutDelay(int delay_ms) override; + void AssociateSendStream(AudioSendStream* send_stream); void SignalNetworkState(NetworkState state); bool DeliverRtcp(const uint8_t* packet, size_t length); @@ -58,25 +71,20 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream, const PacketTime& packet_time); const webrtc::AudioReceiveStream::Config& config() const; - // AudioMixer::Source - AudioFrameInfo GetAudioFrameWithInfo(int sample_rate_hz, - AudioFrame* audio_frame) override; - int PreferredSampleRate() const override; - int Ssrc() const override; - private: VoiceEngine* voice_engine() const; AudioState* audio_state() const; int SetVoiceEnginePlayout(bool playout); - rtc::ThreadChecker thread_checker_; + rtc::ThreadChecker worker_thread_checker_; + rtc::ThreadChecker module_process_thread_checker_; RemoteBitrateEstimator* const remote_bitrate_estimator_; const webrtc::AudioReceiveStream::Config config_; rtc::scoped_refptr audio_state_; std::unique_ptr rtp_header_parser_; std::unique_ptr channel_proxy_; - bool playing_ ACCESS_ON(thread_checker_) = false; + bool playing_ ACCESS_ON(worker_thread_checker_) = false; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioReceiveStream); }; diff --git a/webrtc/call/BUILD.gn b/webrtc/call/BUILD.gn index acad72d461..382ae2e66b 100644 --- a/webrtc/call/BUILD.gn +++ b/webrtc/call/BUILD.gn @@ -16,6 +16,8 @@ rtc_source_set("call_interfaces") { "audio_state.h", "call.h", "flexfec_receive_stream.h", + "syncable.cc", + "syncable.h", ] } diff --git a/webrtc/call/call.cc b/webrtc/call/call.cc index 88faf871a9..6aa564e95e 100644 --- a/webrtc/call/call.cc +++ b/webrtc/call/call.cc @@ -55,7 +55,6 @@ #include "webrtc/video/video_receive_stream.h" #include "webrtc/video/video_send_stream.h" #include "webrtc/video/vie_remb.h" -#include "webrtc/voice_engine/include/voe_codec.h" namespace webrtc { @@ -146,15 +145,6 @@ class Call : public webrtc::Call, void ConfigureSync(const std::string& sync_group) EXCLUSIVE_LOCKS_REQUIRED(receive_crit_); - VoiceEngine* voice_engine() { - internal::AudioState* audio_state = - static_cast(config_.audio_state.get()); - if (audio_state) - return audio_state->voice_engine(); - else - return nullptr; - } - rtc::Optional ParseRtpPacket(const uint8_t* packet, size_t length, const PacketTime& packet_time) @@ -648,8 +638,8 @@ webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream( } VideoReceiveStream* receive_stream = new VideoReceiveStream( num_cpu_cores_, protected_by_flexfec, congestion_controller_.get(), - &packet_router_, std::move(configuration), voice_engine(), - module_process_thread_.get(), call_stats_.get(), &remb_); + &packet_router_, std::move(configuration), module_process_thread_.get(), + call_stats_.get(), &remb_); const webrtc::VideoReceiveStream::Config& config = receive_stream->config(); { @@ -1019,7 +1009,7 @@ void Call::OnAllocationLimitsChanged(uint32_t min_send_bitrate_bps, void Call::ConfigureSync(const std::string& sync_group) { // Set sync only if there was no previous one. - if (voice_engine() == nullptr || sync_group.empty()) + if (sync_group.empty()) return; AudioReceiveStream* sync_audio_stream = nullptr; @@ -1056,11 +1046,11 @@ void Call::ConfigureSync(const std::string& sync_group) { "the current implementation."; } // Only sync the first A/V pair within this sync group. - if (sync_audio_stream != nullptr && num_synced_streams == 1) { - video_stream->SetSyncChannel(voice_engine(), - sync_audio_stream->config().voe_channel_id); + if (num_synced_streams == 1) { + // sync_audio_stream may be null and that's ok. + video_stream->SetSync(sync_audio_stream); } else { - video_stream->SetSyncChannel(voice_engine(), -1); + video_stream->SetSync(nullptr); } } } diff --git a/webrtc/call/syncable.cc b/webrtc/call/syncable.cc new file mode 100644 index 0000000000..c254e4f755 --- /dev/null +++ b/webrtc/call/syncable.cc @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/call/syncable.h" + +namespace webrtc { + +Syncable::~Syncable() = default; + +} // namespace webrtc diff --git a/webrtc/call/syncable.h b/webrtc/call/syncable.h new file mode 100644 index 0000000000..3459f9eb81 --- /dev/null +++ b/webrtc/call/syncable.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// Syncable is used by RtpStreamsSynchronizer in VideoReceiveStream, and +// implemented by AudioReceiveStream. + +#ifndef WEBRTC_CALL_SYNCABLE_H_ +#define WEBRTC_CALL_SYNCABLE_H_ + +#include + +#include "webrtc/base/optional.h" + +namespace webrtc { + +class Syncable { + public: + struct Info { + int64_t latest_receive_time_ms = 0; + uint32_t latest_received_capture_timestamp = 0; + uint32_t capture_time_ntp_secs = 0; + uint32_t capture_time_ntp_frac = 0; + uint32_t capture_time_source_clock = 0; + int current_delay_ms = 0; + }; + + virtual ~Syncable(); + + virtual int id() const = 0; + virtual rtc::Optional GetInfo() const = 0; + virtual uint32_t GetPlayoutTimestamp() const = 0; + virtual void SetMinimumPlayoutDelay(int delay_ms) = 0; +}; +} // namespace webrtc + +#endif // WEBRTC_CALL_SYNCABLE_H_ diff --git a/webrtc/test/mock_voe_channel_proxy.h b/webrtc/test/mock_voe_channel_proxy.h index 44f3c7cbf5..dc2c532b1a 100644 --- a/webrtc/test/mock_voe_channel_proxy.h +++ b/webrtc/test/mock_voe_channel_proxy.h @@ -73,6 +73,12 @@ class MockVoEChannelProxy : public voe::ChannelProxy { MOCK_METHOD1(AssociateSendChannel, void(const ChannelProxy& send_channel_proxy)); MOCK_METHOD0(DisassociateSendChannel, void()); + MOCK_CONST_METHOD2(GetRtpRtcp, void(RtpRtcp** rtp_rtcp, + RtpReceiver** rtp_receiver)); + MOCK_CONST_METHOD2(GetDelayEstimate, void(int* jitter_buffer_delay_ms, + int* playout_buffer_delay_ms)); + MOCK_CONST_METHOD0(GetPlayoutTimestamp, uint32_t()); + MOCK_METHOD1(SetMinimumPlayoutDelay, void(int delay_ms)); }; } // namespace test } // namespace webrtc diff --git a/webrtc/video/rtp_streams_synchronizer.cc b/webrtc/video/rtp_streams_synchronizer.cc index 0d026b310a..1edb9b8e4f 100644 --- a/webrtc/video/rtp_streams_synchronizer.cc +++ b/webrtc/video/rtp_streams_synchronizer.cc @@ -14,83 +14,48 @@ #include "webrtc/base/logging.h" #include "webrtc/base/timeutils.h" #include "webrtc/base/trace_event.h" -#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h" -#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" +#include "webrtc/call/syncable.h" #include "webrtc/modules/video_coding/video_coding_impl.h" -#include "webrtc/system_wrappers/include/clock.h" -#include "webrtc/video/stream_synchronization.h" -#include "webrtc/video_frame.h" -#include "webrtc/voice_engine/include/voe_video_sync.h" namespace webrtc { namespace { bool UpdateMeasurements(StreamSynchronization::Measurements* stream, - RtpRtcp* rtp_rtcp, - RtpReceiver* receiver) { - if (!receiver->Timestamp(&stream->latest_timestamp)) - return false; - if (!receiver->LastReceivedTimeMs(&stream->latest_receive_time_ms)) - return false; - - uint32_t ntp_secs = 0; - uint32_t ntp_frac = 0; - uint32_t rtp_timestamp = 0; - if (rtp_rtcp->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr, - &rtp_timestamp) != 0) { - return false; - } - + const Syncable::Info& info) { + RTC_DCHECK(stream); + stream->latest_timestamp = info.latest_received_capture_timestamp; + stream->latest_receive_time_ms = info.latest_receive_time_ms; bool new_rtcp_sr = false; - if (!stream->rtp_to_ntp.UpdateMeasurements(ntp_secs, ntp_frac, rtp_timestamp, + if (!stream->rtp_to_ntp.UpdateMeasurements(info.capture_time_ntp_secs, + info.capture_time_ntp_frac, + info.capture_time_source_clock, &new_rtcp_sr)) { return false; } - return true; } } // namespace -RtpStreamsSynchronizer::RtpStreamsSynchronizer( - vcm::VideoReceiver* video_receiver, - RtpStreamReceiver* rtp_stream_receiver) - : clock_(Clock::GetRealTimeClock()), - video_receiver_(video_receiver), - video_rtp_receiver_(rtp_stream_receiver->GetRtpReceiver()), - video_rtp_rtcp_(rtp_stream_receiver->rtp_rtcp()), - voe_channel_id_(-1), - voe_sync_interface_(nullptr), - audio_rtp_receiver_(nullptr), - audio_rtp_rtcp_(nullptr), +RtpStreamsSynchronizer::RtpStreamsSynchronizer(Syncable* syncable_video) + : syncable_video_(syncable_video), + syncable_audio_(nullptr), sync_(), last_sync_time_(rtc::TimeNanos()) { + RTC_DCHECK(syncable_video); process_thread_checker_.DetachFromThread(); } -void RtpStreamsSynchronizer::ConfigureSync(int voe_channel_id, - VoEVideoSync* voe_sync_interface) { - if (voe_channel_id != -1) - RTC_DCHECK(voe_sync_interface); - +void RtpStreamsSynchronizer::ConfigureSync(Syncable* syncable_audio) { rtc::CritScope lock(&crit_); - if (voe_channel_id_ == voe_channel_id && - voe_sync_interface_ == voe_sync_interface) { + if (syncable_audio == syncable_audio_) { // This prevents expensive no-ops. return; } - voe_channel_id_ = voe_channel_id; - voe_sync_interface_ = voe_sync_interface; - audio_rtp_rtcp_ = nullptr; - audio_rtp_receiver_ = nullptr; + syncable_audio_ = syncable_audio; sync_.reset(nullptr); - - if (voe_channel_id_ != -1) { - voe_sync_interface_->GetRtpRtcp(voe_channel_id_, &audio_rtp_rtcp_, - &audio_rtp_receiver_); - RTC_DCHECK(audio_rtp_rtcp_); - RTC_DCHECK(audio_rtp_receiver_); - sync_.reset(new StreamSynchronization(video_rtp_rtcp_->SSRC(), - voe_channel_id_)); + if (syncable_audio_) { + sync_.reset(new StreamSynchronization(syncable_video_->id(), + syncable_audio_->id())); } } @@ -103,35 +68,22 @@ int64_t RtpStreamsSynchronizer::TimeUntilNextProcess() { void RtpStreamsSynchronizer::Process() { RTC_DCHECK_RUN_ON(&process_thread_checker_); - - const int current_video_delay_ms = video_receiver_->Delay(); last_sync_time_ = rtc::TimeNanos(); rtc::CritScope lock(&crit_); - if (voe_channel_id_ == -1) { + if (!syncable_audio_) { return; } - RTC_DCHECK(voe_sync_interface_); RTC_DCHECK(sync_.get()); - int audio_jitter_buffer_delay_ms = 0; - int playout_buffer_delay_ms = 0; - if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_, - &audio_jitter_buffer_delay_ms, - &playout_buffer_delay_ms) != 0) { + rtc::Optional audio_info = syncable_audio_->GetInfo(); + if (!audio_info || !UpdateMeasurements(&audio_measurement_, *audio_info)) { return; } - const int current_audio_delay_ms = audio_jitter_buffer_delay_ms + - playout_buffer_delay_ms; int64_t last_video_receive_ms = video_measurement_.latest_receive_time_ms; - if (!UpdateMeasurements(&video_measurement_, video_rtp_rtcp_, - video_rtp_receiver_)) { - return; - } - - if (!UpdateMeasurements(&audio_measurement_, audio_rtp_rtcp_, - audio_rtp_receiver_)) { + rtc::Optional video_info = syncable_video_->GetInfo(); + if (!video_info || !UpdateMeasurements(&video_measurement_, *video_info)) { return; } @@ -147,41 +99,38 @@ void RtpStreamsSynchronizer::Process() { return; } - TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay", current_video_delay_ms); - TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay", current_audio_delay_ms); + TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay", + video_info->current_delay_ms); + TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay", + audio_info->current_delay_ms); TRACE_COUNTER1("webrtc", "SyncRelativeDelay", relative_delay_ms); int target_audio_delay_ms = 0; - int target_video_delay_ms = current_video_delay_ms; + int target_video_delay_ms = video_info->current_delay_ms; // Calculate the necessary extra audio delay and desired total video // delay to get the streams in sync. if (!sync_->ComputeDelays(relative_delay_ms, - current_audio_delay_ms, + audio_info->current_delay_ms, &target_audio_delay_ms, &target_video_delay_ms)) { return; } - if (voe_sync_interface_->SetMinimumPlayoutDelay( - voe_channel_id_, target_audio_delay_ms) == -1) { - LOG(LS_ERROR) << "Error setting voice delay."; - } - video_receiver_->SetMinimumPlayoutDelay(target_video_delay_ms); + syncable_audio_->SetMinimumPlayoutDelay(target_audio_delay_ms); + syncable_video_->SetMinimumPlayoutDelay(target_video_delay_ms); } bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs( - const VideoFrame& frame, + uint32_t timestamp, + int64_t render_time_ms, int64_t* stream_offset_ms, double* estimated_freq_khz) const { rtc::CritScope lock(&crit_); - if (voe_channel_id_ == -1) - return false; - - uint32_t playout_timestamp = 0; - if (voe_sync_interface_->GetPlayoutTimestamp(voe_channel_id_, - playout_timestamp) != 0) { + if (!syncable_audio_) { return false; } + uint32_t playout_timestamp = syncable_audio_->GetPlayoutTimestamp(); + int64_t latest_audio_ntp; if (!audio_measurement_.rtp_to_ntp.Estimate(playout_timestamp, &latest_audio_ntp)) { @@ -189,13 +138,11 @@ bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs( } int64_t latest_video_ntp; - if (!video_measurement_.rtp_to_ntp.Estimate(frame.timestamp(), - &latest_video_ntp)) { + if (!video_measurement_.rtp_to_ntp.Estimate(timestamp, &latest_video_ntp)) { return false; } - int64_t time_to_render_ms = - frame.render_time_ms() - clock_->TimeInMilliseconds(); + int64_t time_to_render_ms = render_time_ms - rtc::TimeMillis(); if (time_to_render_ms > 0) latest_video_ntp += time_to_render_ms; diff --git a/webrtc/video/rtp_streams_synchronizer.h b/webrtc/video/rtp_streams_synchronizer.h index bc24d6f807..afa15015d4 100644 --- a/webrtc/video/rtp_streams_synchronizer.h +++ b/webrtc/video/rtp_streams_synchronizer.h @@ -19,14 +19,11 @@ #include "webrtc/base/criticalsection.h" #include "webrtc/base/thread_checker.h" #include "webrtc/modules/include/module.h" -#include "webrtc/video/rtp_stream_receiver.h" #include "webrtc/video/stream_synchronization.h" namespace webrtc { -class Clock; -class VideoFrame; -class VoEVideoSync; +class Syncable; namespace vcm { class VideoReceiver; @@ -34,11 +31,9 @@ class VideoReceiver; class RtpStreamsSynchronizer : public Module { public: - RtpStreamsSynchronizer(vcm::VideoReceiver* vcm, - RtpStreamReceiver* rtp_stream_receiver); + explicit RtpStreamsSynchronizer(Syncable* syncable_video); - void ConfigureSync(int voe_channel_id, - VoEVideoSync* voe_sync_interface); + void ConfigureSync(Syncable* syncable_audio); // Implements Module. int64_t TimeUntilNextProcess() override; @@ -48,21 +43,16 @@ class RtpStreamsSynchronizer : public Module { // video |frame|. Returns true on success, false otherwise. // The estimated frequency is the frequency used in the RTP to NTP timestamp // conversion. - bool GetStreamSyncOffsetInMs(const VideoFrame& frame, + bool GetStreamSyncOffsetInMs(uint32_t timestamp, + int64_t render_time_ms, int64_t* stream_offset_ms, double* estimated_freq_khz) const; private: - Clock* const clock_; - vcm::VideoReceiver* const video_receiver_; - RtpReceiver* const video_rtp_receiver_; - RtpRtcp* const video_rtp_rtcp_; + Syncable* syncable_video_; rtc::CriticalSection crit_; - int voe_channel_id_ GUARDED_BY(crit_); - VoEVideoSync* voe_sync_interface_ GUARDED_BY(crit_); - RtpReceiver* audio_rtp_receiver_ GUARDED_BY(crit_); - RtpRtcp* audio_rtp_rtcp_ GUARDED_BY(crit_); + Syncable* syncable_audio_ GUARDED_BY(crit_); std::unique_ptr sync_ GUARDED_BY(crit_); StreamSynchronization::Measurements audio_measurement_ GUARDED_BY(crit_); StreamSynchronization::Measurements video_measurement_ GUARDED_BY(crit_); diff --git a/webrtc/video/stream_synchronization.cc b/webrtc/video/stream_synchronization.cc index 145922c3a0..32a2d722f5 100644 --- a/webrtc/video/stream_synchronization.cc +++ b/webrtc/video/stream_synchronization.cc @@ -26,10 +26,10 @@ static const int kFilterLength = 4; // Minimum difference between audio and video to warrant a change. static const int kMinDeltaMs = 30; -StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc, - int audio_channel_id) - : video_primary_ssrc_(video_primary_ssrc), - audio_channel_id_(audio_channel_id), +StreamSynchronization::StreamSynchronization(int video_stream_id, + int audio_stream_id) + : video_stream_id_(video_stream_id), + audio_stream_id_(audio_stream_id), base_target_delay_ms_(0), avg_diff_ms_(0) { } @@ -72,7 +72,7 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms, int current_video_delay_ms = *total_video_delay_target_ms; LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms << " current diff: " << relative_delay_ms - << " for channel " << audio_channel_id_; + << " for stream " << audio_stream_id_; // Calculate the difference between the lowest possible video delay and // the current audio delay. int current_diff_ms = current_video_delay_ms - current_audio_delay_ms + @@ -166,9 +166,9 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms, channel_delay_.last_audio_delay_ms = new_audio_delay_ms; LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms - << " for video primary SSRC " << video_primary_ssrc_ + << " for video stream " << video_stream_id_ << " and audio delay " << channel_delay_.extra_audio_delay_ms - << " for audio channel " << audio_channel_id_; + << " for audio stream " << audio_stream_id_; // Return values. *total_video_delay_target_ms = new_video_delay_ms; diff --git a/webrtc/video/stream_synchronization.h b/webrtc/video/stream_synchronization.h index 0b1ad4fd3a..d32966bc53 100644 --- a/webrtc/video/stream_synchronization.h +++ b/webrtc/video/stream_synchronization.h @@ -27,7 +27,7 @@ class StreamSynchronization { uint32_t latest_timestamp; }; - StreamSynchronization(uint32_t video_primary_ssrc, int audio_channel_id); + StreamSynchronization(int video_stream_id, int audio_stream_id); bool ComputeDelays(int relative_delay_ms, int current_audio_delay_ms, @@ -53,8 +53,8 @@ class StreamSynchronization { }; SynchronizationDelays channel_delay_; - const uint32_t video_primary_ssrc_; - const int audio_channel_id_; + const int video_stream_id_; + const int audio_stream_id_; int base_target_delay_ms_; int avg_diff_ms_; }; diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index 5fd436d6a4..25418c4d8f 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -22,6 +22,8 @@ #include "webrtc/common_video/h264/profile_level_id.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/congestion_controller/include/congestion_controller.h" +#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h" +#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" #include "webrtc/modules/utility/include/process_thread.h" #include "webrtc/modules/video_coding/frame_object.h" #include "webrtc/modules/video_coding/include/video_coding.h" @@ -33,7 +35,6 @@ #include "webrtc/video/call_stats.h" #include "webrtc/video/receive_statistics_proxy.h" #include "webrtc/video_receive_stream.h" -#include "webrtc/voice_engine/include/voe_video_sync.h" namespace webrtc { @@ -190,7 +191,6 @@ VideoReceiveStream::VideoReceiveStream( CongestionController* congestion_controller, PacketRouter* packet_router, VideoReceiveStream::Config config, - webrtc::VoiceEngine* voice_engine, ProcessThread* process_thread, CallStats* call_stats, VieRemb* remb) @@ -220,7 +220,7 @@ VideoReceiveStream::VideoReceiveStream( this, // KeyFrameRequestSender this, // OnCompleteFrameCallback timing_.get()), - rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_), + rtp_stream_sync_(this), jitter_buffer_experiment_( field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") == "Enabled") { @@ -230,6 +230,8 @@ VideoReceiveStream::VideoReceiveStream( RTC_DCHECK(congestion_controller_); RTC_DCHECK(call_stats_); + module_process_thread_checker_.DetachFromThread(); + RTC_DCHECK(!config_.decoders.empty()); std::set decoder_payload_types; for (const Decoder& decoder : config_.decoders) { @@ -254,6 +256,7 @@ VideoReceiveStream::VideoReceiveStream( } VideoReceiveStream::~VideoReceiveStream() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString(); Stop(); @@ -265,6 +268,7 @@ VideoReceiveStream::~VideoReceiveStream() { } void VideoReceiveStream::SignalNetworkState(NetworkState state) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); rtp_stream_receiver_.SignalNetworkState(state); } @@ -281,21 +285,17 @@ bool VideoReceiveStream::DeliverRtp(const uint8_t* packet, bool VideoReceiveStream::OnRecoveredPacket(const uint8_t* packet, size_t length) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); return rtp_stream_receiver_.OnRecoveredPacket(packet, length); } -void VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine, - int audio_channel_id) { - if (voice_engine && audio_channel_id != -1) { - VoEVideoSync* voe_sync_interface = VoEVideoSync::GetInterface(voice_engine); - rtp_stream_sync_.ConfigureSync(audio_channel_id, voe_sync_interface); - voe_sync_interface->Release(); - } else { - rtp_stream_sync_.ConfigureSync(-1, nullptr); - } +void VideoReceiveStream::SetSync(Syncable* audio_syncable) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + rtp_stream_sync_.ConfigureSync(audio_syncable); } void VideoReceiveStream::Start() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); if (decode_thread_.IsRunning()) return; @@ -346,6 +346,7 @@ void VideoReceiveStream::Start() { } void VideoReceiveStream::Stop() { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); rtp_stream_receiver_.StopReceive(); // TriggerDecoderShutdown will release any waiting decoder thread and make it // stop immediately, instead of waiting for a timeout. Needs to be called @@ -407,7 +408,9 @@ void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) { // function itself, another in GetChannel() and a third in // GetPlayoutTimestamp. Seems excessive. Anyhow, I'm assuming the function // succeeds most of the time, which leads to grabbing a fourth lock. - if (rtp_stream_sync_.GetStreamSyncOffsetInMs(video_frame, &sync_offset_ms, + if (rtp_stream_sync_.GetStreamSyncOffsetInMs(video_frame.timestamp(), + video_frame.render_time_ms(), + &sync_offset_ms, &estimated_freq_khz)) { // TODO(tommi): OnSyncOffsetUpdated grabs a lock. stats_proxy_.OnSyncOffsetUpdated(sync_offset_ms, estimated_freq_khz); @@ -461,6 +464,46 @@ void VideoReceiveStream::OnCompleteFrame( rtp_stream_receiver_.FrameContinuous(last_continuous_pid); } +int VideoReceiveStream::id() const { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + return config_.rtp.remote_ssrc; +} + +rtc::Optional VideoReceiveStream::GetInfo() const { + RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + Syncable::Info info; + + RtpReceiver* rtp_receiver = rtp_stream_receiver_.GetRtpReceiver(); + RTC_DCHECK(rtp_receiver); + if (!rtp_receiver->Timestamp(&info.latest_received_capture_timestamp)) + return rtc::Optional(); + if (!rtp_receiver->LastReceivedTimeMs(&info.latest_receive_time_ms)) + return rtc::Optional(); + + RtpRtcp* rtp_rtcp = rtp_stream_receiver_.rtp_rtcp(); + RTC_DCHECK(rtp_rtcp); + if (rtp_rtcp->RemoteNTP(&info.capture_time_ntp_secs, + &info.capture_time_ntp_frac, + nullptr, + nullptr, + &info.capture_time_source_clock) != 0) { + return rtc::Optional(); + } + + info.current_delay_ms = video_receiver_.Delay(); + return rtc::Optional(info); +} + +uint32_t VideoReceiveStream::GetPlayoutTimestamp() const { + RTC_NOTREACHED(); + return 0; +} + +void VideoReceiveStream::SetMinimumPlayoutDelay(int delay_ms) { + RTC_DCHECK_RUN_ON(&module_process_thread_checker_); + video_receiver_.SetMinimumPlayoutDelay(delay_ms); +} + bool VideoReceiveStream::DecodeThreadFunction(void* ptr) { static_cast(ptr)->Decode(); return true; @@ -489,6 +532,5 @@ void VideoReceiveStream::Decode() { video_receiver_.Decode(kMaxDecodeWaitTimeMs); } } - } // namespace internal } // namespace webrtc diff --git a/webrtc/video/video_receive_stream.h b/webrtc/video/video_receive_stream.h index 9960dc9589..063a4e405f 100644 --- a/webrtc/video/video_receive_stream.h +++ b/webrtc/video/video_receive_stream.h @@ -14,6 +14,8 @@ #include #include +#include "webrtc/base/thread_checker.h" +#include "webrtc/call/syncable.h" #include "webrtc/common_video/include/incoming_video_stream.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/rtp_rtcp/include/flexfec_receiver.h" @@ -34,7 +36,6 @@ class CongestionController; class IvfFileWriter; class ProcessThread; class RTPFragmentationHeader; -class VoiceEngine; class VieRemb; class VCMTiming; class VCMJitterEstimator; @@ -46,14 +47,14 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, public EncodedImageCallback, public NackSender, public KeyFrameRequestSender, - public video_coding::OnCompleteFrameCallback { + public video_coding::OnCompleteFrameCallback, + public Syncable { public: VideoReceiveStream(int num_cpu_cores, bool protected_by_flexfec, CongestionController* congestion_controller, PacketRouter* packet_router, VideoReceiveStream::Config config, - webrtc::VoiceEngine* voice_engine, ProcessThread* process_thread, CallStats* call_stats, VieRemb* remb); @@ -69,7 +70,7 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, bool OnRecoveredPacket(const uint8_t* packet, size_t length); - void SetSyncChannel(VoiceEngine* voice_engine, int audio_channel_id); + void SetSync(Syncable* audio_syncable); // Implements webrtc::VideoReceiveStream. void Start() override; @@ -104,10 +105,19 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, void OnCompleteFrame( std::unique_ptr frame) override; + // Implements Syncable. + int id() const override; + rtc::Optional GetInfo() const override; + uint32_t GetPlayoutTimestamp() const override; + void SetMinimumPlayoutDelay(int delay_ms) override; + private: static bool DecodeThreadFunction(void* ptr); void Decode(); + rtc::ThreadChecker worker_thread_checker_; + rtc::ThreadChecker module_process_thread_checker_; + TransportAdapter transport_adapter_; const VideoReceiveStream::Config config_; const int num_cpu_cores_; diff --git a/webrtc/voice_engine/channel_proxy.cc b/webrtc/voice_engine/channel_proxy.cc index 59b920f8e1..b3c3a98507 100644 --- a/webrtc/voice_engine/channel_proxy.cc +++ b/webrtc/voice_engine/channel_proxy.cc @@ -256,6 +256,42 @@ void ChannelProxy::DisassociateSendChannel() { channel()->set_associate_send_channel(ChannelOwner(nullptr)); } +void ChannelProxy::GetRtpRtcp(RtpRtcp** rtp_rtcp, + RtpReceiver** rtp_receiver) const { + // Called on Call's module_process_thread_. + RTC_DCHECK(rtp_rtcp); + RTC_DCHECK(rtp_receiver); + int error = channel()->GetRtpRtcp(rtp_rtcp, rtp_receiver); + RTC_DCHECK_EQ(0, error); +} + +void ChannelProxy::GetDelayEstimate(int* jitter_buffer_delay_ms, + int* playout_buffer_delay_ms) const { + // Called on Call's module_process_thread_. + RTC_DCHECK(jitter_buffer_delay_ms); + RTC_DCHECK(playout_buffer_delay_ms); + bool error = channel()->GetDelayEstimate(jitter_buffer_delay_ms, + playout_buffer_delay_ms); + RTC_DCHECK(error); +} + +uint32_t ChannelProxy::GetPlayoutTimestamp() const { + // Called on video capture thread. + unsigned int timestamp = 0; + int error = channel()->GetPlayoutTimestamp(timestamp); + RTC_DCHECK(!error || timestamp == 0); + return timestamp; +} + +void ChannelProxy::SetMinimumPlayoutDelay(int delay_ms) { + // Called on Call's module_process_thread_. + // Limit to range accepted by both VoE and ACM, so we're at least getting as + // close as possible, instead of failing. + delay_ms = std::max(0, std::min(delay_ms, 10000)); + int error = channel()->SetMinimumPlayoutDelay(delay_ms); + RTC_DCHECK_EQ(0, error); +} + void ChannelProxy::SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats) { RTC_DCHECK(thread_checker_.CalledOnValidThread()); channel()->SetRtcpRttStats(rtcp_rtt_stats); diff --git a/webrtc/voice_engine/channel_proxy.h b/webrtc/voice_engine/channel_proxy.h index 4bd76a4bed..7d18a74d57 100644 --- a/webrtc/voice_engine/channel_proxy.h +++ b/webrtc/voice_engine/channel_proxy.h @@ -29,6 +29,8 @@ class PacketRouter; class RtcEventLog; class RtcpRttStats; class RtpPacketSender; +class RtpReceiver; +class RtpRtcp; class Transport; class TransportFeedbackObserver; @@ -99,6 +101,12 @@ class ChannelProxy { virtual void SetTransportOverhead(int transport_overhead_per_packet); virtual void AssociateSendChannel(const ChannelProxy& send_channel_proxy); virtual void DisassociateSendChannel(); + virtual void GetRtpRtcp(RtpRtcp** rtp_rtcp, + RtpReceiver** rtp_receiver) const; + virtual void GetDelayEstimate(int* jitter_buffer_delay_ms, + int* playout_buffer_delay_ms) const; + virtual uint32_t GetPlayoutTimestamp() const; + virtual void SetMinimumPlayoutDelay(int delay_ms); virtual void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats);