From dea374a467f196e5ec7dadc45e41e93e654f8472 Mon Sep 17 00:00:00 2001 From: Ranveer Aggarwal Date: Sat, 23 Jan 2021 12:27:19 +0530 Subject: [PATCH] Deliver packet info to source listeners when audio playout is disabled. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Normally, packet/frame info is delivered to AudioReceiveStream's source_tracker_ when an audio frame is pulled out of the stream (as a side-effect of GetAudioFrameWithInfo). When playout is muted, though, packets are thrown away in ChannelReceive::OnReceivedPayloadData, so AudioRtpReceiver stops seeing updates to its RtpSources and any related information (e.g. CSRCs and associated timestamps, levels). Skipping the playout path here has a downside of being misaligned with whatever playout delay would normally be, but it allows clients that want to consume RtpSource information to be able to do so while playout is muted. Bug: None Change-Id: Id00566b645de4196c2341611cd9e8b94b35aa157 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/203500 Reviewed-by: Henrik Lundin Reviewed-by: Jakob Ivarsson Reviewed-by: Noah Richards Reviewed-by: Per Åhgren Commit-Queue: Ranveer Aggarwal‎ Cr-Commit-Position: refs/heads/master@{#33236} --- audio/audio_receive_stream.cc | 10 ++++++++-- audio/audio_receive_stream.h | 2 +- audio/audio_receive_stream_unittest.cc | 1 + audio/channel_receive.cc | 24 ++++++++++++++++++++++++ audio/channel_receive.h | 5 +++++ audio/mock_voe_channel_proxy.h | 1 + 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc index 03dd4c0eef..e037ddc644 100644 --- a/audio/audio_receive_stream.cc +++ b/audio/audio_receive_stream.cc @@ -24,6 +24,7 @@ #include "audio/conversion.h" #include "call/rtp_config.h" #include "call/rtp_stream_receiver_controller_interface.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/strings/string_builder.h" @@ -118,8 +119,8 @@ AudioReceiveStream::AudioReceiveStream( webrtc::RtcEventLog* event_log, std::unique_ptr channel_receive) : audio_state_(audio_state), - channel_receive_(std::move(channel_receive)), - source_tracker_(clock) { + source_tracker_(clock), + channel_receive_(std::move(channel_receive)) { RTC_LOG(LS_INFO) << "AudioReceiveStream: " << config.rtp.remote_ssrc; RTC_DCHECK(config.decoder_factory); RTC_DCHECK(config.rtcp_send_transport); @@ -133,6 +134,11 @@ AudioReceiveStream::AudioReceiveStream( // Configure bandwidth estimation. channel_receive_->RegisterReceiverCongestionControlObjects(packet_router); + // When output is muted, ChannelReceive will directly notify the source + // tracker of "delivered" frames, so RtpReceiver information will continue to + // be updated. + channel_receive_->SetSourceTracker(&source_tracker_); + // Register with transport. rtp_stream_receiver_ = receiver_controller->CreateReceiver( config.rtp.remote_ssrc, channel_receive_.get()); diff --git a/audio/audio_receive_stream.h b/audio/audio_receive_stream.h index 0ccd21e9cb..545a1205e7 100644 --- a/audio/audio_receive_stream.h +++ b/audio/audio_receive_stream.h @@ -112,8 +112,8 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream, SequenceChecker module_process_thread_checker_; webrtc::AudioReceiveStream::Config config_; rtc::scoped_refptr audio_state_; - const std::unique_ptr channel_receive_; SourceTracker source_tracker_; + const std::unique_ptr channel_receive_; AudioSendStream* associated_send_stream_ = nullptr; bool playing_ RTC_GUARDED_BY(worker_thread_checker_) = false; diff --git a/audio/audio_receive_stream_unittest.cc b/audio/audio_receive_stream_unittest.cc index fcd691ea80..99e3a56e1b 100644 --- a/audio/audio_receive_stream_unittest.cc +++ b/audio/audio_receive_stream_unittest.cc @@ -106,6 +106,7 @@ struct ConfigHelper { })); EXPECT_CALL(*channel_receive_, SetDepacketizerToDecoderFrameTransformer(_)) .Times(1); + EXPECT_CALL(*channel_receive_, SetSourceTracker(_)); stream_config_.rtp.local_ssrc = kLocalSsrc; stream_config_.rtp.remote_ssrc = kRemoteSsrc; diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index 1316ccb03b..8b6c9cf870 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -162,6 +162,8 @@ class ChannelReceive : public ChannelReceiveInterface { int PreferredSampleRate() const override; + void SetSourceTracker(SourceTracker* source_tracker) override; + // Associate to a send channel. // Used for obtaining RTT for a receive-only channel. void SetAssociatedSendChannel(const ChannelSendInterface* channel) override; @@ -219,6 +221,7 @@ class ChannelReceive : public ChannelReceiveInterface { std::unique_ptr rtp_receive_statistics_; std::unique_ptr rtp_rtcp_; const uint32_t remote_ssrc_; + SourceTracker* source_tracker_ = nullptr; // Info for GetSyncInfo is updated on network or worker thread, and queried on // the worker thread. @@ -233,6 +236,7 @@ class ChannelReceive : public ChannelReceiveInterface { AudioSinkInterface* audio_sink_ = nullptr; AudioLevel _outputAudioLevel; + Clock* const clock_; RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_); // Timestamp of the audio pulled from NetEq. @@ -287,6 +291,21 @@ void ChannelReceive::OnReceivedPayloadData( if (!Playing()) { // Avoid inserting into NetEQ when we are not playing. Count the // packet as discarded. + + // If we have a source_tracker_, tell it that the frame has been + // "delivered". Normally, this happens in AudioReceiveStream when audio + // frames are pulled out, but when playout is muted, nothing is pulling + // frames. The downside of this approach is that frames delivered this way + // won't be delayed for playout, and therefore will be unsynchronized with + // (a) audio delay when playing and (b) any audio/video synchronization. But + // the alternative is that muting playout also stops the SourceTracker from + // updating RtpSource information. + if (source_tracker_) { + RtpPacketInfos::vector_type packet_vector = { + RtpPacketInfo(rtpHeader, clock_->TimeInMilliseconds())}; + source_tracker_->OnFrameDelivered(RtpPacketInfos(packet_vector)); + } + return; } @@ -442,6 +461,10 @@ int ChannelReceive::PreferredSampleRate() const { acm_receiver_.last_output_sample_rate_hz()); } +void ChannelReceive::SetSourceTracker(SourceTracker* source_tracker) { + source_tracker_ = source_tracker; +} + ChannelReceive::ChannelReceive( Clock* clock, ProcessThread* module_process_thread, @@ -469,6 +492,7 @@ ChannelReceive::ChannelReceive( jitter_buffer_max_packets, jitter_buffer_fast_playout)), _outputAudioLevel(), + clock_(clock), ntp_estimator_(clock), playout_timestamp_rtp_(0), playout_delay_ms_(0), diff --git a/audio/channel_receive.h b/audio/channel_receive.h index eef2db425c..32bc813cc5 100644 --- a/audio/channel_receive.h +++ b/audio/channel_receive.h @@ -28,6 +28,7 @@ #include "call/rtp_packet_sink_interface.h" #include "call/syncable.h" #include "modules/audio_coding/include/audio_coding_module_typedefs.h" +#include "modules/rtp_rtcp/source/source_tracker.h" #include "system_wrappers/include/clock.h" // TODO(solenberg, nisse): This file contains a few NOLINT marks, to silence @@ -135,6 +136,10 @@ class ChannelReceiveInterface : public RtpPacketSinkInterface { virtual int PreferredSampleRate() const = 0; + // Sets the source tracker to notify about "delivered" packets when output is + // muted. + virtual void SetSourceTracker(SourceTracker* source_tracker) = 0; + // Associate to a send channel. // Used for obtaining RTT for a receive-only channel. virtual void SetAssociatedSendChannel( diff --git a/audio/mock_voe_channel_proxy.h b/audio/mock_voe_channel_proxy.h index 52e5b2fc83..7f140d400d 100644 --- a/audio/mock_voe_channel_proxy.h +++ b/audio/mock_voe_channel_proxy.h @@ -59,6 +59,7 @@ class MockChannelReceive : public voe::ChannelReceiveInterface { (int sample_rate_hz, AudioFrame*), (override)); MOCK_METHOD(int, PreferredSampleRate, (), (const, override)); + MOCK_METHOD(void, SetSourceTracker, (SourceTracker*), (override)); MOCK_METHOD(void, SetAssociatedSendChannel, (const voe::ChannelSendInterface*),