admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stop using VoEVideoSync in Call/VideoReceiveStream.

Browse Source 
BUG=webrtc:4690

Review-Url: https://codereview.webrtc.org/2452163004
Cr-Commit-Position: refs/heads/master@{#16375}
This commit is contained in:
solenberg 2017-01-31 03:58:40 -08:00 committed by Commit bot
parent 63b14b7d15
commit 3ebbcb528b
15 changed files with 343 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
webrtc/audio/audio_receive_stream.cc
View File

@ -21,6 +21,8 @@
#include "webrtc/base/logging.h"
#include "webrtc/base/timeutils.h"
#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/voice_engine/channel_proxy.h"
#include "webrtc/voice_engine/include/voe_base.h"
#include "webrtc/voice_engine/include/voe_codec.h"
@ -81,6 +83,8 @@ AudioReceiveStream::AudioReceiveStream(
RTC_DCHECK(remote_bitrate_estimator);
RTC_DCHECK(rtp_header_parser_);
module_process_thread_checker_.DetachFromThread();
VoiceEngineImpl* voe_impl = static_cast<VoiceEngineImpl*>(voice_engine());
channel_proxy_ = voe_impl->GetChannelProxy(config_.voe_channel_id);
channel_proxy_->SetRtcEventLog(event_log);
@ -125,7 +129,7 @@ AudioReceiveStream::AudioReceiveStream(
}
AudioReceiveStream::~AudioReceiveStream() {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
LOG(LS_INFO) << "~AudioReceiveStream: " << config_.ToString();
if (playing_) {
Stop();
@ -138,7 +142,7 @@ AudioReceiveStream::~AudioReceiveStream() {
}
void AudioReceiveStream::Start() {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (playing_) {
return;
}
@ -159,7 +163,7 @@ void AudioReceiveStream::Start() {
}
void AudioReceiveStream::Stop() {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (!playing_) {
return;
}
@ -170,7 +174,7 @@ void AudioReceiveStream::Stop() {
}
webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
webrtc::AudioReceiveStream::Stats stats;
stats.remote_ssrc = config_.rtp.remote_ssrc;
ScopedVoEInterface<VoECodec> codec(voice_engine());
@ -220,22 +224,78 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const {
}
void AudioReceiveStream::SetSink(std::unique_ptr<AudioSinkInterface> sink) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
channel_proxy_->SetSink(std::move(sink));
}
void AudioReceiveStream::SetGain(float gain) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
channel_proxy_->SetChannelOutputVolumeScaling(gain);
}
const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const {
RTC_DCHECK_RUN_ON(&thread_checker_);
return config_;
AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo(
int sample_rate_hz,
AudioFrame* audio_frame) {
return channel_proxy_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame);
}
int AudioReceiveStream::Ssrc() const {
return config_.rtp.remote_ssrc;
}
int AudioReceiveStream::PreferredSampleRate() const {
return channel_proxy_->NeededFrequency();
}
int AudioReceiveStream::id() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return config_.rtp.remote_ssrc;
}
rtc::Optional<Syncable::Info> AudioReceiveStream::GetInfo() const {
RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
Syncable::Info info;
RtpRtcp* rtp_rtcp = nullptr;
RtpReceiver* rtp_receiver = nullptr;
channel_proxy_->GetRtpRtcp(&rtp_rtcp, &rtp_receiver);
RTC_DCHECK(rtp_rtcp);
RTC_DCHECK(rtp_receiver);
if (!rtp_receiver->Timestamp(&info.latest_received_capture_timestamp)) {
return rtc::Optional<Syncable::Info>();
}
if (!rtp_receiver->LastReceivedTimeMs(&info.latest_receive_time_ms)) {
return rtc::Optional<Syncable::Info>();
}
if (rtp_rtcp->RemoteNTP(&info.capture_time_ntp_secs,
&info.capture_time_ntp_frac,
nullptr,
nullptr,
&info.capture_time_source_clock) != 0) {
return rtc::Optional<Syncable::Info>();
}
int jitter_buffer_delay_ms = 0;
int playout_buffer_delay_ms = 0;
channel_proxy_->GetDelayEstimate(&jitter_buffer_delay_ms,
&playout_buffer_delay_ms);
info.current_delay_ms = jitter_buffer_delay_ms + playout_buffer_delay_ms;
return rtc::Optional<Syncable::Info>(info);
}
uint32_t AudioReceiveStream::GetPlayoutTimestamp() const {
// Called on video capture thread.
return channel_proxy_->GetPlayoutTimestamp();
}
void AudioReceiveStream::SetMinimumPlayoutDelay(int delay_ms) {
RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
return channel_proxy_->SetMinimumPlayoutDelay(delay_ms);
}
void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (send_stream) {
VoiceEngineImpl* voe_impl = static_cast<VoiceEngineImpl*>(voice_engine());
std::unique_ptr<voe::ChannelProxy> send_channel_proxy =
@ -247,7 +307,7 @@ void AudioReceiveStream::AssociateSendStream(AudioSendStream* send_stream) {
}
void AudioReceiveStream::SignalNetworkState(NetworkState state) {
RTC_DCHECK_RUN_ON(&thread_checker_);
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
}
bool AudioReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) {
@ -286,24 +346,9 @@ bool AudioReceiveStream::DeliverRtp(const uint8_t* packet,
return channel_proxy_->ReceivedRTPPacket(packet, length, packet_time);
}
AudioMixer::Source::AudioFrameInfo AudioReceiveStream::GetAudioFrameWithInfo(
int sample_rate_hz,
AudioFrame* audio_frame) {
return channel_proxy_->GetAudioFrameWithInfo(sample_rate_hz, audio_frame);
}
int AudioReceiveStream::PreferredSampleRate() const {
return channel_proxy_->NeededFrequency();
}
int AudioReceiveStream::Ssrc() const {
return config_.rtp.remote_ssrc;
}
internal::AudioState* AudioReceiveStream::audio_state() const {
auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get());
RTC_DCHECK(audio_state);
return audio_state;
const webrtc::AudioReceiveStream::Config& AudioReceiveStream::config() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return config_;
}
VoiceEngine* AudioReceiveStream::voice_engine() const {
@ -312,6 +357,12 @@ VoiceEngine* AudioReceiveStream::voice_engine() const {
return voice_engine;
}
internal::AudioState* AudioReceiveStream::audio_state() const {
auto* audio_state = static_cast<internal::AudioState*>(audio_state_.get());
RTC_DCHECK(audio_state);
return audio_state;
}
int AudioReceiveStream::SetVoiceEnginePlayout(bool playout) {
ScopedVoEInterface<VoEBase> base(voice_engine());
if (playout) {
@ -320,6 +371,5 @@ int AudioReceiveStream::SetVoiceEnginePlayout(bool playout) {
return base->StopPlayout(config_.voe_channel_id);
}
}
} // namespace internal
} // namespace webrtc

30
webrtc/audio/audio_receive_stream.h
View File

@ -18,13 +18,13 @@
#include "webrtc/base/constructormagic.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/call/audio_receive_stream.h"
#include "webrtc/call/audio_state.h"
#include "webrtc/call/syncable.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
namespace webrtc {
class PacketRouter;
class RemoteBitrateEstimator;
class RtcEventLog;
class PacketRouter;
namespace voe {
class ChannelProxy;
@ -34,7 +34,8 @@ namespace internal {
class AudioSendStream;
class AudioReceiveStream final : public webrtc::AudioReceiveStream,
public AudioMixer::Source {
public AudioMixer::Source,
public Syncable {
public:
AudioReceiveStream(PacketRouter* packet_router,
RemoteBitrateEstimator* remote_bitrate_estimator,
@ -50,6 +51,18 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream,
void SetSink(std::unique_ptr<AudioSinkInterface> sink) override;
void SetGain(float gain) override;
// AudioMixer::Source
AudioFrameInfo GetAudioFrameWithInfo(int sample_rate_hz,
AudioFrame* audio_frame) override;
int Ssrc() const override;
int PreferredSampleRate() const override;
// Syncable
int id() const override;
rtc::Optional<Syncable::Info> GetInfo() const override;
uint32_t GetPlayoutTimestamp() const override;
void SetMinimumPlayoutDelay(int delay_ms) override;
void AssociateSendStream(AudioSendStream* send_stream);
void SignalNetworkState(NetworkState state);
bool DeliverRtcp(const uint8_t* packet, size_t length);
@ -58,25 +71,20 @@ class AudioReceiveStream final : public webrtc::AudioReceiveStream,
const PacketTime& packet_time);
const webrtc::AudioReceiveStream::Config& config() const;
// AudioMixer::Source
AudioFrameInfo GetAudioFrameWithInfo(int sample_rate_hz,
AudioFrame* audio_frame) override;
int PreferredSampleRate() const override;
int Ssrc() const override;
private:
VoiceEngine* voice_engine() const;
AudioState* audio_state() const;
int SetVoiceEnginePlayout(bool playout);
rtc::ThreadChecker thread_checker_;
rtc::ThreadChecker worker_thread_checker_;
rtc::ThreadChecker module_process_thread_checker_;
RemoteBitrateEstimator* const remote_bitrate_estimator_;
const webrtc::AudioReceiveStream::Config config_;
rtc::scoped_refptr<webrtc::AudioState> audio_state_;
std::unique_ptr<RtpHeaderParser> rtp_header_parser_;
std::unique_ptr<voe::ChannelProxy> channel_proxy_;
bool playing_ ACCESS_ON(thread_checker_) = false;
bool playing_ ACCESS_ON(worker_thread_checker_) = false;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioReceiveStream);
};

2
webrtc/call/BUILD.gn
View File

@ -16,6 +16,8 @@ rtc_source_set("call_interfaces") {
"audio_state.h",
"call.h",
"flexfec_receive_stream.h",
"syncable.cc",
"syncable.h",
]
}

24
webrtc/call/call.cc
View File

@ -55,7 +55,6 @@
#include "webrtc/video/video_receive_stream.h"
#include "webrtc/video/video_send_stream.h"
#include "webrtc/video/vie_remb.h"
#include "webrtc/voice_engine/include/voe_codec.h"
namespace webrtc {
@ -146,15 +145,6 @@ class Call : public webrtc::Call,
void ConfigureSync(const std::string& sync_group)
EXCLUSIVE_LOCKS_REQUIRED(receive_crit_);
VoiceEngine* voice_engine() {
internal::AudioState* audio_state =
static_cast<internal::AudioState*>(config_.audio_state.get());
if (audio_state)
return audio_state->voice_engine();
else
return nullptr;
}
rtc::Optional<RtpPacketReceived> ParseRtpPacket(const uint8_t* packet,
size_t length,
const PacketTime& packet_time)
@ -648,8 +638,8 @@ webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream(
}
VideoReceiveStream* receive_stream = new VideoReceiveStream(
num_cpu_cores_, protected_by_flexfec, congestion_controller_.get(),
&packet_router_, std::move(configuration), voice_engine(),
module_process_thread_.get(), call_stats_.get(), &remb_);
&packet_router_, std::move(configuration), module_process_thread_.get(),
call_stats_.get(), &remb_);
const webrtc::VideoReceiveStream::Config& config = receive_stream->config();
{
@ -1019,7 +1009,7 @@ void Call::OnAllocationLimitsChanged(uint32_t min_send_bitrate_bps,
void Call::ConfigureSync(const std::string& sync_group) {
// Set sync only if there was no previous one.
if (voice_engine() == nullptr || sync_group.empty())
if (sync_group.empty())
return;
AudioReceiveStream* sync_audio_stream = nullptr;
@ -1056,11 +1046,11 @@ void Call::ConfigureSync(const std::string& sync_group) {
"the current implementation.";
}
// Only sync the first A/V pair within this sync group.
if (sync_audio_stream != nullptr && num_synced_streams == 1) {
video_stream->SetSyncChannel(voice_engine(),
sync_audio_stream->config().voe_channel_id);
if (num_synced_streams == 1) {
// sync_audio_stream may be null and that's ok.
video_stream->SetSync(sync_audio_stream);
} else {
video_stream->SetSyncChannel(voice_engine(), -1);
video_stream->SetSync(nullptr);
}
}
}

17
webrtc/call/syncable.cc Normal file
View File

@ -0,0 +1,17 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/call/syncable.h"
namespace webrtc {
Syncable::~Syncable() = default;
} // namespace webrtc

43
webrtc/call/syncable.h Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// Syncable is used by RtpStreamsSynchronizer in VideoReceiveStream, and
// implemented by AudioReceiveStream.
#ifndef WEBRTC_CALL_SYNCABLE_H_
#define WEBRTC_CALL_SYNCABLE_H_
#include <stdint.h>
#include "webrtc/base/optional.h"
namespace webrtc {
class Syncable {
public:
struct Info {
int64_t latest_receive_time_ms = 0;
uint32_t latest_received_capture_timestamp = 0;
uint32_t capture_time_ntp_secs = 0;
uint32_t capture_time_ntp_frac = 0;
uint32_t capture_time_source_clock = 0;
int current_delay_ms = 0;
};
virtual ~Syncable();
virtual int id() const = 0;
virtual rtc::Optional<Info> GetInfo() const = 0;
virtual uint32_t GetPlayoutTimestamp() const = 0;
virtual void SetMinimumPlayoutDelay(int delay_ms) = 0;
};
} // namespace webrtc
#endif // WEBRTC_CALL_SYNCABLE_H_

6
webrtc/test/mock_voe_channel_proxy.h
View File

@ -73,6 +73,12 @@ class MockVoEChannelProxy : public voe::ChannelProxy {
MOCK_METHOD1(AssociateSendChannel,
void(const ChannelProxy& send_channel_proxy));
MOCK_METHOD0(DisassociateSendChannel, void());
MOCK_CONST_METHOD2(GetRtpRtcp, void(RtpRtcp** rtp_rtcp,
RtpReceiver** rtp_receiver));
MOCK_CONST_METHOD2(GetDelayEstimate, void(int* jitter_buffer_delay_ms,
int* playout_buffer_delay_ms));
MOCK_CONST_METHOD0(GetPlayoutTimestamp, uint32_t());
MOCK_METHOD1(SetMinimumPlayoutDelay, void(int delay_ms));
};
} // namespace test
} // namespace webrtc

129
webrtc/video/rtp_streams_synchronizer.cc
View File

@ -14,83 +14,48 @@
#include "webrtc/base/logging.h"
#include "webrtc/base/timeutils.h"
#include "webrtc/base/trace_event.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/call/syncable.h"
#include "webrtc/modules/video_coding/video_coding_impl.h"
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc/video/stream_synchronization.h"
#include "webrtc/video_frame.h"
#include "webrtc/voice_engine/include/voe_video_sync.h"
namespace webrtc {
namespace {
bool UpdateMeasurements(StreamSynchronization::Measurements* stream,
RtpRtcp* rtp_rtcp,
RtpReceiver* receiver) {
if (!receiver->Timestamp(&stream->latest_timestamp))
return false;
if (!receiver->LastReceivedTimeMs(&stream->latest_receive_time_ms))
return false;
uint32_t ntp_secs = 0;
uint32_t ntp_frac = 0;
uint32_t rtp_timestamp = 0;
if (rtp_rtcp->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr,
&rtp_timestamp) != 0) {
return false;
}
const Syncable::Info& info) {
RTC_DCHECK(stream);
stream->latest_timestamp = info.latest_received_capture_timestamp;
stream->latest_receive_time_ms = info.latest_receive_time_ms;
bool new_rtcp_sr = false;
if (!stream->rtp_to_ntp.UpdateMeasurements(ntp_secs, ntp_frac, rtp_timestamp,
if (!stream->rtp_to_ntp.UpdateMeasurements(info.capture_time_ntp_secs,
info.capture_time_ntp_frac,
info.capture_time_source_clock,
&new_rtcp_sr)) {
return false;
}
return true;
}
} // namespace
RtpStreamsSynchronizer::RtpStreamsSynchronizer(
vcm::VideoReceiver* video_receiver,
RtpStreamReceiver* rtp_stream_receiver)
: clock_(Clock::GetRealTimeClock()),
video_receiver_(video_receiver),
video_rtp_receiver_(rtp_stream_receiver->GetRtpReceiver()),
video_rtp_rtcp_(rtp_stream_receiver->rtp_rtcp()),
voe_channel_id_(-1),
voe_sync_interface_(nullptr),
audio_rtp_receiver_(nullptr),
audio_rtp_rtcp_(nullptr),
RtpStreamsSynchronizer::RtpStreamsSynchronizer(Syncable* syncable_video)
: syncable_video_(syncable_video),
syncable_audio_(nullptr),
sync_(),
last_sync_time_(rtc::TimeNanos()) {
RTC_DCHECK(syncable_video);
process_thread_checker_.DetachFromThread();
}
void RtpStreamsSynchronizer::ConfigureSync(int voe_channel_id,
VoEVideoSync* voe_sync_interface) {
if (voe_channel_id != -1)
RTC_DCHECK(voe_sync_interface);
void RtpStreamsSynchronizer::ConfigureSync(Syncable* syncable_audio) {
rtc::CritScope lock(&crit_);
if (voe_channel_id_ == voe_channel_id &&
voe_sync_interface_ == voe_sync_interface) {
if (syncable_audio == syncable_audio_) {
// This prevents expensive no-ops.
return;
}
voe_channel_id_ = voe_channel_id;
voe_sync_interface_ = voe_sync_interface;
audio_rtp_rtcp_ = nullptr;
audio_rtp_receiver_ = nullptr;
syncable_audio_ = syncable_audio;
sync_.reset(nullptr);
if (voe_channel_id_ != -1) {
voe_sync_interface_->GetRtpRtcp(voe_channel_id_, &audio_rtp_rtcp_,
&audio_rtp_receiver_);
RTC_DCHECK(audio_rtp_rtcp_);
RTC_DCHECK(audio_rtp_receiver_);
sync_.reset(new StreamSynchronization(video_rtp_rtcp_->SSRC(),
voe_channel_id_));
if (syncable_audio_) {
sync_.reset(new StreamSynchronization(syncable_video_->id(),
syncable_audio_->id()));
}
}
@ -103,35 +68,22 @@ int64_t RtpStreamsSynchronizer::TimeUntilNextProcess() {
void RtpStreamsSynchronizer::Process() {
RTC_DCHECK_RUN_ON(&process_thread_checker_);
const int current_video_delay_ms = video_receiver_->Delay();
last_sync_time_ = rtc::TimeNanos();
rtc::CritScope lock(&crit_);
if (voe_channel_id_ == -1) {
if (!syncable_audio_) {
return;
}
RTC_DCHECK(voe_sync_interface_);
RTC_DCHECK(sync_.get());
int audio_jitter_buffer_delay_ms = 0;
int playout_buffer_delay_ms = 0;
if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_,
&audio_jitter_buffer_delay_ms,
&playout_buffer_delay_ms) != 0) {
rtc::Optional<Syncable::Info> audio_info = syncable_audio_->GetInfo();
if (!audio_info || !UpdateMeasurements(&audio_measurement_, *audio_info)) {
return;
}
const int current_audio_delay_ms = audio_jitter_buffer_delay_ms +
playout_buffer_delay_ms;
int64_t last_video_receive_ms = video_measurement_.latest_receive_time_ms;
if (!UpdateMeasurements(&video_measurement_, video_rtp_rtcp_,
video_rtp_receiver_)) {
return;
}
if (!UpdateMeasurements(&audio_measurement_, audio_rtp_rtcp_,
audio_rtp_receiver_)) {
rtc::Optional<Syncable::Info> video_info = syncable_video_->GetInfo();
if (!video_info || !UpdateMeasurements(&video_measurement_, *video_info)) {
return;
}
@ -147,41 +99,38 @@ void RtpStreamsSynchronizer::Process() {
return;
}
TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay", current_video_delay_ms);
TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay", current_audio_delay_ms);
TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay",
video_info->current_delay_ms);
TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay",
audio_info->current_delay_ms);
TRACE_COUNTER1("webrtc", "SyncRelativeDelay", relative_delay_ms);
int target_audio_delay_ms = 0;
int target_video_delay_ms = current_video_delay_ms;
int target_video_delay_ms = video_info->current_delay_ms;
// Calculate the necessary extra audio delay and desired total video
// delay to get the streams in sync.
if (!sync_->ComputeDelays(relative_delay_ms,
current_audio_delay_ms,
audio_info->current_delay_ms,
&target_audio_delay_ms,
&target_video_delay_ms)) {
return;
}
if (voe_sync_interface_->SetMinimumPlayoutDelay(
voe_channel_id_, target_audio_delay_ms) == -1) {
LOG(LS_ERROR) << "Error setting voice delay.";
}
video_receiver_->SetMinimumPlayoutDelay(target_video_delay_ms);
syncable_audio_->SetMinimumPlayoutDelay(target_audio_delay_ms);
syncable_video_->SetMinimumPlayoutDelay(target_video_delay_ms);
}
bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs(
const VideoFrame& frame,
uint32_t timestamp,
int64_t render_time_ms,
int64_t* stream_offset_ms,
double* estimated_freq_khz) const {
rtc::CritScope lock(&crit_);
if (voe_channel_id_ == -1)
return false;
uint32_t playout_timestamp = 0;
if (voe_sync_interface_->GetPlayoutTimestamp(voe_channel_id_,
playout_timestamp) != 0) {
if (!syncable_audio_) {
return false;
}
uint32_t playout_timestamp = syncable_audio_->GetPlayoutTimestamp();
int64_t latest_audio_ntp;
if (!audio_measurement_.rtp_to_ntp.Estimate(playout_timestamp,
&latest_audio_ntp)) {
@ -189,13 +138,11 @@ bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs(
}
int64_t latest_video_ntp;
if (!video_measurement_.rtp_to_ntp.Estimate(frame.timestamp(),
&latest_video_ntp)) {
if (!video_measurement_.rtp_to_ntp.Estimate(timestamp, &latest_video_ntp)) {
return false;
}
int64_t time_to_render_ms =
frame.render_time_ms() - clock_->TimeInMilliseconds();
int64_t time_to_render_ms = render_time_ms - rtc::TimeMillis();
if (time_to_render_ms > 0)
latest_video_ntp += time_to_render_ms;

24
webrtc/video/rtp_streams_synchronizer.h
View File

@ -19,14 +19,11 @@
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/modules/include/module.h"
#include "webrtc/video/rtp_stream_receiver.h"
#include "webrtc/video/stream_synchronization.h"
namespace webrtc {
class Clock;
class VideoFrame;
class VoEVideoSync;
class Syncable;
namespace vcm {
class VideoReceiver;
@ -34,11 +31,9 @@ class VideoReceiver;
class RtpStreamsSynchronizer : public Module {
public:
RtpStreamsSynchronizer(vcm::VideoReceiver* vcm,
RtpStreamReceiver* rtp_stream_receiver);
explicit RtpStreamsSynchronizer(Syncable* syncable_video);
void ConfigureSync(int voe_channel_id,
VoEVideoSync* voe_sync_interface);
void ConfigureSync(Syncable* syncable_audio);
// Implements Module.
int64_t TimeUntilNextProcess() override;
@ -48,21 +43,16 @@ class RtpStreamsSynchronizer : public Module {
// video |frame|. Returns true on success, false otherwise.
// The estimated frequency is the frequency used in the RTP to NTP timestamp
// conversion.
bool GetStreamSyncOffsetInMs(const VideoFrame& frame,
bool GetStreamSyncOffsetInMs(uint32_t timestamp,
int64_t render_time_ms,
int64_t* stream_offset_ms,
double* estimated_freq_khz) const;
private:
Clock* const clock_;
vcm::VideoReceiver* const video_receiver_;
RtpReceiver* const video_rtp_receiver_;
RtpRtcp* const video_rtp_rtcp_;
Syncable* syncable_video_;
rtc::CriticalSection crit_;
int voe_channel_id_ GUARDED_BY(crit_);
VoEVideoSync* voe_sync_interface_ GUARDED_BY(crit_);
RtpReceiver* audio_rtp_receiver_ GUARDED_BY(crit_);
RtpRtcp* audio_rtp_rtcp_ GUARDED_BY(crit_);
Syncable* syncable_audio_ GUARDED_BY(crit_);
std::unique_ptr<StreamSynchronization> sync_ GUARDED_BY(crit_);
StreamSynchronization::Measurements audio_measurement_ GUARDED_BY(crit_);
StreamSynchronization::Measurements video_measurement_ GUARDED_BY(crit_);

14
webrtc/video/stream_synchronization.cc
View File

@ -26,10 +26,10 @@ static const int kFilterLength = 4;
// Minimum difference between audio and video to warrant a change.
static const int kMinDeltaMs = 30;
StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc,
int audio_channel_id)
: video_primary_ssrc_(video_primary_ssrc),
audio_channel_id_(audio_channel_id),
StreamSynchronization::StreamSynchronization(int video_stream_id,
int audio_stream_id)
: video_stream_id_(video_stream_id),
audio_stream_id_(audio_stream_id),
base_target_delay_ms_(0),
avg_diff_ms_(0) {
}
@ -72,7 +72,7 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
int current_video_delay_ms = *total_video_delay_target_ms;
LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
<< " current diff: " << relative_delay_ms
<< " for channel " << audio_channel_id_;
<< " for stream " << audio_stream_id_;
// Calculate the difference between the lowest possible video delay and
// the current audio delay.
int current_diff_ms = current_video_delay_ms - current_audio_delay_ms +
@ -166,9 +166,9 @@ bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
channel_delay_.last_audio_delay_ms = new_audio_delay_ms;
LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
<< " for video primary SSRC " << video_primary_ssrc_
<< " for video stream " << video_stream_id_
<< " and audio delay " << channel_delay_.extra_audio_delay_ms
<< " for audio channel " << audio_channel_id_;
<< " for audio stream " << audio_stream_id_;
// Return values.
*total_video_delay_target_ms = new_video_delay_ms;

6
webrtc/video/stream_synchronization.h
View File

@ -27,7 +27,7 @@ class StreamSynchronization {
uint32_t latest_timestamp;
};
StreamSynchronization(uint32_t video_primary_ssrc, int audio_channel_id);
StreamSynchronization(int video_stream_id, int audio_stream_id);
bool ComputeDelays(int relative_delay_ms,
int current_audio_delay_ms,
@ -53,8 +53,8 @@ class StreamSynchronization {
};
SynchronizationDelays channel_delay_;
const uint32_t video_primary_ssrc_;
const int audio_channel_id_;
const int video_stream_id_;
const int audio_stream_id_;
int base_target_delay_ms_;
int avg_diff_ms_;
};

70
webrtc/video/video_receive_stream.cc
View File

@ -22,6 +22,8 @@
#include "webrtc/common_video/h264/profile_level_id.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/congestion_controller/include/congestion_controller.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/modules/utility/include/process_thread.h"
#include "webrtc/modules/video_coding/frame_object.h"
#include "webrtc/modules/video_coding/include/video_coding.h"
@ -33,7 +35,6 @@
#include "webrtc/video/call_stats.h"
#include "webrtc/video/receive_statistics_proxy.h"
#include "webrtc/video_receive_stream.h"
#include "webrtc/voice_engine/include/voe_video_sync.h"
namespace webrtc {
@ -190,7 +191,6 @@ VideoReceiveStream::VideoReceiveStream(
CongestionController* congestion_controller,
PacketRouter* packet_router,
VideoReceiveStream::Config config,
webrtc::VoiceEngine* voice_engine,
ProcessThread* process_thread,
CallStats* call_stats,
VieRemb* remb)
@ -220,7 +220,7 @@ VideoReceiveStream::VideoReceiveStream(
this, // KeyFrameRequestSender
this, // OnCompleteFrameCallback
timing_.get()),
rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_),
rtp_stream_sync_(this),
jitter_buffer_experiment_(
field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") ==
"Enabled") {
@ -230,6 +230,8 @@ VideoReceiveStream::VideoReceiveStream(
RTC_DCHECK(congestion_controller_);
RTC_DCHECK(call_stats_);
module_process_thread_checker_.DetachFromThread();
RTC_DCHECK(!config_.decoders.empty());
std::set<int> decoder_payload_types;
for (const Decoder& decoder : config_.decoders) {
@ -254,6 +256,7 @@ VideoReceiveStream::VideoReceiveStream(
}
VideoReceiveStream::~VideoReceiveStream() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString();
Stop();
@ -265,6 +268,7 @@ VideoReceiveStream::~VideoReceiveStream() {
}
void VideoReceiveStream::SignalNetworkState(NetworkState state) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
rtp_stream_receiver_.SignalNetworkState(state);
}
@ -281,21 +285,17 @@ bool VideoReceiveStream::DeliverRtp(const uint8_t* packet,
bool VideoReceiveStream::OnRecoveredPacket(const uint8_t* packet,
size_t length) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return rtp_stream_receiver_.OnRecoveredPacket(packet, length);
}
void VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine,
int audio_channel_id) {
if (voice_engine && audio_channel_id != -1) {
VoEVideoSync* voe_sync_interface = VoEVideoSync::GetInterface(voice_engine);
rtp_stream_sync_.ConfigureSync(audio_channel_id, voe_sync_interface);
voe_sync_interface->Release();
} else {
rtp_stream_sync_.ConfigureSync(-1, nullptr);
}
void VideoReceiveStream::SetSync(Syncable* audio_syncable) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
rtp_stream_sync_.ConfigureSync(audio_syncable);
}
void VideoReceiveStream::Start() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
if (decode_thread_.IsRunning())
return;
@ -346,6 +346,7 @@ void VideoReceiveStream::Start() {
}
void VideoReceiveStream::Stop() {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
rtp_stream_receiver_.StopReceive();
// TriggerDecoderShutdown will release any waiting decoder thread and make it
// stop immediately, instead of waiting for a timeout. Needs to be called
@ -407,7 +408,9 @@ void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) {
// function itself, another in GetChannel() and a third in
// GetPlayoutTimestamp. Seems excessive. Anyhow, I'm assuming the function
// succeeds most of the time, which leads to grabbing a fourth lock.
if (rtp_stream_sync_.GetStreamSyncOffsetInMs(video_frame, &sync_offset_ms,
if (rtp_stream_sync_.GetStreamSyncOffsetInMs(video_frame.timestamp(),
video_frame.render_time_ms(),
&sync_offset_ms,
&estimated_freq_khz)) {
// TODO(tommi): OnSyncOffsetUpdated grabs a lock.
stats_proxy_.OnSyncOffsetUpdated(sync_offset_ms, estimated_freq_khz);
@ -461,6 +464,46 @@ void VideoReceiveStream::OnCompleteFrame(
rtp_stream_receiver_.FrameContinuous(last_continuous_pid);
}
int VideoReceiveStream::id() const {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
return config_.rtp.remote_ssrc;
}
rtc::Optional<Syncable::Info> VideoReceiveStream::GetInfo() const {
RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
Syncable::Info info;
RtpReceiver* rtp_receiver = rtp_stream_receiver_.GetRtpReceiver();
RTC_DCHECK(rtp_receiver);
if (!rtp_receiver->Timestamp(&info.latest_received_capture_timestamp))
return rtc::Optional<Syncable::Info>();
if (!rtp_receiver->LastReceivedTimeMs(&info.latest_receive_time_ms))
return rtc::Optional<Syncable::Info>();
RtpRtcp* rtp_rtcp = rtp_stream_receiver_.rtp_rtcp();
RTC_DCHECK(rtp_rtcp);
if (rtp_rtcp->RemoteNTP(&info.capture_time_ntp_secs,
&info.capture_time_ntp_frac,
nullptr,
nullptr,
&info.capture_time_source_clock) != 0) {
return rtc::Optional<Syncable::Info>();
}
info.current_delay_ms = video_receiver_.Delay();
return rtc::Optional<Syncable::Info>(info);
}
uint32_t VideoReceiveStream::GetPlayoutTimestamp() const {
RTC_NOTREACHED();
return 0;
}
void VideoReceiveStream::SetMinimumPlayoutDelay(int delay_ms) {
RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
video_receiver_.SetMinimumPlayoutDelay(delay_ms);
}
bool VideoReceiveStream::DecodeThreadFunction(void* ptr) {
static_cast<VideoReceiveStream*>(ptr)->Decode();
return true;
@ -489,6 +532,5 @@ void VideoReceiveStream::Decode() {
video_receiver_.Decode(kMaxDecodeWaitTimeMs);
}
}
} // namespace internal
} // namespace webrtc

18
webrtc/video/video_receive_stream.h
View File

@ -14,6 +14,8 @@
#include <memory>
#include <vector>
#include "webrtc/base/thread_checker.h"
#include "webrtc/call/syncable.h"
#include "webrtc/common_video/include/incoming_video_stream.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/rtp_rtcp/include/flexfec_receiver.h"
@ -34,7 +36,6 @@ class CongestionController;
class IvfFileWriter;
class ProcessThread;
class RTPFragmentationHeader;
class VoiceEngine;
class VieRemb;
class VCMTiming;
class VCMJitterEstimator;
@ -46,14 +47,14 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream,
public EncodedImageCallback,
public NackSender,
public KeyFrameRequestSender,
public video_coding::OnCompleteFrameCallback {
public video_coding::OnCompleteFrameCallback,
public Syncable {
public:
VideoReceiveStream(int num_cpu_cores,
bool protected_by_flexfec,
CongestionController* congestion_controller,
PacketRouter* packet_router,
VideoReceiveStream::Config config,
webrtc::VoiceEngine* voice_engine,
ProcessThread* process_thread,
CallStats* call_stats,
VieRemb* remb);
@ -69,7 +70,7 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream,
bool OnRecoveredPacket(const uint8_t* packet, size_t length);
void SetSyncChannel(VoiceEngine* voice_engine, int audio_channel_id);
void SetSync(Syncable* audio_syncable);
// Implements webrtc::VideoReceiveStream.
void Start() override;
@ -104,10 +105,19 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream,
void OnCompleteFrame(
std::unique_ptr<video_coding::FrameObject> frame) override;
// Implements Syncable.
int id() const override;
rtc::Optional<Syncable::Info> GetInfo() const override;
uint32_t GetPlayoutTimestamp() const override;
void SetMinimumPlayoutDelay(int delay_ms) override;
private:
static bool DecodeThreadFunction(void* ptr);
void Decode();
rtc::ThreadChecker worker_thread_checker_;
rtc::ThreadChecker module_process_thread_checker_;
TransportAdapter transport_adapter_;
const VideoReceiveStream::Config config_;
const int num_cpu_cores_;

36
webrtc/voice_engine/channel_proxy.cc
View File

@ -256,6 +256,42 @@ void ChannelProxy::DisassociateSendChannel() {
channel()->set_associate_send_channel(ChannelOwner(nullptr));
}
void ChannelProxy::GetRtpRtcp(RtpRtcp** rtp_rtcp,
RtpReceiver** rtp_receiver) const {
// Called on Call's module_process_thread_.
RTC_DCHECK(rtp_rtcp);
RTC_DCHECK(rtp_receiver);
int error = channel()->GetRtpRtcp(rtp_rtcp, rtp_receiver);
RTC_DCHECK_EQ(0, error);
}
void ChannelProxy::GetDelayEstimate(int* jitter_buffer_delay_ms,
int* playout_buffer_delay_ms) const {
// Called on Call's module_process_thread_.
RTC_DCHECK(jitter_buffer_delay_ms);
RTC_DCHECK(playout_buffer_delay_ms);
bool error = channel()->GetDelayEstimate(jitter_buffer_delay_ms,
playout_buffer_delay_ms);
RTC_DCHECK(error);
}
uint32_t ChannelProxy::GetPlayoutTimestamp() const {
// Called on video capture thread.
unsigned int timestamp = 0;
int error = channel()->GetPlayoutTimestamp(timestamp);
RTC_DCHECK(!error || timestamp == 0);
return timestamp;
}
void ChannelProxy::SetMinimumPlayoutDelay(int delay_ms) {
// Called on Call's module_process_thread_.
// Limit to range accepted by both VoE and ACM, so we're at least getting as
// close as possible, instead of failing.
delay_ms = std::max(0, std::min(delay_ms, 10000));
int error = channel()->SetMinimumPlayoutDelay(delay_ms);
RTC_DCHECK_EQ(0, error);
}
void ChannelProxy::SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats) {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
channel()->SetRtcpRttStats(rtcp_rtt_stats);

8
webrtc/voice_engine/channel_proxy.h
View File

@ -29,6 +29,8 @@ class PacketRouter;
class RtcEventLog;
class RtcpRttStats;
class RtpPacketSender;
class RtpReceiver;
class RtpRtcp;
class Transport;
class TransportFeedbackObserver;
@ -99,6 +101,12 @@ class ChannelProxy {
virtual void SetTransportOverhead(int transport_overhead_per_packet);
virtual void AssociateSendChannel(const ChannelProxy& send_channel_proxy);
virtual void DisassociateSendChannel();
virtual void GetRtpRtcp(RtpRtcp** rtp_rtcp,
RtpReceiver** rtp_receiver) const;
virtual void GetDelayEstimate(int* jitter_buffer_delay_ms,
int* playout_buffer_delay_ms) const;
virtual uint32_t GetPlayoutTimestamp() const;
virtual void SetMinimumPlayoutDelay(int delay_ms);
virtual void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats);