From fb45c6c103eb5fe594a54f4291726e8242f0ec38 Mon Sep 17 00:00:00 2001 From: brandtr Date: Fri, 27 Jan 2017 06:47:55 -0800 Subject: [PATCH] Inform jitter buffer about FlexFEC protection. This CL introduces a way for the VideoReceiveStreams to check whether they are protected by any FlexfecReceiveStreams. This is done in the VideoReceiveStream::Start() method, which then propagates this information down to the jitter buffer adaptation logic. BUG=webrtc:5654 Review-Url: https://codereview.webrtc.org/2649973005 Cr-Commit-Position: refs/heads/master@{#16328} --- webrtc/call/call.cc | 14 ++++++-- webrtc/call/call.h | 3 ++ webrtc/media/engine/webrtcvideoengine2.cc | 11 +++--- webrtc/test/call_test.cc | 20 +++++------ webrtc/video/video_receive_stream.cc | 43 ++++++++++++----------- webrtc/video/video_receive_stream.h | 2 ++ 6 files changed, 55 insertions(+), 38 deletions(-) diff --git a/webrtc/call/call.cc b/webrtc/call/call.cc index 9915a77258..88faf871a9 100644 --- a/webrtc/call/call.cc +++ b/webrtc/call/call.cc @@ -638,10 +638,18 @@ webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream( webrtc::VideoReceiveStream::Config configuration) { TRACE_EVENT0("webrtc", "Call::CreateVideoReceiveStream"); RTC_DCHECK(configuration_thread_checker_.CalledOnValidThread()); + + bool protected_by_flexfec = false; + { + ReadLockScoped read_lock(*receive_crit_); + protected_by_flexfec = + flexfec_receive_ssrcs_media_.find(configuration.rtp.remote_ssrc) != + flexfec_receive_ssrcs_media_.end(); + } VideoReceiveStream* receive_stream = new VideoReceiveStream( - num_cpu_cores_, congestion_controller_.get(), &packet_router_, - std::move(configuration), voice_engine(), module_process_thread_.get(), - call_stats_.get(), &remb_); + num_cpu_cores_, protected_by_flexfec, congestion_controller_.get(), + &packet_router_, std::move(configuration), voice_engine(), + module_process_thread_.get(), call_stats_.get(), &remb_); const webrtc::VideoReceiveStream::Config& config = receive_stream->config(); { diff --git a/webrtc/call/call.h b/webrtc/call/call.h index fdb6cff785..ec73bf7714 100644 --- a/webrtc/call/call.h +++ b/webrtc/call/call.h @@ -119,6 +119,9 @@ class Call { virtual void DestroyVideoReceiveStream( VideoReceiveStream* receive_stream) = 0; + // In order for a created VideoReceiveStream to be aware that it is + // protected by a FlexfecReceiveStream, the latter should be created before + // the former. virtual FlexfecReceiveStream* CreateFlexfecReceiveStream( const FlexfecReceiveStream::Config& config) = 0; virtual void DestroyFlexfecReceiveStream( diff --git a/webrtc/media/engine/webrtcvideoengine2.cc b/webrtc/media/engine/webrtcvideoengine2.cc index 40f486dc0f..4301bd0a93 100644 --- a/webrtc/media/engine/webrtcvideoengine2.cc +++ b/webrtc/media/engine/webrtcvideoengine2.cc @@ -2278,19 +2278,20 @@ void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetRecvParameters( } void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RecreateWebRtcStream() { + if (stream_) { + call_->DestroyVideoReceiveStream(stream_); + stream_ = nullptr; + } if (flexfec_stream_) { call_->DestroyFlexfecReceiveStream(flexfec_stream_); flexfec_stream_ = nullptr; } - if (stream_) { - call_->DestroyVideoReceiveStream(stream_); - } - stream_ = call_->CreateVideoReceiveStream(config_.Copy()); - stream_->Start(); if (IsFlexfecFieldTrialEnabled() && flexfec_config_.IsCompleteAndEnabled()) { flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_); flexfec_stream_->Start(); } + stream_ = call_->CreateVideoReceiveStream(config_.Copy()); + stream_->Start(); } void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders( diff --git a/webrtc/test/call_test.cc b/webrtc/test/call_test.cc index 9521d3430d..c1c7355b1b 100644 --- a/webrtc/test/call_test.cc +++ b/webrtc/test/call_test.cc @@ -100,6 +100,10 @@ void CallTest::RunBaseTest(BaseTest* test) { test->ModifyFlexfecConfigs(&flexfec_receive_configs_); } + if (num_flexfec_streams_ > 0) { + CreateFlexfecStreams(); + test->OnFlexfecStreamsCreated(flexfec_receive_streams_); + } if (num_video_streams_ > 0) { CreateVideoStreams(); test->OnVideoStreamsCreated(video_send_stream_, video_receive_streams_); @@ -108,10 +112,6 @@ void CallTest::RunBaseTest(BaseTest* test) { CreateAudioStreams(); test->OnAudioStreamsCreated(audio_send_stream_, audio_receive_streams_); } - if (num_flexfec_streams_ > 0) { - CreateFlexfecStreams(); - test->OnFlexfecStreamsCreated(flexfec_receive_streams_); - } if (num_video_streams_ > 0) { int width = kDefaultWidth; @@ -352,18 +352,18 @@ void CallTest::CreateFlexfecStreams() { } void CallTest::DestroyStreams() { - if (video_send_stream_) - sender_call_->DestroyVideoSendStream(video_send_stream_); - video_send_stream_ = nullptr; - for (VideoReceiveStream* video_recv_stream : video_receive_streams_) - receiver_call_->DestroyVideoReceiveStream(video_recv_stream); - if (audio_send_stream_) sender_call_->DestroyAudioSendStream(audio_send_stream_); audio_send_stream_ = nullptr; for (AudioReceiveStream* audio_recv_stream : audio_receive_streams_) receiver_call_->DestroyAudioReceiveStream(audio_recv_stream); + if (video_send_stream_) + sender_call_->DestroyVideoSendStream(video_send_stream_); + video_send_stream_ = nullptr; + for (VideoReceiveStream* video_recv_stream : video_receive_streams_) + receiver_call_->DestroyVideoReceiveStream(video_recv_stream); + for (FlexfecReceiveStream* flexfec_recv_stream : flexfec_receive_streams_) receiver_call_->DestroyFlexfecReceiveStream(flexfec_recv_stream); diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index 79bebebb04..5fd436d6a4 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -186,6 +186,7 @@ namespace internal { VideoReceiveStream::VideoReceiveStream( int num_cpu_cores, + bool protected_by_flexfec, CongestionController* congestion_controller, PacketRouter* packet_router, VideoReceiveStream::Config config, @@ -196,6 +197,7 @@ VideoReceiveStream::VideoReceiveStream( : transport_adapter_(config.rtcp_send_transport), config_(std::move(config)), num_cpu_cores_(num_cpu_cores), + protected_by_flexfec_(protected_by_flexfec), process_thread_(process_thread), clock_(Clock::GetRealTimeClock()), decode_thread_(DecodeThreadFunction, this, "DecodingThread"), @@ -204,21 +206,20 @@ VideoReceiveStream::VideoReceiveStream( timing_(new VCMTiming(clock_)), video_receiver_(clock_, nullptr, this, timing_.get(), this, this), stats_proxy_(&config_, clock_), - rtp_stream_receiver_( - &video_receiver_, - congestion_controller_->GetRemoteBitrateEstimator( - UseSendSideBwe(config_)), - &transport_adapter_, - call_stats_->rtcp_rtt_stats(), - packet_router, - remb, - &config_, - &stats_proxy_, - process_thread_, - this, // NackSender - this, // KeyFrameRequestSender - this, // OnCompleteFrameCallback - timing_.get()), + rtp_stream_receiver_(&video_receiver_, + congestion_controller_->GetRemoteBitrateEstimator( + UseSendSideBwe(config_)), + &transport_adapter_, + call_stats_->rtcp_rtt_stats(), + packet_router, + remb, + &config_, + &stats_proxy_, + process_thread_, + this, // NackSender + this, // KeyFrameRequestSender + this, // OnCompleteFrameCallback + timing_.get()), rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_), jitter_buffer_experiment_( field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") == @@ -297,12 +298,15 @@ void VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine, void VideoReceiveStream::Start() { if (decode_thread_.IsRunning()) return; + + bool protected_by_fec = + protected_by_flexfec_ || rtp_stream_receiver_.IsUlpfecEnabled(); + if (jitter_buffer_experiment_) { frame_buffer_->Start(); call_stats_->RegisterStatsObserver(&rtp_stream_receiver_); - if (rtp_stream_receiver_.IsRetransmissionsEnabled() && - rtp_stream_receiver_.IsUlpfecEnabled()) { + if (rtp_stream_receiver_.IsRetransmissionsEnabled() && protected_by_fec) { frame_buffer_->SetProtectionMode(kProtectionNackFEC); } } @@ -331,9 +335,8 @@ void VideoReceiveStream::Start() { video_stream_decoder_.reset(new VideoStreamDecoder( &video_receiver_, &rtp_stream_receiver_, &rtp_stream_receiver_, - rtp_stream_receiver_.IsRetransmissionsEnabled(), - rtp_stream_receiver_.IsUlpfecEnabled(), &stats_proxy_, renderer, - config_.pre_render_callback)); + rtp_stream_receiver_.IsRetransmissionsEnabled(), protected_by_fec, + &stats_proxy_, renderer, config_.pre_render_callback)); // Register the channel to receive stats updates. call_stats_->RegisterStatsObserver(video_stream_decoder_.get()); // Start the decode thread diff --git a/webrtc/video/video_receive_stream.h b/webrtc/video/video_receive_stream.h index 146d56255c..9960dc9589 100644 --- a/webrtc/video/video_receive_stream.h +++ b/webrtc/video/video_receive_stream.h @@ -49,6 +49,7 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, public video_coding::OnCompleteFrameCallback { public: VideoReceiveStream(int num_cpu_cores, + bool protected_by_flexfec, CongestionController* congestion_controller, PacketRouter* packet_router, VideoReceiveStream::Config config, @@ -110,6 +111,7 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, TransportAdapter transport_adapter_; const VideoReceiveStream::Config config_; const int num_cpu_cores_; + const bool protected_by_flexfec_; ProcessThread* const process_thread_; Clock* const clock_;