From 444682acf9804c5fcbddaded9e900ba3cc6921fc Mon Sep 17 00:00:00 2001 From: qiangchen Date: Tue, 24 Nov 2015 18:07:56 -0800 Subject: [PATCH] Remove frame time scheduing in IncomingVideoStream This is part of the project that makes RTC rendering more smooth. We've already finished the developement of the frame selection algorithm in WebMediaPlayerMS, where we managed a frame pool, and based on the vsync interval, we actively select the best frame to render in order to maximize the rendering smoothness. Thus the frame timeline control in IncomingVideoStream is no longer needed, because with sophisticated frame selection algorithm in WebMediaPlayerMS, the time control in IncomingVideoStream will do nothing but add some extra delay. BUG=514873 Review URL: https://codereview.webrtc.org/1419673014 Cr-Commit-Position: refs/heads/master@{#10781} --- talk/app/webrtc/peerconnectioninterface.h | 5 +- talk/app/webrtc/webrtcsession.cc | 2 + talk/media/base/mediachannel.h | 12 +- talk/media/webrtc/webrtcvideoengine2.cc | 11 +- talk/media/webrtc/webrtcvideoengine2.h | 6 +- .../include/incoming_video_stream.h | 6 +- webrtc/common_video/incoming_video_stream.cc | 114 ++++++++++-------- .../modules/video_render/video_render_impl.cc | 3 +- .../video_render_internal_impl.cc | 3 +- webrtc/video/video_receive_stream.cc | 3 +- webrtc/video_renderer.h | 5 + 11 files changed, 109 insertions(+), 61 deletions(-) diff --git a/talk/app/webrtc/peerconnectioninterface.h b/talk/app/webrtc/peerconnectioninterface.h index b7814e5830..89dd5f1bd6 100644 --- a/talk/app/webrtc/peerconnectioninterface.h +++ b/talk/app/webrtc/peerconnectioninterface.h @@ -256,7 +256,7 @@ class PeerConnectionInterface : public rtc::RefCountInterface { int ice_connection_receiving_timeout; ContinualGatheringPolicy continual_gathering_policy; std::vector> certificates; - + bool disable_prerenderer_smoothing; RTCConfiguration() : type(kAll), bundle_policy(kBundlePolicyBalanced), @@ -265,7 +265,8 @@ class PeerConnectionInterface : public rtc::RefCountInterface { audio_jitter_buffer_max_packets(kAudioJitterBufferMaxPackets), audio_jitter_buffer_fast_accelerate(false), ice_connection_receiving_timeout(kUndefined), - continual_gathering_policy(GATHER_ONCE) {} + continual_gathering_policy(GATHER_ONCE), + disable_prerenderer_smoothing(false) {} }; struct RTCOfferAnswerOptions { diff --git a/talk/app/webrtc/webrtcsession.cc b/talk/app/webrtc/webrtcsession.cc index 420477dbed..737379e875 100644 --- a/talk/app/webrtc/webrtcsession.cc +++ b/talk/app/webrtc/webrtcsession.cc @@ -593,6 +593,8 @@ bool WebRtcSession::Initialize( const PeerConnectionInterface::RTCConfiguration& rtc_configuration) { bundle_policy_ = rtc_configuration.bundle_policy; rtcp_mux_policy_ = rtc_configuration.rtcp_mux_policy; + video_options_.disable_prerenderer_smoothing = + rtc::Optional(rtc_configuration.disable_prerenderer_smoothing); transport_controller_->SetSslMaxProtocolVersion(options.ssl_max_version); // Obtain a certificate from RTCConfiguration if any were provided (optional). diff --git a/talk/media/base/mediachannel.h b/talk/media/base/mediachannel.h index fb828ef6c6..3f777b3bbc 100644 --- a/talk/media/base/mediachannel.h +++ b/talk/media/base/mediachannel.h @@ -267,6 +267,8 @@ struct VideoOptions { change.unsignalled_recv_stream_limit); SetFrom(&use_simulcast_adapter, change.use_simulcast_adapter); SetFrom(&screencast_min_bitrate, change.screencast_min_bitrate); + SetFrom(&disable_prerenderer_smoothing, + change.disable_prerenderer_smoothing); } bool operator==(const VideoOptions& o) const { @@ -293,7 +295,8 @@ struct VideoOptions { suspend_below_min_bitrate == o.suspend_below_min_bitrate && unsignalled_recv_stream_limit == o.unsignalled_recv_stream_limit && use_simulcast_adapter == o.use_simulcast_adapter && - screencast_min_bitrate == o.screencast_min_bitrate; + screencast_min_bitrate == o.screencast_min_bitrate && + disable_prerenderer_smoothing == o.disable_prerenderer_smoothing; } std::string ToString() const { @@ -379,6 +382,13 @@ struct VideoOptions { rtc::Optional use_simulcast_adapter; // Force screencast to use a minimum bitrate rtc::Optional screencast_min_bitrate; + // Set to true if the renderer has an algorithm of frame selection. + // If the value is true, then WebRTC will hand over a frame as soon as + // possible without delay, and rendering smoothness is completely the duty + // of the renderer; + // If the value is false, then WebRTC is responsible to delay frame release + // in order to increase rendering smoothness. + rtc::Optional disable_prerenderer_smoothing; private: template diff --git a/talk/media/webrtc/webrtcvideoengine2.cc b/talk/media/webrtc/webrtcvideoengine2.cc index 2d648b73a3..2f07687711 100644 --- a/talk/media/webrtc/webrtcvideoengine2.cc +++ b/talk/media/webrtc/webrtcvideoengine2.cc @@ -1225,7 +1225,7 @@ bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp, receive_streams_[ssrc] = new WebRtcVideoReceiveStream( call_, sp, config, external_decoder_factory_, default_stream, - recv_codecs_); + recv_codecs_, options_.disable_prerenderer_smoothing.value_or(false)); return true; } @@ -2335,7 +2335,8 @@ WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( const webrtc::VideoReceiveStream::Config& config, WebRtcVideoDecoderFactory* external_decoder_factory, bool default_stream, - const std::vector& recv_codecs) + const std::vector& recv_codecs, + bool disable_prerenderer_smoothing) : call_(call), ssrcs_(sp.ssrcs), ssrc_groups_(sp.ssrc_groups), @@ -2343,6 +2344,7 @@ WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( default_stream_(default_stream), config_(config), external_decoder_factory_(external_decoder_factory), + disable_prerenderer_smoothing_(disable_prerenderer_smoothing), renderer_(NULL), last_width_(-1), last_height_(-1), @@ -2558,6 +2560,11 @@ bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsTextureSupported() const { return true; } +bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::SmoothsRenderedFrames() + const { + return disable_prerenderer_smoothing_; +} + bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsDefaultStream() const { return default_stream_; } diff --git a/talk/media/webrtc/webrtcvideoengine2.h b/talk/media/webrtc/webrtcvideoengine2.h index 85e79281f8..c5dc88c034 100644 --- a/talk/media/webrtc/webrtcvideoengine2.h +++ b/talk/media/webrtc/webrtcvideoengine2.h @@ -394,7 +394,8 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, const webrtc::VideoReceiveStream::Config& config, WebRtcVideoDecoderFactory* external_decoder_factory, bool default_stream, - const std::vector& recv_codecs); + const std::vector& recv_codecs, + bool disable_prerenderer_smoothing); ~WebRtcVideoReceiveStream(); const std::vector& GetSsrcs() const; @@ -409,6 +410,7 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, void RenderFrame(const webrtc::VideoFrame& frame, int time_to_render_ms) override; bool IsTextureSupported() const override; + bool SmoothsRenderedFrames() const override; bool IsDefaultStream() const; void SetRenderer(cricket::VideoRenderer* renderer); @@ -449,6 +451,8 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, WebRtcVideoDecoderFactory* const external_decoder_factory_; std::vector allocated_decoders_; + const bool disable_prerenderer_smoothing_; + rtc::CriticalSection renderer_lock_; cricket::VideoRenderer* renderer_ GUARDED_BY(renderer_lock_); int last_width_ GUARDED_BY(renderer_lock_); diff --git a/webrtc/common_video/include/incoming_video_stream.h b/webrtc/common_video/include/incoming_video_stream.h index 93bc1baf5b..3fa1424c99 100644 --- a/webrtc/common_video/include/incoming_video_stream.h +++ b/webrtc/common_video/include/incoming_video_stream.h @@ -19,6 +19,7 @@ namespace webrtc { class CriticalSectionWrapper; class EventTimerWrapper; class PlatformThread; +class VideoRenderer; class VideoRenderCallback { public: @@ -31,7 +32,7 @@ class VideoRenderCallback { class IncomingVideoStream : public VideoRenderCallback { public: - explicit IncomingVideoStream(uint32_t stream_id); + IncomingVideoStream(uint32_t stream_id, bool disable_prerenderer_smoothing); ~IncomingVideoStream(); // Get callback to deliver frames to the module. @@ -72,7 +73,10 @@ class IncomingVideoStream : public VideoRenderCallback { enum { kEventMaxWaitTimeMs = 100 }; enum { kFrameRatePeriodMs = 1000 }; + void DeliverFrame(const VideoFrame& video_frame); + uint32_t const stream_id_; + const bool disable_prerenderer_smoothing_; // Critsects in allowed to enter order. const rtc::scoped_ptr stream_critsect_; const rtc::scoped_ptr thread_critsect_; diff --git a/webrtc/common_video/incoming_video_stream.cc b/webrtc/common_video/incoming_video_stream.cc index 78fc38695c..010a57ed5b 100644 --- a/webrtc/common_video/incoming_video_stream.cc +++ b/webrtc/common_video/incoming_video_stream.cc @@ -28,11 +28,14 @@ #include "webrtc/system_wrappers/include/event_wrapper.h" #include "webrtc/system_wrappers/include/tick_util.h" #include "webrtc/system_wrappers/include/trace.h" +#include "webrtc/video_renderer.h" namespace webrtc { -IncomingVideoStream::IncomingVideoStream(uint32_t stream_id) +IncomingVideoStream::IncomingVideoStream(uint32_t stream_id, + bool disable_prerenderer_smoothing) : stream_id_(stream_id), + disable_prerenderer_smoothing_(disable_prerenderer_smoothing), stream_critsect_(CriticalSectionWrapper::CreateCriticalSection()), thread_critsect_(CriticalSectionWrapper::CreateCriticalSection()), buffer_critsect_(CriticalSectionWrapper::CreateCriticalSection()), @@ -49,8 +52,7 @@ IncomingVideoStream::IncomingVideoStream(uint32_t stream_id) temp_frame_(), start_image_(), timeout_image_(), - timeout_time_() { -} + timeout_time_() {} IncomingVideoStream::~IncomingVideoStream() { Stop(); @@ -80,11 +82,15 @@ int32_t IncomingVideoStream::RenderFrame(const uint32_t stream_id, last_rate_calculation_time_ms_ = now_ms; } - // Insert frame. - CriticalSectionScoped csB(buffer_critsect_.get()); - if (render_buffers_->AddFrame(video_frame) == 1) - deliver_buffer_event_->Set(); - + // Hand over or insert frame. + if (disable_prerenderer_smoothing_) { + DeliverFrame(video_frame); + } else { + CriticalSectionScoped csB(buffer_critsect_.get()); + if (render_buffers_->AddFrame(video_frame) == 1) { + deliver_buffer_event_->Set(); + } + } return 0; } @@ -128,22 +134,23 @@ int32_t IncomingVideoStream::Start() { return 0; } - CriticalSectionScoped csT(thread_critsect_.get()); - assert(incoming_render_thread_ == NULL); + if (!disable_prerenderer_smoothing_) { + CriticalSectionScoped csT(thread_critsect_.get()); + assert(incoming_render_thread_ == NULL); - incoming_render_thread_ = PlatformThread::CreateThread( - IncomingVideoStreamThreadFun, this, "IncomingVideoStreamThread"); - if (!incoming_render_thread_) { - return -1; + incoming_render_thread_ = PlatformThread::CreateThread( + IncomingVideoStreamThreadFun, this, "IncomingVideoStreamThread"); + if (!incoming_render_thread_) { + return -1; + } + + if (incoming_render_thread_->Start()) { + } else { + return -1; + } + incoming_render_thread_->SetPriority(kRealtimePriority); + deliver_buffer_event_->StartTimer(false, kEventStartupTimeMs); } - - if (incoming_render_thread_->Start()) { - } else { - return -1; - } - incoming_render_thread_->SetPriority(kRealtimePriority); - deliver_buffer_event_->StartTimer(false, kEventStartupTimeMs); - running_ = true; return 0; } @@ -205,6 +212,7 @@ bool IncomingVideoStream::IncomingVideoStreamProcess() { // Terminating return false; } + // Get a new frame to render and the time for the frame after this one. VideoFrame frame_to_render; uint32_t wait_time; @@ -220,37 +228,41 @@ bool IncomingVideoStream::IncomingVideoStreamProcess() { } deliver_buffer_event_->StartTimer(false, wait_time); - if (frame_to_render.IsZeroSize()) { - if (render_callback_) { - if (last_render_time_ms_ == 0 && !start_image_.IsZeroSize()) { - // We have not rendered anything and have a start image. - temp_frame_.CopyFrame(start_image_); - render_callback_->RenderFrame(stream_id_, temp_frame_); - } else if (!timeout_image_.IsZeroSize() && - last_render_time_ms_ + timeout_time_ < - TickTime::MillisecondTimestamp()) { - // Render a timeout image. - temp_frame_.CopyFrame(timeout_image_); - render_callback_->RenderFrame(stream_id_, temp_frame_); - } - } - - // No frame. - return true; - } - - // Send frame for rendering. - if (external_callback_) { - external_callback_->RenderFrame(stream_id_, frame_to_render); - } else if (render_callback_) { - render_callback_->RenderFrame(stream_id_, frame_to_render); - } - - // We're done with this frame. - if (!frame_to_render.IsZeroSize()) - last_render_time_ms_ = frame_to_render.render_time_ms(); + DeliverFrame(frame_to_render); } return true; } +void IncomingVideoStream::DeliverFrame(const VideoFrame& video_frame) { + CriticalSectionScoped cs(thread_critsect_.get()); + if (video_frame.IsZeroSize()) { + if (render_callback_) { + if (last_render_time_ms_ == 0 && !start_image_.IsZeroSize()) { + // We have not rendered anything and have a start image. + temp_frame_.CopyFrame(start_image_); + render_callback_->RenderFrame(stream_id_, temp_frame_); + } else if (!timeout_image_.IsZeroSize() && + last_render_time_ms_ + timeout_time_ < + TickTime::MillisecondTimestamp()) { + // Render a timeout image. + temp_frame_.CopyFrame(timeout_image_); + render_callback_->RenderFrame(stream_id_, temp_frame_); + } + } + + // No frame. + return; + } + + // Send frame for rendering. + if (external_callback_) { + external_callback_->RenderFrame(stream_id_, video_frame); + } else if (render_callback_) { + render_callback_->RenderFrame(stream_id_, video_frame); + } + + // We're done with this frame. + last_render_time_ms_ = video_frame.render_time_ms(); +} + } // namespace webrtc diff --git a/webrtc/modules/video_render/video_render_impl.cc b/webrtc/modules/video_render/video_render_impl.cc index daa64bc0da..d2a074b4c4 100644 --- a/webrtc/modules/video_render/video_render_impl.cc +++ b/webrtc/modules/video_render/video_render_impl.cc @@ -197,7 +197,8 @@ ModuleVideoRenderImpl::AddIncomingRenderStream(const uint32_t streamId, } // Create platform independant code - IncomingVideoStream* ptrIncomingStream = new IncomingVideoStream(streamId); + IncomingVideoStream* ptrIncomingStream = + new IncomingVideoStream(streamId, false); ptrIncomingStream->SetRenderCallback(ptrRenderCallback); VideoRenderCallback* moduleCallback = ptrIncomingStream->ModuleCallback(); diff --git a/webrtc/modules/video_render/video_render_internal_impl.cc b/webrtc/modules/video_render/video_render_internal_impl.cc index b6c2441425..1fed26e9c4 100644 --- a/webrtc/modules/video_render/video_render_internal_impl.cc +++ b/webrtc/modules/video_render/video_render_internal_impl.cc @@ -420,7 +420,8 @@ ModuleVideoRenderImpl::AddIncomingRenderStream(const uint32_t streamId, } // Create platform independant code - IncomingVideoStream* ptrIncomingStream = new IncomingVideoStream(streamId); + IncomingVideoStream* ptrIncomingStream = + new IncomingVideoStream(streamId, false); ptrIncomingStream->SetRenderCallback(ptrRenderCallback); VideoRenderCallback* moduleCallback = ptrIncomingStream->ModuleCallback(); diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index 683e89aa97..1c0ef78290 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -272,7 +272,8 @@ VideoReceiveStream::VideoReceiveStream( RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec)); } - incoming_video_stream_.reset(new IncomingVideoStream(0)); + incoming_video_stream_.reset(new IncomingVideoStream( + 0, config.renderer ? config.renderer->SmoothsRenderedFrames() : false)); incoming_video_stream_->SetExpectedRenderDelay(config.render_delay_ms); incoming_video_stream_->SetExternalCallback(this); vie_channel_->SetIncomingVideoStream(incoming_video_stream_.get()); diff --git a/webrtc/video_renderer.h b/webrtc/video_renderer.h index fedd28b22e..621e705825 100644 --- a/webrtc/video_renderer.h +++ b/webrtc/video_renderer.h @@ -25,6 +25,11 @@ class VideoRenderer { virtual bool IsTextureSupported() const = 0; + // This function returns true if WebRTC should not delay frames for + // smoothness. In general, this case means the renderer can schedule frames to + // optimize smoothness. + virtual bool SmoothsRenderedFrames() const { return false; } + protected: virtual ~VideoRenderer() {} };