diff --git a/talk/app/webrtc/peerconnectioninterface.h b/talk/app/webrtc/peerconnectioninterface.h index b7814e5830..89dd5f1bd6 100644 --- a/talk/app/webrtc/peerconnectioninterface.h +++ b/talk/app/webrtc/peerconnectioninterface.h @@ -256,7 +256,7 @@ class PeerConnectionInterface : public rtc::RefCountInterface { int ice_connection_receiving_timeout; ContinualGatheringPolicy continual_gathering_policy; std::vector> certificates; - + bool disable_prerenderer_smoothing; RTCConfiguration() : type(kAll), bundle_policy(kBundlePolicyBalanced), @@ -265,7 +265,8 @@ class PeerConnectionInterface : public rtc::RefCountInterface { audio_jitter_buffer_max_packets(kAudioJitterBufferMaxPackets), audio_jitter_buffer_fast_accelerate(false), ice_connection_receiving_timeout(kUndefined), - continual_gathering_policy(GATHER_ONCE) {} + continual_gathering_policy(GATHER_ONCE), + disable_prerenderer_smoothing(false) {} }; struct RTCOfferAnswerOptions { diff --git a/talk/app/webrtc/webrtcsession.cc b/talk/app/webrtc/webrtcsession.cc index 420477dbed..737379e875 100644 --- a/talk/app/webrtc/webrtcsession.cc +++ b/talk/app/webrtc/webrtcsession.cc @@ -593,6 +593,8 @@ bool WebRtcSession::Initialize( const PeerConnectionInterface::RTCConfiguration& rtc_configuration) { bundle_policy_ = rtc_configuration.bundle_policy; rtcp_mux_policy_ = rtc_configuration.rtcp_mux_policy; + video_options_.disable_prerenderer_smoothing = + rtc::Optional(rtc_configuration.disable_prerenderer_smoothing); transport_controller_->SetSslMaxProtocolVersion(options.ssl_max_version); // Obtain a certificate from RTCConfiguration if any were provided (optional). diff --git a/talk/media/base/mediachannel.h b/talk/media/base/mediachannel.h index fb828ef6c6..3f777b3bbc 100644 --- a/talk/media/base/mediachannel.h +++ b/talk/media/base/mediachannel.h @@ -267,6 +267,8 @@ struct VideoOptions { change.unsignalled_recv_stream_limit); SetFrom(&use_simulcast_adapter, change.use_simulcast_adapter); SetFrom(&screencast_min_bitrate, change.screencast_min_bitrate); + SetFrom(&disable_prerenderer_smoothing, + change.disable_prerenderer_smoothing); } bool operator==(const VideoOptions& o) const { @@ -293,7 +295,8 @@ struct VideoOptions { suspend_below_min_bitrate == o.suspend_below_min_bitrate && unsignalled_recv_stream_limit == o.unsignalled_recv_stream_limit && use_simulcast_adapter == o.use_simulcast_adapter && - screencast_min_bitrate == o.screencast_min_bitrate; + screencast_min_bitrate == o.screencast_min_bitrate && + disable_prerenderer_smoothing == o.disable_prerenderer_smoothing; } std::string ToString() const { @@ -379,6 +382,13 @@ struct VideoOptions { rtc::Optional use_simulcast_adapter; // Force screencast to use a minimum bitrate rtc::Optional screencast_min_bitrate; + // Set to true if the renderer has an algorithm of frame selection. + // If the value is true, then WebRTC will hand over a frame as soon as + // possible without delay, and rendering smoothness is completely the duty + // of the renderer; + // If the value is false, then WebRTC is responsible to delay frame release + // in order to increase rendering smoothness. + rtc::Optional disable_prerenderer_smoothing; private: template diff --git a/talk/media/webrtc/webrtcvideoengine2.cc b/talk/media/webrtc/webrtcvideoengine2.cc index 2d648b73a3..2f07687711 100644 --- a/talk/media/webrtc/webrtcvideoengine2.cc +++ b/talk/media/webrtc/webrtcvideoengine2.cc @@ -1225,7 +1225,7 @@ bool WebRtcVideoChannel2::AddRecvStream(const StreamParams& sp, receive_streams_[ssrc] = new WebRtcVideoReceiveStream( call_, sp, config, external_decoder_factory_, default_stream, - recv_codecs_); + recv_codecs_, options_.disable_prerenderer_smoothing.value_or(false)); return true; } @@ -2335,7 +2335,8 @@ WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( const webrtc::VideoReceiveStream::Config& config, WebRtcVideoDecoderFactory* external_decoder_factory, bool default_stream, - const std::vector& recv_codecs) + const std::vector& recv_codecs, + bool disable_prerenderer_smoothing) : call_(call), ssrcs_(sp.ssrcs), ssrc_groups_(sp.ssrc_groups), @@ -2343,6 +2344,7 @@ WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( default_stream_(default_stream), config_(config), external_decoder_factory_(external_decoder_factory), + disable_prerenderer_smoothing_(disable_prerenderer_smoothing), renderer_(NULL), last_width_(-1), last_height_(-1), @@ -2558,6 +2560,11 @@ bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsTextureSupported() const { return true; } +bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::SmoothsRenderedFrames() + const { + return disable_prerenderer_smoothing_; +} + bool WebRtcVideoChannel2::WebRtcVideoReceiveStream::IsDefaultStream() const { return default_stream_; } diff --git a/talk/media/webrtc/webrtcvideoengine2.h b/talk/media/webrtc/webrtcvideoengine2.h index 85e79281f8..c5dc88c034 100644 --- a/talk/media/webrtc/webrtcvideoengine2.h +++ b/talk/media/webrtc/webrtcvideoengine2.h @@ -394,7 +394,8 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, const webrtc::VideoReceiveStream::Config& config, WebRtcVideoDecoderFactory* external_decoder_factory, bool default_stream, - const std::vector& recv_codecs); + const std::vector& recv_codecs, + bool disable_prerenderer_smoothing); ~WebRtcVideoReceiveStream(); const std::vector& GetSsrcs() const; @@ -409,6 +410,7 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, void RenderFrame(const webrtc::VideoFrame& frame, int time_to_render_ms) override; bool IsTextureSupported() const override; + bool SmoothsRenderedFrames() const override; bool IsDefaultStream() const; void SetRenderer(cricket::VideoRenderer* renderer); @@ -449,6 +451,8 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, WebRtcVideoDecoderFactory* const external_decoder_factory_; std::vector allocated_decoders_; + const bool disable_prerenderer_smoothing_; + rtc::CriticalSection renderer_lock_; cricket::VideoRenderer* renderer_ GUARDED_BY(renderer_lock_); int last_width_ GUARDED_BY(renderer_lock_); diff --git a/webrtc/common_video/include/incoming_video_stream.h b/webrtc/common_video/include/incoming_video_stream.h index 93bc1baf5b..3fa1424c99 100644 --- a/webrtc/common_video/include/incoming_video_stream.h +++ b/webrtc/common_video/include/incoming_video_stream.h @@ -19,6 +19,7 @@ namespace webrtc { class CriticalSectionWrapper; class EventTimerWrapper; class PlatformThread; +class VideoRenderer; class VideoRenderCallback { public: @@ -31,7 +32,7 @@ class VideoRenderCallback { class IncomingVideoStream : public VideoRenderCallback { public: - explicit IncomingVideoStream(uint32_t stream_id); + IncomingVideoStream(uint32_t stream_id, bool disable_prerenderer_smoothing); ~IncomingVideoStream(); // Get callback to deliver frames to the module. @@ -72,7 +73,10 @@ class IncomingVideoStream : public VideoRenderCallback { enum { kEventMaxWaitTimeMs = 100 }; enum { kFrameRatePeriodMs = 1000 }; + void DeliverFrame(const VideoFrame& video_frame); + uint32_t const stream_id_; + const bool disable_prerenderer_smoothing_; // Critsects in allowed to enter order. const rtc::scoped_ptr stream_critsect_; const rtc::scoped_ptr thread_critsect_; diff --git a/webrtc/common_video/incoming_video_stream.cc b/webrtc/common_video/incoming_video_stream.cc index 78fc38695c..010a57ed5b 100644 --- a/webrtc/common_video/incoming_video_stream.cc +++ b/webrtc/common_video/incoming_video_stream.cc @@ -28,11 +28,14 @@ #include "webrtc/system_wrappers/include/event_wrapper.h" #include "webrtc/system_wrappers/include/tick_util.h" #include "webrtc/system_wrappers/include/trace.h" +#include "webrtc/video_renderer.h" namespace webrtc { -IncomingVideoStream::IncomingVideoStream(uint32_t stream_id) +IncomingVideoStream::IncomingVideoStream(uint32_t stream_id, + bool disable_prerenderer_smoothing) : stream_id_(stream_id), + disable_prerenderer_smoothing_(disable_prerenderer_smoothing), stream_critsect_(CriticalSectionWrapper::CreateCriticalSection()), thread_critsect_(CriticalSectionWrapper::CreateCriticalSection()), buffer_critsect_(CriticalSectionWrapper::CreateCriticalSection()), @@ -49,8 +52,7 @@ IncomingVideoStream::IncomingVideoStream(uint32_t stream_id) temp_frame_(), start_image_(), timeout_image_(), - timeout_time_() { -} + timeout_time_() {} IncomingVideoStream::~IncomingVideoStream() { Stop(); @@ -80,11 +82,15 @@ int32_t IncomingVideoStream::RenderFrame(const uint32_t stream_id, last_rate_calculation_time_ms_ = now_ms; } - // Insert frame. - CriticalSectionScoped csB(buffer_critsect_.get()); - if (render_buffers_->AddFrame(video_frame) == 1) - deliver_buffer_event_->Set(); - + // Hand over or insert frame. + if (disable_prerenderer_smoothing_) { + DeliverFrame(video_frame); + } else { + CriticalSectionScoped csB(buffer_critsect_.get()); + if (render_buffers_->AddFrame(video_frame) == 1) { + deliver_buffer_event_->Set(); + } + } return 0; } @@ -128,22 +134,23 @@ int32_t IncomingVideoStream::Start() { return 0; } - CriticalSectionScoped csT(thread_critsect_.get()); - assert(incoming_render_thread_ == NULL); + if (!disable_prerenderer_smoothing_) { + CriticalSectionScoped csT(thread_critsect_.get()); + assert(incoming_render_thread_ == NULL); - incoming_render_thread_ = PlatformThread::CreateThread( - IncomingVideoStreamThreadFun, this, "IncomingVideoStreamThread"); - if (!incoming_render_thread_) { - return -1; + incoming_render_thread_ = PlatformThread::CreateThread( + IncomingVideoStreamThreadFun, this, "IncomingVideoStreamThread"); + if (!incoming_render_thread_) { + return -1; + } + + if (incoming_render_thread_->Start()) { + } else { + return -1; + } + incoming_render_thread_->SetPriority(kRealtimePriority); + deliver_buffer_event_->StartTimer(false, kEventStartupTimeMs); } - - if (incoming_render_thread_->Start()) { - } else { - return -1; - } - incoming_render_thread_->SetPriority(kRealtimePriority); - deliver_buffer_event_->StartTimer(false, kEventStartupTimeMs); - running_ = true; return 0; } @@ -205,6 +212,7 @@ bool IncomingVideoStream::IncomingVideoStreamProcess() { // Terminating return false; } + // Get a new frame to render and the time for the frame after this one. VideoFrame frame_to_render; uint32_t wait_time; @@ -220,37 +228,41 @@ bool IncomingVideoStream::IncomingVideoStreamProcess() { } deliver_buffer_event_->StartTimer(false, wait_time); - if (frame_to_render.IsZeroSize()) { - if (render_callback_) { - if (last_render_time_ms_ == 0 && !start_image_.IsZeroSize()) { - // We have not rendered anything and have a start image. - temp_frame_.CopyFrame(start_image_); - render_callback_->RenderFrame(stream_id_, temp_frame_); - } else if (!timeout_image_.IsZeroSize() && - last_render_time_ms_ + timeout_time_ < - TickTime::MillisecondTimestamp()) { - // Render a timeout image. - temp_frame_.CopyFrame(timeout_image_); - render_callback_->RenderFrame(stream_id_, temp_frame_); - } - } - - // No frame. - return true; - } - - // Send frame for rendering. - if (external_callback_) { - external_callback_->RenderFrame(stream_id_, frame_to_render); - } else if (render_callback_) { - render_callback_->RenderFrame(stream_id_, frame_to_render); - } - - // We're done with this frame. - if (!frame_to_render.IsZeroSize()) - last_render_time_ms_ = frame_to_render.render_time_ms(); + DeliverFrame(frame_to_render); } return true; } +void IncomingVideoStream::DeliverFrame(const VideoFrame& video_frame) { + CriticalSectionScoped cs(thread_critsect_.get()); + if (video_frame.IsZeroSize()) { + if (render_callback_) { + if (last_render_time_ms_ == 0 && !start_image_.IsZeroSize()) { + // We have not rendered anything and have a start image. + temp_frame_.CopyFrame(start_image_); + render_callback_->RenderFrame(stream_id_, temp_frame_); + } else if (!timeout_image_.IsZeroSize() && + last_render_time_ms_ + timeout_time_ < + TickTime::MillisecondTimestamp()) { + // Render a timeout image. + temp_frame_.CopyFrame(timeout_image_); + render_callback_->RenderFrame(stream_id_, temp_frame_); + } + } + + // No frame. + return; + } + + // Send frame for rendering. + if (external_callback_) { + external_callback_->RenderFrame(stream_id_, video_frame); + } else if (render_callback_) { + render_callback_->RenderFrame(stream_id_, video_frame); + } + + // We're done with this frame. + last_render_time_ms_ = video_frame.render_time_ms(); +} + } // namespace webrtc diff --git a/webrtc/modules/video_render/video_render_impl.cc b/webrtc/modules/video_render/video_render_impl.cc index daa64bc0da..d2a074b4c4 100644 --- a/webrtc/modules/video_render/video_render_impl.cc +++ b/webrtc/modules/video_render/video_render_impl.cc @@ -197,7 +197,8 @@ ModuleVideoRenderImpl::AddIncomingRenderStream(const uint32_t streamId, } // Create platform independant code - IncomingVideoStream* ptrIncomingStream = new IncomingVideoStream(streamId); + IncomingVideoStream* ptrIncomingStream = + new IncomingVideoStream(streamId, false); ptrIncomingStream->SetRenderCallback(ptrRenderCallback); VideoRenderCallback* moduleCallback = ptrIncomingStream->ModuleCallback(); diff --git a/webrtc/modules/video_render/video_render_internal_impl.cc b/webrtc/modules/video_render/video_render_internal_impl.cc index b6c2441425..1fed26e9c4 100644 --- a/webrtc/modules/video_render/video_render_internal_impl.cc +++ b/webrtc/modules/video_render/video_render_internal_impl.cc @@ -420,7 +420,8 @@ ModuleVideoRenderImpl::AddIncomingRenderStream(const uint32_t streamId, } // Create platform independant code - IncomingVideoStream* ptrIncomingStream = new IncomingVideoStream(streamId); + IncomingVideoStream* ptrIncomingStream = + new IncomingVideoStream(streamId, false); ptrIncomingStream->SetRenderCallback(ptrRenderCallback); VideoRenderCallback* moduleCallback = ptrIncomingStream->ModuleCallback(); diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index 683e89aa97..1c0ef78290 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -272,7 +272,8 @@ VideoReceiveStream::VideoReceiveStream( RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec)); } - incoming_video_stream_.reset(new IncomingVideoStream(0)); + incoming_video_stream_.reset(new IncomingVideoStream( + 0, config.renderer ? config.renderer->SmoothsRenderedFrames() : false)); incoming_video_stream_->SetExpectedRenderDelay(config.render_delay_ms); incoming_video_stream_->SetExternalCallback(this); vie_channel_->SetIncomingVideoStream(incoming_video_stream_.get()); diff --git a/webrtc/video_renderer.h b/webrtc/video_renderer.h index fedd28b22e..621e705825 100644 --- a/webrtc/video_renderer.h +++ b/webrtc/video_renderer.h @@ -25,6 +25,11 @@ class VideoRenderer { virtual bool IsTextureSupported() const = 0; + // This function returns true if WebRTC should not delay frames for + // smoothness. In general, this case means the renderer can schedule frames to + // optimize smoothness. + virtual bool SmoothsRenderedFrames() const { return false; } + protected: virtual ~VideoRenderer() {} };