diff --git a/webrtc/modules/video_coding/main/interface/video_coding.h b/webrtc/modules/video_coding/main/interface/video_coding.h index 2b3c94bff2..50d0c1ad20 100644 --- a/webrtc/modules/video_coding/main/interface/video_coding.h +++ b/webrtc/modules/video_coding/main/interface/video_coding.h @@ -376,6 +376,18 @@ public: virtual int32_t RegisterReceiveStatisticsCallback( VCMReceiveStatisticsCallback* receiveStats) = 0; + // Register a decoder timing callback which will be called to deliver + // information about the timing of the decoder in the receiving side of the + // VCM, for instance the current and maximum frame decode latency. + // + // Input: + // - decoderTiming : The callback object to register. + // + // Return value : VCM_OK, on success. + // < 0, on error. + virtual int32_t RegisterDecoderTimingCallback( + VCMDecoderTimingCallback* decoderTiming) = 0; + // Register a frame type request callback. This callback will be called when the // module needs to request specific frame types from the send side. // diff --git a/webrtc/modules/video_coding/main/interface/video_coding_defines.h b/webrtc/modules/video_coding/main/interface/video_coding_defines.h index ae1acd3100..fab91afd32 100644 --- a/webrtc/modules/video_coding/main/interface/video_coding_defines.h +++ b/webrtc/modules/video_coding/main/interface/video_coding_defines.h @@ -118,6 +118,21 @@ class VCMReceiveStatisticsCallback { } }; +// Callback class used for informing the user of decode timing info. +class VCMDecoderTimingCallback { + public: + virtual void OnDecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms) = 0; + + protected: + virtual ~VCMDecoderTimingCallback() {} +}; + // Callback class used for telling the user about how to configure the FEC, // and the rates sent the last second is returned to the VCM. class VCMProtectionCallback { diff --git a/webrtc/modules/video_coding/main/source/timing.cc b/webrtc/modules/video_coding/main/source/timing.cc index 97fdac98bc..98a69e962f 100644 --- a/webrtc/modules/video_coding/main/source/timing.cc +++ b/webrtc/modules/video_coding/main/source/timing.cc @@ -36,6 +36,7 @@ VCMTiming::VCMTiming(Clock* clock, min_playout_delay_ms_(0), jitter_delay_ms_(0), current_delay_ms_(0), + last_decode_ms_(0), prev_frame_timestamp_(0) { if (master_timing == NULL) { master_ = true; @@ -158,7 +159,7 @@ int32_t VCMTiming::StopDecodeTimer(uint32_t time_stamp, timing_id_), "Codec timer error: %d", time_diff_ms); assert(false); } - + last_decode_ms_ = time_diff_ms; if (master_) { WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, VCMId(vcm_id_, timing_id_), @@ -262,4 +263,21 @@ uint32_t VCMTiming::TargetDelayInternal() const { jitter_delay_ms_ + MaxDecodeTimeMs() + render_delay_ms_); } +void VCMTiming::GetTimings(int* decode_ms, + int* max_decode_ms, + int* current_delay_ms, + int* target_delay_ms, + int* jitter_buffer_ms, + int* min_playout_delay_ms, + int* render_delay_ms) const { + CriticalSectionScoped cs(crit_sect_); + *decode_ms = last_decode_ms_; + *max_decode_ms = MaxDecodeTimeMs(); + *current_delay_ms = current_delay_ms_; + *target_delay_ms = TargetDelayInternal(); + *jitter_buffer_ms = jitter_delay_ms_; + *min_playout_delay_ms = min_playout_delay_ms_; + *render_delay_ms = render_delay_ms_; +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/timing.h b/webrtc/modules/video_coding/main/source/timing.h index 3148a7a3cc..eb251b7115 100644 --- a/webrtc/modules/video_coding/main/source/timing.h +++ b/webrtc/modules/video_coding/main/source/timing.h @@ -82,6 +82,15 @@ class VCMTiming { // certain amount of processing time. bool EnoughTimeToDecode(uint32_t available_processing_time_ms) const; + // Return current timing information. + void GetTimings(int* decode_ms, + int* max_decode_ms, + int* current_delay_ms, + int* target_delay_ms, + int* jitter_buffer_ms, + int* min_playout_delay_ms, + int* render_delay_ms) const; + enum { kDefaultRenderDelayMs = 10 }; enum { kDelayMaxChangeMsPerS = 100 }; @@ -102,6 +111,7 @@ class VCMTiming { uint32_t min_playout_delay_ms_; uint32_t jitter_delay_ms_; uint32_t current_delay_ms_; + int last_decode_ms_; uint32_t prev_frame_timestamp_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.cc b/webrtc/modules/video_coding/main/source/video_coding_impl.cc index 922ecb4334..42cb5ad8ea 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.cc +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.cc @@ -237,6 +237,11 @@ class VideoCodingModuleImpl : public VideoCodingModule { return receiver_->RegisterReceiveStatisticsCallback(receiveStats); } + virtual int32_t RegisterDecoderTimingCallback( + VCMDecoderTimingCallback* decoderTiming) OVERRIDE { + return receiver_->RegisterDecoderTimingCallback(decoderTiming); + } + virtual int32_t RegisterFrameTypeCallback( VCMFrameTypeCallback* frameTypeCallback) OVERRIDE { return receiver_->RegisterFrameTypeCallback(frameTypeCallback); diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.h b/webrtc/modules/video_coding/main/source/video_coding_impl.h index 0605ae3078..795c6ce4b5 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.h +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.h @@ -137,6 +137,8 @@ class VideoReceiver { int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback); int32_t RegisterReceiveStatisticsCallback( VCMReceiveStatisticsCallback* receiveStats); + int32_t RegisterDecoderTimingCallback( + VCMDecoderTimingCallback* decoderTiming); int32_t RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback); int32_t RegisterPacketRequestCallback(VCMPacketRequestCallback* callback); int RegisterRenderBufferSizeCallback(VCMRenderBufferSizeCallback* callback); @@ -200,6 +202,7 @@ class VideoReceiver { VCMDecodedFrameCallback _dualDecodedFrameCallback; VCMFrameTypeCallback* _frameTypeCallback; VCMReceiveStatisticsCallback* _receiveStatsCallback; + VCMDecoderTimingCallback* _decoderTimingCallback; VCMPacketRequestCallback* _packetRequestCallback; VCMRenderBufferSizeCallback* render_buffer_callback_; VCMGenericDecoder* _decoder; diff --git a/webrtc/modules/video_coding/main/source/video_receiver.cc b/webrtc/modules/video_coding/main/source/video_receiver.cc index 7528aabfdf..cea8b448d5 100644 --- a/webrtc/modules/video_coding/main/source/video_receiver.cc +++ b/webrtc/modules/video_coding/main/source/video_receiver.cc @@ -40,6 +40,7 @@ VideoReceiver::VideoReceiver(const int32_t id, _dualDecodedFrameCallback(_dualTiming, clock_), _frameTypeCallback(NULL), _receiveStatsCallback(NULL), + _decoderTimingCallback(NULL), _packetRequestCallback(NULL), render_buffer_callback_(NULL), _decoder(NULL), @@ -85,6 +86,30 @@ int32_t VideoReceiver::Process() { _receiveStatsCallback->OnReceiveStatisticsUpdate(bitRate, frameRate); } + if (_decoderTimingCallback != NULL) { + int decode_ms; + int max_decode_ms; + int current_delay_ms; + int target_delay_ms; + int jitter_buffer_ms; + int min_playout_delay_ms; + int render_delay_ms; + _timing.GetTimings(&decode_ms, + &max_decode_ms, + ¤t_delay_ms, + &target_delay_ms, + &jitter_buffer_ms, + &min_playout_delay_ms, + &render_delay_ms); + _decoderTimingCallback->OnDecoderTiming(decode_ms, + max_decode_ms, + current_delay_ms, + target_delay_ms, + jitter_buffer_ms, + min_playout_delay_ms, + render_delay_ms); + } + // Size of render buffer. if (render_buffer_callback_) { int buffer_size_ms = _receiver.RenderBufferSizeMs(); @@ -255,6 +280,7 @@ int32_t VideoReceiver::InitializeReceiver() { _receiverInited = true; _frameTypeCallback = NULL; _receiveStatsCallback = NULL; + _decoderTimingCallback = NULL; _packetRequestCallback = NULL; _keyRequestMode = kKeyOnError; _scheduleKeyRequest = false; @@ -278,6 +304,13 @@ int32_t VideoReceiver::RegisterReceiveStatisticsCallback( return VCM_OK; } +int32_t VideoReceiver::RegisterDecoderTimingCallback( + VCMDecoderTimingCallback* decoderTiming) { + CriticalSectionScoped cs(process_crit_sect_.get()); + _decoderTimingCallback = decoderTiming; + return VCM_OK; +} + // Register an externally defined decoder/render object. // Can be a decoder only or a decoder coupled with a renderer. int32_t VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder, diff --git a/webrtc/video_engine/include/vie_codec.h b/webrtc/video_engine/include/vie_codec.h index 33ada28419..92d79b001e 100644 --- a/webrtc/video_engine/include/vie_codec.h +++ b/webrtc/video_engine/include/vie_codec.h @@ -62,6 +62,18 @@ class WEBRTC_DLLEXPORT ViEDecoderObserver { const unsigned int framerate, const unsigned int bitrate) = 0; + // Called periodically with decoder timing information. All values are + // "current" snapshots unless decorated with a min_/max_ prefix. + // TODO(fischman): drop the do-nothing default impl. when + // WebRtcDecoderObserver is updated. + virtual void DecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms) {} + // This method is called when the decoder needs a new key frame from encoder // on the sender. virtual void RequestNewKeyFrame(const int video_channel) = 0; diff --git a/webrtc/video_engine/test/android/jni/vie_android_java_api.cc b/webrtc/video_engine/test/android/jni/vie_android_java_api.cc index b04a34e9f8..4e7d96b4f3 100644 --- a/webrtc/video_engine/test/android/jni/vie_android_java_api.cc +++ b/webrtc/video_engine/test/android/jni/vie_android_java_api.cc @@ -188,7 +188,17 @@ class VideoCallbackAndroid: public ViEDecoderObserver, packetLossRate, _frameRateO, _bitRateO); webrtcGlobalVM->DetachCurrentThread(); } - ; + + virtual void DecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms) + { + // TODO(fischman): consider plumbing this through to Java. + } virtual void IncomingCodecChanged(const int videoChannel, const webrtc::VideoCodec& videoCodec) @@ -207,12 +217,10 @@ class VideoCallbackAndroid: public ViEDecoderObserver, videoCodec.width, videoCodec.height); webrtcGlobalVM->DetachCurrentThread(); } - ; virtual void RequestNewKeyFrame(const int videoChannel) { } - ; virtual void OutgoingRate(const int videoChannel, const unsigned int framerate, @@ -223,7 +231,6 @@ class VideoCallbackAndroid: public ViEDecoderObserver, //__android_log_print(ANDROID_LOG_DEBUG, WEBRTC_LOG_TAG, // "SendRate frameRate %d bitrate %d\n",frameRate,bitrate); } - ; public: VideoEngineData& _vieData; diff --git a/webrtc/video_engine/test/auto_test/source/vie_autotest_codec.cc b/webrtc/video_engine/test/auto_test/source/vie_autotest_codec.cc index 84ae14cb35..3fa2bc42eb 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_autotest_codec.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_autotest_codec.cc @@ -28,12 +28,12 @@ #include "webrtc/video_engine/test/libvietest/include/tb_video_channel.h" #include "webrtc/voice_engine/include/voe_base.h" -class TestCodecObserver - : public webrtc::ViEEncoderObserver, - public webrtc::ViEDecoderObserver { - public: +class TestCodecObserver : public webrtc::ViEEncoderObserver, + public webrtc::ViEDecoderObserver { + public: int incoming_codec_called_; int incoming_rate_called_; + int decoder_timing_called_; int outgoing_rate_called_; unsigned char last_payload_type_; @@ -51,6 +51,7 @@ class TestCodecObserver TestCodecObserver() : incoming_codec_called_(0), incoming_rate_called_(0), + decoder_timing_called_(0), outgoing_rate_called_(0), last_payload_type_(0), last_width_(0), @@ -80,6 +81,17 @@ class TestCodecObserver last_incoming_bitrate_ += bitrate; } + virtual void DecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms) { + ++decoder_timing_called_; + // TODO(fischman): anything useful to be done with the data here? + } + virtual void OutgoingRate(const int video_channel, const unsigned int framerate, const unsigned int bitrate) { @@ -257,6 +269,7 @@ void ViEAutoTest::ViECodecStandardTest() { EXPECT_GT(codec_observer.incoming_codec_called_, 0); EXPECT_GT(codec_observer.incoming_rate_called_, 0); + EXPECT_GT(codec_observer.decoder_timing_called_, 0); EXPECT_GT(codec_observer.outgoing_rate_called_, 0); EXPECT_EQ(0, base->StopReceive(video_channel)); diff --git a/webrtc/video_engine/test/auto_test/source/vie_autotest_custom_call.cc b/webrtc/video_engine/test/auto_test/source/vie_autotest_custom_call.cc index 103e422ddb..8d7613d690 100644 --- a/webrtc/video_engine/test/auto_test/source/vie_autotest_custom_call.cc +++ b/webrtc/video_engine/test/auto_test/source/vie_autotest_custom_call.cc @@ -89,6 +89,23 @@ class ViEAutotestDecoderObserver : public webrtc::ViEDecoderObserver { std::cout << "Received FR: " << framerate << " BR: " << bitrate << std::endl; } + + virtual void DecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms) { + std::cout << "Decoder timing: DecodeMS: " << decode_ms + << ", MaxDecodeMS: " << max_decode_ms + << ", CurrentDelayMS: " << current_delay_ms + << ", TargetDelayMS: " << target_delay_ms + << ", JitterBufferMS: " << jitter_buffer_ms + << ", MinPlayoutDelayMS: " << min_playout_delay_ms + << ", RenderDelayMS: " << render_delay_ms; + } + void IncomingCodecChanged(const int video_channel, const webrtc::VideoCodec& codec) {} void RequestNewKeyFrame(const int video_channel) { diff --git a/webrtc/video_engine/vie_channel.cc b/webrtc/video_engine/vie_channel.cc index dabccefa01..b766a64321 100644 --- a/webrtc/video_engine/vie_channel.cc +++ b/webrtc/video_engine/vie_channel.cc @@ -189,6 +189,11 @@ int32_t ViEChannel::Init() { "%s: VCM::RegisterReceiveStatisticsCallback failure", __FUNCTION__); } + if (vcm_.RegisterDecoderTimingCallback(this) != 0) { + WEBRTC_TRACE(kTraceWarning, kTraceVideo, ViEId(engine_id_, channel_id_), + "%s: VCM::RegisterDecoderTimingCallback failure", + __FUNCTION__); + } if (vcm_.SetRenderDelay(kViEDefaultRenderDelayMs) != 0) { WEBRTC_TRACE(kTraceWarning, kTraceVideo, ViEId(engine_id_, channel_id_), "%s: VCM::SetRenderDelay failure", __FUNCTION__); @@ -1651,6 +1656,25 @@ int32_t ViEChannel::OnReceiveStatisticsUpdate(const uint32_t bit_rate, return 0; } +void ViEChannel::OnDecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms) { + CriticalSectionScoped cs(callback_cs_.get()); + if (!codec_observer_) + return; + codec_observer_->DecoderTiming(decode_ms, + max_decode_ms, + current_delay_ms, + target_delay_ms, + jitter_buffer_ms, + min_playout_delay_ms, + render_delay_ms); +} + int32_t ViEChannel::RequestKeyFrame() { WEBRTC_TRACE(kTraceStream, kTraceVideo, ViEId(engine_id_, channel_id_), "%s", __FUNCTION__); diff --git a/webrtc/video_engine/vie_channel.h b/webrtc/video_engine/vie_channel.h index 40ceb1cd6a..f774311624 100644 --- a/webrtc/video_engine/vie_channel.h +++ b/webrtc/video_engine/vie_channel.h @@ -53,6 +53,7 @@ class ViEChannel : public VCMFrameTypeCallback, public VCMReceiveCallback, public VCMReceiveStatisticsCallback, + public VCMDecoderTimingCallback, public VCMPacketRequestCallback, public RtcpFeedback, public RtpFeedback, @@ -282,10 +283,19 @@ class ViEChannel // Implements VCMReceiveCallback. virtual void IncomingCodecChanged(const VideoCodec& codec); - // Implements VideoReceiveStatisticsCallback. + // Implements VCMReceiveStatisticsCallback. virtual int32_t OnReceiveStatisticsUpdate(const uint32_t bit_rate, const uint32_t frame_rate); + // Implements VCMDecoderTimingCallback. + virtual void OnDecoderTiming(int decode_ms, + int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms); + // Implements VideoFrameTypeCallback. virtual int32_t RequestKeyFrame();