diff --git a/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h b/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h index 6c926d4794..b3af479421 100644 --- a/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h +++ b/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h @@ -48,6 +48,8 @@ class MockVideoEncoder : public VideoEncoder { class MockDecodedImageCallback : public DecodedImageCallback { public: MOCK_METHOD1(Decoded, int32_t(VideoFrame& decodedImage)); + MOCK_METHOD2(Decoded, int32_t(VideoFrame& decodedImage, + int64_t decode_time_ms)); MOCK_METHOD1(ReceivedDecodedReferenceFrame, int32_t(const uint64_t pictureId)); MOCK_METHOD1(ReceivedDecodedFrame, diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.h b/webrtc/modules/video_coding/codecs/test/videoprocessor.h index 0b094ae73e..582d903343 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor.h +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.h @@ -13,6 +13,7 @@ #include +#include "webrtc/base/checks.h" #include "webrtc/common_video/libyuv/include/scaler.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" @@ -248,6 +249,11 @@ class VideoProcessorImpl : public VideoProcessor { : video_processor_(vp) { } int32_t Decoded(webrtc::VideoFrame& image) override; + int32_t Decoded( + webrtc::VideoFrame& image, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1; + } private: VideoProcessorImpl* video_processor_; diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h index e4fc986545..2577badd83 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h @@ -14,6 +14,7 @@ #include #include +#include "webrtc/base/checks.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/common.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" @@ -123,7 +124,7 @@ class Vp8TestDecodedImageCallback : public DecodedImageCallback { Vp8TestDecodedImageCallback() : decoded_frames_(0) { } - virtual int32_t Decoded(VideoFrame& decoded_image) { + int32_t Decoded(VideoFrame& decoded_image) override { for (int i = 0; i < decoded_image.width(); ++i) { EXPECT_NEAR(kColorY, decoded_image.buffer(kYPlane)[i], 1); } @@ -136,6 +137,10 @@ class Vp8TestDecodedImageCallback : public DecodedImageCallback { decoded_frames_++; return 0; } + int32_t Decoded(VideoFrame& decoded_image, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1; + } int DecodedFrames() { return decoded_frames_; } diff --git a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index 5ec674f16a..7650a250ce 100644 --- a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -11,6 +11,7 @@ #include #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/checks.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" @@ -78,7 +79,11 @@ class Vp8UnitTestDecodeCompleteCallback : public webrtc::DecodedImageCallback { public: explicit Vp8UnitTestDecodeCompleteCallback(VideoFrame* frame) : decoded_frame_(frame), decode_complete(false) {} - int Decoded(webrtc::VideoFrame& frame); + int32_t Decoded(VideoFrame& frame) override; + int32_t Decoded(VideoFrame& frame, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1; + } bool DecodeComplete(); private: diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc index 5843d83fa7..ea7db5a7f6 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc @@ -9,6 +9,7 @@ */ #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/checks.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/common_video/interface/video_image.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" @@ -68,7 +69,11 @@ class Vp8SequenceCoderDecodeCallback : public webrtc::DecodedImageCallback { public: explicit Vp8SequenceCoderDecodeCallback(FILE* decoded_file) : decoded_file_(decoded_file) {} - int Decoded(webrtc::VideoFrame& frame); + int32_t Decoded(webrtc::VideoFrame& frame) override; + int32_t Decoded(webrtc::VideoFrame& frame, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1;; + } bool DecodeComplete(); private: diff --git a/webrtc/modules/video_coding/main/source/codec_timer.cc b/webrtc/modules/video_coding/main/source/codec_timer.cc index a462258813..57985c71e7 100644 --- a/webrtc/modules/video_coding/main/source/codec_timer.cc +++ b/webrtc/modules/video_coding/main/source/codec_timer.cc @@ -28,13 +28,6 @@ _history() Reset(); } -int32_t VCMCodecTimer::StopTimer(int64_t startTimeMs, int64_t nowMs) -{ - const int32_t timeDiff = static_cast(nowMs - startTimeMs); - MaxFilter(timeDiff, nowMs); - return timeDiff; -} - void VCMCodecTimer::Reset() { _filteredMax = 0; diff --git a/webrtc/modules/video_coding/main/source/codec_timer.h b/webrtc/modules/video_coding/main/source/codec_timer.h index cb7e813ba0..a7abeb850e 100644 --- a/webrtc/modules/video_coding/main/source/codec_timer.h +++ b/webrtc/modules/video_coding/main/source/codec_timer.h @@ -35,8 +35,8 @@ class VCMCodecTimer public: VCMCodecTimer(); - // Updates and returns the max filtered decode time. - int32_t StopTimer(int64_t startTimeMs, int64_t nowMs); + // Updates the max filtered decode time. + void MaxFilter(int32_t newDecodeTimeMs, int64_t nowMs); // Empty the list of timers. void Reset(); @@ -46,7 +46,6 @@ public: private: void UpdateMaxHistory(int32_t decodeTime, int64_t now); - void MaxFilter(int32_t newTime, int64_t nowMs); void ProcessHistory(int64_t nowMs); int32_t _filteredMax; diff --git a/webrtc/modules/video_coding/main/source/generic_decoder.cc b/webrtc/modules/video_coding/main/source/generic_decoder.cc index 8b2d3974de..4f4a09b058 100644 --- a/webrtc/modules/video_coding/main/source/generic_decoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_decoder.cc @@ -47,6 +47,11 @@ VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() } int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) { + return Decoded(decodedImage, -1); +} + +int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, + int64_t decode_time_ms) { // TODO(holmer): We should improve this so that we can handle multiple // callbacks from one call to Decode(). VCMFrameInformation* frameInfo; @@ -63,10 +68,15 @@ int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) { return WEBRTC_VIDEO_CODEC_OK; } + const int64_t now_ms = _clock->TimeInMilliseconds(); + if (decode_time_ms < 0) { + decode_time_ms = + static_cast(now_ms - frameInfo->decodeStartTimeMs); + } _timing.StopDecodeTimer( decodedImage.timestamp(), - frameInfo->decodeStartTimeMs, - _clock->TimeInMilliseconds(), + decode_time_ms, + now_ms, frameInfo->renderTimeMs); if (callback != NULL) diff --git a/webrtc/modules/video_coding/main/source/generic_decoder.h b/webrtc/modules/video_coding/main/source/generic_decoder.h index c1298bb3af..7788425d44 100644 --- a/webrtc/modules/video_coding/main/source/generic_decoder.h +++ b/webrtc/modules/video_coding/main/source/generic_decoder.h @@ -41,6 +41,7 @@ public: VCMReceiveCallback* UserReceiveCallback(); virtual int32_t Decoded(VideoFrame& decodedImage); + virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms); virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId); virtual int32_t ReceivedDecodedFrame(const uint64_t pictureId); diff --git a/webrtc/modules/video_coding/main/source/timing.cc b/webrtc/modules/video_coding/main/source/timing.cc index 8d59135876..d8ab96d844 100644 --- a/webrtc/modules/video_coding/main/source/timing.cc +++ b/webrtc/modules/video_coding/main/source/timing.cc @@ -165,13 +165,13 @@ void VCMTiming::UpdateCurrentDelay(int64_t render_time_ms, } int32_t VCMTiming::StopDecodeTimer(uint32_t time_stamp, - int64_t start_time_ms, + int32_t decode_time_ms, int64_t now_ms, int64_t render_time_ms) { CriticalSectionScoped cs(crit_sect_); - int32_t time_diff_ms = codec_timer_.StopTimer(start_time_ms, now_ms); - assert(time_diff_ms >= 0); - last_decode_ms_ = time_diff_ms; + codec_timer_.MaxFilter(decode_time_ms, now_ms); + assert(decode_time_ms >= 0); + last_decode_ms_ = decode_time_ms; // Update stats. ++num_decoded_frames_; diff --git a/webrtc/modules/video_coding/main/source/timing.h b/webrtc/modules/video_coding/main/source/timing.h index d3b8fa673f..41dec8ce7f 100644 --- a/webrtc/modules/video_coding/main/source/timing.h +++ b/webrtc/modules/video_coding/main/source/timing.h @@ -58,7 +58,7 @@ class VCMTiming { // Stops the decoder timer, should be called when the decoder returns a frame // or when the decoded frame callback is called. int32_t StopDecodeTimer(uint32_t time_stamp, - int64_t start_time_ms, + int32_t decode_time_ms, int64_t now_ms, int64_t render_time_ms); diff --git a/webrtc/modules/video_coding/main/source/timing_unittest.cc b/webrtc/modules/video_coding/main/source/timing_unittest.cc index 694a600c2a..8ab2aed92e 100644 --- a/webrtc/modules/video_coding/main/source/timing_unittest.cc +++ b/webrtc/modules/video_coding/main/source/timing_unittest.cc @@ -85,8 +85,10 @@ TEST(ReceiverTiming, Tests) { for (int i = 0; i < 10; i++) { int64_t startTimeMs = clock.TimeInMilliseconds(); clock.AdvanceTimeMilliseconds(10); - timing.StopDecodeTimer(timeStamp, startTimeMs, - clock.TimeInMilliseconds(), timing.RenderTimeMs( + timing.StopDecodeTimer(timeStamp, + clock.TimeInMilliseconds() - startTimeMs, + clock.TimeInMilliseconds(), + timing.RenderTimeMs( timeStamp, clock.TimeInMilliseconds())); timeStamp += 90000 / 25; clock.AdvanceTimeMilliseconds(1000 / 25 - 10); diff --git a/webrtc/video/video_decoder_unittest.cc b/webrtc/video/video_decoder_unittest.cc index be09b191ac..fd1605df5c 100644 --- a/webrtc/video/video_decoder_unittest.cc +++ b/webrtc/video/video_decoder_unittest.cc @@ -11,6 +11,7 @@ #include "webrtc/video_decoder.h" #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/checks.h" #include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h" namespace webrtc { @@ -148,6 +149,11 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsRegisterDecodeCompleteCallback) { class FakeDecodedImageCallback : public DecodedImageCallback { int32_t Decoded(VideoFrame& decodedImage) override { return 0; } + int32_t Decoded( + webrtc::VideoFrame& decodedImage, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1; + } } callback, callback2; VideoCodec codec = {}; diff --git a/webrtc/video_decoder.h b/webrtc/video_decoder.h index 2822677868..3d0410429f 100644 --- a/webrtc/video_decoder.h +++ b/webrtc/video_decoder.h @@ -29,6 +29,16 @@ class DecodedImageCallback { virtual ~DecodedImageCallback() {} virtual int32_t Decoded(VideoFrame& decodedImage) = 0; + // Provides an alternative interface that allows the decoder to specify the + // decode time excluding waiting time for any previous pending frame to + // return. This is necessary for breaking positive feedback in the delay + // estimation when the decoder has a single output buffer. + // TODO(perkj): Remove default implementation when chromium has been updated. + virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) { + // The default implementation ignores custom decode time value. + return Decoded(decodedImage); + } + virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId) { return -1; }