From 327d8babc81913410dbb3125a62be3ff0e46e9f4 Mon Sep 17 00:00:00 2001 From: Per Date: Tue, 10 Nov 2015 14:00:27 +0100 Subject: [PATCH] Add DecodedImageCallback::Decoded() function with custom decode time value. On Android, we would like to use MediaCodec output buffers to hold decoded frames until they can be rendered to a texture. There can only be one texture buffer used at the same time and therefore the calculated decode time in VCMTiming will be wrong since that calculation will also include the time where the decoder waited for the upper layers (that depend on network jitter and actual render time) to release the frame. This new method will be used in https://codereview.webrtc.org/1422963003/ BUG=webrtc:4993 R=stefan@webrtc.org TBR=mflodman@webrtc.org Review URL: https://codereview.webrtc.org/1414693006 . Cr-Commit-Position: refs/heads/master@{#10576} --- .../interface/mock/mock_video_codec_interface.h | 2 ++ .../video_coding/codecs/test/videoprocessor.h | 6 ++++++ .../video_coding/codecs/vp8/simulcast_unittest.h | 7 ++++++- .../codecs/vp8/test/vp8_impl_unittest.cc | 7 ++++++- .../video_coding/codecs/vp8/vp8_sequence_coder.cc | 7 ++++++- .../video_coding/main/source/codec_timer.cc | 7 ------- .../modules/video_coding/main/source/codec_timer.h | 5 ++--- .../video_coding/main/source/generic_decoder.cc | 14 ++++++++++++-- .../video_coding/main/source/generic_decoder.h | 1 + webrtc/modules/video_coding/main/source/timing.cc | 8 ++++---- webrtc/modules/video_coding/main/source/timing.h | 2 +- .../video_coding/main/source/timing_unittest.cc | 6 ++++-- webrtc/video/video_decoder_unittest.cc | 6 ++++++ webrtc/video_decoder.h | 10 ++++++++++ 14 files changed, 66 insertions(+), 22 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h b/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h index 6c926d4794..b3af479421 100644 --- a/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h +++ b/webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h @@ -48,6 +48,8 @@ class MockVideoEncoder : public VideoEncoder { class MockDecodedImageCallback : public DecodedImageCallback { public: MOCK_METHOD1(Decoded, int32_t(VideoFrame& decodedImage)); + MOCK_METHOD2(Decoded, int32_t(VideoFrame& decodedImage, + int64_t decode_time_ms)); MOCK_METHOD1(ReceivedDecodedReferenceFrame, int32_t(const uint64_t pictureId)); MOCK_METHOD1(ReceivedDecodedFrame, diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.h b/webrtc/modules/video_coding/codecs/test/videoprocessor.h index 0b094ae73e..582d903343 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor.h +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.h @@ -13,6 +13,7 @@ #include +#include "webrtc/base/checks.h" #include "webrtc/common_video/libyuv/include/scaler.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" @@ -248,6 +249,11 @@ class VideoProcessorImpl : public VideoProcessor { : video_processor_(vp) { } int32_t Decoded(webrtc::VideoFrame& image) override; + int32_t Decoded( + webrtc::VideoFrame& image, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1; + } private: VideoProcessorImpl* video_processor_; diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h index e4fc986545..2577badd83 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h @@ -14,6 +14,7 @@ #include #include +#include "webrtc/base/checks.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/common.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" @@ -123,7 +124,7 @@ class Vp8TestDecodedImageCallback : public DecodedImageCallback { Vp8TestDecodedImageCallback() : decoded_frames_(0) { } - virtual int32_t Decoded(VideoFrame& decoded_image) { + int32_t Decoded(VideoFrame& decoded_image) override { for (int i = 0; i < decoded_image.width(); ++i) { EXPECT_NEAR(kColorY, decoded_image.buffer(kYPlane)[i], 1); } @@ -136,6 +137,10 @@ class Vp8TestDecodedImageCallback : public DecodedImageCallback { decoded_frames_++; return 0; } + int32_t Decoded(VideoFrame& decoded_image, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1; + } int DecodedFrames() { return decoded_frames_; } diff --git a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index 5ec674f16a..7650a250ce 100644 --- a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -11,6 +11,7 @@ #include #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/checks.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" @@ -78,7 +79,11 @@ class Vp8UnitTestDecodeCompleteCallback : public webrtc::DecodedImageCallback { public: explicit Vp8UnitTestDecodeCompleteCallback(VideoFrame* frame) : decoded_frame_(frame), decode_complete(false) {} - int Decoded(webrtc::VideoFrame& frame); + int32_t Decoded(VideoFrame& frame) override; + int32_t Decoded(VideoFrame& frame, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1; + } bool DecodeComplete(); private: diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc index 5843d83fa7..ea7db5a7f6 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_sequence_coder.cc @@ -9,6 +9,7 @@ */ #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/checks.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/common_video/interface/video_image.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" @@ -68,7 +69,11 @@ class Vp8SequenceCoderDecodeCallback : public webrtc::DecodedImageCallback { public: explicit Vp8SequenceCoderDecodeCallback(FILE* decoded_file) : decoded_file_(decoded_file) {} - int Decoded(webrtc::VideoFrame& frame); + int32_t Decoded(webrtc::VideoFrame& frame) override; + int32_t Decoded(webrtc::VideoFrame& frame, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1;; + } bool DecodeComplete(); private: diff --git a/webrtc/modules/video_coding/main/source/codec_timer.cc b/webrtc/modules/video_coding/main/source/codec_timer.cc index a462258813..57985c71e7 100644 --- a/webrtc/modules/video_coding/main/source/codec_timer.cc +++ b/webrtc/modules/video_coding/main/source/codec_timer.cc @@ -28,13 +28,6 @@ _history() Reset(); } -int32_t VCMCodecTimer::StopTimer(int64_t startTimeMs, int64_t nowMs) -{ - const int32_t timeDiff = static_cast(nowMs - startTimeMs); - MaxFilter(timeDiff, nowMs); - return timeDiff; -} - void VCMCodecTimer::Reset() { _filteredMax = 0; diff --git a/webrtc/modules/video_coding/main/source/codec_timer.h b/webrtc/modules/video_coding/main/source/codec_timer.h index cb7e813ba0..a7abeb850e 100644 --- a/webrtc/modules/video_coding/main/source/codec_timer.h +++ b/webrtc/modules/video_coding/main/source/codec_timer.h @@ -35,8 +35,8 @@ class VCMCodecTimer public: VCMCodecTimer(); - // Updates and returns the max filtered decode time. - int32_t StopTimer(int64_t startTimeMs, int64_t nowMs); + // Updates the max filtered decode time. + void MaxFilter(int32_t newDecodeTimeMs, int64_t nowMs); // Empty the list of timers. void Reset(); @@ -46,7 +46,6 @@ public: private: void UpdateMaxHistory(int32_t decodeTime, int64_t now); - void MaxFilter(int32_t newTime, int64_t nowMs); void ProcessHistory(int64_t nowMs); int32_t _filteredMax; diff --git a/webrtc/modules/video_coding/main/source/generic_decoder.cc b/webrtc/modules/video_coding/main/source/generic_decoder.cc index 8b2d3974de..4f4a09b058 100644 --- a/webrtc/modules/video_coding/main/source/generic_decoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_decoder.cc @@ -47,6 +47,11 @@ VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() } int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) { + return Decoded(decodedImage, -1); +} + +int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, + int64_t decode_time_ms) { // TODO(holmer): We should improve this so that we can handle multiple // callbacks from one call to Decode(). VCMFrameInformation* frameInfo; @@ -63,10 +68,15 @@ int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) { return WEBRTC_VIDEO_CODEC_OK; } + const int64_t now_ms = _clock->TimeInMilliseconds(); + if (decode_time_ms < 0) { + decode_time_ms = + static_cast(now_ms - frameInfo->decodeStartTimeMs); + } _timing.StopDecodeTimer( decodedImage.timestamp(), - frameInfo->decodeStartTimeMs, - _clock->TimeInMilliseconds(), + decode_time_ms, + now_ms, frameInfo->renderTimeMs); if (callback != NULL) diff --git a/webrtc/modules/video_coding/main/source/generic_decoder.h b/webrtc/modules/video_coding/main/source/generic_decoder.h index c1298bb3af..7788425d44 100644 --- a/webrtc/modules/video_coding/main/source/generic_decoder.h +++ b/webrtc/modules/video_coding/main/source/generic_decoder.h @@ -41,6 +41,7 @@ public: VCMReceiveCallback* UserReceiveCallback(); virtual int32_t Decoded(VideoFrame& decodedImage); + virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms); virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId); virtual int32_t ReceivedDecodedFrame(const uint64_t pictureId); diff --git a/webrtc/modules/video_coding/main/source/timing.cc b/webrtc/modules/video_coding/main/source/timing.cc index 8d59135876..d8ab96d844 100644 --- a/webrtc/modules/video_coding/main/source/timing.cc +++ b/webrtc/modules/video_coding/main/source/timing.cc @@ -165,13 +165,13 @@ void VCMTiming::UpdateCurrentDelay(int64_t render_time_ms, } int32_t VCMTiming::StopDecodeTimer(uint32_t time_stamp, - int64_t start_time_ms, + int32_t decode_time_ms, int64_t now_ms, int64_t render_time_ms) { CriticalSectionScoped cs(crit_sect_); - int32_t time_diff_ms = codec_timer_.StopTimer(start_time_ms, now_ms); - assert(time_diff_ms >= 0); - last_decode_ms_ = time_diff_ms; + codec_timer_.MaxFilter(decode_time_ms, now_ms); + assert(decode_time_ms >= 0); + last_decode_ms_ = decode_time_ms; // Update stats. ++num_decoded_frames_; diff --git a/webrtc/modules/video_coding/main/source/timing.h b/webrtc/modules/video_coding/main/source/timing.h index d3b8fa673f..41dec8ce7f 100644 --- a/webrtc/modules/video_coding/main/source/timing.h +++ b/webrtc/modules/video_coding/main/source/timing.h @@ -58,7 +58,7 @@ class VCMTiming { // Stops the decoder timer, should be called when the decoder returns a frame // or when the decoded frame callback is called. int32_t StopDecodeTimer(uint32_t time_stamp, - int64_t start_time_ms, + int32_t decode_time_ms, int64_t now_ms, int64_t render_time_ms); diff --git a/webrtc/modules/video_coding/main/source/timing_unittest.cc b/webrtc/modules/video_coding/main/source/timing_unittest.cc index 694a600c2a..8ab2aed92e 100644 --- a/webrtc/modules/video_coding/main/source/timing_unittest.cc +++ b/webrtc/modules/video_coding/main/source/timing_unittest.cc @@ -85,8 +85,10 @@ TEST(ReceiverTiming, Tests) { for (int i = 0; i < 10; i++) { int64_t startTimeMs = clock.TimeInMilliseconds(); clock.AdvanceTimeMilliseconds(10); - timing.StopDecodeTimer(timeStamp, startTimeMs, - clock.TimeInMilliseconds(), timing.RenderTimeMs( + timing.StopDecodeTimer(timeStamp, + clock.TimeInMilliseconds() - startTimeMs, + clock.TimeInMilliseconds(), + timing.RenderTimeMs( timeStamp, clock.TimeInMilliseconds())); timeStamp += 90000 / 25; clock.AdvanceTimeMilliseconds(1000 / 25 - 10); diff --git a/webrtc/video/video_decoder_unittest.cc b/webrtc/video/video_decoder_unittest.cc index be09b191ac..fd1605df5c 100644 --- a/webrtc/video/video_decoder_unittest.cc +++ b/webrtc/video/video_decoder_unittest.cc @@ -11,6 +11,7 @@ #include "webrtc/video_decoder.h" #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/checks.h" #include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h" namespace webrtc { @@ -148,6 +149,11 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsRegisterDecodeCompleteCallback) { class FakeDecodedImageCallback : public DecodedImageCallback { int32_t Decoded(VideoFrame& decodedImage) override { return 0; } + int32_t Decoded( + webrtc::VideoFrame& decodedImage, int64_t decode_time_ms) override { + RTC_NOTREACHED(); + return -1; + } } callback, callback2; VideoCodec codec = {}; diff --git a/webrtc/video_decoder.h b/webrtc/video_decoder.h index 2822677868..3d0410429f 100644 --- a/webrtc/video_decoder.h +++ b/webrtc/video_decoder.h @@ -29,6 +29,16 @@ class DecodedImageCallback { virtual ~DecodedImageCallback() {} virtual int32_t Decoded(VideoFrame& decodedImage) = 0; + // Provides an alternative interface that allows the decoder to specify the + // decode time excluding waiting time for any previous pending frame to + // return. This is necessary for breaking positive feedback in the delay + // estimation when the decoder has a single output buffer. + // TODO(perkj): Remove default implementation when chromium has been updated. + virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) { + // The default implementation ignores custom decode time value. + return Decoded(decodedImage); + } + virtual int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId) { return -1; }