From 111e9814665c2aadc19ec3b2d31f36883c1508b6 Mon Sep 17 00:00:00 2001 From: Johannes Kron Date: Mon, 26 Oct 2020 13:54:40 +0100 Subject: [PATCH] Signaling for low-latency renderer algorithm This feature is active if and only if the RTP header extension playout-delay is used with min playout delay=0 and max playout delay>0. In this case, a maximum composition delay will be calculated and attached to the video frame as a signal to use the low-latency renderer algorithm, which is landed in a separate CL in Chromium. The maximum composition delay is specified in number of frames and is calculated based on the max playout delay. The feature can be completetly disabled by specifying the field trial WebRTC-LowLatencyRenderer/enabled:false/ Bug: chromium:1138888 Change-Id: I05f461982d0632bd6e09e5d7ec1a8985dccdc61b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/190141 Reviewed-by: Niels Moller Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Johannes Kron Cr-Commit-Position: refs/heads/master@{#32493} --- api/video/video_frame.h | 11 ++++++ modules/video_coding/encoded_frame.h | 2 ++ modules/video_coding/frame_buffer2.cc | 5 +++ modules/video_coding/frame_buffer2.h | 2 ++ modules/video_coding/generic_decoder.cc | 28 ++++++++++++++- modules/video_coding/generic_decoder.h | 11 ++++++ .../video_coding/generic_decoder_unittest.cc | 24 +++++++++++++ modules/video_coding/timestamp_map.cc | 9 +++++ modules/video_coding/timestamp_map.h | 1 + modules/video_coding/timing.cc | 26 ++++++++++++-- modules/video_coding/timing.h | 11 ++++++ video/video_receive_stream2.cc | 23 +++++++++++++ video/video_receive_stream2.h | 10 ++++++ video/video_receive_stream2_unittest.cc | 34 +++++++++++++++++++ 14 files changed, 193 insertions(+), 4 deletions(-) diff --git a/api/video/video_frame.h b/api/video/video_frame.h index 08c939d916..e62aae8e5d 100644 --- a/api/video/video_frame.h +++ b/api/video/video_frame.h @@ -186,6 +186,16 @@ class RTC_EXPORT VideoFrame { color_space_ = color_space; } + // max_composition_delay_in_frames() is used in an experiment of a low-latency + // renderer algorithm see crbug.com/1138888. + absl::optional max_composition_delay_in_frames() const { + return max_composition_delay_in_frames_; + } + void set_max_composition_delay_in_frames( + absl::optional max_composition_delay_in_frames) { + max_composition_delay_in_frames_ = max_composition_delay_in_frames; + } + // Get render time in milliseconds. // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). int64_t render_time_ms() const; @@ -255,6 +265,7 @@ class RTC_EXPORT VideoFrame { int64_t timestamp_us_; VideoRotation rotation_; absl::optional color_space_; + absl::optional max_composition_delay_in_frames_; // Updated since the last frame area. If present it means that the bounding // box of all the changes is within the rectangular area and is close to it. // If absent, it means that there's no information about the change at all and diff --git a/modules/video_coding/encoded_frame.h b/modules/video_coding/encoded_frame.h index 3e2994072c..ddab6cedf8 100644 --- a/modules/video_coding/encoded_frame.h +++ b/modules/video_coding/encoded_frame.h @@ -34,6 +34,8 @@ class RTC_EXPORT VCMEncodedFrame : protected EncodedImage { _renderTimeMs = renderTimeMs; } + VideoPlayoutDelay PlayoutDelay() const { return playout_delay_; } + void SetPlayoutDelay(VideoPlayoutDelay playout_delay) { playout_delay_ = playout_delay; } diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index fd65d7e25d..afce787664 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -348,6 +348,11 @@ void FrameBuffer::Clear() { ClearFramesAndHistory(); } +int FrameBuffer::Size() { + MutexLock lock(&mutex_); + return frames_.size(); +} + void FrameBuffer::UpdateRtt(int64_t rtt_ms) { MutexLock lock(&mutex_); jitter_estimator_.UpdateRtt(rtt_ms); diff --git a/modules/video_coding/frame_buffer2.h b/modules/video_coding/frame_buffer2.h index 746773d632..2ed21c4f70 100644 --- a/modules/video_coding/frame_buffer2.h +++ b/modules/video_coding/frame_buffer2.h @@ -84,6 +84,8 @@ class FrameBuffer { // Clears the FrameBuffer, removing all the buffered frames. void Clear(); + int Size(); + private: struct FrameInfo { FrameInfo(); diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index b6e8203c0a..79057926fc 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -13,6 +13,7 @@ #include #include +#include #include "api/video/video_timing.h" #include "modules/video_coding/include/video_error_codes.h" @@ -31,12 +32,18 @@ VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing, : _clock(clock), _timing(timing), _timestampMap(kDecoderFrameMemoryLength), - _extra_decode_time("t", absl::nullopt) { + _extra_decode_time("t", absl::nullopt), + low_latency_renderer_enabled_("enabled", true), + low_latency_renderer_include_predecode_buffer_("include_predecode_buffer", + true) { ntp_offset_ = _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); ParseFieldTrial({&_extra_decode_time}, field_trial::FindFullName("WebRTC-SlowDownDecoder")); + ParseFieldTrial({&low_latency_renderer_enabled_, + &low_latency_renderer_include_predecode_buffer_}, + field_trial::FindFullName("WebRTC-LowLatencyRenderer")); } VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {} @@ -85,9 +92,11 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, // TODO(holmer): We should improve this so that we can handle multiple // callbacks from one call to Decode(). VCMFrameInformation* frameInfo; + int timestamp_map_size = 0; { MutexLock lock(&lock_); frameInfo = _timestampMap.Pop(decodedImage.timestamp()); + timestamp_map_size = _timestampMap.Size(); } if (frameInfo == NULL) { @@ -101,6 +110,22 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, decodedImage.set_packet_infos(frameInfo->packet_infos); decodedImage.set_rotation(frameInfo->rotation); + if (low_latency_renderer_enabled_ && frameInfo->playout_delay.min_ms == 0 && + frameInfo->playout_delay.max_ms > 0) { + absl::optional max_composition_delay_in_frames = + _timing->MaxCompositionDelayInFrames(); + if (max_composition_delay_in_frames) { + // Subtract frames that are in flight. + if (low_latency_renderer_include_predecode_buffer_) { + *max_composition_delay_in_frames -= timestamp_map_size; + *max_composition_delay_in_frames = + std::max(0, *max_composition_delay_in_frames); + } + decodedImage.set_max_composition_delay_in_frames( + max_composition_delay_in_frames); + } + } + RTC_DCHECK(frameInfo->decodeStart); const Timestamp now = _clock->CurrentTime(); const TimeDelta decode_time = decode_time_ms @@ -224,6 +249,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { _frameInfos[_nextFrameInfoIdx].decodeStart = now; _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); + _frameInfos[_nextFrameInfoIdx].playout_delay = frame.PlayoutDelay(); _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); _frameInfos[_nextFrameInfoIdx].ntp_time_ms = frame.EncodedImage().ntp_time_ms_; diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h index b89d3f4368..8481fdc15d 100644 --- a/modules/video_coding/generic_decoder.h +++ b/modules/video_coding/generic_decoder.h @@ -35,6 +35,7 @@ struct VCMFrameInformation { void* userData; VideoRotation rotation; VideoContentType content_type; + PlayoutDelay playout_delay; EncodedImage::Timing timing; int64_t ntp_time_ms; RtpPacketInfos packet_infos; @@ -75,6 +76,16 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { int64_t ntp_offset_; // Set by the field trial WebRTC-SlowDownDecoder to simulate a slow decoder. FieldTrialOptional _extra_decode_time; + + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter |enabled| + // determines if the low-latency renderer algorithm should be used for the + // case min playout delay=0 and max playout delay>0. + FieldTrialParameter low_latency_renderer_enabled_; + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter + // |include_predecode_buffer| determines if the predecode buffer should be + // taken into account when calculating maximum number of frames in composition + // queue. + FieldTrialParameter low_latency_renderer_include_predecode_buffer_; }; class VCMGenericDecoder { diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc index dbceb187be..a4cc5b0ded 100644 --- a/modules/video_coding/generic_decoder_unittest.cc +++ b/modules/video_coding/generic_decoder_unittest.cc @@ -115,5 +115,29 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) { EXPECT_EQ(decoded_frame->packet_infos().size(), 3U); } +TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) { + VCMEncodedFrame encoded_frame; + generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); + absl::optional decoded_frame = user_callback_.WaitForFrame(10); + ASSERT_TRUE(decoded_frame.has_value()); + EXPECT_FALSE(decoded_frame->max_composition_delay_in_frames()); +} + +TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) { + VCMEncodedFrame encoded_frame; + // VideoReceiveStream2 would set MaxCompositionDelayInFrames if playout delay + // is specified as X,Y, where X=0, Y>0. + const VideoPlayoutDelay kPlayoutDelay = {0, 50}; + constexpr int kMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps. + encoded_frame.SetPlayoutDelay(kPlayoutDelay); + timing_.SetMaxCompositionDelayInFrames( + absl::make_optional(kMaxCompositionDelayInFrames)); + generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); + absl::optional decoded_frame = user_callback_.WaitForFrame(10); + ASSERT_TRUE(decoded_frame.has_value()); + EXPECT_EQ(kMaxCompositionDelayInFrames, + decoded_frame->max_composition_delay_in_frames()); +} + } // namespace video_coding } // namespace webrtc diff --git a/modules/video_coding/timestamp_map.cc b/modules/video_coding/timestamp_map.cc index d93293704d..d79075ff21 100644 --- a/modules/video_coding/timestamp_map.cc +++ b/modules/video_coding/timestamp_map.cc @@ -60,4 +60,13 @@ VCMFrameInformation* VCMTimestampMap::Pop(uint32_t timestamp) { bool VCMTimestampMap::IsEmpty() const { return (next_add_idx_ == next_pop_idx_); } + +size_t VCMTimestampMap::Size() const { + // The maximum number of elements in the list is |capacity_| - 1. The list is + // empty if the add and pop indices are equal. + return next_add_idx_ >= next_pop_idx_ + ? next_add_idx_ - next_pop_idx_ + : next_add_idx_ + capacity_ - next_pop_idx_; +} + } // namespace webrtc diff --git a/modules/video_coding/timestamp_map.h b/modules/video_coding/timestamp_map.h index c85666c9aa..cfa12573ec 100644 --- a/modules/video_coding/timestamp_map.h +++ b/modules/video_coding/timestamp_map.h @@ -24,6 +24,7 @@ class VCMTimestampMap { void Add(uint32_t timestamp, VCMFrameInformation* data); VCMFrameInformation* Pop(uint32_t timestamp); + size_t Size() const; private: struct TimestampDataTuple { diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc index f046edf497..f1c66b89c9 100644 --- a/modules/video_coding/timing.cc +++ b/modules/video_coding/timing.cc @@ -14,8 +14,10 @@ #include +#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/time/timestamp_extrapolator.h" #include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { @@ -31,7 +33,10 @@ VCMTiming::VCMTiming(Clock* clock, VCMTiming* master_timing) current_delay_ms_(0), prev_frame_timestamp_(0), timing_frame_info_(), - num_decoded_frames_(0) { + num_decoded_frames_(0), + low_latency_renderer_enabled_("enabled", true) { + ParseFieldTrial({&low_latency_renderer_enabled_}, + field_trial::FindFullName("WebRTC-LowLatencyRenderer")); if (master_timing == NULL) { master_ = true; ts_extrapolator_ = new TimestampExtrapolator(clock_->TimeInMilliseconds()); @@ -177,8 +182,12 @@ int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp, int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp, int64_t now_ms) const { - if (min_playout_delay_ms_ == 0 && max_playout_delay_ms_ == 0) { - // Render as soon as possible. + constexpr int kLowLatencyRendererMaxPlayoutDelayMs = 500; + if (min_playout_delay_ms_ == 0 && + (max_playout_delay_ms_ == 0 || + (low_latency_renderer_enabled_ && + max_playout_delay_ms_ <= kLowLatencyRendererMaxPlayoutDelayMs))) { + // Render as soon as possible or with low-latency renderer algorithm. return 0; } int64_t estimated_complete_time_ms = @@ -246,4 +255,15 @@ absl::optional VCMTiming::GetTimingFrameInfo() { return timing_frame_info_; } +void VCMTiming::SetMaxCompositionDelayInFrames( + absl::optional max_composition_delay_in_frames) { + MutexLock lock(&mutex_); + max_composition_delay_in_frames_ = max_composition_delay_in_frames; +} + +absl::optional VCMTiming::MaxCompositionDelayInFrames() const { + MutexLock lock(&mutex_); + return max_composition_delay_in_frames_; +} + } // namespace webrtc diff --git a/modules/video_coding/timing.h b/modules/video_coding/timing.h index 75b8e7d99d..5c7cfbc8b4 100644 --- a/modules/video_coding/timing.h +++ b/modules/video_coding/timing.h @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "api/video/video_timing.h" #include "modules/video_coding/codec_timer.h" +#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" @@ -100,6 +101,10 @@ class VCMTiming { void SetTimingFrameInfo(const TimingFrameInfo& info); absl::optional GetTimingFrameInfo(); + void SetMaxCompositionDelayInFrames( + absl::optional max_composition_delay_in_frames); + absl::optional MaxCompositionDelayInFrames() const; + enum { kDefaultRenderDelayMs = 10 }; enum { kDelayMaxChangeMsPerS = 100 }; @@ -128,6 +133,12 @@ class VCMTiming { uint32_t prev_frame_timestamp_ RTC_GUARDED_BY(mutex_); absl::optional timing_frame_info_ RTC_GUARDED_BY(mutex_); size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_); + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter enabled + // determines if the low-latency renderer algorithm should be used for the + // case min playout delay=0 and max playout delay>0. + FieldTrialParameter low_latency_renderer_enabled_ + RTC_GUARDED_BY(mutex_); + absl::optional max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); }; } // namespace webrtc diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index c2440f8e1c..8cc14e57c1 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -231,6 +231,9 @@ VideoReceiveStream2::VideoReceiveStream2( max_wait_for_frame_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() .MaxWaitForFrameMs() .value_or(kMaxWaitForFrameMs)), + low_latency_renderer_enabled_("enabled", true), + low_latency_renderer_include_predecode_buffer_("include_predecode_buffer", + true), decode_queue_(task_queue_factory_->CreateTaskQueue( "DecodingQueue", TaskQueueFactory::Priority::HIGH)) { @@ -271,6 +274,10 @@ VideoReceiveStream2::VideoReceiveStream2( rtp_receive_statistics_->EnableRetransmitDetection(config.rtp.remote_ssrc, true); } + + ParseFieldTrial({&low_latency_renderer_enabled_, + &low_latency_renderer_include_predecode_buffer_}, + field_trial::FindFullName("WebRTC-LowLatencyRenderer")); } VideoReceiveStream2::~VideoReceiveStream2() { @@ -778,6 +785,22 @@ void VideoReceiveStream2::UpdatePlayoutDelays() const { syncable_minimum_playout_delay_ms_}); if (minimum_delay_ms >= 0) { timing_->set_min_playout_delay(minimum_delay_ms); + if (frame_minimum_playout_delay_ms_ == 0 && + frame_maximum_playout_delay_ms_ > 0 && low_latency_renderer_enabled_) { + // TODO(kron): Estimate frame rate from video stream. + constexpr double kFrameRate = 60.0; + // Convert playout delay in ms to number of frames. + int max_composition_delay_in_frames = std::lrint( + static_cast(frame_maximum_playout_delay_ms_ * kFrameRate) / + rtc::kNumMillisecsPerSec); + if (low_latency_renderer_include_predecode_buffer_) { + // Subtract frames in buffer. + max_composition_delay_in_frames = std::max( + max_composition_delay_in_frames - frame_buffer_->Size(), 0); + } + timing_->SetMaxCompositionDelayInFrames( + absl::make_optional(max_composition_delay_in_frames)); + } } const int maximum_delay_ms = frame_maximum_playout_delay_ms_; diff --git a/video/video_receive_stream2.h b/video/video_receive_stream2.h index 9b152f420a..e8e3edc3d1 100644 --- a/video/video_receive_stream2.h +++ b/video/video_receive_stream2.h @@ -257,6 +257,16 @@ class VideoReceiveStream2 : public webrtc::VideoReceiveStream, bool keyframe_generation_requested_ RTC_GUARDED_BY(worker_sequence_checker_) = false; + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter |enabled| + // determines if the low-latency renderer algorithm should be used for the + // case min playout delay=0 and max playout delay>0. + FieldTrialParameter low_latency_renderer_enabled_; + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter + // |include_predecode_buffer| determines if the predecode buffer should be + // taken into account when calculating maximum number of frames in composition + // queue. + FieldTrialParameter low_latency_renderer_include_predecode_buffer_; + // Defined last so they are destroyed before all other members. rtc::TaskQueue decode_queue_; diff --git a/video/video_receive_stream2_unittest.cc b/video/video_receive_stream2_unittest.cc index 435975062c..3f10686db7 100644 --- a/video/video_receive_stream2_unittest.cc +++ b/video/video_receive_stream2_unittest.cc @@ -230,6 +230,40 @@ TEST_F(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMinValue) { EXPECT_EQ(default_min_playout_latency, timing_->min_playout_delay()); } +TEST_F(VideoReceiveStream2Test, MaxCompositionDelayNotSetByDefault) { + // Default with no playout delay set. + std::unique_ptr test_frame0(new FrameObjectFake()); + test_frame0->id.picture_id = 0; + video_receive_stream_->OnCompleteFrame(std::move(test_frame0)); + EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); + + // Max composition delay not set for playout delay 0,0. + std::unique_ptr test_frame1(new FrameObjectFake()); + test_frame1->id.picture_id = 1; + test_frame1->SetPlayoutDelay({0, 0}); + video_receive_stream_->OnCompleteFrame(std::move(test_frame1)); + EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); + + // Max composition delay not set for playout delay X,Y, where X,Y>0. + std::unique_ptr test_frame2(new FrameObjectFake()); + test_frame2->id.picture_id = 2; + test_frame2->SetPlayoutDelay({10, 30}); + video_receive_stream_->OnCompleteFrame(std::move(test_frame2)); + EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); +} + +TEST_F(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) { + // Max composition delay set if playout delay X,Y, where X=0,Y>0. + const VideoPlayoutDelay kPlayoutDelayMs = {0, 50}; + const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps. + std::unique_ptr test_frame(new FrameObjectFake()); + test_frame->id.picture_id = 0; + test_frame->SetPlayoutDelay(kPlayoutDelayMs); + video_receive_stream_->OnCompleteFrame(std::move(test_frame)); + EXPECT_EQ(kExpectedMaxCompositionDelayInFrames, + timing_->MaxCompositionDelayInFrames()); +} + class VideoReceiveStream2TestWithFakeDecoder : public ::testing::Test { public: VideoReceiveStream2TestWithFakeDecoder()