diff --git a/api/video/video_frame.h b/api/video/video_frame.h index 08c939d916..e62aae8e5d 100644 --- a/api/video/video_frame.h +++ b/api/video/video_frame.h @@ -186,6 +186,16 @@ class RTC_EXPORT VideoFrame { color_space_ = color_space; } + // max_composition_delay_in_frames() is used in an experiment of a low-latency + // renderer algorithm see crbug.com/1138888. + absl::optional max_composition_delay_in_frames() const { + return max_composition_delay_in_frames_; + } + void set_max_composition_delay_in_frames( + absl::optional max_composition_delay_in_frames) { + max_composition_delay_in_frames_ = max_composition_delay_in_frames; + } + // Get render time in milliseconds. // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). int64_t render_time_ms() const; @@ -255,6 +265,7 @@ class RTC_EXPORT VideoFrame { int64_t timestamp_us_; VideoRotation rotation_; absl::optional color_space_; + absl::optional max_composition_delay_in_frames_; // Updated since the last frame area. If present it means that the bounding // box of all the changes is within the rectangular area and is close to it. // If absent, it means that there's no information about the change at all and diff --git a/modules/video_coding/encoded_frame.h b/modules/video_coding/encoded_frame.h index 3e2994072c..ddab6cedf8 100644 --- a/modules/video_coding/encoded_frame.h +++ b/modules/video_coding/encoded_frame.h @@ -34,6 +34,8 @@ class RTC_EXPORT VCMEncodedFrame : protected EncodedImage { _renderTimeMs = renderTimeMs; } + VideoPlayoutDelay PlayoutDelay() const { return playout_delay_; } + void SetPlayoutDelay(VideoPlayoutDelay playout_delay) { playout_delay_ = playout_delay; } diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index fd65d7e25d..afce787664 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -348,6 +348,11 @@ void FrameBuffer::Clear() { ClearFramesAndHistory(); } +int FrameBuffer::Size() { + MutexLock lock(&mutex_); + return frames_.size(); +} + void FrameBuffer::UpdateRtt(int64_t rtt_ms) { MutexLock lock(&mutex_); jitter_estimator_.UpdateRtt(rtt_ms); diff --git a/modules/video_coding/frame_buffer2.h b/modules/video_coding/frame_buffer2.h index 746773d632..2ed21c4f70 100644 --- a/modules/video_coding/frame_buffer2.h +++ b/modules/video_coding/frame_buffer2.h @@ -84,6 +84,8 @@ class FrameBuffer { // Clears the FrameBuffer, removing all the buffered frames. void Clear(); + int Size(); + private: struct FrameInfo { FrameInfo(); diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index b6e8203c0a..79057926fc 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -13,6 +13,7 @@ #include #include +#include #include "api/video/video_timing.h" #include "modules/video_coding/include/video_error_codes.h" @@ -31,12 +32,18 @@ VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing, : _clock(clock), _timing(timing), _timestampMap(kDecoderFrameMemoryLength), - _extra_decode_time("t", absl::nullopt) { + _extra_decode_time("t", absl::nullopt), + low_latency_renderer_enabled_("enabled", true), + low_latency_renderer_include_predecode_buffer_("include_predecode_buffer", + true) { ntp_offset_ = _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); ParseFieldTrial({&_extra_decode_time}, field_trial::FindFullName("WebRTC-SlowDownDecoder")); + ParseFieldTrial({&low_latency_renderer_enabled_, + &low_latency_renderer_include_predecode_buffer_}, + field_trial::FindFullName("WebRTC-LowLatencyRenderer")); } VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {} @@ -85,9 +92,11 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, // TODO(holmer): We should improve this so that we can handle multiple // callbacks from one call to Decode(). VCMFrameInformation* frameInfo; + int timestamp_map_size = 0; { MutexLock lock(&lock_); frameInfo = _timestampMap.Pop(decodedImage.timestamp()); + timestamp_map_size = _timestampMap.Size(); } if (frameInfo == NULL) { @@ -101,6 +110,22 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, decodedImage.set_packet_infos(frameInfo->packet_infos); decodedImage.set_rotation(frameInfo->rotation); + if (low_latency_renderer_enabled_ && frameInfo->playout_delay.min_ms == 0 && + frameInfo->playout_delay.max_ms > 0) { + absl::optional max_composition_delay_in_frames = + _timing->MaxCompositionDelayInFrames(); + if (max_composition_delay_in_frames) { + // Subtract frames that are in flight. + if (low_latency_renderer_include_predecode_buffer_) { + *max_composition_delay_in_frames -= timestamp_map_size; + *max_composition_delay_in_frames = + std::max(0, *max_composition_delay_in_frames); + } + decodedImage.set_max_composition_delay_in_frames( + max_composition_delay_in_frames); + } + } + RTC_DCHECK(frameInfo->decodeStart); const Timestamp now = _clock->CurrentTime(); const TimeDelta decode_time = decode_time_ms @@ -224,6 +249,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { _frameInfos[_nextFrameInfoIdx].decodeStart = now; _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); + _frameInfos[_nextFrameInfoIdx].playout_delay = frame.PlayoutDelay(); _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); _frameInfos[_nextFrameInfoIdx].ntp_time_ms = frame.EncodedImage().ntp_time_ms_; diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h index b89d3f4368..8481fdc15d 100644 --- a/modules/video_coding/generic_decoder.h +++ b/modules/video_coding/generic_decoder.h @@ -35,6 +35,7 @@ struct VCMFrameInformation { void* userData; VideoRotation rotation; VideoContentType content_type; + PlayoutDelay playout_delay; EncodedImage::Timing timing; int64_t ntp_time_ms; RtpPacketInfos packet_infos; @@ -75,6 +76,16 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { int64_t ntp_offset_; // Set by the field trial WebRTC-SlowDownDecoder to simulate a slow decoder. FieldTrialOptional _extra_decode_time; + + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter |enabled| + // determines if the low-latency renderer algorithm should be used for the + // case min playout delay=0 and max playout delay>0. + FieldTrialParameter low_latency_renderer_enabled_; + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter + // |include_predecode_buffer| determines if the predecode buffer should be + // taken into account when calculating maximum number of frames in composition + // queue. + FieldTrialParameter low_latency_renderer_include_predecode_buffer_; }; class VCMGenericDecoder { diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc index dbceb187be..a4cc5b0ded 100644 --- a/modules/video_coding/generic_decoder_unittest.cc +++ b/modules/video_coding/generic_decoder_unittest.cc @@ -115,5 +115,29 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) { EXPECT_EQ(decoded_frame->packet_infos().size(), 3U); } +TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) { + VCMEncodedFrame encoded_frame; + generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); + absl::optional decoded_frame = user_callback_.WaitForFrame(10); + ASSERT_TRUE(decoded_frame.has_value()); + EXPECT_FALSE(decoded_frame->max_composition_delay_in_frames()); +} + +TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) { + VCMEncodedFrame encoded_frame; + // VideoReceiveStream2 would set MaxCompositionDelayInFrames if playout delay + // is specified as X,Y, where X=0, Y>0. + const VideoPlayoutDelay kPlayoutDelay = {0, 50}; + constexpr int kMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps. + encoded_frame.SetPlayoutDelay(kPlayoutDelay); + timing_.SetMaxCompositionDelayInFrames( + absl::make_optional(kMaxCompositionDelayInFrames)); + generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); + absl::optional decoded_frame = user_callback_.WaitForFrame(10); + ASSERT_TRUE(decoded_frame.has_value()); + EXPECT_EQ(kMaxCompositionDelayInFrames, + decoded_frame->max_composition_delay_in_frames()); +} + } // namespace video_coding } // namespace webrtc diff --git a/modules/video_coding/timestamp_map.cc b/modules/video_coding/timestamp_map.cc index d93293704d..d79075ff21 100644 --- a/modules/video_coding/timestamp_map.cc +++ b/modules/video_coding/timestamp_map.cc @@ -60,4 +60,13 @@ VCMFrameInformation* VCMTimestampMap::Pop(uint32_t timestamp) { bool VCMTimestampMap::IsEmpty() const { return (next_add_idx_ == next_pop_idx_); } + +size_t VCMTimestampMap::Size() const { + // The maximum number of elements in the list is |capacity_| - 1. The list is + // empty if the add and pop indices are equal. + return next_add_idx_ >= next_pop_idx_ + ? next_add_idx_ - next_pop_idx_ + : next_add_idx_ + capacity_ - next_pop_idx_; +} + } // namespace webrtc diff --git a/modules/video_coding/timestamp_map.h b/modules/video_coding/timestamp_map.h index c85666c9aa..cfa12573ec 100644 --- a/modules/video_coding/timestamp_map.h +++ b/modules/video_coding/timestamp_map.h @@ -24,6 +24,7 @@ class VCMTimestampMap { void Add(uint32_t timestamp, VCMFrameInformation* data); VCMFrameInformation* Pop(uint32_t timestamp); + size_t Size() const; private: struct TimestampDataTuple { diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc index f046edf497..f1c66b89c9 100644 --- a/modules/video_coding/timing.cc +++ b/modules/video_coding/timing.cc @@ -14,8 +14,10 @@ #include +#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/time/timestamp_extrapolator.h" #include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { @@ -31,7 +33,10 @@ VCMTiming::VCMTiming(Clock* clock, VCMTiming* master_timing) current_delay_ms_(0), prev_frame_timestamp_(0), timing_frame_info_(), - num_decoded_frames_(0) { + num_decoded_frames_(0), + low_latency_renderer_enabled_("enabled", true) { + ParseFieldTrial({&low_latency_renderer_enabled_}, + field_trial::FindFullName("WebRTC-LowLatencyRenderer")); if (master_timing == NULL) { master_ = true; ts_extrapolator_ = new TimestampExtrapolator(clock_->TimeInMilliseconds()); @@ -177,8 +182,12 @@ int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp, int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp, int64_t now_ms) const { - if (min_playout_delay_ms_ == 0 && max_playout_delay_ms_ == 0) { - // Render as soon as possible. + constexpr int kLowLatencyRendererMaxPlayoutDelayMs = 500; + if (min_playout_delay_ms_ == 0 && + (max_playout_delay_ms_ == 0 || + (low_latency_renderer_enabled_ && + max_playout_delay_ms_ <= kLowLatencyRendererMaxPlayoutDelayMs))) { + // Render as soon as possible or with low-latency renderer algorithm. return 0; } int64_t estimated_complete_time_ms = @@ -246,4 +255,15 @@ absl::optional VCMTiming::GetTimingFrameInfo() { return timing_frame_info_; } +void VCMTiming::SetMaxCompositionDelayInFrames( + absl::optional max_composition_delay_in_frames) { + MutexLock lock(&mutex_); + max_composition_delay_in_frames_ = max_composition_delay_in_frames; +} + +absl::optional VCMTiming::MaxCompositionDelayInFrames() const { + MutexLock lock(&mutex_); + return max_composition_delay_in_frames_; +} + } // namespace webrtc diff --git a/modules/video_coding/timing.h b/modules/video_coding/timing.h index 75b8e7d99d..5c7cfbc8b4 100644 --- a/modules/video_coding/timing.h +++ b/modules/video_coding/timing.h @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "api/video/video_timing.h" #include "modules/video_coding/codec_timer.h" +#include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" @@ -100,6 +101,10 @@ class VCMTiming { void SetTimingFrameInfo(const TimingFrameInfo& info); absl::optional GetTimingFrameInfo(); + void SetMaxCompositionDelayInFrames( + absl::optional max_composition_delay_in_frames); + absl::optional MaxCompositionDelayInFrames() const; + enum { kDefaultRenderDelayMs = 10 }; enum { kDelayMaxChangeMsPerS = 100 }; @@ -128,6 +133,12 @@ class VCMTiming { uint32_t prev_frame_timestamp_ RTC_GUARDED_BY(mutex_); absl::optional timing_frame_info_ RTC_GUARDED_BY(mutex_); size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_); + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter enabled + // determines if the low-latency renderer algorithm should be used for the + // case min playout delay=0 and max playout delay>0. + FieldTrialParameter low_latency_renderer_enabled_ + RTC_GUARDED_BY(mutex_); + absl::optional max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); }; } // namespace webrtc diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index c2440f8e1c..8cc14e57c1 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -231,6 +231,9 @@ VideoReceiveStream2::VideoReceiveStream2( max_wait_for_frame_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() .MaxWaitForFrameMs() .value_or(kMaxWaitForFrameMs)), + low_latency_renderer_enabled_("enabled", true), + low_latency_renderer_include_predecode_buffer_("include_predecode_buffer", + true), decode_queue_(task_queue_factory_->CreateTaskQueue( "DecodingQueue", TaskQueueFactory::Priority::HIGH)) { @@ -271,6 +274,10 @@ VideoReceiveStream2::VideoReceiveStream2( rtp_receive_statistics_->EnableRetransmitDetection(config.rtp.remote_ssrc, true); } + + ParseFieldTrial({&low_latency_renderer_enabled_, + &low_latency_renderer_include_predecode_buffer_}, + field_trial::FindFullName("WebRTC-LowLatencyRenderer")); } VideoReceiveStream2::~VideoReceiveStream2() { @@ -778,6 +785,22 @@ void VideoReceiveStream2::UpdatePlayoutDelays() const { syncable_minimum_playout_delay_ms_}); if (minimum_delay_ms >= 0) { timing_->set_min_playout_delay(minimum_delay_ms); + if (frame_minimum_playout_delay_ms_ == 0 && + frame_maximum_playout_delay_ms_ > 0 && low_latency_renderer_enabled_) { + // TODO(kron): Estimate frame rate from video stream. + constexpr double kFrameRate = 60.0; + // Convert playout delay in ms to number of frames. + int max_composition_delay_in_frames = std::lrint( + static_cast(frame_maximum_playout_delay_ms_ * kFrameRate) / + rtc::kNumMillisecsPerSec); + if (low_latency_renderer_include_predecode_buffer_) { + // Subtract frames in buffer. + max_composition_delay_in_frames = std::max( + max_composition_delay_in_frames - frame_buffer_->Size(), 0); + } + timing_->SetMaxCompositionDelayInFrames( + absl::make_optional(max_composition_delay_in_frames)); + } } const int maximum_delay_ms = frame_maximum_playout_delay_ms_; diff --git a/video/video_receive_stream2.h b/video/video_receive_stream2.h index 9b152f420a..e8e3edc3d1 100644 --- a/video/video_receive_stream2.h +++ b/video/video_receive_stream2.h @@ -257,6 +257,16 @@ class VideoReceiveStream2 : public webrtc::VideoReceiveStream, bool keyframe_generation_requested_ RTC_GUARDED_BY(worker_sequence_checker_) = false; + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter |enabled| + // determines if the low-latency renderer algorithm should be used for the + // case min playout delay=0 and max playout delay>0. + FieldTrialParameter low_latency_renderer_enabled_; + // Set by the field trial WebRTC-LowLatencyRenderer. The parameter + // |include_predecode_buffer| determines if the predecode buffer should be + // taken into account when calculating maximum number of frames in composition + // queue. + FieldTrialParameter low_latency_renderer_include_predecode_buffer_; + // Defined last so they are destroyed before all other members. rtc::TaskQueue decode_queue_; diff --git a/video/video_receive_stream2_unittest.cc b/video/video_receive_stream2_unittest.cc index 435975062c..3f10686db7 100644 --- a/video/video_receive_stream2_unittest.cc +++ b/video/video_receive_stream2_unittest.cc @@ -230,6 +230,40 @@ TEST_F(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMinValue) { EXPECT_EQ(default_min_playout_latency, timing_->min_playout_delay()); } +TEST_F(VideoReceiveStream2Test, MaxCompositionDelayNotSetByDefault) { + // Default with no playout delay set. + std::unique_ptr test_frame0(new FrameObjectFake()); + test_frame0->id.picture_id = 0; + video_receive_stream_->OnCompleteFrame(std::move(test_frame0)); + EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); + + // Max composition delay not set for playout delay 0,0. + std::unique_ptr test_frame1(new FrameObjectFake()); + test_frame1->id.picture_id = 1; + test_frame1->SetPlayoutDelay({0, 0}); + video_receive_stream_->OnCompleteFrame(std::move(test_frame1)); + EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); + + // Max composition delay not set for playout delay X,Y, where X,Y>0. + std::unique_ptr test_frame2(new FrameObjectFake()); + test_frame2->id.picture_id = 2; + test_frame2->SetPlayoutDelay({10, 30}); + video_receive_stream_->OnCompleteFrame(std::move(test_frame2)); + EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); +} + +TEST_F(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) { + // Max composition delay set if playout delay X,Y, where X=0,Y>0. + const VideoPlayoutDelay kPlayoutDelayMs = {0, 50}; + const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps. + std::unique_ptr test_frame(new FrameObjectFake()); + test_frame->id.picture_id = 0; + test_frame->SetPlayoutDelay(kPlayoutDelayMs); + video_receive_stream_->OnCompleteFrame(std::move(test_frame)); + EXPECT_EQ(kExpectedMaxCompositionDelayInFrames, + timing_->MaxCompositionDelayInFrames()); +} + class VideoReceiveStream2TestWithFakeDecoder : public ::testing::Test { public: VideoReceiveStream2TestWithFakeDecoder()