Signaling for low-latency renderer algorithm

This feature is active if and only if the RTP header extension
playout-delay is used with min playout delay=0 and max playout delay>0.

In this case, a maximum composition delay will be calculated and attached
to the video frame as a signal to use the low-latency renderer algorithm,
which is landed in a separate CL in Chromium.

The maximum composition delay is specified in number of frames and is
calculated based on the max playout delay.

The feature can be completetly disabled by specifying the field trial
WebRTC-LowLatencyRenderer/enabled:false/

Bug: chromium:1138888
Change-Id: I05f461982d0632bd6e09e5d7ec1a8985dccdc61b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/190141
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32493}

api/video/video_frame.h

@@ -186,6 +186,16 @@ class RTC_EXPORT VideoFrame {
     color_space_ = color_space;
   }
 
+  // max_composition_delay_in_frames() is used in an experiment of a low-latency
+  // renderer algorithm see crbug.com/1138888.
+  absl::optional<int32_t> max_composition_delay_in_frames() const {
+    return max_composition_delay_in_frames_;
+  }
+  void set_max_composition_delay_in_frames(
+      absl::optional<int32_t> max_composition_delay_in_frames) {
+    max_composition_delay_in_frames_ = max_composition_delay_in_frames;
+  }
+
   // Get render time in milliseconds.
   // TODO(nisse): Deprecated. Migrate all users to timestamp_us().
   int64_t render_time_ms() const;
@@ -255,6 +265,7 @@ class RTC_EXPORT VideoFrame {
   int64_t timestamp_us_;
   VideoRotation rotation_;
   absl::optional<ColorSpace> color_space_;
+  absl::optional<int32_t> max_composition_delay_in_frames_;
   // Updated since the last frame area. If present it means that the bounding
   // box of all the changes is within the rectangular area and is close to it.
   // If absent, it means that there's no information about the change at all and

modules/video_coding/encoded_frame.h

@@ -34,6 +34,8 @@ class RTC_EXPORT VCMEncodedFrame : protected EncodedImage {
     _renderTimeMs = renderTimeMs;
   }
 
+  VideoPlayoutDelay PlayoutDelay() const { return playout_delay_; }
+
   void SetPlayoutDelay(VideoPlayoutDelay playout_delay) {
     playout_delay_ = playout_delay;
   }

modules/video_coding/frame_buffer2.cc

@@ -348,6 +348,11 @@ void FrameBuffer::Clear() {
   ClearFramesAndHistory();
 }
 
+int FrameBuffer::Size() {
+  MutexLock lock(&mutex_);
+  return frames_.size();
+}
+
 void FrameBuffer::UpdateRtt(int64_t rtt_ms) {
   MutexLock lock(&mutex_);
   jitter_estimator_.UpdateRtt(rtt_ms);

modules/video_coding/frame_buffer2.h

@@ -84,6 +84,8 @@ class FrameBuffer {
   // Clears the FrameBuffer, removing all the buffered frames.
   void Clear();
 
+  int Size();
+
  private:
   struct FrameInfo {
     FrameInfo();

modules/video_coding/generic_decoder.cc

@@ -13,6 +13,7 @@
 #include <stddef.h>
 
 #include <algorithm>
+#include <cmath>
 
 #include "api/video/video_timing.h"
 #include "modules/video_coding/include/video_error_codes.h"
@@ -31,12 +32,18 @@ VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing,
     : _clock(clock),
       _timing(timing),
       _timestampMap(kDecoderFrameMemoryLength),
-      _extra_decode_time("t", absl::nullopt) {
+      _extra_decode_time("t", absl::nullopt),
+      low_latency_renderer_enabled_("enabled", true),
+      low_latency_renderer_include_predecode_buffer_("include_predecode_buffer",
+                                                     true) {
   ntp_offset_ =
       _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds();
 
   ParseFieldTrial({&_extra_decode_time},
                   field_trial::FindFullName("WebRTC-SlowDownDecoder"));
+  ParseFieldTrial({&low_latency_renderer_enabled_,
+                   &low_latency_renderer_include_predecode_buffer_},
+                  field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
 }
 
 VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {}
@@ -85,9 +92,11 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
   // TODO(holmer): We should improve this so that we can handle multiple
   // callbacks from one call to Decode().
   VCMFrameInformation* frameInfo;
+  int timestamp_map_size = 0;
   {
     MutexLock lock(&lock_);
     frameInfo = _timestampMap.Pop(decodedImage.timestamp());
+    timestamp_map_size = _timestampMap.Size();
   }
 
   if (frameInfo == NULL) {
@@ -101,6 +110,22 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
   decodedImage.set_packet_infos(frameInfo->packet_infos);
   decodedImage.set_rotation(frameInfo->rotation);
 
+  if (low_latency_renderer_enabled_ && frameInfo->playout_delay.min_ms == 0 &&
+      frameInfo->playout_delay.max_ms > 0) {
+    absl::optional<int> max_composition_delay_in_frames =
+        _timing->MaxCompositionDelayInFrames();
+    if (max_composition_delay_in_frames) {
+      // Subtract frames that are in flight.
+      if (low_latency_renderer_include_predecode_buffer_) {
+        *max_composition_delay_in_frames -= timestamp_map_size;
+        *max_composition_delay_in_frames =
+            std::max(0, *max_composition_delay_in_frames);
+      }
+      decodedImage.set_max_composition_delay_in_frames(
+          max_composition_delay_in_frames);
+    }
+  }
+
   RTC_DCHECK(frameInfo->decodeStart);
   const Timestamp now = _clock->CurrentTime();
   const TimeDelta decode_time = decode_time_ms
@@ -224,6 +249,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
   _frameInfos[_nextFrameInfoIdx].decodeStart = now;
   _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
   _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
+  _frameInfos[_nextFrameInfoIdx].playout_delay = frame.PlayoutDelay();
   _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing();
   _frameInfos[_nextFrameInfoIdx].ntp_time_ms =
       frame.EncodedImage().ntp_time_ms_;

modules/video_coding/generic_decoder.h

@@ -35,6 +35,7 @@ struct VCMFrameInformation {
   void* userData;
   VideoRotation rotation;
   VideoContentType content_type;
+  PlayoutDelay playout_delay;
   EncodedImage::Timing timing;
   int64_t ntp_time_ms;
   RtpPacketInfos packet_infos;
@@ -75,6 +76,16 @@ class VCMDecodedFrameCallback : public DecodedImageCallback {
   int64_t ntp_offset_;
   // Set by the field trial WebRTC-SlowDownDecoder to simulate a slow decoder.
   FieldTrialOptional<TimeDelta> _extra_decode_time;
+
+  // Set by the field trial WebRTC-LowLatencyRenderer. The parameter |enabled|
+  // determines if the low-latency renderer algorithm should be used for the
+  // case min playout delay=0 and max playout delay>0.
+  FieldTrialParameter<bool> low_latency_renderer_enabled_;
+  // Set by the field trial WebRTC-LowLatencyRenderer. The parameter
+  // |include_predecode_buffer| determines if the predecode buffer should be
+  // taken into account when calculating maximum number of frames in composition
+  // queue.
+  FieldTrialParameter<bool> low_latency_renderer_include_predecode_buffer_;
 };
 
 class VCMGenericDecoder {

modules/video_coding/generic_decoder_unittest.cc

@@ -115,5 +115,29 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) {
   EXPECT_EQ(decoded_frame->packet_infos().size(), 3U);
 }
 
+TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) {
+  VCMEncodedFrame encoded_frame;
+  generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
+  absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
+  ASSERT_TRUE(decoded_frame.has_value());
+  EXPECT_FALSE(decoded_frame->max_composition_delay_in_frames());
+}
+
+TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) {
+  VCMEncodedFrame encoded_frame;
+  // VideoReceiveStream2 would set MaxCompositionDelayInFrames if playout delay
+  // is specified as X,Y, where X=0, Y>0.
+  const VideoPlayoutDelay kPlayoutDelay = {0, 50};
+  constexpr int kMaxCompositionDelayInFrames = 3;  // ~50 ms at 60 fps.
+  encoded_frame.SetPlayoutDelay(kPlayoutDelay);
+  timing_.SetMaxCompositionDelayInFrames(
+      absl::make_optional(kMaxCompositionDelayInFrames));
+  generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
+  absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
+  ASSERT_TRUE(decoded_frame.has_value());
+  EXPECT_EQ(kMaxCompositionDelayInFrames,
+            decoded_frame->max_composition_delay_in_frames());
+}
+
 }  // namespace video_coding
 }  // namespace webrtc

modules/video_coding/timestamp_map.cc

@@ -60,4 +60,13 @@ VCMFrameInformation* VCMTimestampMap::Pop(uint32_t timestamp) {
 bool VCMTimestampMap::IsEmpty() const {
   return (next_add_idx_ == next_pop_idx_);
 }
+
+size_t VCMTimestampMap::Size() const {
+  // The maximum number of elements in the list is |capacity_| - 1. The list is
+  // empty if the add and pop indices are equal.
+  return next_add_idx_ >= next_pop_idx_
+             ? next_add_idx_ - next_pop_idx_
+             : next_add_idx_ + capacity_ - next_pop_idx_;
+}
+
 }  // namespace webrtc

modules/video_coding/timestamp_map.h

@@ -24,6 +24,7 @@ class VCMTimestampMap {
 
   void Add(uint32_t timestamp, VCMFrameInformation* data);
   VCMFrameInformation* Pop(uint32_t timestamp);
+  size_t Size() const;
 
  private:
   struct TimestampDataTuple {

modules/video_coding/timing.cc

@@ -14,8 +14,10 @@
 
 #include <algorithm>
 
+#include "rtc_base/experiments/field_trial_parser.h"
 #include "rtc_base/time/timestamp_extrapolator.h"
 #include "system_wrappers/include/clock.h"
+#include "system_wrappers/include/field_trial.h"
 
 namespace webrtc {
 
@@ -31,7 +33,10 @@ VCMTiming::VCMTiming(Clock* clock, VCMTiming* master_timing)
       current_delay_ms_(0),
       prev_frame_timestamp_(0),
       timing_frame_info_(),
-      num_decoded_frames_(0) {
+      num_decoded_frames_(0),
+      low_latency_renderer_enabled_("enabled", true) {
+  ParseFieldTrial({&low_latency_renderer_enabled_},
+                  field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
   if (master_timing == NULL) {
     master_ = true;
     ts_extrapolator_ = new TimestampExtrapolator(clock_->TimeInMilliseconds());
@@ -177,8 +182,12 @@ int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp,
 
 int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp,
                                         int64_t now_ms) const {
-  if (min_playout_delay_ms_ == 0 && max_playout_delay_ms_ == 0) {
-    // Render as soon as possible.
+  constexpr int kLowLatencyRendererMaxPlayoutDelayMs = 500;
+  if (min_playout_delay_ms_ == 0 &&
+      (max_playout_delay_ms_ == 0 ||
+       (low_latency_renderer_enabled_ &&
+        max_playout_delay_ms_ <= kLowLatencyRendererMaxPlayoutDelayMs))) {
+    // Render as soon as possible or with low-latency renderer algorithm.
     return 0;
   }
   int64_t estimated_complete_time_ms =
@@ -246,4 +255,15 @@ absl::optional<TimingFrameInfo> VCMTiming::GetTimingFrameInfo() {
   return timing_frame_info_;
 }
 
+void VCMTiming::SetMaxCompositionDelayInFrames(
+    absl::optional<int> max_composition_delay_in_frames) {
+  MutexLock lock(&mutex_);
+  max_composition_delay_in_frames_ = max_composition_delay_in_frames;
+}
+
+absl::optional<int> VCMTiming::MaxCompositionDelayInFrames() const {
+  MutexLock lock(&mutex_);
+  return max_composition_delay_in_frames_;
+}
+
 }  // namespace webrtc

modules/video_coding/timing.h

@@ -16,6 +16,7 @@
 #include "absl/types/optional.h"
 #include "api/video/video_timing.h"
 #include "modules/video_coding/codec_timer.h"
+#include "rtc_base/experiments/field_trial_parser.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/thread_annotations.h"
 
@@ -100,6 +101,10 @@ class VCMTiming {
   void SetTimingFrameInfo(const TimingFrameInfo& info);
   absl::optional<TimingFrameInfo> GetTimingFrameInfo();
 
+  void SetMaxCompositionDelayInFrames(
+      absl::optional<int> max_composition_delay_in_frames);
+  absl::optional<int> MaxCompositionDelayInFrames() const;
+
   enum { kDefaultRenderDelayMs = 10 };
   enum { kDelayMaxChangeMsPerS = 100 };
 
@@ -128,6 +133,12 @@ class VCMTiming {
   uint32_t prev_frame_timestamp_ RTC_GUARDED_BY(mutex_);
   absl::optional<TimingFrameInfo> timing_frame_info_ RTC_GUARDED_BY(mutex_);
   size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_);
+  // Set by the field trial WebRTC-LowLatencyRenderer. The parameter enabled
+  // determines if the low-latency renderer algorithm should be used for the
+  // case min playout delay=0 and max playout delay>0.
+  FieldTrialParameter<bool> low_latency_renderer_enabled_
+      RTC_GUARDED_BY(mutex_);
+  absl::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_);
 };
 }  // namespace webrtc
 

video/video_receive_stream2.cc

@@ -231,6 +231,9 @@ VideoReceiveStream2::VideoReceiveStream2(
       max_wait_for_frame_ms_(KeyframeIntervalSettings::ParseFromFieldTrials()
                                  .MaxWaitForFrameMs()
                                  .value_or(kMaxWaitForFrameMs)),
+      low_latency_renderer_enabled_("enabled", true),
+      low_latency_renderer_include_predecode_buffer_("include_predecode_buffer",
+                                                     true),
       decode_queue_(task_queue_factory_->CreateTaskQueue(
           "DecodingQueue",
           TaskQueueFactory::Priority::HIGH)) {
@@ -271,6 +274,10 @@ VideoReceiveStream2::VideoReceiveStream2(
     rtp_receive_statistics_->EnableRetransmitDetection(config.rtp.remote_ssrc,
                                                        true);
   }
+
+  ParseFieldTrial({&low_latency_renderer_enabled_,
+                   &low_latency_renderer_include_predecode_buffer_},
+                  field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
 }
 
 VideoReceiveStream2::~VideoReceiveStream2() {
@@ -778,6 +785,22 @@ void VideoReceiveStream2::UpdatePlayoutDelays() const {
                 syncable_minimum_playout_delay_ms_});
   if (minimum_delay_ms >= 0) {
     timing_->set_min_playout_delay(minimum_delay_ms);
+    if (frame_minimum_playout_delay_ms_ == 0 &&
+        frame_maximum_playout_delay_ms_ > 0 && low_latency_renderer_enabled_) {
+      // TODO(kron): Estimate frame rate from video stream.
+      constexpr double kFrameRate = 60.0;
+      // Convert playout delay in ms to number of frames.
+      int max_composition_delay_in_frames = std::lrint(
+          static_cast<double>(frame_maximum_playout_delay_ms_ * kFrameRate) /
+          rtc::kNumMillisecsPerSec);
+      if (low_latency_renderer_include_predecode_buffer_) {
+        // Subtract frames in buffer.
+        max_composition_delay_in_frames = std::max<int16_t>(
+            max_composition_delay_in_frames - frame_buffer_->Size(), 0);
+      }
+      timing_->SetMaxCompositionDelayInFrames(
+          absl::make_optional(max_composition_delay_in_frames));
+    }
   }
 
   const int maximum_delay_ms = frame_maximum_playout_delay_ms_;

video/video_receive_stream2.h

@@ -257,6 +257,16 @@ class VideoReceiveStream2 : public webrtc::VideoReceiveStream,
   bool keyframe_generation_requested_ RTC_GUARDED_BY(worker_sequence_checker_) =
       false;
 
+  // Set by the field trial WebRTC-LowLatencyRenderer. The parameter |enabled|
+  // determines if the low-latency renderer algorithm should be used for the
+  // case min playout delay=0 and max playout delay>0.
+  FieldTrialParameter<bool> low_latency_renderer_enabled_;
+  // Set by the field trial WebRTC-LowLatencyRenderer. The parameter
+  // |include_predecode_buffer| determines if the predecode buffer should be
+  // taken into account when calculating maximum number of frames in composition
+  // queue.
+  FieldTrialParameter<bool> low_latency_renderer_include_predecode_buffer_;
+
   // Defined last so they are destroyed before all other members.
   rtc::TaskQueue decode_queue_;
 

video/video_receive_stream2_unittest.cc

@@ -230,6 +230,40 @@ TEST_F(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMinValue) {
   EXPECT_EQ(default_min_playout_latency, timing_->min_playout_delay());
 }
 
+TEST_F(VideoReceiveStream2Test, MaxCompositionDelayNotSetByDefault) {
+  // Default with no playout delay set.
+  std::unique_ptr<FrameObjectFake> test_frame0(new FrameObjectFake());
+  test_frame0->id.picture_id = 0;
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
+  EXPECT_FALSE(timing_->MaxCompositionDelayInFrames());
+
+  // Max composition delay not set for playout delay 0,0.
+  std::unique_ptr<FrameObjectFake> test_frame1(new FrameObjectFake());
+  test_frame1->id.picture_id = 1;
+  test_frame1->SetPlayoutDelay({0, 0});
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
+  EXPECT_FALSE(timing_->MaxCompositionDelayInFrames());
+
+  // Max composition delay not set for playout delay X,Y, where X,Y>0.
+  std::unique_ptr<FrameObjectFake> test_frame2(new FrameObjectFake());
+  test_frame2->id.picture_id = 2;
+  test_frame2->SetPlayoutDelay({10, 30});
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
+  EXPECT_FALSE(timing_->MaxCompositionDelayInFrames());
+}
+
+TEST_F(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) {
+  // Max composition delay set if playout delay X,Y, where X=0,Y>0.
+  const VideoPlayoutDelay kPlayoutDelayMs = {0, 50};
+  const int kExpectedMaxCompositionDelayInFrames = 3;  // ~50 ms at 60 fps.
+  std::unique_ptr<FrameObjectFake> test_frame(new FrameObjectFake());
+  test_frame->id.picture_id = 0;
+  test_frame->SetPlayoutDelay(kPlayoutDelayMs);
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame));
+  EXPECT_EQ(kExpectedMaxCompositionDelayInFrames,
+            timing_->MaxCompositionDelayInFrames());
+}
+
 class VideoReceiveStream2TestWithFakeDecoder : public ::testing::Test {
  public:
   VideoReceiveStream2TestWithFakeDecoder()