Set is_steady_state_refresh_frame in FrameEncodeMetadataWriter

Set is_steady_state_refresh_frame true if the update rectangle of the
corresponding VideoFrame is empty. Set it to false otherwise.

Rename FillTimingInfo to FillMetadataAndTimingInfo.

Bug: chromium:328598314
Change-Id: I7a3e89b180432473b087e849fce636ce1b329637
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/355780
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42551}

video/frame_encode_metadata_writer.cc

@@ -105,6 +105,7 @@ void FrameEncodeMetadataWriter::OnEncodeStarted(const VideoFrame& frame) {
   metadata.timestamp_us = frame.timestamp_us();
   metadata.rotation = frame.rotation();
   metadata.color_space = frame.color_space();
+  metadata.is_steady_state_refresh_frame = frame.update_rect().IsEmpty();
   metadata.packet_infos = frame.packet_infos();
   for (size_t si = 0; si < num_spatial_layers_; ++si) {
     RTC_DCHECK(timing_frames_info_[si].frames.empty() ||
@@ -136,8 +137,16 @@ void FrameEncodeMetadataWriter::OnEncodeStarted(const VideoFrame& frame) {
   }
 }
 
+// Deprecated, remove once downstream projects are updated.
+// https://crbug.com/328598314.
 void FrameEncodeMetadataWriter::FillTimingInfo(size_t simulcast_svc_idx,
                                                EncodedImage* encoded_image) {
+  FillMetadataAndTimingInfo(simulcast_svc_idx, encoded_image);
+}
+
+void FrameEncodeMetadataWriter::FillMetadataAndTimingInfo(
+    size_t simulcast_svc_idx,
+    EncodedImage* encoded_image) {
   MutexLock lock(&lock_);
   absl::optional<size_t> outlier_frame_size;
   absl::optional<int64_t> encode_start_ms;
@@ -256,6 +265,8 @@ FrameEncodeMetadataWriter::ExtractEncodeStartTimeAndFillMetadata(
       encoded_image->ntp_time_ms_ = metadata_list->front().ntp_time_ms;
       encoded_image->rotation_ = metadata_list->front().rotation;
       encoded_image->SetColorSpace(metadata_list->front().color_space);
+      encoded_image->SetIsSteadyStateRefreshFrame(
+          metadata_list->front().is_steady_state_refresh_frame);
       encoded_image->SetPacketInfos(metadata_list->front().packet_infos);
       metadata_list->pop_front();
     } else {

video/frame_encode_metadata_writer.h

@@ -34,6 +34,11 @@ class FrameEncodeMetadataWriter {
 
   void OnEncodeStarted(const VideoFrame& frame);
 
+  void FillMetadataAndTimingInfo(size_t simulcast_svc_idx,
+                                 EncodedImage* encoded_image);
+
+  // Deprecated, remove once downstream projects are updated.
+  // https://crbug.com/328598314.
   void FillTimingInfo(size_t simulcast_svc_idx, EncodedImage* encoded_image);
 
   void UpdateBitstream(const CodecSpecificInfo* codec_specific_info,
@@ -55,6 +60,7 @@ class FrameEncodeMetadataWriter {
     int64_t timestamp_us = 0;
     VideoRotation rotation = kVideoRotation_0;
     absl::optional<ColorSpace> color_space;
+    bool is_steady_state_refresh_frame = false;
     RtpPacketInfos packet_infos;
   };
   struct TimingFramesLayerInfo {

video/frame_encode_metadata_writer_unittest.cc

@@ -113,7 +113,7 @@ std::vector<std::vector<FrameType>> GetTimingFrames(
         continue;
       }
 
-      encode_timer.FillTimingInfo(si, &image);
+      encode_timer.FillMetadataAndTimingInfo(si, &image);
 
       if (IsTimingFrame(image)) {
         result[si].push_back(FrameType::kTiming);
@@ -217,14 +217,14 @@ TEST(FrameEncodeMetadataWriterTest, NoTimingFrameIfNoEncodeStartTime) {
                          .set_video_frame_buffer(kFrameBuffer)
                          .build();
   encode_timer.OnEncodeStarted(frame);
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   EXPECT_TRUE(IsTimingFrame(image));
 
   // New frame, now skip OnEncodeStarted. Should not result in timing frame.
   image.capture_time_ms_ = ++timestamp;
   image.SetRtpTimestamp(static_cast<uint32_t>(timestamp * 90));
   image.timing_ = EncodedImage::Timing();
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   EXPECT_FALSE(IsTimingFrame(image));
 }
 
@@ -256,7 +256,7 @@ TEST(FrameEncodeMetadataWriterTest, NotifiesAboutDroppedFrames) {
   encode_timer.OnEncodeStarted(frame);
 
   EXPECT_EQ(0u, sink.GetNumFramesDropped());
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
 
   image.capture_time_ms_ = kTimestampMs2;
   image.SetRtpTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
@@ -274,7 +274,7 @@ TEST(FrameEncodeMetadataWriterTest, NotifiesAboutDroppedFrames) {
   frame.set_rtp_timestamp(image.capture_time_ms_ * 90);
   frame.set_timestamp_us(image.capture_time_ms_ * 1000);
   encode_timer.OnEncodeStarted(frame);
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   EXPECT_EQ(1u, sink.GetNumFramesDropped());
 
   image.capture_time_ms_ = kTimestampMs4;
@@ -283,7 +283,7 @@ TEST(FrameEncodeMetadataWriterTest, NotifiesAboutDroppedFrames) {
   frame.set_rtp_timestamp(image.capture_time_ms_ * 90);
   frame.set_timestamp_us(image.capture_time_ms_ * 1000);
   encode_timer.OnEncodeStarted(frame);
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   EXPECT_EQ(1u, sink.GetNumFramesDropped());
 }
 
@@ -308,7 +308,7 @@ TEST(FrameEncodeMetadataWriterTest, RestoresCaptureTimestamps) {
                          .build();
   encode_timer.OnEncodeStarted(frame);
   image.capture_time_ms_ = 0;  // Incorrect timestamp.
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   EXPECT_EQ(kTimestampMs, image.capture_time_ms_);
 }
 
@@ -332,7 +332,7 @@ TEST(FrameEncodeMetadataWriterTest, CopiesRotation) {
                          .set_video_frame_buffer(kFrameBuffer)
                          .build();
   encode_timer.OnEncodeStarted(frame);
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   EXPECT_EQ(kVideoRotation_180, image.rotation_);
 }
 
@@ -358,7 +358,7 @@ TEST(FrameEncodeMetadataWriterTest, SetsContentType) {
                          .set_video_frame_buffer(kFrameBuffer)
                          .build();
   encode_timer.OnEncodeStarted(frame);
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   EXPECT_EQ(VideoContentType::SCREENSHARE, image.content_type_);
 }
 
@@ -384,11 +384,46 @@ TEST(FrameEncodeMetadataWriterTest, CopiesColorSpace) {
                          .set_video_frame_buffer(kFrameBuffer)
                          .build();
   encode_timer.OnEncodeStarted(frame);
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   ASSERT_NE(image.ColorSpace(), nullptr);
   EXPECT_EQ(color_space, *image.ColorSpace());
 }
 
+TEST(FrameEncodeMetadataWriterTest, SetsIsSteadyStateRefreshFrame) {
+  EncodedImage image;
+  const int64_t kTimestampMs = 123456;
+  FakeEncodedImageCallback sink;
+
+  FrameEncodeMetadataWriter encode_timer(&sink);
+  encode_timer.OnEncoderInit(VideoCodec());
+  // Any non-zero bitrate needed to be set before the first frame.
+  VideoBitrateAllocation bitrate_allocation;
+  bitrate_allocation.SetBitrate(0, 0, 500000);
+  encode_timer.OnSetRates(bitrate_allocation, 30);
+
+  image.SetRtpTimestamp(static_cast<uint32_t>(kTimestampMs * 90));
+  VideoFrame not_refresh_frame = VideoFrame::Builder()
+                                     .set_timestamp_ms(kTimestampMs)
+                                     .set_rtp_timestamp(kTimestampMs * 90)
+                                     .set_video_frame_buffer(kFrameBuffer)
+                                     .build();
+  encode_timer.OnEncodeStarted(not_refresh_frame);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
+  EXPECT_FALSE(image.IsSteadyStateRefreshFrame());
+
+  VideoFrame::UpdateRect empty_update_rect;
+  empty_update_rect.MakeEmptyUpdate();
+  VideoFrame refresh_frame = VideoFrame::Builder()
+                                 .set_timestamp_ms(kTimestampMs)
+                                 .set_rtp_timestamp(kTimestampMs * 90)
+                                 .set_update_rect(empty_update_rect)
+                                 .set_video_frame_buffer(kFrameBuffer)
+                                 .build();
+  encode_timer.OnEncodeStarted(refresh_frame);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
+  EXPECT_TRUE(image.IsSteadyStateRefreshFrame());
+}
+
 TEST(FrameEncodeMetadataWriterTest, CopiesPacketInfos) {
   EncodedImage image;
   const int64_t kTimestampMs = 123456;
@@ -410,7 +445,7 @@ TEST(FrameEncodeMetadataWriterTest, CopiesPacketInfos) {
                          .set_video_frame_buffer(kFrameBuffer)
                          .build();
   encode_timer.OnEncodeStarted(frame);
-  encode_timer.FillTimingInfo(0, &image);
+  encode_timer.FillMetadataAndTimingInfo(0, &image);
   EXPECT_EQ(image.PacketInfos().size(), 3U);
 }
 

video/video_stream_encoder.cc

@@ -2078,7 +2078,9 @@ EncodedImage VideoStreamEncoder::AugmentEncodedImage(
   // simulcast and spatial indices.
   int stream_idx = encoded_image.SpatialIndex().value_or(
       encoded_image.SimulcastIndex().value_or(0));
-  frame_encode_metadata_writer_.FillTimingInfo(stream_idx, &image_copy);
+
+  frame_encode_metadata_writer_.FillMetadataAndTimingInfo(stream_idx,
+                                                          &image_copy);
   frame_encode_metadata_writer_.UpdateBitstream(codec_specific_info,
                                                 &image_copy);
   VideoCodecType codec_type = codec_specific_info