admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Set is_steady_state_refresh_frame in FrameEncodeMetadataWriter

Browse Source 
Set is_steady_state_refresh_frame true if the update rectangle of the
corresponding VideoFrame is empty. Set it to false otherwise.

Rename FillTimingInfo to FillMetadataAndTimingInfo.

Bug: chromium:328598314
Change-Id: I7a3e89b180432473b087e849fce636ce1b329637
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/355780
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42551}
This commit is contained in:
Johannes Kron 2024-06-26 18:16:02 +02:00 committed by WebRTC LUCI CQ
parent 7235dd0e2b
commit 529707576a
4 changed files with 66 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
video/frame_encode_metadata_writer.cc
View File

@ -105,6 +105,7 @@ void FrameEncodeMetadataWriter::OnEncodeStarted(const VideoFrame& frame) {
metadata.timestamp_us = frame.timestamp_us();
metadata.rotation = frame.rotation();
metadata.color_space = frame.color_space();
metadata.is_steady_state_refresh_frame = frame.update_rect().IsEmpty();
metadata.packet_infos = frame.packet_infos();
for (size_t si = 0; si < num_spatial_layers_; ++si) {
RTC_DCHECK(timing_frames_info_[si].frames.empty() ||
@ -136,8 +137,16 @@ void FrameEncodeMetadataWriter::OnEncodeStarted(const VideoFrame& frame) {
}
}
// Deprecated, remove once downstream projects are updated.
// https://crbug.com/328598314.
void FrameEncodeMetadataWriter::FillTimingInfo(size_t simulcast_svc_idx,
EncodedImage* encoded_image) {
FillMetadataAndTimingInfo(simulcast_svc_idx, encoded_image);
}
void FrameEncodeMetadataWriter::FillMetadataAndTimingInfo(
size_t simulcast_svc_idx,
EncodedImage* encoded_image) {
MutexLock lock(&lock_);
absl::optional<size_t> outlier_frame_size;
absl::optional<int64_t> encode_start_ms;
@ -256,6 +265,8 @@ FrameEncodeMetadataWriter::ExtractEncodeStartTimeAndFillMetadata(
encoded_image->ntp_time_ms_ = metadata_list->front().ntp_time_ms;
encoded_image->rotation_ = metadata_list->front().rotation;
encoded_image->SetColorSpace(metadata_list->front().color_space);
encoded_image->SetIsSteadyStateRefreshFrame(
metadata_list->front().is_steady_state_refresh_frame);
encoded_image->SetPacketInfos(metadata_list->front().packet_infos);
metadata_list->pop_front();
} else {

6
video/frame_encode_metadata_writer.h
View File

@ -34,6 +34,11 @@ class FrameEncodeMetadataWriter {
void OnEncodeStarted(const VideoFrame& frame);
void FillMetadataAndTimingInfo(size_t simulcast_svc_idx,
EncodedImage* encoded_image);
// Deprecated, remove once downstream projects are updated.
// https://crbug.com/328598314.
void FillTimingInfo(size_t simulcast_svc_idx, EncodedImage* encoded_image);
void UpdateBitstream(const CodecSpecificInfo* codec_specific_info,
@ -55,6 +60,7 @@ class FrameEncodeMetadataWriter {
int64_t timestamp_us = 0;
VideoRotation rotation = kVideoRotation_0;
absl::optional<ColorSpace> color_space;
bool is_steady_state_refresh_frame = false;
RtpPacketInfos packet_infos;
};
struct TimingFramesLayerInfo {

57
video/frame_encode_metadata_writer_unittest.cc
View File

@ -113,7 +113,7 @@ std::vector<std::vector<FrameType>> GetTimingFrames(
continue;
}
encode_timer.FillTimingInfo(si, &image);
encode_timer.FillMetadataAndTimingInfo(si, &image);
if (IsTimingFrame(image)) {
result[si].push_back(FrameType::kTiming);
@ -217,14 +217,14 @@ TEST(FrameEncodeMetadataWriterTest, NoTimingFrameIfNoEncodeStartTime) {
.set_video_frame_buffer(kFrameBuffer)
.build();
encode_timer.OnEncodeStarted(frame);
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_TRUE(IsTimingFrame(image));
// New frame, now skip OnEncodeStarted. Should not result in timing frame.
image.capture_time_ms_ = ++timestamp;
image.SetRtpTimestamp(static_cast<uint32_t>(timestamp * 90));
image.timing_ = EncodedImage::Timing();
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_FALSE(IsTimingFrame(image));
}
@ -256,7 +256,7 @@ TEST(FrameEncodeMetadataWriterTest, NotifiesAboutDroppedFrames) {
encode_timer.OnEncodeStarted(frame);
EXPECT_EQ(0u, sink.GetNumFramesDropped());
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
image.capture_time_ms_ = kTimestampMs2;
image.SetRtpTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
@ -274,7 +274,7 @@ TEST(FrameEncodeMetadataWriterTest, NotifiesAboutDroppedFrames) {
frame.set_rtp_timestamp(image.capture_time_ms_ * 90);
frame.set_timestamp_us(image.capture_time_ms_ * 1000);
encode_timer.OnEncodeStarted(frame);
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_EQ(1u, sink.GetNumFramesDropped());
image.capture_time_ms_ = kTimestampMs4;
@ -283,7 +283,7 @@ TEST(FrameEncodeMetadataWriterTest, NotifiesAboutDroppedFrames) {
frame.set_rtp_timestamp(image.capture_time_ms_ * 90);
frame.set_timestamp_us(image.capture_time_ms_ * 1000);
encode_timer.OnEncodeStarted(frame);
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_EQ(1u, sink.GetNumFramesDropped());
}
@ -308,7 +308,7 @@ TEST(FrameEncodeMetadataWriterTest, RestoresCaptureTimestamps) {
.build();
encode_timer.OnEncodeStarted(frame);
image.capture_time_ms_ = 0; // Incorrect timestamp.
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_EQ(kTimestampMs, image.capture_time_ms_);
}
@ -332,7 +332,7 @@ TEST(FrameEncodeMetadataWriterTest, CopiesRotation) {
.set_video_frame_buffer(kFrameBuffer)
.build();
encode_timer.OnEncodeStarted(frame);
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_EQ(kVideoRotation_180, image.rotation_);
}
@ -358,7 +358,7 @@ TEST(FrameEncodeMetadataWriterTest, SetsContentType) {
.set_video_frame_buffer(kFrameBuffer)
.build();
encode_timer.OnEncodeStarted(frame);
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_EQ(VideoContentType::SCREENSHARE, image.content_type_);
}
@ -384,11 +384,46 @@ TEST(FrameEncodeMetadataWriterTest, CopiesColorSpace) {
.set_video_frame_buffer(kFrameBuffer)
.build();
encode_timer.OnEncodeStarted(frame);
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
ASSERT_NE(image.ColorSpace(), nullptr);
EXPECT_EQ(color_space, *image.ColorSpace());
}
TEST(FrameEncodeMetadataWriterTest, SetsIsSteadyStateRefreshFrame) {
EncodedImage image;
const int64_t kTimestampMs = 123456;
FakeEncodedImageCallback sink;
FrameEncodeMetadataWriter encode_timer(&sink);
encode_timer.OnEncoderInit(VideoCodec());
// Any non-zero bitrate needed to be set before the first frame.
VideoBitrateAllocation bitrate_allocation;
bitrate_allocation.SetBitrate(0, 0, 500000);
encode_timer.OnSetRates(bitrate_allocation, 30);
image.SetRtpTimestamp(static_cast<uint32_t>(kTimestampMs * 90));
VideoFrame not_refresh_frame = VideoFrame::Builder()
.set_timestamp_ms(kTimestampMs)
.set_rtp_timestamp(kTimestampMs * 90)
.set_video_frame_buffer(kFrameBuffer)
.build();
encode_timer.OnEncodeStarted(not_refresh_frame);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_FALSE(image.IsSteadyStateRefreshFrame());
VideoFrame::UpdateRect empty_update_rect;
empty_update_rect.MakeEmptyUpdate();
VideoFrame refresh_frame = VideoFrame::Builder()
.set_timestamp_ms(kTimestampMs)
.set_rtp_timestamp(kTimestampMs * 90)
.set_update_rect(empty_update_rect)
.set_video_frame_buffer(kFrameBuffer)
.build();
encode_timer.OnEncodeStarted(refresh_frame);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_TRUE(image.IsSteadyStateRefreshFrame());
}
TEST(FrameEncodeMetadataWriterTest, CopiesPacketInfos) {
EncodedImage image;
const int64_t kTimestampMs = 123456;
@ -410,7 +445,7 @@ TEST(FrameEncodeMetadataWriterTest, CopiesPacketInfos) {
.set_video_frame_buffer(kFrameBuffer)
.build();
encode_timer.OnEncodeStarted(frame);
encode_timer.FillTimingInfo(0, &image);
encode_timer.FillMetadataAndTimingInfo(0, &image);
EXPECT_EQ(image.PacketInfos().size(), 3U);
}

4
video/video_stream_encoder.cc
View File

@ -2078,7 +2078,9 @@ EncodedImage VideoStreamEncoder::AugmentEncodedImage(
// simulcast and spatial indices.
int stream_idx = encoded_image.SpatialIndex().value_or(
encoded_image.SimulcastIndex().value_or(0));
frame_encode_metadata_writer_.FillTimingInfo(stream_idx, &image_copy);
frame_encode_metadata_writer_.FillMetadataAndTimingInfo(stream_idx,
&image_copy);
frame_encode_metadata_writer_.UpdateBitstream(codec_specific_info,
&image_copy);
VideoCodecType codec_type = codec_specific_info