From 7fe7091f8a75364a689f7a09a1a8ffe566082961 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Thu, 6 Oct 2022 11:28:30 +0200 Subject: [PATCH] [DVQA] Annotate per frame stats with relevant frame id Bug: b/240540204 Change-Id: Ic12a5778ecdbf7a0b8bd9a54f3d927289c49c34a Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/277802 Reviewed-by: Andrey Logvin Commit-Queue: Artem Titov Cr-Commit-Position: refs/heads/main@{#38307} --- ..._video_quality_analyzer_frame_in_flight.cc | 6 +- ...t_video_quality_analyzer_frame_in_flight.h | 3 + ...ideo_quality_analyzer_frames_comparator.cc | 66 +++--- ...quality_analyzer_frames_comparator_test.cc | 195 +++++++++++++++--- ...quality_analyzer_internal_shared_objects.h | 4 +- 5 files changed, 221 insertions(+), 53 deletions(-) diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc index b6031fdec9..968c6c6555 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "api/units/data_size.h" #include "api/units/timestamp.h" +#include "api/video/video_frame.h" #include "api/video/video_frame_type.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" @@ -54,6 +55,7 @@ void FrameInFlight::SetFrameId(uint16_t id) { if (frame_) { frame_->set_id(id); } + frame_id_ = id; } std::vector FrameInFlight::GetPeersWhichDidntReceive() const { @@ -172,7 +174,9 @@ bool FrameInFlight::IsDropped(size_t peer) const { } FrameStats FrameInFlight::GetStatsForPeer(size_t peer) const { - FrameStats stats(captured_time_); + RTC_DCHECK_NE(frame_id_, VideoFrame::kNotSetId) + << "Frame id isn't initialized"; + FrameStats stats(frame_id_, captured_time_); stats.pre_encode_time = pre_encode_time_; stats.encoded_time = encoded_time_; stats.target_encode_bitrate = target_encode_bitrate_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h index 9ce2e010a8..92c031a442 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h @@ -141,6 +141,9 @@ class FrameInFlight { // object to decide when it should be deleted. std::set expected_receivers_; absl::optional frame_; + // Store frame id separately because `frame_` can be removed when we have too + // much memory consuption. + uint16_t frame_id_ = VideoFrame::kNotSetId; // Frame events timestamp. Timestamp captured_time_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc index ae311f418f..4048efc17a 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -11,6 +11,8 @@ #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" #include +#include +#include #include #include @@ -32,15 +34,22 @@ namespace { constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150); constexpr int kMaxActiveComparisons = 10; +constexpr char kFrameIdMetadataKey[] = "frame_id"; -SamplesStatsCounter::StatsSample StatsSample(double value, - Timestamp sampling_time) { - return SamplesStatsCounter::StatsSample{value, sampling_time}; +SamplesStatsCounter::StatsSample StatsSample( + double value, + Timestamp sampling_time, + std::map metadata) { + return SamplesStatsCounter::StatsSample{value, sampling_time, + std::move(metadata)}; } -SamplesStatsCounter::StatsSample StatsSample(TimeDelta duration, - Timestamp sampling_time) { - return SamplesStatsCounter::StatsSample{duration.ms(), sampling_time}; +SamplesStatsCounter::StatsSample StatsSample( + TimeDelta duration, + Timestamp sampling_time, + std::map metadata) { + return SamplesStatsCounter::StatsSample{duration.ms(), sampling_time, + std::move(metadata)}; } FrameComparison ValidateFrameComparison(FrameComparison comparison) { @@ -224,7 +233,7 @@ void DefaultVideoQualityAnalyzerFramesComparator::Stop( // to last freeze end as time between freezes. stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(StatsSample( last_rendered_frame_time - stream_last_freeze_end_time_.at(stats_key), - Now())); + Now(), /*metadata=*/{})); } } } @@ -298,8 +307,10 @@ void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( MutexLock lock(&mutex_); RTC_CHECK_EQ(state_, State::kActive) << "Frames comparator has to be started before it will be used"; - stream_stats_.at(stats_key).skipped_between_rendered.AddSample( - StatsSample(skipped_between_rendered, Now())); + stream_stats_.at(stats_key).skipped_between_rendered.AddSample(StatsSample( + skipped_between_rendered, Now(), + /*metadata=*/ + {{kFrameIdMetadataKey, std::to_string(frame_stats.frame_id)}})); AddComparisonInternal(std::move(stats_key), std::move(captured), std::move(rendered), type, std::move(frame_stats)); } @@ -312,7 +323,7 @@ void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal( FrameStats frame_stats) { cpu_measurer_.StartExcludingCpuThreadTime(); frames_comparator_stats_.comparisons_queue_size.AddSample( - StatsSample(comparisons_.size(), Now())); + StatsSample(comparisons_.size(), Now(), /*metadata=*/{})); // If there too many computations waiting in the queue, we won't provide // frames itself to make future computations lighter. if (comparisons_.size() >= kMaxActiveComparisons) { @@ -408,11 +419,16 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( frames_comparator_stats_.memory_overloaded_comparisons_done++; } + std::map metadata; + metadata.emplace(kFrameIdMetadataKey, std::to_string(frame_stats.frame_id)); + if (psnr > 0) { - stats->psnr.AddSample(StatsSample(psnr, frame_stats.rendered_time)); + stats->psnr.AddSample( + StatsSample(psnr, frame_stats.rendered_time, metadata)); } if (ssim > 0) { - stats->ssim.AddSample(StatsSample(ssim, frame_stats.received_time)); + stats->ssim.AddSample( + StatsSample(ssim, frame_stats.received_time, metadata)); } // Compute dropped phase for dropped frame @@ -436,12 +452,12 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( if (frame_stats.encoded_time.IsFinite()) { stats->encode_time_ms.AddSample( StatsSample(frame_stats.encoded_time - frame_stats.pre_encode_time, - frame_stats.encoded_time)); + frame_stats.encoded_time, metadata)); stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); stats->total_encoded_images_payload += frame_stats.encoded_image_size.bytes(); stats->target_encode_bitrate.AddSample(StatsSample( - frame_stats.target_encode_bitrate, frame_stats.encoded_time)); + frame_stats.target_encode_bitrate, frame_stats.encoded_time, metadata)); // Stats sliced on encoded frame type. if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) { @@ -455,18 +471,18 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( stats->resolution_of_rendered_frame.AddSample( StatsSample(*comparison.frame_stats.rendered_frame_width * *comparison.frame_stats.rendered_frame_height, - frame_stats.rendered_time)); + frame_stats.rendered_time, metadata)); stats->total_delay_incl_transport_ms.AddSample( StatsSample(frame_stats.rendered_time - frame_stats.captured_time, - frame_stats.received_time)); + frame_stats.received_time, metadata)); stats->receive_to_render_time_ms.AddSample( StatsSample(frame_stats.rendered_time - frame_stats.received_time, - frame_stats.rendered_time)); + frame_stats.rendered_time, metadata)); } if (frame_stats.decode_start_time.IsFinite()) { stats->transport_time_ms.AddSample( StatsSample(frame_stats.decode_start_time - frame_stats.encoded_time, - frame_stats.decode_start_time)); + frame_stats.decode_start_time, metadata)); // Stats sliced on decoded frame type. if (frame_stats.pre_decoded_frame_type == @@ -474,39 +490,39 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( ++stats->num_recv_key_frames; stats->recv_key_frame_size_bytes.AddSample( StatsSample(frame_stats.pre_decoded_image_size.bytes(), - frame_stats.decode_start_time)); + frame_stats.decode_start_time, metadata)); } else if (frame_stats.pre_decoded_frame_type == VideoFrameType::kVideoFrameDelta) { stats->recv_delta_frame_size_bytes.AddSample( StatsSample(frame_stats.pre_decoded_image_size.bytes(), - frame_stats.decode_start_time)); + frame_stats.decode_start_time, metadata)); } } if (frame_stats.decode_end_time.IsFinite()) { stats->decode_time_ms.AddSample(StatsSample( frame_stats.decode_end_time - frame_stats.decode_start_time, - frame_stats.decode_end_time)); + frame_stats.decode_end_time, metadata)); } if (frame_stats.prev_frame_rendered_time.IsFinite() && frame_stats.rendered_time.IsFinite()) { TimeDelta time_between_rendered_frames = frame_stats.rendered_time - frame_stats.prev_frame_rendered_time; - stats->time_between_rendered_frames_ms.AddSample( - StatsSample(time_between_rendered_frames, frame_stats.rendered_time)); + stats->time_between_rendered_frames_ms.AddSample(StatsSample( + time_between_rendered_frames, frame_stats.rendered_time, metadata)); TimeDelta average_time_between_rendered_frames = TimeDelta::Millis( stats->time_between_rendered_frames_ms.GetAverage()); if (time_between_rendered_frames > std::max(kFreezeThreshold + average_time_between_rendered_frames, 3 * average_time_between_rendered_frames)) { stats->freeze_time_ms.AddSample(StatsSample( - time_between_rendered_frames, frame_stats.rendered_time)); + time_between_rendered_frames, frame_stats.rendered_time, metadata)); auto freeze_end_it = stream_last_freeze_end_time_.find(comparison.stats_key); RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end()); stats->time_between_freezes_ms.AddSample(StatsSample( frame_stats.prev_frame_rendered_time - freeze_end_it->second, - frame_stats.rendered_time)); + frame_stats.rendered_time, metadata)); freeze_end_it->second = frame_stats.rendered_time; } } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc index bd0f75d0b4..3f9d1a4b5b 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc @@ -11,6 +11,7 @@ #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" #include +#include #include #include "api/test/create_frame_generator.h" @@ -25,8 +26,10 @@ namespace webrtc { namespace { +using ::testing::Contains; using ::testing::Eq; using ::testing::IsEmpty; +using ::testing::Pair; using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; @@ -70,8 +73,9 @@ StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id, Timestamp time) { } FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + uint16_t frame_id, Timestamp captured_time) { - FrameStats frame_stats(captured_time); + FrameStats frame_stats(frame_id, captured_time); frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); frame_stats.received_time = captured_time + TimeDelta::Millis(30); @@ -88,7 +92,7 @@ FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( } FrameStats ShiftStatsOn(const FrameStats& stats, TimeDelta delta) { - FrameStats frame_stats(stats.captured_time + delta); + FrameStats frame_stats(stats.frame_id, stats.captured_time + delta); frame_stats.pre_encode_time = stats.pre_encode_time + delta; frame_stats.encoded_time = stats.encoded_time + delta; frame_stats.received_time = stats.received_time + delta; @@ -105,10 +109,18 @@ FrameStats ShiftStatsOn(const FrameStats& stats, TimeDelta delta) { } double GetFirstOrDie(const SamplesStatsCounter& counter) { - EXPECT_TRUE(!counter.IsEmpty()) << "Counter has to be not empty"; + EXPECT_FALSE(counter.IsEmpty()) << "Counter has to be not empty"; return counter.GetSamples()[0]; } +void AssertFirstMetadataHasField(const SamplesStatsCounter& counter, + const std::string& field_name, + const std::string& field_value) { + EXPECT_FALSE(counter.IsEmpty()) << "Coutner has to be not empty"; + EXPECT_THAT(counter.GetTimedSamples()[0].metadata, + Contains(Pair(field_name, field_value))); +} + std::string ToString(const SamplesStatsCounter& counter) { rtc::StringBuilder out; for (const StatsSample& s : counter.GetTimedSamples()) { @@ -141,8 +153,8 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, size_t peers_count = 2; InternalStatsKey stats_key(stream, sender, receiver); - FrameStats frame_stats = - FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time); + FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/1, stream_start_time); comparator.Start(/*max_threads_count=*/1); comparator.EnsureStatsForStream(stream, sender, peers_count, @@ -165,8 +177,9 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, GetFirstOrDie(stats.at(stats_key).resolution_of_rendered_frame), 100.0); } -TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, - MultiFrameStatsPresentedAfterAddingTwoComparisonWith10msDelay) { +TEST( + DefaultVideoQualityAnalyzerFramesComparatorTest, + MultiFrameStatsPresentedWithMetadataAfterAddingTwoComparisonWith10msDelay) { DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; DefaultVideoQualityAnalyzerFramesComparator comparator( Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); @@ -178,10 +191,10 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, size_t peers_count = 2; InternalStatsKey stats_key(stream, sender, receiver); - FrameStats frame_stats1 = - FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time); + FrameStats frame_stats1 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/1, stream_start_time); FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( - stream_start_time + TimeDelta::Millis(15)); + /*frame_id=*/2, stream_start_time + TimeDelta::Millis(15)); frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time; comparator.Start(/*max_threads_count=*/1); @@ -200,10 +213,11 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, std::map stats = comparator.stream_stats(); EXPECT_DOUBLE_EQ( GetFirstOrDie(stats.at(stats_key).time_between_rendered_frames_ms), 15.0); + AssertFirstMetadataHasField( + stats.at(stats_key).time_between_rendered_frames_ms, "frame_id", "2"); EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(), 2.0 / 15 * 1000) << "There should be 2 events with interval of 15 ms"; - ; } TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, @@ -232,26 +246,30 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, // * 6th - all of them set std::vector stats; // 1st stat - FrameStats frame_stats(stream_start_time); + FrameStats frame_stats(/*frame_id=*/1, stream_start_time); stats.push_back(frame_stats); // 2nd stat frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 2; frame_stats.pre_encode_time = frame_stats.captured_time + TimeDelta::Millis(10); stats.push_back(frame_stats); // 3rd stat frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 3; frame_stats.encoded_time = frame_stats.captured_time + TimeDelta::Millis(20); frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time); stats.push_back(frame_stats); // 4th stat frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 4; frame_stats.received_time = frame_stats.captured_time + TimeDelta::Millis(30); frame_stats.decode_start_time = frame_stats.captured_time + TimeDelta::Millis(40); stats.push_back(frame_stats); // 5th stat frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 5; frame_stats.decode_end_time = frame_stats.captured_time + TimeDelta::Millis(50); frame_stats.used_decoder = @@ -259,6 +277,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, stats.push_back(frame_stats); // 6th stat frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 6; frame_stats.rendered_time = frame_stats.captured_time + TimeDelta::Millis(60); frame_stats.rendered_frame_width = 10; frame_stats.rendered_frame_height = 10; @@ -330,7 +349,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); comparator.Start(/*max_threads_count=*/1); comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, @@ -387,7 +406,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); @@ -447,7 +466,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -515,7 +534,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -583,7 +602,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -656,7 +675,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -734,7 +753,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -814,7 +833,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); comparator.Start(/*max_threads_count=*/1); comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, @@ -871,7 +890,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); @@ -931,7 +950,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -999,7 +1018,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -1073,7 +1092,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -1151,7 +1170,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, InternalStatsKey stats_key(stream, sender, receiver); // Frame captured - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -1233,7 +1252,7 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, // Frame captured VideoFrame frame = CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time); - FrameStats frame_stats(captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); // Frame pre encoded frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); // Frame encoded @@ -1299,7 +1318,131 @@ TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, EXPECT_EQ(stats.decoders, std::vector{*frame_stats.used_decoder}); } + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, AllStatsHaveMetadataSet) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + VideoFrame frame = + CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + // Frame rendered + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + frame_stats.rendered_frame_width = 200; + frame_stats.rendered_frame_height = 100; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/frame, + /*rendered=*/frame, FrameComparisonType::kRegular, + frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + AssertFirstMetadataHasField(stats.psnr, "frame_id", "1"); + AssertFirstMetadataHasField(stats.ssim, "frame_id", "1"); + AssertFirstMetadataHasField(stats.transport_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.total_delay_incl_transport_ms, "frame_id", + "1"); + AssertFirstMetadataHasField(stats.encode_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.decode_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.receive_to_render_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.resolution_of_rendered_frame, "frame_id", + "1"); + AssertFirstMetadataHasField(stats.target_encode_bitrate, "frame_id", "1"); + AssertFirstMetadataHasField(stats.recv_key_frame_size_bytes, "frame_id", "1"); + + expectEmpty(stats.recv_delta_frame_size_bytes); +} // Regular frame end + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + FreezeStatsPresentedWithMetadataAfterAddFrameWithSkippedAndDelay) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + + // Add 5 frames which were rendered with 30 fps (~30ms between frames) + // Frame ids are in [1..5] and last frame is with 120ms offset from first. + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + for (int i = 0; i < 5; ++i) { + FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/i + 1, stream_start_time + TimeDelta::Millis(30 * i)); + frame_stats.prev_frame_rendered_time = prev_frame_rendered_time; + prev_frame_rendered_time = frame_stats.rendered_time; + + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats); + } + + // Next frame was rendered with 4 frames skipped and delay 300ms after last + // frame. + FrameStats freeze_frame_stats = + FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/10, stream_start_time + TimeDelta::Millis(120 + 300)); + freeze_frame_stats.prev_frame_rendered_time = prev_frame_rendered_time; + + comparator.AddComparison(stats_key, + /*skipped_between_rendered=*/4, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, freeze_frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + StreamStats stats = comparator.stream_stats().at(stats_key); + ASSERT_THAT(GetFirstOrDie(stats.skipped_between_rendered), Eq(4)); + AssertFirstMetadataHasField(stats.skipped_between_rendered, "frame_id", "10"); + ASSERT_THAT(GetFirstOrDie(stats.freeze_time_ms), Eq(300)); + AssertFirstMetadataHasField(stats.freeze_time_ms, "frame_id", "10"); + // 180ms is time from the stream start to the rendered time of the last frame + // among first 5 frames which were received before freeze. + ASSERT_THAT(GetFirstOrDie(stats.time_between_freezes_ms), Eq(180)); + AssertFirstMetadataHasField(stats.time_between_freezes_ms, "frame_id", "10"); +} // Stats validation tests end. } // namespace diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h index 033fa7514f..93d45e4e26 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h @@ -42,8 +42,10 @@ bool operator==(const InternalStatsKey& a, const InternalStatsKey& b); // Final stats computed for frame after it went through the whole video // pipeline from capturing to rendering or dropping. struct FrameStats { - explicit FrameStats(Timestamp captured_time) : captured_time(captured_time) {} + FrameStats(uint16_t frame_id, Timestamp captured_time) + : frame_id(frame_id), captured_time(captured_time) {} + uint16_t frame_id; // Frame events timestamp. Timestamp captured_time; Timestamp pre_encode_time = Timestamp::MinusInfinity();