From 649c8186c7e7be64261e64e630d80986a8112794 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Mon, 20 Mar 2023 18:35:50 +0100 Subject: [PATCH] [DVQA] Make harmonic fps precomputed Also add assertion on it for some DVQA tests Bug: webrtc:14995, b/271542055 Change-Id: Ie35a85832b6d860885366fb613700bdef3db38f0 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/297820 Commit-Queue: Artem Titov Reviewed-by: Mirko Bonadei Cr-Commit-Position: refs/heads/main@{#39611} --- .../video/default_video_quality_analyzer.cc | 22 +------------- ...ideo_quality_analyzer_frames_comparator.cc | 29 +++++++++++++++++-- ...lt_video_quality_analyzer_shared_objects.h | 2 ++ .../default_video_quality_analyzer_test.cc | 8 ++--- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 96269a6f75..1b9642381e 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -1158,26 +1158,6 @@ void DefaultVideoQualityAnalyzer::ReportResults( {MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)}, {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}}; - double sum_squared_interframe_delays_secs = 0; - double video_duration_ms = 0; - for (const SamplesStatsCounter::StatsSample& sample : - stats.time_between_rendered_frames_ms.GetTimedSamples()) { - double interframe_delay_ms = sample.value; - const double interframe_delays_secs = interframe_delay_ms / 1000.0; - // Sum of squared inter frame intervals is used to calculate the harmonic - // frame rate metric. The metric aims to reflect overall experience related - // to smoothness of video playback and includes both freezes and pauses. - sum_squared_interframe_delays_secs += - interframe_delays_secs * interframe_delays_secs; - - video_duration_ms += sample.value; - } - double harmonic_framerate_fps = 0; - if (sum_squared_interframe_delays_secs > 0.0) { - harmonic_framerate_fps = - video_duration_ms / 1000.0 / sum_squared_interframe_delays_secs; - } - metrics_logger_->LogMetric( "psnr_dB", test_case_name, stats.psnr, Unit::kUnitless, ImprovementDirection::kBiggerIsBetter, metric_metadata); @@ -1197,7 +1177,7 @@ void DefaultVideoQualityAnalyzer::ReportResults( stats.time_between_rendered_frames_ms, Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( - "harmonic_framerate", test_case_name, harmonic_framerate_fps, + "harmonic_framerate", test_case_name, stats.harmonic_framerate_fps, Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( "encode_frame_rate", test_case_name, diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc index fc91dd107d..7ea2a2a90c 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -37,6 +37,7 @@ using ::webrtc::webrtc_pc_e2e::SampleMetadataKey; constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150); constexpr int kMaxActiveComparisons = 10; +constexpr int kMillisInSecond = 1000; SamplesStatsCounter::StatsSample StatsSample( double value, @@ -235,13 +236,35 @@ void DefaultVideoQualityAnalyzerFramesComparator::Stop( Now(), /*metadata=*/{})); } - // Freeze Time: - // If there were no freezes on a video stream, add only one sample with - // value 0 (0ms freezes time). for (auto& [key, stream_stats] : stream_stats_) { + // Freeze Time: + // If there were no freezes on a video stream, add only one sample with + // value 0 (0ms freezes time). if (stream_stats.freeze_time_ms.IsEmpty()) { stream_stats.freeze_time_ms.AddSample(0); } + + // Harmonic framerate (fps): + // sum of interframe delays / squared sum of interframe delays. + // The metric aims to reflect overall experience related to smoothness of + // video playback and includes both freezes and pauses. + double sum_squared_interframe_delays_secs = 0; + double sum_interframe_delays_ms = 0; + for (const SamplesStatsCounter::StatsSample& sample : + stream_stats.time_between_rendered_frames_ms.GetTimedSamples()) { + double interframe_delay_ms = sample.value; + const double interframe_delays_secs = + interframe_delay_ms / static_cast(kMillisInSecond); + sum_squared_interframe_delays_secs += + interframe_delays_secs * interframe_delays_secs; + + sum_interframe_delays_ms += interframe_delay_ms; + } + if (sum_squared_interframe_delays_secs > 0.0) { + stream_stats.harmonic_framerate_fps = + sum_interframe_delays_ms / static_cast(kMillisInSecond) / + sum_squared_interframe_delays_secs; + } } } } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h index 17e7e0ea2e..e8ded4f3d8 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -164,6 +164,8 @@ struct StreamStats { std::vector encoders; // Vectors of decoders used for this stream by receiving client. std::vector decoders; + + double harmonic_framerate_fps = 0; }; struct AnalyzerStats { diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc index 534e56c3bf..4222fcc1ca 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -2147,7 +2147,7 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest, ExpectRateIs(bob_stream_stats.capture_frame_rate, 13.559322); // TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0) ExpectRateIs(bob_stream_stats.encode_frame_rate, 13.559322); - // TODO(bugs.webrtc.org/14995): Assert on harmonic fps + EXPECT_DOUBLE_EQ(bob_stream_stats.harmonic_framerate_fps, 20); StreamStats charlie_stream_stats = streams_stats.at(StatsKey("alice_video", "charlie")); @@ -2159,7 +2159,7 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest, ElementsAre(1050.0)); EXPECT_THAT(GetTimeSortedValues(charlie_stream_stats.time_between_freezes_ms), ElementsAre(950.0, 950.0)); - // TODO(bugs.webrtc.org/14995): Assert on harmonic fps + EXPECT_NEAR(charlie_stream_stats.harmonic_framerate_fps, 2.463465, 1e-6); } TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest, @@ -2217,7 +2217,7 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest, ElementsAre(950.0)); ExpectRateIs(bob_stream_stats.capture_frame_rate, 21.052631); ExpectRateIs(bob_stream_stats.encode_frame_rate, 21.052631); - // TODO(bugs.webrtc.org/14995): Assert on harmonic fps + EXPECT_DOUBLE_EQ(bob_stream_stats.harmonic_framerate_fps, 20); StreamStats charlie_stream_stats = streams_stats.at(StatsKey("alice_video", "charlie")); @@ -2229,7 +2229,7 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest, ElementsAre(1050.0)); EXPECT_THAT(GetTimeSortedValues(charlie_stream_stats.time_between_freezes_ms), ElementsAre(950.0, 950.0)); - // TODO(bugs.webrtc.org/14995): Assert on harmonic fps + EXPECT_NEAR(charlie_stream_stats.harmonic_framerate_fps, 2.463465, 1e-6); } TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,