admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[DVQA] Make harmonic fps precomputed

Browse Source 
Also add assertion on it for some DVQA tests

Bug: webrtc:14995, b/271542055
Change-Id: Ie35a85832b6d860885366fb613700bdef3db38f0
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/297820
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39611}
This commit is contained in:
Artem Titov 2023-03-20 18:35:50 +01:00 committed by WebRTC LUCI CQ
parent a2d85e4565
commit 649c8186c7
4 changed files with 33 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -1158,26 +1158,6 @@ void DefaultVideoQualityAnalyzer::ReportResults(
{MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)},
{MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}};
double sum_squared_interframe_delays_secs = 0;
double video_duration_ms = 0;
for (const SamplesStatsCounter::StatsSample& sample :
stats.time_between_rendered_frames_ms.GetTimedSamples()) {
double interframe_delay_ms = sample.value;
const double interframe_delays_secs = interframe_delay_ms / 1000.0;
// Sum of squared inter frame intervals is used to calculate the harmonic
// frame rate metric. The metric aims to reflect overall experience related
// to smoothness of video playback and includes both freezes and pauses.
sum_squared_interframe_delays_secs +=
interframe_delays_secs * interframe_delays_secs;
video_duration_ms += sample.value;
}
double harmonic_framerate_fps = 0;
if (sum_squared_interframe_delays_secs > 0.0) {
harmonic_framerate_fps =
video_duration_ms / 1000.0 / sum_squared_interframe_delays_secs;
}
metrics_logger_->LogMetric(
"psnr_dB", test_case_name, stats.psnr, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter, metric_metadata);
@ -1197,7 +1177,7 @@ void DefaultVideoQualityAnalyzer::ReportResults(
stats.time_between_rendered_frames_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"harmonic_framerate", test_case_name, harmonic_framerate_fps,
"harmonic_framerate", test_case_name, stats.harmonic_framerate_fps,
Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"encode_frame_rate", test_case_name,

29
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
View File

@ -37,6 +37,7 @@ using ::webrtc::webrtc_pc_e2e::SampleMetadataKey;
constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150);
constexpr int kMaxActiveComparisons = 10;
constexpr int kMillisInSecond = 1000;
SamplesStatsCounter::StatsSample StatsSample(
double value,
@ -235,13 +236,35 @@ void DefaultVideoQualityAnalyzerFramesComparator::Stop(
Now(), /*metadata=*/{}));
}
// Freeze Time:
// If there were no freezes on a video stream, add only one sample with
// value 0 (0ms freezes time).
for (auto& [key, stream_stats] : stream_stats_) {
// Freeze Time:
// If there were no freezes on a video stream, add only one sample with
// value 0 (0ms freezes time).
if (stream_stats.freeze_time_ms.IsEmpty()) {
stream_stats.freeze_time_ms.AddSample(0);
}
// Harmonic framerate (fps):
// sum of interframe delays / squared sum of interframe delays.
// The metric aims to reflect overall experience related to smoothness of
// video playback and includes both freezes and pauses.
double sum_squared_interframe_delays_secs = 0;
double sum_interframe_delays_ms = 0;
for (const SamplesStatsCounter::StatsSample& sample :
stream_stats.time_between_rendered_frames_ms.GetTimedSamples()) {
double interframe_delay_ms = sample.value;
const double interframe_delays_secs =
interframe_delay_ms / static_cast<double>(kMillisInSecond);
sum_squared_interframe_delays_secs +=
interframe_delays_secs * interframe_delays_secs;
sum_interframe_delays_ms += interframe_delay_ms;
}
if (sum_squared_interframe_delays_secs > 0.0) {
stream_stats.harmonic_framerate_fps =
sum_interframe_delays_ms / static_cast<double>(kMillisInSecond) /
sum_squared_interframe_delays_secs;
}
}
}
}

2
test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
View File

@ -164,6 +164,8 @@ struct StreamStats {
std::vector<StreamCodecInfo> encoders;
// Vectors of decoders used for this stream by receiving client.
std::vector<StreamCodecInfo> decoders;
double harmonic_framerate_fps = 0;
};
struct AnalyzerStats {

8
test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
View File

@ -2147,7 +2147,7 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
ExpectRateIs(bob_stream_stats.capture_frame_rate, 13.559322);
// TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0)
ExpectRateIs(bob_stream_stats.encode_frame_rate, 13.559322);
// TODO(bugs.webrtc.org/14995): Assert on harmonic fps
EXPECT_DOUBLE_EQ(bob_stream_stats.harmonic_framerate_fps, 20);
StreamStats charlie_stream_stats =
streams_stats.at(StatsKey("alice_video", "charlie"));
@ -2159,7 +2159,7 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
ElementsAre(1050.0));
EXPECT_THAT(GetTimeSortedValues(charlie_stream_stats.time_between_freezes_ms),
ElementsAre(950.0, 950.0));
// TODO(bugs.webrtc.org/14995): Assert on harmonic fps
EXPECT_NEAR(charlie_stream_stats.harmonic_framerate_fps, 2.463465, 1e-6);
}
TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
@ -2217,7 +2217,7 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
ElementsAre(950.0));
ExpectRateIs(bob_stream_stats.capture_frame_rate, 21.052631);
ExpectRateIs(bob_stream_stats.encode_frame_rate, 21.052631);
// TODO(bugs.webrtc.org/14995): Assert on harmonic fps
EXPECT_DOUBLE_EQ(bob_stream_stats.harmonic_framerate_fps, 20);
StreamStats charlie_stream_stats =
streams_stats.at(StatsKey("alice_video", "charlie"));
@ -2229,7 +2229,7 @@ TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
ElementsAre(1050.0));
EXPECT_THAT(GetTimeSortedValues(charlie_stream_stats.time_between_freezes_ms),
ElementsAre(950.0, 950.0));
// TODO(bugs.webrtc.org/14995): Assert on harmonic fps
EXPECT_NEAR(charlie_stream_stats.harmonic_framerate_fps, 2.463465, 1e-6);
}
TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,