From 182044184e9b5a17ecb994c2bb1860fdf4eb48e0 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Sun, 25 Sep 2022 01:47:04 +0200 Subject: [PATCH] Migrate Call-level tests on SamplesStatsCounter and new perf metrics API Bug: b/246095034 Change-Id: I86ff4fb8dffa6a888409f69a590fd4aa156b738b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276623 Reviewed-by: Mirko Bonadei Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Artem Titov Cr-Commit-Position: refs/heads/main@{#38200} --- video/BUILD.gn | 4 +- video/video_analyzer.cc | 182 +++++++++++++++++++++------------------- video/video_analyzer.h | 66 +++++++-------- 3 files changed, 134 insertions(+), 118 deletions(-) diff --git a/video/BUILD.gn b/video/BUILD.gn index 2c8e26844c..76367fd53e 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -483,6 +483,8 @@ if (rtc_include_tests) { "../api/rtc_event_log:rtc_event_log_factory", "../api/task_queue", "../api/task_queue:default_task_queue_factory", + "../api/test/metrics:global_metrics_logger_and_exporter", + "../api/test/metrics:metric", "../api/video:builtin_video_bitrate_allocator_factory", "../api/video:video_bitrate_allocator_factory", "../api/video:video_frame", @@ -520,7 +522,6 @@ if (rtc_include_tests) { "../system_wrappers", "../test:fake_video_codecs", "../test:fileutils", - "../test:perf_test", "../test:platform_video_capturer", "../test:rtp_test_utils", "../test:test_common", @@ -534,6 +535,7 @@ if (rtc_include_tests) { "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/flags:flag", "//third_party/abseil-cpp/absl/flags:parse", + "//third_party/abseil-cpp/absl/strings", ] if (is_mac || is_ios) { diff --git a/video/video_analyzer.cc b/video/video_analyzer.cc index 66b5ecba7f..3077a77b2c 100644 --- a/video/video_analyzer.cc +++ b/video/video_analyzer.cc @@ -17,6 +17,9 @@ #include "absl/algorithm/container.h" #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/strings/string_view.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/metrics/metric.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h" #include "modules/rtp_rtcp/source/rtp_packet.h" @@ -30,7 +33,6 @@ #include "test/call_test.h" #include "test/testsupport/file_utils.h" #include "test/testsupport/frame_writer.h" -#include "test/testsupport/perf_test.h" #include "test/testsupport/test_artifacts.h" ABSL_FLAG(bool, @@ -41,6 +43,12 @@ ABSL_FLAG(bool, namespace webrtc { namespace { + +using ::webrtc::test::GetGlobalMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Metric; +using ::webrtc::test::Unit; + constexpr TimeDelta kSendStatsPollingInterval = TimeDelta::Seconds(1); constexpr size_t kMaxComparisons = 10; // How often is keep alive message printed. @@ -53,6 +61,7 @@ constexpr int kKeepAliveIntervalIterations = bool IsFlexfec(int payload_type) { return payload_type == test::CallTest::kFlexfecPayloadType; } + } // namespace VideoAnalyzer::VideoAnalyzer(test::LayerFilteringTransport* transport, @@ -609,8 +618,6 @@ bool VideoAnalyzer::FrameProcessed() { } void VideoAnalyzer::PrintResults() { - using ::webrtc::test::ImproveDirection; - StopMeasuringCpuProcessTime(); int dropped_frames_diff; { @@ -619,36 +626,39 @@ void VideoAnalyzer::PrintResults() { dropped_frames_before_rendering_ + frames_.size(); } MutexLock lock(&comparison_lock_); - PrintResult("psnr", psnr_, "dB", ImproveDirection::kBiggerIsBetter); - PrintResult("ssim", ssim_, "unitless", ImproveDirection::kBiggerIsBetter); - PrintResult("sender_time", sender_time_, "ms", - ImproveDirection::kSmallerIsBetter); - PrintResult("receiver_time", receiver_time_, "ms", - ImproveDirection::kSmallerIsBetter); - PrintResult("network_time", network_time_, "ms", - ImproveDirection::kSmallerIsBetter); - PrintResult("total_delay_incl_network", end_to_end_, "ms", - ImproveDirection::kSmallerIsBetter); - PrintResult("time_between_rendered_frames", rendered_delta_, "ms", - ImproveDirection::kSmallerIsBetter); - PrintResult("encode_frame_rate", encode_frame_rate_, "fps", - ImproveDirection::kBiggerIsBetter); - PrintResult("encode_time", encode_time_ms_, "ms", - ImproveDirection::kSmallerIsBetter); - PrintResult("media_bitrate", media_bitrate_bps_, "bps", - ImproveDirection::kNone); - PrintResult("fec_bitrate", fec_bitrate_bps_, "bps", ImproveDirection::kNone); - PrintResult("send_bandwidth", send_bandwidth_bps_, "bps", - ImproveDirection::kNone); - PrintResult("pixels_per_frame", pixels_, "count", - ImproveDirection::kBiggerIsBetter); + PrintResult("psnr_dB", psnr_, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + PrintResult("ssim", ssim_, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + PrintResult("sender_time", sender_time_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + PrintResult("receiver_time", receiver_time_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + PrintResult("network_time", network_time_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + PrintResult("total_delay_incl_network", end_to_end_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + PrintResult("time_between_rendered_frames", rendered_delta_, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + PrintResult("encode_frame_rate_fps", encode_frame_rate_, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter); + PrintResult("encode_time", encode_time_ms_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + PrintResult("media_bitrate", media_bitrate_bps_ / 1000.0, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + PrintResult("fec_bitrate", fec_bitrate_bps_ / 1000.0, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + PrintResult("send_bandwidth", send_bandwidth_bps_ / 1000.0, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + PrintResult("pixels_per_frame", pixels_, Unit::kCount, + ImprovementDirection::kBiggerIsBetter); - test::PrintResult("decode_frame_rate", "", test_label_.c_str(), - decode_frame_rate_, "fps", false, - ImproveDirection::kBiggerIsBetter); - test::PrintResult("render_frame_rate", "", test_label_.c_str(), - render_frame_rate_, "fps", false, - ImproveDirection::kBiggerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "decode_frame_rate_fps", test_label_, decode_frame_rate_, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "render_frame_rate_fps", test_label_, render_frame_rate_, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter); // Record the time from the last freeze until the last rendered frame to // ensure we cover the full timespan of the session. Otherwise the metric @@ -656,8 +666,8 @@ void VideoAnalyzer::PrintResults() { time_between_freezes_.AddSample(last_render_time_ - last_unfreeze_time_ms_); // Freeze metrics. - PrintResult("time_between_freezes", time_between_freezes_, "ms", - ImproveDirection::kBiggerIsBetter); + PrintResult("time_between_freezes", time_between_freezes_, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter); const double freeze_count_double = static_cast(freeze_count_); const double total_freezes_duration_ms_double = @@ -666,10 +676,10 @@ void VideoAnalyzer::PrintResults() { static_cast(total_frames_duration_ms_); if (total_frames_duration_ms_double > 0) { - test::PrintResult( - "freeze_duration_ratio", "", test_label_.c_str(), + GetGlobalMetricsLogger()->LogSingleValueMetric( + "freeze_duration_ratio", test_label_, total_freezes_duration_ms_double / total_frames_duration_ms_double, - "unitless", false, ImproveDirection::kSmallerIsBetter); + Unit::kUnitless, ImprovementDirection::kSmallerIsBetter); RTC_DCHECK_LE(total_freezes_duration_ms_double, total_frames_duration_ms_double); @@ -677,47 +687,52 @@ void VideoAnalyzer::PrintResults() { const double total_frames_duration_min = total_frames_duration_ms_double / ms_per_minute; if (total_frames_duration_min > 0) { - test::PrintResult("freeze_count_per_minute", "", test_label_.c_str(), - freeze_count_double / total_frames_duration_min, - "unitless", false, ImproveDirection::kSmallerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "freeze_count_per_minute", test_label_, + freeze_count_double / total_frames_duration_min, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter); } } - test::PrintResult("freeze_duration_average", "", test_label_.c_str(), - freeze_count_double > 0 - ? total_freezes_duration_ms_double / freeze_count_double - : 0, - "ms", false, ImproveDirection::kSmallerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "freeze_duration_average", test_label_, + freeze_count_double > 0 + ? total_freezes_duration_ms_double / freeze_count_double + : 0, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); if (1000 * sum_squared_frame_durations_ > 0) { - test::PrintResult( - "harmonic_frame_rate", "", test_label_.c_str(), + GetGlobalMetricsLogger()->LogSingleValueMetric( + "harmonic_frame_rate_fps", test_label_, total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_), - "fps", false, ImproveDirection::kBiggerIsBetter); + Unit::kHertz, ImprovementDirection::kBiggerIsBetter); } if (worst_frame_) { - test::PrintResult("min_psnr", "", test_label_.c_str(), worst_frame_->psnr, - "dB", false, ImproveDirection::kBiggerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "min_psnr_dB", test_label_, worst_frame_->psnr, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); } if (receive_stream_ != nullptr) { PrintResultWithExternalMean("decode_time", mean_decode_time_ms_, - decode_time_ms_, "ms", - ImproveDirection::kSmallerIsBetter); + decode_time_ms_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); } dropped_frames_ += dropped_frames_diff; - test::PrintResult("dropped_frames", "", test_label_.c_str(), dropped_frames_, - "count", false, ImproveDirection::kSmallerIsBetter); - test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(), - "%", false, ImproveDirection::kSmallerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "dropped_frames", test_label_, dropped_frames_, Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter); #if defined(WEBRTC_WIN) // On Linux and Mac in Resident Set some unused pages may be counted. // Therefore this metric will depend on order in which tests are run and // will be flaky. - PrintResult("memory_usage", memory_usage_, "sizeInBytes", - ImproveDirection::kSmallerIsBetter); + PrintResult("memory_usage", memory_usage_, Unit::kBytes, + ImprovementDirection::kSmallerIsBetter); #endif // Saving only the worst frame for manual analysis. Intention here is to @@ -735,19 +750,19 @@ void VideoAnalyzer::PrintResults() { } if (audio_receive_stream_ != nullptr) { - PrintResult("audio_expand_rate", audio_expand_rate_, "unitless", - ImproveDirection::kSmallerIsBetter); - PrintResult("audio_accelerate_rate", audio_accelerate_rate_, "unitless", - ImproveDirection::kSmallerIsBetter); - PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, "ms", - ImproveDirection::kNone); + PrintResult("audio_expand_rate", audio_expand_rate_, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter); + PrintResult("audio_accelerate_rate", audio_accelerate_rate_, + Unit::kUnitless, ImprovementDirection::kSmallerIsBetter); + PrintResult("audio_jitter_buffer", audio_jitter_buffer_ms_, + Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter); } // Disable quality check for quick test, as quality checks may fail // because too few samples were collected. if (!is_quick_test_enabled_) { - EXPECT_GT(*psnr_.GetMean(), avg_psnr_threshold_); - EXPECT_GT(*ssim_.GetMean(), avg_ssim_threshold_); + EXPECT_GT(psnr_.GetAverage(), avg_psnr_threshold_); + EXPECT_GT(ssim_.GetAverage(), avg_ssim_threshold_); } } @@ -820,32 +835,31 @@ void VideoAnalyzer::PerformFrameComparison( encoded_frame_size_.AddSample(comparison.encoded_frame_size); } -void VideoAnalyzer::PrintResult( - const char* result_type, - Statistics stats, - const char* unit, - webrtc::test::ImproveDirection improve_direction) { - test::PrintResultMeanAndError( - result_type, "", test_label_.c_str(), stats.GetMean().value_or(0), - stats.GetStandardDeviation().value_or(0), unit, false, improve_direction); +void VideoAnalyzer::PrintResult(absl::string_view result_type, + const SamplesStatsCounter& stats, + Unit unit, + ImprovementDirection improvement_direction) { + GetGlobalMetricsLogger()->LogMetric(result_type, test_label_, stats, unit, + improvement_direction); } void VideoAnalyzer::PrintResultWithExternalMean( - const char* result_type, + absl::string_view result_type, double mean, - Statistics stats, - const char* unit, - webrtc::test::ImproveDirection improve_direction) { + const SamplesStatsCounter& stats, + Unit unit, + ImprovementDirection improvement_direction) { // If the true mean is different than the sample mean, the sample variance is // too low. The sample variance given a known mean is obtained by adding the // squared error between the true mean and the sample mean. double compensated_variance = - stats.Size() > 0 - ? *stats.GetVariance() + pow(mean - *stats.GetMean(), 2.0) - : 0.0; - test::PrintResultMeanAndError(result_type, "", test_label_.c_str(), mean, - std::sqrt(compensated_variance), unit, false, - improve_direction); + stats.IsEmpty() + ? 0.0 + : stats.GetVariance() + pow(mean - stats.GetAverage(), 2.0); + GetGlobalMetricsLogger()->LogMetric( + result_type, test_label_, + Metric::Stats{.mean = mean, .stddev = std::sqrt(compensated_variance)}, + unit, improvement_direction); } void VideoAnalyzer::PrintSamplesToFile() { diff --git a/video/video_analyzer.h b/video/video_analyzer.h index 725dacfdd8..2cee5e1b92 100644 --- a/video/video_analyzer.h +++ b/video/video_analyzer.h @@ -16,7 +16,10 @@ #include #include +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" #include "api/task_queue/task_queue_base.h" +#include "api/test/metrics/metric.h" #include "api/video/video_source_interface.h" #include "modules/rtp_rtcp/source/rtp_packet.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" @@ -27,7 +30,6 @@ #include "rtc_base/time_utils.h" #include "test/layer_filtering_transport.h" #include "test/rtp_file_writer.h" -#include "test/testsupport/perf_test.h" namespace webrtc { @@ -35,8 +37,6 @@ class VideoAnalyzer : public PacketReceiver, public Transport, public rtc::VideoSinkInterface { public: - using Statistics = webrtc_impl::RunningStatistics; - VideoAnalyzer(test::LayerFilteringTransport* transport, const std::string& test_label, double avg_psnr_threshold, @@ -205,16 +205,16 @@ class VideoAnalyzer : public PacketReceiver, void PrintResults() RTC_LOCKS_EXCLUDED(lock_, comparison_lock_); void PerformFrameComparison(const FrameComparison& comparison) RTC_LOCKS_EXCLUDED(comparison_lock_); - void PrintResult(const char* result_type, - Statistics stats, - const char* unit, - webrtc::test::ImproveDirection improve_direction); + void PrintResult(absl::string_view result_type, + const SamplesStatsCounter& stats, + webrtc::test::Unit unit, + webrtc::test::ImprovementDirection improvement_direction); void PrintResultWithExternalMean( - const char* result_type, + absl::string_view result_type, double mean, - Statistics stats, - const char* unit, - webrtc::test::ImproveDirection improve_direction); + const SamplesStatsCounter& stats, + webrtc::test::Unit unit, + webrtc::test::ImprovementDirection improvement_direction); void PrintSamplesToFile(void) RTC_LOCKS_EXCLUDED(comparison_lock_); void AddCapturedFrameForComparison(const VideoFrame& video_frame) RTC_LOCKS_EXCLUDED(lock_, comparison_lock_); @@ -235,32 +235,32 @@ class VideoAnalyzer : public PacketReceiver, Mutex comparison_lock_; std::vector samples_ RTC_GUARDED_BY(comparison_lock_); - Statistics sender_time_ RTC_GUARDED_BY(comparison_lock_); - Statistics receiver_time_ RTC_GUARDED_BY(comparison_lock_); - Statistics network_time_ RTC_GUARDED_BY(comparison_lock_); - Statistics psnr_ RTC_GUARDED_BY(comparison_lock_); - Statistics ssim_ RTC_GUARDED_BY(comparison_lock_); - Statistics end_to_end_ RTC_GUARDED_BY(comparison_lock_); - Statistics rendered_delta_ RTC_GUARDED_BY(comparison_lock_); - Statistics encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_); - Statistics encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_); - Statistics encode_time_ms_ RTC_GUARDED_BY(comparison_lock_); - Statistics encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter sender_time_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter receiver_time_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter network_time_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter psnr_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter ssim_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter end_to_end_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter rendered_delta_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter encode_time_ms_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_); double mean_decode_time_ms_ RTC_GUARDED_BY(comparison_lock_); - Statistics decode_time_ms_ RTC_GUARDED_BY(comparison_lock_); - Statistics decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_); - Statistics media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); - Statistics fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); - Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_); - Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_); - Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_); - Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_); - Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_); - Statistics pixels_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter decode_time_ms_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter memory_usage_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter pixels_ RTC_GUARDED_BY(comparison_lock_); // Rendered frame with worst PSNR is saved for further analysis. absl::optional worst_frame_ RTC_GUARDED_BY(comparison_lock_); // Freeze metrics. - Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_); + SamplesStatsCounter time_between_freezes_ RTC_GUARDED_BY(comparison_lock_); uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_); uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_); uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_);