diff --git a/audio/BUILD.gn b/audio/BUILD.gn index 4a904aaf28..bb31ca0e83 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -224,6 +224,7 @@ if (rtc_include_tests) { "../api:peer_connection_quality_test_fixture_api", "../api:simulated_network_api", "../api:time_controller", + "../api/test/metrics:global_metrics_logger_and_exporter", "../call:simulated_network", "../common_audio", "../system_wrappers", diff --git a/audio/test/pc_low_bandwidth_audio_test.cc b/audio/test/pc_low_bandwidth_audio_test.cc index 0364670b91..e05116ea67 100644 --- a/audio/test/pc_low_bandwidth_audio_test.cc +++ b/audio/test/pc_low_bandwidth_audio_test.cc @@ -15,6 +15,7 @@ #include "absl/strings/string_view.h" #include "api/test/create_network_emulation_manager.h" #include "api/test/create_peerconnection_quality_test_fixture.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/test/network_emulation_manager.h" #include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/simulated_network.h" @@ -88,7 +89,8 @@ CreateTestFixture(absl::string_view test_case_name, bob_configurer); fixture->AddQualityMetricsReporter( std::make_unique( - network_links.first, network_links.second)); + network_links.first, network_links.second, + test::GetGlobalMetricsLogger())); return fixture; } diff --git a/pc/BUILD.gn b/pc/BUILD.gn index 9d3125828b..2da600e974 100644 --- a/pc/BUILD.gn +++ b/pc/BUILD.gn @@ -2708,6 +2708,7 @@ if (rtc_include_tests && !build_with_chromium) { "../api:peer_connection_quality_test_fixture_api", "../api:simulated_network_api", "../api:time_controller", + "../api/test/metrics:global_metrics_logger_and_exporter", "../api/video_codecs:video_codecs_api", "../call:simulated_network", "../modules/video_coding:webrtc_vp9", diff --git a/pc/test/svc_e2e_tests.cc b/pc/test/svc_e2e_tests.cc index ad70bf7889..de0e050b68 100644 --- a/pc/test/svc_e2e_tests.cc +++ b/pc/test/svc_e2e_tests.cc @@ -17,6 +17,7 @@ #include "api/test/create_peer_connection_quality_test_frame_generator.h" #include "api/test/create_peerconnection_quality_test_fixture.h" #include "api/test/frame_generator_interface.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/test/network_emulation_manager.h" #include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/simulated_network.h" @@ -185,6 +186,7 @@ class SvcVideoQualityAnalyzer : public DefaultVideoQualityAnalyzer { explicit SvcVideoQualityAnalyzer(webrtc::Clock* clock) : DefaultVideoQualityAnalyzer(clock, + test::GetGlobalMetricsLogger(), DefaultVideoQualityAnalyzerOptions{ .compute_psnr = false, .compute_ssim = false, diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index c725d035fb..2157d5c7b0 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -581,6 +581,7 @@ if (!build_with_chromium) { "../..:test_support", "../../../api:create_frame_generator", "../../../api:rtp_packet_info", + "../../../api/test/metrics:global_metrics_logger_and_exporter", "../../../api/video:encoded_image", "../../../api/video:video_frame", "../../../common_video", @@ -700,6 +701,7 @@ if (!build_with_chromium) { "../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/units:time_delta", "../../../api/units:timestamp", + "../../../rtc_base:checks", "../../../rtc_base:criticalsection", "../../../rtc_base:logging", "../../../rtc_base:rtc_numerics", @@ -749,6 +751,7 @@ if (!build_with_chromium) { "../../../api/units:data_size", "../../../api/units:time_delta", "../../../api/units:timestamp", + "../../../rtc_base:checks", "../../../rtc_base:criticalsection", "../../../rtc_base:rtc_numerics", "../../../rtc_base/synchronization:mutex", @@ -887,6 +890,7 @@ if (!build_with_chromium) { "../../../api/test/metrics:metrics_logger", "../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/units:data_size", + "../../../rtc_base:checks", "../../../rtc_base:criticalsection", "../../../rtc_base:rtc_event", "../../../rtc_base/synchronization:mutex", @@ -918,6 +922,7 @@ if (!build_with_chromium) { "../../../api/units:data_size", "../../../api/units:timestamp", "../../../rtc_base", + "../../../rtc_base:checks", "../../../rtc_base:ip_address", "../../../rtc_base:rtc_event", "../../../rtc_base:stringutils", @@ -945,6 +950,7 @@ if (!build_with_chromium) { "../../../api/test/metrics:metrics_logger", "../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/units:timestamp", + "../../../rtc_base:checks", "../../../rtc_base:criticalsection", "../../../rtc_base:rtc_event", "../../../rtc_base:rtc_numerics", diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc index 83ae8f2655..4a0061ed13 100644 --- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc @@ -13,6 +13,7 @@ #include "api/stats/rtc_stats.h" #include "api/stats/rtcstats_objects.h" #include "api/test/metrics/metric.h" +#include "rtc_base/checks.h" #include "rtc_base/logging.h" namespace webrtc { @@ -21,6 +22,12 @@ namespace webrtc_pc_e2e { using ::webrtc::test::ImprovementDirection; using ::webrtc::test::Unit; +DefaultAudioQualityAnalyzer::DefaultAudioQualityAnalyzer( + test::MetricsLogger* const metrics_logger) + : metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + void DefaultAudioQualityAnalyzer::Start(std::string test_case_name, TrackIdStreamInfoMap* analyzer_helper) { test_case_name_ = std::move(test_case_name); @@ -116,48 +123,29 @@ std::string DefaultAudioQualityAnalyzer::GetTestCaseName( } void DefaultAudioQualityAnalyzer::Stop() { - using ::webrtc::test::ImproveDirection; MutexLock lock(&lock_); for (auto& item : streams_stats_) { - if (metrics_logger_ == nullptr) { - ReportResult("expand_rate", item.first, item.second.expand_rate, - "unitless", ImproveDirection::kSmallerIsBetter); - ReportResult("accelerate_rate", item.first, item.second.accelerate_rate, - "unitless", ImproveDirection::kSmallerIsBetter); - ReportResult("preemptive_rate", item.first, item.second.preemptive_rate, - "unitless", ImproveDirection::kSmallerIsBetter); - ReportResult("speech_expand_rate", item.first, - item.second.speech_expand_rate, "unitless", - ImproveDirection::kSmallerIsBetter); - ReportResult("average_jitter_buffer_delay_ms", item.first, - item.second.average_jitter_buffer_delay_ms, "ms", - ImproveDirection::kNone); - ReportResult("preferred_buffer_size_ms", item.first, - item.second.preferred_buffer_size_ms, "ms", - ImproveDirection::kNone); - } else { - metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first), - item.second.expand_rate, Unit::kUnitless, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first), - item.second.accelerate_rate, Unit::kUnitless, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first), - item.second.preemptive_rate, Unit::kUnitless, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric( - "speech_expand_rate", GetTestCaseName(item.first), - item.second.speech_expand_rate, Unit::kUnitless, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric( - "average_jitter_buffer_delay_ms", GetTestCaseName(item.first), - item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogMetric( - "preferred_buffer_size_ms", GetTestCaseName(item.first), - item.second.preferred_buffer_size_ms, Unit::kMilliseconds, - ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first), + item.second.expand_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first), + item.second.accelerate_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first), + item.second.preemptive_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("speech_expand_rate", + GetTestCaseName(item.first), + item.second.speech_expand_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "average_jitter_buffer_delay_ms", GetTestCaseName(item.first), + item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogMetric( + "preferred_buffer_size_ms", GetTestCaseName(item.first), + item.second.preferred_buffer_size_ms, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter); } } @@ -167,18 +155,5 @@ DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const { return streams_stats_; } -void DefaultAudioQualityAnalyzer::ReportResult( - const std::string& metric_name, - const std::string& stream_label, - const SamplesStatsCounter& counter, - const std::string& unit, - webrtc::test::ImproveDirection improve_direction) const { - test::PrintResultMeanAndError( - metric_name, /*modifier=*/"", GetTestCaseName(stream_label), - counter.IsEmpty() ? 0 : counter.GetAverage(), - counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit, - /*important=*/false, improve_direction); -} - } // namespace webrtc_pc_e2e } // namespace webrtc diff --git a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h index c08e740628..f4084468d7 100644 --- a/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h +++ b/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h @@ -21,7 +21,6 @@ #include "api/test/track_id_stream_info_map.h" #include "api/units/time_delta.h" #include "rtc_base/synchronization/mutex.h" -#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -37,11 +36,8 @@ struct AudioStreamStats { class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { public: - DefaultAudioQualityAnalyzer() - : DefaultAudioQualityAnalyzer(/*metrics_logger=*/nullptr) {} explicit DefaultAudioQualityAnalyzer( - test::MetricsLogger* const metrics_logger) - : metrics_logger_(metrics_logger) {} + test::MetricsLogger* const metrics_logger); void Start(std::string test_case_name, TrackIdStreamInfoMap* analyzer_helper) override; @@ -66,11 +62,6 @@ class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { }; std::string GetTestCaseName(const std::string& stream_label) const; - void ReportResult(const std::string& metric_name, - const std::string& stream_label, - const SamplesStatsCounter& counter, - const std::string& unit, - webrtc::test::ImproveDirection improve_direction) const; test::MetricsLogger* const metrics_logger_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 95a73b35ec..123071bb7e 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -23,6 +23,7 @@ #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" #include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/platform_thread.h" #include "rtc_base/strings/string_builder.h" @@ -124,13 +125,6 @@ SamplesStatsCounter::StatsSample StatsSample(double value, } // namespace -DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( - webrtc::Clock* clock, - DefaultVideoQualityAnalyzerOptions options) - : DefaultVideoQualityAnalyzer(clock, - /*metrics_logger=*/nullptr, - std::move(options)) {} - DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( webrtc::Clock* clock, test::MetricsLogger* metrics_logger, @@ -138,7 +132,9 @@ DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( : options_(options), clock_(clock), metrics_logger_(metrics_logger), - frames_comparator_(clock, cpu_measurer_, options) {} + frames_comparator_(clock, cpu_measurer_, options) { + RTC_CHECK(metrics_logger_); +} DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { Stop(); @@ -891,22 +887,14 @@ void DefaultVideoQualityAnalyzer:: } void DefaultVideoQualityAnalyzer::ReportResults() { - using ::webrtc::test::ImproveDirection; - MutexLock lock(&mutex_); for (auto& item : frames_comparator_.stream_stats()) { ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second, stream_frame_counters_.at(item.first)); } - if (metrics_logger_ == nullptr) { - test::PrintResult("cpu_usage", "", test_label_.c_str(), - GetCpuUsagePercent(), "%", false, - ImproveDirection::kSmallerIsBetter); - } else { - metrics_logger_->LogSingleValueMetric( - "cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless, - ImprovementDirection::kSmallerIsBetter); - } + metrics_logger_->LogSingleValueMetric("cpu_usage_%", test_label_, + GetCpuUsagePercent(), Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter); LogFrameCounters("Global", frame_counters_); if (!unknown_sender_frame_counters_.empty()) { RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:"; @@ -943,7 +931,6 @@ void DefaultVideoQualityAnalyzer::ReportResults( const std::string& test_case_name, const StreamStats& stats, const FrameCounters& frame_counters) { - using ::webrtc::test::ImproveDirection; TimeDelta test_duration = Now() - start_time_; double sum_squared_interframe_delays_secs = 0; @@ -972,199 +959,101 @@ void DefaultVideoQualityAnalyzer::ReportResults( video_duration.seconds() / sum_squared_interframe_delays_secs; } - if (metrics_logger_ == nullptr) { - // Report results through old performance metrics API. - ReportResult("psnr", test_case_name, stats.psnr, "dB", - ImproveDirection::kBiggerIsBetter); - ReportResult("ssim", test_case_name, stats.ssim, "unitless", - ImproveDirection::kBiggerIsBetter); - ReportResult("transport_time", test_case_name, stats.transport_time_ms, - "ms", ImproveDirection::kSmallerIsBetter); - ReportResult("total_delay_incl_transport", test_case_name, - stats.total_delay_incl_transport_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("time_between_rendered_frames", test_case_name, - stats.time_between_rendered_frames_ms, "ms", - ImproveDirection::kSmallerIsBetter); - test::PrintResult("harmonic_framerate", "", test_case_name, - harmonic_framerate_fps, "Hz", /*important=*/false, - ImproveDirection::kBiggerIsBetter); - test::PrintResult("encode_frame_rate", "", test_case_name, - stats.encode_frame_rate.IsEmpty() - ? 0 - : stats.encode_frame_rate.GetEventsPerSecond(), - "Hz", /*important=*/false, - ImproveDirection::kBiggerIsBetter); - ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("time_between_freezes", test_case_name, - stats.time_between_freezes_ms, "ms", - ImproveDirection::kBiggerIsBetter); - ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("pixels_per_frame", test_case_name, - stats.resolution_of_rendered_frame, "count", - ImproveDirection::kBiggerIsBetter); - test::PrintResult("min_psnr", "", test_case_name, - stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("receive_to_render_time", test_case_name, - stats.receive_to_render_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - test::PrintResult("dropped_frames", "", test_case_name, - frame_counters.dropped, "count", - /*important=*/false, ImproveDirection::kSmallerIsBetter); - test::PrintResult("frames_in_flight", "", test_case_name, - frame_counters.captured - frame_counters.rendered - - frame_counters.dropped, - "count", - /*important=*/false, ImproveDirection::kSmallerIsBetter); - test::PrintResult("rendered_frames", "", test_case_name, - frame_counters.rendered, "count", /*important=*/false, - ImproveDirection::kBiggerIsBetter); - ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered, - "count", ImproveDirection::kSmallerIsBetter); - ReportResult("target_encode_bitrate", test_case_name, - stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond", - ImproveDirection::kNone); - test::PrintResult("actual_encode_bitrate", "", test_case_name, - static_cast(stats.total_encoded_images_payload) / - test_duration.seconds(), - "bytesPerSecond", /*important=*/false, - ImproveDirection::kNone); + metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr, + Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim, + Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("transport_time", test_case_name, + stats.transport_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name, + stats.total_delay_incl_transport_ms, + Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name, + stats.time_between_rendered_frames_ms, + Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric("harmonic_framerate", test_case_name, + harmonic_framerate_fps, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "encode_frame_rate", test_case_name, + stats.encode_frame_rate.IsEmpty() + ? 0 + : stats.encode_frame_rate.GetEventsPerSecond(), + Unit::kHertz, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("encode_time", test_case_name, + stats.encode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("time_between_freezes", test_case_name, + stats.time_between_freezes_ms, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("freeze_time_ms", test_case_name, + stats.freeze_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("pixels_per_frame", test_case_name, + stats.resolution_of_rendered_frame, Unit::kCount, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "min_psnr_dB", test_case_name, + stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("decode_time", test_case_name, + stats.decode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "receive_to_render_time", test_case_name, stats.receive_to_render_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric("dropped_frames", test_case_name, + frame_counters.dropped, Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric( + "frames_in_flight", test_case_name, + frame_counters.captured - frame_counters.rendered - + frame_counters.dropped, + Unit::kCount, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric("rendered_frames", test_case_name, + frame_counters.rendered, Unit::kCount, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("max_skipped", test_case_name, + stats.skipped_between_rendered, Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("target_encode_bitrate", test_case_name, + stats.target_encode_bitrate / 1000, + Unit::kKilobitsPerSecond, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "actual_encode_bitrate", test_case_name, + static_cast(stats.total_encoded_images_payload) / + test_duration.seconds() * kBitsInByte / 1000, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); - if (options_.report_detailed_frame_stats) { - test::PrintResult("num_encoded_frames", "", test_case_name, - frame_counters.encoded, "count", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - test::PrintResult("num_decoded_frames", "", test_case_name, - frame_counters.decoded, "count", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - test::PrintResult("num_send_key_frames", "", test_case_name, - stats.num_send_key_frames, "count", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - test::PrintResult("num_recv_key_frames", "", test_case_name, - stats.num_recv_key_frames, "count", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - - ReportResult("recv_key_frame_size_bytes", test_case_name, - stats.recv_key_frame_size_bytes, "count", - ImproveDirection::kBiggerIsBetter); - ReportResult("recv_delta_frame_size_bytes", test_case_name, - stats.recv_delta_frame_size_bytes, "count", - ImproveDirection::kBiggerIsBetter); - } - } else { - metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr, - Unit::kUnitless, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim, - Unit::kUnitless, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("transport_time", test_case_name, - stats.transport_time_ms, Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name, - stats.total_delay_incl_transport_ms, - Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name, - stats.time_between_rendered_frames_ms, - Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); + if (options_.report_detailed_frame_stats) { metrics_logger_->LogSingleValueMetric( - "harmonic_framerate", test_case_name, harmonic_framerate_fps, - Unit::kHertz, ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogSingleValueMetric( - "encode_frame_rate", test_case_name, - stats.encode_frame_rate.IsEmpty() - ? 0 - : stats.encode_frame_rate.GetEventsPerSecond(), - Unit::kHertz, ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("encode_time", test_case_name, - stats.encode_time_ms, Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric( - "time_between_freezes", test_case_name, stats.time_between_freezes_ms, - Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("freeze_time_ms", test_case_name, - stats.freeze_time_ms, Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("pixels_per_frame", test_case_name, - stats.resolution_of_rendered_frame, Unit::kCount, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogSingleValueMetric( - "min_psnr_dB", test_case_name, - stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("decode_time", test_case_name, - stats.decode_time_ms, Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("receive_to_render_time", test_case_name, - stats.receive_to_render_time_ms, - Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogSingleValueMetric( - "dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogSingleValueMetric( - "frames_in_flight", test_case_name, - frame_counters.captured - frame_counters.rendered - - frame_counters.dropped, - Unit::kCount, ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogSingleValueMetric( - "rendered_frames", test_case_name, frame_counters.rendered, + "num_encoded_frames", test_case_name, frame_counters.encoded, Unit::kCount, ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("max_skipped", test_case_name, - stats.skipped_between_rendered, Unit::kCount, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("target_encode_bitrate", test_case_name, - stats.target_encode_bitrate / 1000, - Unit::kKilobitsPerSecond, - ImprovementDirection::kNeitherIsBetter); metrics_logger_->LogSingleValueMetric( - "actual_encode_bitrate", test_case_name, - static_cast(stats.total_encoded_images_payload) / - test_duration.seconds() * kBitsInByte / 1000, - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + "num_decoded_frames", test_case_name, frame_counters.decoded, + Unit::kCount, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "num_send_key_frames", test_case_name, stats.num_send_key_frames, + Unit::kCount, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "num_recv_key_frames", test_case_name, stats.num_recv_key_frames, + Unit::kCount, ImprovementDirection::kBiggerIsBetter); - if (options_.report_detailed_frame_stats) { - metrics_logger_->LogSingleValueMetric( - "num_encoded_frames", test_case_name, frame_counters.encoded, - Unit::kCount, ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogSingleValueMetric( - "num_decoded_frames", test_case_name, frame_counters.decoded, - Unit::kCount, ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogSingleValueMetric( - "num_send_key_frames", test_case_name, stats.num_send_key_frames, - Unit::kCount, ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogSingleValueMetric( - "num_recv_key_frames", test_case_name, stats.num_recv_key_frames, - Unit::kCount, ImprovementDirection::kBiggerIsBetter); - - metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name, - stats.recv_key_frame_size_bytes, Unit::kCount, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name, - stats.recv_delta_frame_size_bytes, - Unit::kCount, - ImprovementDirection::kBiggerIsBetter); - } + metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name, + stats.recv_key_frame_size_bytes, Unit::kCount, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name, + stats.recv_delta_frame_size_bytes, Unit::kCount, + ImprovementDirection::kBiggerIsBetter); } } -void DefaultVideoQualityAnalyzer::ReportResult( - const std::string& metric_name, - const std::string& test_case_name, - const SamplesStatsCounter& counter, - const std::string& unit, - webrtc::test::ImproveDirection improve_direction) { - test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit, - /*important=*/false, improve_direction); -} - std::string DefaultVideoQualityAnalyzer::GetTestCaseName( const std::string& stream_label) const { return test_label_ + "/" + stream_label; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index e26d7b4810..2b16ecad44 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -41,15 +41,11 @@ #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" #include "test/pc/e2e/analyzer/video/names_collection.h" -#include "test/testsupport/perf_test.h" namespace webrtc { class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { public: - explicit DefaultVideoQualityAnalyzer( - webrtc::Clock* clock, - DefaultVideoQualityAnalyzerOptions options = {}); DefaultVideoQualityAnalyzer(webrtc::Clock* clock, test::MetricsLogger* metrics_logger, DefaultVideoQualityAnalyzerOptions options = {}); @@ -131,13 +127,6 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { const StreamStats& stats, const FrameCounters& frame_counters) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - // Report result for single metric for specified stream. - static void ReportResult(const std::string& metric_name, - const std::string& test_case_name, - const SamplesStatsCounter& counter, - const std::string& unit, - webrtc::test::ImproveDirection improve_direction = - webrtc::test::ImproveDirection::kNone); // Returns name of current test case for reporting. std::string GetTestCaseName(const std::string& stream_label) const; Timestamp Now(); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc index 2396053044..7a276b0854 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -18,6 +18,7 @@ #include "api/rtp_packet_info.h" #include "api/rtp_packet_infos.h" #include "api/test/create_frame_generator.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" @@ -148,6 +149,7 @@ TEST(DefaultVideoQualityAnalyzerTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -199,6 +201,7 @@ TEST(DefaultVideoQualityAnalyzerTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -280,6 +283,7 @@ TEST(DefaultVideoQualityAnalyzerTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -330,6 +334,7 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -388,6 +393,7 @@ TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -444,6 +450,7 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) { constexpr char kCharlie[] = "charlie"; DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kAlice, kBob, kCharlie}, kAnalyzerMaxThreadsCount); @@ -574,6 +581,7 @@ TEST(DefaultVideoQualityAnalyzerTest, constexpr char kCharlie[] = "charlie"; DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kAlice, kBob, kCharlie}, kAnalyzerMaxThreadsCount); @@ -633,6 +641,7 @@ TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) { analyzer_options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), analyzer_options); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -692,6 +701,7 @@ TEST(DefaultVideoQualityAnalyzerTest, analyzer_options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), analyzer_options); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -750,6 +760,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -813,6 +824,7 @@ TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) { constexpr int kTwoThirdFrames = 2 * kOneThirdFrames; DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount); @@ -958,6 +970,7 @@ TEST(DefaultVideoQualityAnalyzerTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); constexpr char kAlice[] = "alice"; constexpr char kBob[] = "bob"; @@ -1020,7 +1033,8 @@ TEST(DefaultVideoQualityAnalyzerTest, DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); options.enable_receive_own_stream = true; - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, kAnalyzerMaxThreadsCount); @@ -1115,7 +1129,8 @@ TEST(DefaultVideoQualityAnalyzerTest, DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); options.enable_receive_own_stream = true; - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, kAnalyzerMaxThreadsCount); @@ -1208,6 +1223,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), AnalyzerOptionsForTest()); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, @@ -1278,7 +1294,8 @@ TEST(DefaultVideoQualityAnalyzerTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, kAnalyzerMaxThreadsCount); @@ -1370,7 +1387,8 @@ TEST( DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); options.enable_receive_own_stream = true; - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{kSenderPeerName, kReceiverPeerName}, kAnalyzerMaxThreadsCount); @@ -1474,7 +1492,8 @@ TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1518,7 +1537,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1561,7 +1581,8 @@ TEST(DefaultVideoQualityAnalyzerTest, DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); options.enable_receive_own_stream = true; - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1604,7 +1625,8 @@ TEST(DefaultVideoQualityAnalyzerTest, DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); options.enable_receive_own_stream = true; - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1647,7 +1669,8 @@ TEST(DefaultVideoQualityAnalyzerTest, DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); options.enable_receive_own_stream = true; - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1702,7 +1725,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1742,7 +1766,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1784,7 +1809,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1827,7 +1853,8 @@ TEST(DefaultVideoQualityAnalyzerTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -1873,7 +1900,8 @@ TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) { /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob", "charlie"}, kAnalyzerMaxThreadsCount); @@ -1934,7 +1962,8 @@ TEST(DefaultVideoQualityAnalyzerTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob", "charlie"}, kAnalyzerMaxThreadsCount); @@ -1989,7 +2018,8 @@ TEST(DefaultVideoQualityAnalyzerTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); @@ -2026,7 +2056,8 @@ TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest, /*num_squares=*/absl::nullopt); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); - DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); analyzer.Start("test_case", std::vector{"alice", "bob"}, kAnalyzerMaxThreadsCount); diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc index 0270805351..32398ea0a0 100644 --- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc @@ -16,6 +16,7 @@ #include "api/units/data_rate.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" +#include "rtc_base/checks.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -30,6 +31,13 @@ SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) { } // namespace +VideoQualityMetricsReporter::VideoQualityMetricsReporter( + Clock* const clock, + test::MetricsLogger* const metrics_logger) + : clock_(clock), metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + void VideoQualityMetricsReporter::Start( absl::string_view test_case_name, const TrackIdStreamInfoMap* /*reporter_helper*/) { @@ -122,37 +130,18 @@ std::string VideoQualityMetricsReporter::GetTestCaseName( void VideoQualityMetricsReporter::ReportVideoBweResults( const std::string& test_case_name, const VideoBweStats& video_bwe_stats) { - if (metrics_logger_ == nullptr) { - ReportResult("available_send_bandwidth", test_case_name, - video_bwe_stats.available_send_bandwidth, "bytesPerSecond"); - ReportResult("transmission_bitrate", test_case_name, - video_bwe_stats.transmission_bitrate, "bytesPerSecond"); - ReportResult("retransmission_bitrate", test_case_name, - video_bwe_stats.retransmission_bitrate, "bytesPerSecond"); - } else { - metrics_logger_->LogMetric( - "available_send_bandwidth", test_case_name, - BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth), - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogMetric( - "transmission_bitrate", test_case_name, - BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate), - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogMetric( - "retransmission_bitrate", test_case_name, - BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate), - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); - } -} - -void VideoQualityMetricsReporter::ReportResult( - const std::string& metric_name, - const std::string& test_case_name, - const SamplesStatsCounter& counter, - const std::string& unit, - webrtc::test::ImproveDirection improve_direction) { - test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit, - /*important=*/false, improve_direction); + metrics_logger_->LogMetric( + "available_send_bandwidth", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogMetric( + "transmission_bitrate", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogMetric( + "retransmission_bitrate", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); } } // namespace webrtc_pc_e2e diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h index 5d3d2adb1a..47e26a5f99 100644 --- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h @@ -22,7 +22,6 @@ #include "api/units/data_size.h" #include "api/units/timestamp.h" #include "rtc_base/synchronization/mutex.h" -#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -36,12 +35,8 @@ struct VideoBweStats { class VideoQualityMetricsReporter : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { public: - explicit VideoQualityMetricsReporter(Clock* const clock) - : VideoQualityMetricsReporter(clock, /*metrics_logger=*/nullptr) {} - explicit VideoQualityMetricsReporter( - Clock* const clock, - test::MetricsLogger* const metrics_logger) - : clock_(clock), metrics_logger_(metrics_logger) {} + VideoQualityMetricsReporter(Clock* const clock, + test::MetricsLogger* const metrics_logger); ~VideoQualityMetricsReporter() override = default; void Start(absl::string_view test_case_name, @@ -63,13 +58,6 @@ class VideoQualityMetricsReporter std::string GetTestCaseName(const std::string& stream_label) const; void ReportVideoBweResults(const std::string& test_case_name, const VideoBweStats& video_bwe_stats); - // Report result for single metric for specified stream. - static void ReportResult(const std::string& metric_name, - const std::string& test_case_name, - const SamplesStatsCounter& counter, - const std::string& unit, - webrtc::test::ImproveDirection improve_direction = - webrtc::test::ImproveDirection::kNone); Timestamp Now() const { return clock_->CurrentTime(); } Clock* const clock_; diff --git a/test/pc/e2e/cross_media_metrics_reporter.cc b/test/pc/e2e/cross_media_metrics_reporter.cc index 2b5373fdad..f14901236c 100644 --- a/test/pc/e2e/cross_media_metrics_reporter.cc +++ b/test/pc/e2e/cross_media_metrics_reporter.cc @@ -16,6 +16,7 @@ #include "api/stats/rtcstats_objects.h" #include "api/test/metrics/metric.h" #include "api/units/timestamp.h" +#include "rtc_base/checks.h" #include "rtc_base/event.h" #include "system_wrappers/include/field_trial.h" @@ -25,6 +26,12 @@ namespace webrtc_pc_e2e { using ::webrtc::test::ImprovementDirection; using ::webrtc::test::Unit; +CrossMediaMetricsReporter::CrossMediaMetricsReporter( + test::MetricsLogger* metrics_logger) + : metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + void CrossMediaMetricsReporter::Start( absl::string_view test_case_name, const TrackIdStreamInfoMap* reporter_helper) { @@ -102,40 +109,19 @@ void CrossMediaMetricsReporter::StopAndReportResults() { MutexLock lock(&mutex_); for (const auto& pair : stats_info_) { const std::string& sync_group = pair.first; - if (metrics_logger_ == nullptr) { - ReportResult("audio_ahead_ms", - GetTestCaseName(pair.second.audio_stream_label, sync_group), - pair.second.audio_ahead_ms, "ms", - webrtc::test::ImproveDirection::kSmallerIsBetter); - ReportResult("video_ahead_ms", - GetTestCaseName(pair.second.video_stream_label, sync_group), - pair.second.video_ahead_ms, "ms", - webrtc::test::ImproveDirection::kSmallerIsBetter); - } else { - metrics_logger_->LogMetric( - "audio_ahead_ms", - GetTestCaseName(pair.second.audio_stream_label, sync_group), - pair.second.audio_ahead_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric( - "video_ahead_ms", - GetTestCaseName(pair.second.video_stream_label, sync_group), - pair.second.video_ahead_ms, Unit::kMilliseconds, - webrtc::test::ImprovementDirection::kSmallerIsBetter); - } + metrics_logger_->LogMetric( + "audio_ahead_ms", + GetTestCaseName(pair.second.audio_stream_label, sync_group), + pair.second.audio_ahead_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "video_ahead_ms", + GetTestCaseName(pair.second.video_stream_label, sync_group), + pair.second.video_ahead_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter); } } -void CrossMediaMetricsReporter::ReportResult( - const std::string& metric_name, - const std::string& test_case_name, - const SamplesStatsCounter& counter, - const std::string& unit, - webrtc::test::ImproveDirection improve_direction) { - test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit, - /*important=*/false, improve_direction); -} - std::string CrossMediaMetricsReporter::GetTestCaseName( const std::string& stream_label, const std::string& sync_group) const { diff --git a/test/pc/e2e/cross_media_metrics_reporter.h b/test/pc/e2e/cross_media_metrics_reporter.h index f43d80b934..42baf43a71 100644 --- a/test/pc/e2e/cross_media_metrics_reporter.h +++ b/test/pc/e2e/cross_media_metrics_reporter.h @@ -22,7 +22,6 @@ #include "api/test/track_id_stream_info_map.h" #include "api/units/timestamp.h" #include "rtc_base/synchronization/mutex.h" -#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -30,10 +29,7 @@ namespace webrtc_pc_e2e { class CrossMediaMetricsReporter : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { public: - CrossMediaMetricsReporter() - : CrossMediaMetricsReporter(/*metrics_logger=*/nullptr) {} - explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger) - : metrics_logger_(metrics_logger) {} + explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger); ~CrossMediaMetricsReporter() override = default; void Start(absl::string_view test_case_name, @@ -52,12 +48,6 @@ class CrossMediaMetricsReporter std::string video_stream_label; }; - static void ReportResult(const std::string& metric_name, - const std::string& test_case_name, - const SamplesStatsCounter& counter, - const std::string& unit, - webrtc::test::ImproveDirection improve_direction = - webrtc::test::ImproveDirection::kNone); std::string GetTestCaseName(const std::string& stream_label, const std::string& sync_group) const; diff --git a/test/pc/e2e/network_quality_metrics_reporter.cc b/test/pc/e2e/network_quality_metrics_reporter.cc index d073b15437..c5d47beb37 100644 --- a/test/pc/e2e/network_quality_metrics_reporter.cc +++ b/test/pc/e2e/network_quality_metrics_reporter.cc @@ -14,9 +14,9 @@ #include "api/stats/rtc_stats.h" #include "api/stats/rtcstats_objects.h" #include "api/test/metrics/metric.h" +#include "rtc_base/checks.h" #include "rtc_base/event.h" #include "system_wrappers/include/field_trial.h" -#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -31,6 +31,17 @@ constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1); // sent/received per stream. If enabled, padding and headers are not included // in bytes sent or received. constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats"; + +} // namespace + +NetworkQualityMetricsReporter::NetworkQualityMetricsReporter( + EmulatedNetworkManagerInterface* alice_network, + EmulatedNetworkManagerInterface* bob_network, + test::MetricsLogger* metrics_logger) + : alice_network_(alice_network), + bob_network_(bob_network), + metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); } void NetworkQualityMetricsReporter::Start( @@ -113,93 +124,49 @@ void NetworkQualityMetricsReporter::ReportStats( const std::string& network_label, std::unique_ptr stats, int64_t packet_loss) { - if (metrics_logger_ == nullptr) { - ReportResult("bytes_sent", network_label, stats->BytesSent().bytes(), - "sizeInBytes"); - ReportResult("packets_sent", network_label, stats->PacketsSent(), - "unitless"); - ReportResult("average_send_rate", network_label, - stats->PacketsSent() >= 2 - ? stats->AverageSendRate().bytes_per_sec() - : 0, - "bytesPerSecond"); - ReportResult("bytes_discarded_no_receiver", network_label, - stats->BytesDropped().bytes(), "sizeInBytes"); - ReportResult("packets_discarded_no_receiver", network_label, - stats->PacketsDropped(), "unitless"); - ReportResult("bytes_received", network_label, - stats->BytesReceived().bytes(), "sizeInBytes"); - ReportResult("packets_received", network_label, stats->PacketsReceived(), - "unitless"); - ReportResult("average_receive_rate", network_label, - stats->PacketsReceived() >= 2 - ? stats->AverageReceiveRate().bytes_per_sec() - : 0, - "bytesPerSecond"); - ReportResult("sent_packets_loss", network_label, packet_loss, "unitless"); - } else { - metrics_logger_->LogSingleValueMetric( - "bytes_sent", GetTestCaseName(network_label), - stats->BytesSent().bytes(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "packets_sent", GetTestCaseName(network_label), stats->PacketsSent(), - Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "average_send_rate", GetTestCaseName(network_label), - stats->PacketsSent() >= 2 ? stats->AverageSendRate().kbps() : 0, - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "bytes_discarded_no_receiver", GetTestCaseName(network_label), - stats->BytesDropped().bytes(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "packets_discarded_no_receiver", GetTestCaseName(network_label), - stats->PacketsDropped(), Unit::kUnitless, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "bytes_received", GetTestCaseName(network_label), - stats->BytesReceived().bytes(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "packets_received", GetTestCaseName(network_label), - stats->PacketsReceived(), Unit::kUnitless, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "average_receive_rate", GetTestCaseName(network_label), - stats->PacketsReceived() >= 2 ? stats->AverageReceiveRate().kbps() : 0, - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "sent_packets_loss", GetTestCaseName(network_label), packet_loss, - Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogSingleValueMetric( + "bytes_sent", GetTestCaseName(network_label), stats->BytesSent().bytes(), + Unit::kBytes, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_sent", GetTestCaseName(network_label), stats->PacketsSent(), + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "average_send_rate", GetTestCaseName(network_label), + stats->PacketsSent() >= 2 ? stats->AverageSendRate().kbps() : 0, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "bytes_discarded_no_receiver", GetTestCaseName(network_label), + stats->BytesDropped().bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_discarded_no_receiver", GetTestCaseName(network_label), + stats->PacketsDropped(), Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "bytes_received", GetTestCaseName(network_label), + stats->BytesReceived().bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_received", GetTestCaseName(network_label), + stats->PacketsReceived(), Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "average_receive_rate", GetTestCaseName(network_label), + stats->PacketsReceived() >= 2 ? stats->AverageReceiveRate().kbps() : 0, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "sent_packets_loss", GetTestCaseName(network_label), packet_loss, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); } void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label, const PCStats& stats) { - if (metrics_logger_ == nullptr) { - ReportResult("payload_bytes_received", pc_label, - stats.payload_received.bytes(), "sizeInBytes"); - ReportResult("payload_bytes_sent", pc_label, stats.payload_sent.bytes(), - "sizeInBytes"); - } else { - metrics_logger_->LogSingleValueMetric( - "payload_bytes_received", pc_label, stats.payload_received.bytes(), - Unit::kBytes, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "payload_bytes_sent", pc_label, stats.payload_sent.bytes(), - Unit::kBytes, ImprovementDirection::kNeitherIsBetter); - } -} - -void NetworkQualityMetricsReporter::ReportResult( - const std::string& metric_name, - const std::string& network_label, - const double value, - const std::string& unit) const { - test::PrintResult(metric_name, /*modifier=*/"", - GetTestCaseName(network_label), value, unit, - /*important=*/false); + metrics_logger_->LogSingleValueMetric( + "payload_bytes_received", pc_label, stats.payload_received.bytes(), + Unit::kBytes, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "payload_bytes_sent", pc_label, stats.payload_sent.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); } std::string NetworkQualityMetricsReporter::GetTestCaseName( diff --git a/test/pc/e2e/network_quality_metrics_reporter.h b/test/pc/e2e/network_quality_metrics_reporter.h index db62c26b2f..9de65b56cb 100644 --- a/test/pc/e2e/network_quality_metrics_reporter.h +++ b/test/pc/e2e/network_quality_metrics_reporter.h @@ -28,17 +28,9 @@ namespace webrtc_pc_e2e { class NetworkQualityMetricsReporter : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { public: - NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network, - EmulatedNetworkManagerInterface* bob_network) - : NetworkQualityMetricsReporter(alice_network, - bob_network, - /*metrics_logger=*/nullptr) {} NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network, EmulatedNetworkManagerInterface* bob_network, - test::MetricsLogger* metrics_logger) - : alice_network_(alice_network), - bob_network_(bob_network), - metrics_logger_(metrics_logger) {} + test::MetricsLogger* metrics_logger); ~NetworkQualityMetricsReporter() override = default; // Network stats must be empty when this method will be invoked. @@ -63,10 +55,6 @@ class NetworkQualityMetricsReporter std::unique_ptr stats, int64_t packet_loss); void ReportPCStats(const std::string& pc_label, const PCStats& stats); - void ReportResult(const std::string& metric_name, - const std::string& network_label, - double value, - const std::string& unit) const; std::string GetTestCaseName(const std::string& network_label) const; std::string test_case_name_; diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index af381763a3..6cf7449a14 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -41,7 +41,6 @@ #include "test/pc/e2e/stats_poller.h" #include "test/pc/e2e/test_peer_factory.h" #include "test/testsupport/file_utils.h" -#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -736,24 +735,12 @@ void PeerConnectionE2EQualityTest::TearDownCall() { } void PeerConnectionE2EQualityTest::ReportGeneralTestResults() { - if (metrics_logger_ == nullptr) { - test::PrintResult(*alice_->params().name + "_connected", "", - test_case_name_, alice_connected_, "unitless", - /*important=*/false, - test::ImproveDirection::kBiggerIsBetter); - test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_, - bob_connected_, "unitless", - /*important=*/false, - test::ImproveDirection::kBiggerIsBetter); - } else { - metrics_logger_->LogSingleValueMetric( - *alice_->params().name + "_connected", test_case_name_, - alice_connected_, Unit::kUnitless, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogSingleValueMetric( - *bob_->params().name + "_connected", test_case_name_, bob_connected_, - Unit::kUnitless, ImprovementDirection::kBiggerIsBetter); - } + metrics_logger_->LogSingleValueMetric( + *alice_->params().name + "_connected", test_case_name_, alice_connected_, + Unit::kUnitless, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + *bob_->params().name + "_connected", test_case_name_, bob_connected_, + Unit::kUnitless, ImprovementDirection::kBiggerIsBetter); } Timestamp PeerConnectionE2EQualityTest::Now() const { diff --git a/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc b/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc index bc7c968026..30b032c2e0 100644 --- a/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc +++ b/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc @@ -29,12 +29,12 @@ #include "api/test/network_emulation_manager.h" #include "api/units/data_rate.h" #include "api/units/timestamp.h" +#include "rtc_base/checks.h" #include "rtc_base/event.h" #include "rtc_base/ip_address.h" #include "rtc_base/strings/string_builder.h" #include "rtc_base/synchronization/mutex.h" #include "system_wrappers/include/field_trial.h" -#include "test/testsupport/perf_test.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -81,6 +81,17 @@ std::map PopulateIpToPeer( } // namespace +StatsBasedNetworkQualityMetricsReporter:: + StatsBasedNetworkQualityMetricsReporter( + std::map> peer_endpoints, + NetworkEmulationManager* network_emulation, + test::MetricsLogger* metrics_logger) + : collector_(std::move(peer_endpoints), network_emulation), + clock_(network_emulation->time_controller()->GetClock()), + metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: NetworkLayerStatsCollector( std::map> peer_endpoints, @@ -241,99 +252,48 @@ void StatsBasedNetworkQualityMetricsReporter::ReportStats( const NetworkLayerStats& network_layer_stats, int64_t packet_loss, const Timestamp& end_time) { - if (metrics_logger_ == nullptr) { - ReportResult("bytes_discarded_no_receiver", pc_label, - network_layer_stats.stats->BytesDropped().bytes(), - "sizeInBytes"); - ReportResult("packets_discarded_no_receiver", pc_label, - network_layer_stats.stats->PacketsDropped(), "unitless"); + metrics_logger_->LogSingleValueMetric( + "bytes_discarded_no_receiver", GetTestCaseName(pc_label), + network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_discarded_no_receiver", GetTestCaseName(pc_label), + network_layer_stats.stats->PacketsDropped(), Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); - ReportResult("payload_bytes_received", pc_label, - pc_stats.payload_received.bytes(), "sizeInBytes"); - ReportResult("payload_bytes_sent", pc_label, pc_stats.payload_sent.bytes(), - "sizeInBytes"); + metrics_logger_->LogSingleValueMetric( + "payload_bytes_received", GetTestCaseName(pc_label), + pc_stats.payload_received.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "payload_bytes_sent", GetTestCaseName(pc_label), + pc_stats.payload_sent.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); - ReportResult("bytes_sent", pc_label, pc_stats.total_sent.bytes(), - "sizeInBytes"); - ReportResult("packets_sent", pc_label, pc_stats.packets_sent, "unitless"); - ReportResult( - "average_send_rate", pc_label, - (pc_stats.total_sent / (end_time - start_time_)).bytes_per_sec(), - "bytesPerSecond"); - ReportResult("bytes_received", pc_label, pc_stats.total_received.bytes(), - "sizeInBytes"); - ReportResult("packets_received", pc_label, pc_stats.packets_received, - "unitless"); - ReportResult( - "average_receive_rate", pc_label, - (pc_stats.total_received / (end_time - start_time_)).bytes_per_sec(), - "bytesPerSecond"); - ReportResult("sent_packets_loss", pc_label, packet_loss, "unitless"); - } else { - metrics_logger_->LogSingleValueMetric( - "bytes_discarded_no_receiver", GetTestCaseName(pc_label), - network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "packets_discarded_no_receiver", GetTestCaseName(pc_label), - network_layer_stats.stats->PacketsDropped(), Unit::kUnitless, - ImprovementDirection::kNeitherIsBetter); - - metrics_logger_->LogSingleValueMetric( - "payload_bytes_received", GetTestCaseName(pc_label), - pc_stats.payload_received.bytes(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "payload_bytes_sent", GetTestCaseName(pc_label), - pc_stats.payload_sent.bytes(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - - metrics_logger_->LogSingleValueMetric( - "bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(), - Unit::kBytes, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent, - Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "average_send_rate", GetTestCaseName(pc_label), - (pc_stats.total_sent / (end_time - start_time_)).kbps(), - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "bytes_received", GetTestCaseName(pc_label), - pc_stats.total_received.bytes(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "packets_received", GetTestCaseName(pc_label), - pc_stats.packets_received, Unit::kUnitless, - ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "average_receive_rate", GetTestCaseName(pc_label), - (pc_stats.total_received / (end_time - start_time_)).kbps(), - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); - metrics_logger_->LogSingleValueMetric( - "sent_packets_loss", GetTestCaseName(pc_label), packet_loss, - Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); - } -} - -void StatsBasedNetworkQualityMetricsReporter::ReportResult( - const std::string& metric_name, - const std::string& network_label, - const double value, - const std::string& unit) const { - test::PrintResult(metric_name, /*modifier=*/"", - GetTestCaseName(network_label), value, unit, - /*important=*/false); -} - -void StatsBasedNetworkQualityMetricsReporter::ReportResult( - const std::string& metric_name, - const std::string& network_label, - const SamplesStatsCounter& value, - const std::string& unit) const { - test::PrintResult(metric_name, /*modifier=*/"", - GetTestCaseName(network_label), value, unit, - /*important=*/false); + metrics_logger_->LogSingleValueMetric( + "bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(), + Unit::kBytes, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "average_send_rate", GetTestCaseName(pc_label), + (pc_stats.total_sent / (end_time - start_time_)).kbps(), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "bytes_received", GetTestCaseName(pc_label), + pc_stats.total_received.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_received", GetTestCaseName(pc_label), pc_stats.packets_received, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "average_receive_rate", GetTestCaseName(pc_label), + (pc_stats.total_received / (end_time - start_time_)).kbps(), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "sent_packets_loss", GetTestCaseName(pc_label), packet_loss, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); } std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName( @@ -360,48 +320,28 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats( log << " " << local_ips[i].ToString() << "\n"; } if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) { - if (metrics_logger_ == nullptr) { - ReportResult("sent_packets_size", peer_name, - stats.stats->SentPacketsSizeCounter(), "sizeInBytes"); - } else { - metrics_logger_->LogMetric( - "sent_packets_size", GetTestCaseName(peer_name), - stats.stats->SentPacketsSizeCounter(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogMetric("sent_packets_size", GetTestCaseName(peer_name), + stats.stats->SentPacketsSizeCounter(), + Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); } if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) { - if (metrics_logger_ == nullptr) { - ReportResult("received_packets_size", peer_name, - stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes"); - } else { - metrics_logger_->LogMetric( - "received_packets_size", GetTestCaseName(peer_name), - stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogMetric( + "received_packets_size", GetTestCaseName(peer_name), + stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); } if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) { - if (metrics_logger_ == nullptr) { - ReportResult("dropped_packets_size", peer_name, - stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes"); - } else { - metrics_logger_->LogMetric( - "dropped_packets_size", GetTestCaseName(peer_name), - stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogMetric( + "dropped_packets_size", GetTestCaseName(peer_name), + stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); } if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) { - if (metrics_logger_ == nullptr) { - ReportResult("sent_packets_queue_wait_time_us", peer_name, - stats.stats->SentPacketsQueueWaitTimeUs(), "unitless"); - } else { - metrics_logger_->LogMetric( - "sent_packets_queue_wait_time_us", GetTestCaseName(peer_name), - stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless, - ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogMetric( + "sent_packets_queue_wait_time_us", GetTestCaseName(peer_name), + stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); } log << "Send statistic:\n" @@ -421,17 +361,11 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats( << " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec() << " avg_rate (bps): " << source_average_send_rate.bps() << "\n"; if (!entry.second->SentPacketsSizeCounter().IsEmpty()) { - if (metrics_logger_ == nullptr) { - ReportResult("sent_packets_size", - peer_name + "/" + entry.first.ToString(), - stats.stats->SentPacketsSizeCounter(), "sizeInBytes"); - } else { - metrics_logger_->LogMetric( - "sent_packets_size", - GetTestCaseName(peer_name + "/" + entry.first.ToString()), - stats.stats->SentPacketsSizeCounter(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogMetric( + "sent_packets_size", + GetTestCaseName(peer_name + "/" + entry.first.ToString()), + stats.stats->SentPacketsSizeCounter(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); } } @@ -454,30 +388,18 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats( << source_average_receive_rate.bytes_per_sec() << " avg_rate (bps): " << source_average_receive_rate.bps() << "\n"; if (!entry.second->ReceivedPacketsSizeCounter().IsEmpty()) { - if (metrics_logger_ == nullptr) { - ReportResult("received_packets_size", - peer_name + "/" + entry.first.ToString(), - stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes"); - } else { - metrics_logger_->LogMetric( - "received_packets_size", - GetTestCaseName(peer_name + "/" + entry.first.ToString()), - stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogMetric( + "received_packets_size", + GetTestCaseName(peer_name + "/" + entry.first.ToString()), + stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); } if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) { - if (metrics_logger_ == nullptr) { - ReportResult("dropped_packets_size", - peer_name + "/" + entry.first.ToString(), - stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes"); - } else { - metrics_logger_->LogMetric( - "dropped_packets_size", - GetTestCaseName(peer_name + "/" + entry.first.ToString()), - stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes, - ImprovementDirection::kNeitherIsBetter); - } + metrics_logger_->LogMetric( + "dropped_packets_size", + GetTestCaseName(peer_name + "/" + entry.first.ToString()), + stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); } } diff --git a/test/pc/e2e/stats_based_network_quality_metrics_reporter.h b/test/pc/e2e/stats_based_network_quality_metrics_reporter.h index e30dda483a..c89a3b27a7 100644 --- a/test/pc/e2e/stats_based_network_quality_metrics_reporter.h +++ b/test/pc/e2e/stats_based_network_quality_metrics_reporter.h @@ -39,19 +39,10 @@ class StatsBasedNetworkQualityMetricsReporter public: // `networks` map peer name to network to report network layer stability stats // and to log network layer metrics. - StatsBasedNetworkQualityMetricsReporter( - std::map> peer_endpoints, - NetworkEmulationManager* network_emulation) - : StatsBasedNetworkQualityMetricsReporter(std::move(peer_endpoints), - network_emulation, - /*metrics_logger=*/nullptr) {} StatsBasedNetworkQualityMetricsReporter( std::map> peer_endpoints, NetworkEmulationManager* network_emulation, - test::MetricsLogger* metrics_logger) - : collector_(std::move(peer_endpoints), network_emulation), - clock_(network_emulation->time_controller()->GetClock()), - metrics_logger_(metrics_logger) {} + test::MetricsLogger* metrics_logger); ~StatsBasedNetworkQualityMetricsReporter() override = default; void AddPeer(absl::string_view peer_name, @@ -111,14 +102,6 @@ class StatsBasedNetworkQualityMetricsReporter const NetworkLayerStats& network_layer_stats, int64_t packet_loss, const Timestamp& end_time); - void ReportResult(const std::string& metric_name, - const std::string& network_label, - double value, - const std::string& unit) const; - void ReportResult(const std::string& metric_name, - const std::string& network_label, - const SamplesStatsCounter& value, - const std::string& unit) const; std::string GetTestCaseName(absl::string_view network_label) const; void LogNetworkLayerStats(const std::string& peer_name, const NetworkLayerStats& stats) const; diff --git a/video/BUILD.gn b/video/BUILD.gn index 82bd9e4b6f..828803bd22 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -585,6 +585,7 @@ if (rtc_include_tests) { "../api:peer_connection_quality_test_fixture_api", "../api:simulated_network_api", "../api:time_controller", + "../api/test/metrics:global_metrics_logger_and_exporter", "../api/video_codecs:video_codecs_api", "../call:simulated_network", "../modules/video_coding:webrtc_vp9", diff --git a/video/pc_full_stack_tests.cc b/video/pc_full_stack_tests.cc index f6e272b2df..3eab854d75 100644 --- a/video/pc_full_stack_tests.cc +++ b/video/pc_full_stack_tests.cc @@ -17,6 +17,7 @@ #include "api/test/create_peer_connection_quality_test_frame_generator.h" #include "api/test/create_peerconnection_quality_test_fixture.h" #include "api/test/frame_generator_interface.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/test/network_emulation_manager.h" #include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/simulated_network.h" @@ -93,7 +94,8 @@ CreateTestFixture(const std::string& test_case_name, bob_configurer); fixture->AddQualityMetricsReporter( std::make_unique( - network_links.first, network_links.second)); + network_links.first, network_links.second, + test::GetGlobalMetricsLogger())); return fixture; }