From b15faaa2640d137d6341b09fd3670813e3aaf99e Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Fri, 7 Oct 2022 00:54:34 +0200 Subject: [PATCH] [PCLF] Annotate video metrics with peer, stream and receiver Bug: b/240540204 Change-Id: I05eddea33a2eb680b59c8247f2acba1e7c7d6a84 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/278260 Reviewed-by: Andrey Logvin Commit-Queue: Artem Titov Cr-Commit-Position: refs/heads/main@{#38316} --- test/pc/e2e/BUILD.gn | 10 + .../video/default_video_quality_analyzer.cc | 126 +++--- .../video/default_video_quality_analyzer.h | 2 +- ...ideo_quality_analyzer_frames_comparator.cc | 16 +- .../video/video_quality_metrics_reporter.cc | 26 +- .../video/video_quality_metrics_reporter.h | 4 +- test/pc/e2e/metric_metadata_keys.h | 37 ++ ...nnection_quality_test_metric_names_test.cc | 387 +++++++++++++----- 8 files changed, 442 insertions(+), 166 deletions(-) create mode 100644 test/pc/e2e/metric_metadata_keys.h diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index f12e8cf1da..ddcf9c98db 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -524,6 +524,7 @@ if (!build_with_chromium) { testonly = true sources = [ "peer_connection_quality_test_metric_names_test.cc" ] deps = [ + ":metric_metadata_keys", ":peerconnection_quality_test", ":stats_based_network_quality_metrics_reporter", "../..:test_support", @@ -741,6 +742,7 @@ if (!build_with_chromium) { "analyzer/video/video_quality_metrics_reporter.h", ] deps = [ + ":metric_metadata_keys", "../..:perf_test", "../../../api:peer_connection_quality_test_fixture_api", "../../../api:rtc_stats_api", @@ -761,6 +763,12 @@ if (!build_with_chromium) { absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } + rtc_library("metric_metadata_keys") { + visibility = [ "*" ] + testonly = true + sources = [ "metric_metadata_keys.h" ] + } + rtc_library("default_video_quality_analyzer") { visibility = [ "*" ] @@ -773,6 +781,7 @@ if (!build_with_chromium) { deps = [ ":default_video_quality_analyzer_internal", ":default_video_quality_analyzer_shared", + ":metric_metadata_keys", "../..:perf_test", "../../../api:array_view", "../../../api:video_quality_analyzer_api", @@ -832,6 +841,7 @@ if (!build_with_chromium) { deps = [ ":default_video_quality_analyzer_shared", + ":metric_metadata_keys", ":multi_reader_queue", "../../../api:array_view", "../../../api:scoped_refptr", diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 08a6fca792..2cb4409e9d 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -11,8 +11,10 @@ #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" #include +#include #include #include +#include #include #include @@ -34,12 +36,14 @@ #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" +#include "test/pc/e2e/metric_metadata_keys.h" namespace webrtc { namespace { using ::webrtc::test::ImprovementDirection; using ::webrtc::test::Unit; +using ::webrtc::webrtc_pc_e2e::MetricMetadataKey; constexpr int kBitsInByte = 8; constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed"; @@ -892,7 +896,7 @@ void DefaultVideoQualityAnalyzer:: void DefaultVideoQualityAnalyzer::ReportResults() { MutexLock lock(&mutex_); for (auto& item : frames_comparator_.stream_stats()) { - ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second, + ReportResults(item.first, item.second, stream_frame_counters_.at(item.first)); } metrics_logger_->LogSingleValueMetric("cpu_usage_%", test_label_, @@ -931,10 +935,15 @@ void DefaultVideoQualityAnalyzer::ReportResults() { } void DefaultVideoQualityAnalyzer::ReportResults( - const std::string& test_case_name, + const InternalStatsKey& key, const StreamStats& stats, const FrameCounters& frame_counters) { TimeDelta test_duration = Now() - start_time_; + std::string test_case_name = GetTestCaseName(ToMetricName(key)); + std::map metric_metadata{ + {MetricMetadataKey::kPeerMetadataKey, peers_->name(key.sender)}, + {MetricMetadataKey::kStreamMetadataKey, streams_.name(key.stream)}, + {MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)}}; double sum_squared_interframe_delays_secs = 0; Timestamp video_start_time = Timestamp::PlusInfinity(); @@ -962,98 +971,105 @@ void DefaultVideoQualityAnalyzer::ReportResults( video_duration.seconds() / sum_squared_interframe_delays_secs; } - metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr, - Unit::kUnitless, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim, - Unit::kUnitless, - ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric( + "psnr_dB", test_case_name, stats.psnr, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "ssim", test_case_name, stats.ssim, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metric_metadata); metrics_logger_->LogMetric("transport_time", test_case_name, stats.transport_time_ms, Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name, - stats.total_delay_incl_transport_ms, - Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name, - stats.time_between_rendered_frames_ms, - Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogSingleValueMetric("harmonic_framerate", test_case_name, - harmonic_framerate_fps, Unit::kHertz, - ImprovementDirection::kBiggerIsBetter); + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "total_delay_incl_transport", test_case_name, + stats.total_delay_incl_transport_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "time_between_rendered_frames", test_case_name, + stats.time_between_rendered_frames_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "harmonic_framerate", test_case_name, harmonic_framerate_fps, + Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( "encode_frame_rate", test_case_name, stats.encode_frame_rate.IsEmpty() ? 0 : stats.encode_frame_rate.GetEventsPerSecond(), - Unit::kHertz, ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("encode_time", test_case_name, - stats.encode_time_ms, Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); + Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "encode_time", test_case_name, stats.encode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); metrics_logger_->LogMetric("time_between_freezes", test_case_name, stats.time_between_freezes_ms, Unit::kMilliseconds, - ImprovementDirection::kBiggerIsBetter); + ImprovementDirection::kBiggerIsBetter, + metric_metadata); metrics_logger_->LogMetric("freeze_time_ms", test_case_name, stats.freeze_time_ms, Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("pixels_per_frame", test_case_name, - stats.resolution_of_rendered_frame, Unit::kCount, - ImprovementDirection::kBiggerIsBetter); + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "pixels_per_frame", test_case_name, stats.resolution_of_rendered_frame, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( "min_psnr_dB", test_case_name, stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("decode_time", test_case_name, - stats.decode_time_ms, Unit::kMilliseconds, - ImprovementDirection::kSmallerIsBetter); + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "decode_time", test_case_name, stats.decode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); metrics_logger_->LogMetric( "receive_to_render_time", test_case_name, stats.receive_to_render_time_ms, - Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogSingleValueMetric("dropped_frames", test_case_name, - frame_counters.dropped, Unit::kCount, - ImprovementDirection::kSmallerIsBetter); + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogSingleValueMetric( + "dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount, + ImprovementDirection::kSmallerIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( "frames_in_flight", test_case_name, frame_counters.captured - frame_counters.rendered - frame_counters.dropped, - Unit::kCount, ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogSingleValueMetric("rendered_frames", test_case_name, - frame_counters.rendered, Unit::kCount, - ImprovementDirection::kBiggerIsBetter); - metrics_logger_->LogMetric("max_skipped", test_case_name, - stats.skipped_between_rendered, Unit::kCount, - ImprovementDirection::kSmallerIsBetter); - metrics_logger_->LogMetric("target_encode_bitrate", test_case_name, - stats.target_encode_bitrate / 1000, - Unit::kKilobitsPerSecond, - ImprovementDirection::kNeitherIsBetter); + Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "rendered_frames", test_case_name, frame_counters.rendered, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "max_skipped", test_case_name, stats.skipped_between_rendered, + Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "target_encode_bitrate", test_case_name, + stats.target_encode_bitrate / 1000, Unit::kKilobitsPerSecond, + ImprovementDirection::kNeitherIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( "actual_encode_bitrate", test_case_name, static_cast(stats.total_encoded_images_payload) / test_duration.seconds() * kBitsInByte / 1000, - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); if (options_.report_detailed_frame_stats) { metrics_logger_->LogSingleValueMetric( "num_encoded_frames", test_case_name, frame_counters.encoded, - Unit::kCount, ImprovementDirection::kBiggerIsBetter); + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( "num_decoded_frames", test_case_name, frame_counters.decoded, - Unit::kCount, ImprovementDirection::kBiggerIsBetter); + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( "num_send_key_frames", test_case_name, stats.num_send_key_frames, - Unit::kCount, ImprovementDirection::kBiggerIsBetter); + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); metrics_logger_->LogSingleValueMetric( "num_recv_key_frames", test_case_name, stats.num_recv_key_frames, - Unit::kCount, ImprovementDirection::kBiggerIsBetter); + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name, stats.recv_key_frame_size_bytes, Unit::kCount, - ImprovementDirection::kBiggerIsBetter); + ImprovementDirection::kBiggerIsBetter, + metric_metadata); metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name, stats.recv_delta_frame_size_bytes, Unit::kCount, - ImprovementDirection::kBiggerIsBetter); + ImprovementDirection::kBiggerIsBetter, + metric_metadata); } } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 5501fb5896..bc3edf7dd2 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -124,7 +124,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { // Report results for all metrics for all streams. void ReportResults(); - void ReportResults(const std::string& test_case_name, + void ReportResults(const InternalStatsKey& key, const StreamStats& stats, const FrameCounters& frame_counters) RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc index 4048efc17a..739ab3d895 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -28,13 +28,15 @@ #include "rtc_tools/frame_analyzer/video_geometry_aligner.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/metric_metadata_keys.h" namespace webrtc { namespace { +using ::webrtc::webrtc_pc_e2e::SampleMetadataKey; + constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150); constexpr int kMaxActiveComparisons = 10; -constexpr char kFrameIdMetadataKey[] = "frame_id"; SamplesStatsCounter::StatsSample StatsSample( double value, @@ -307,10 +309,11 @@ void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( MutexLock lock(&mutex_); RTC_CHECK_EQ(state_, State::kActive) << "Frames comparator has to be started before it will be used"; - stream_stats_.at(stats_key).skipped_between_rendered.AddSample(StatsSample( - skipped_between_rendered, Now(), - /*metadata=*/ - {{kFrameIdMetadataKey, std::to_string(frame_stats.frame_id)}})); + stream_stats_.at(stats_key).skipped_between_rendered.AddSample( + StatsSample(skipped_between_rendered, Now(), + /*metadata=*/ + {{SampleMetadataKey::kFrameIdMetadataKey, + std::to_string(frame_stats.frame_id)}})); AddComparisonInternal(std::move(stats_key), std::move(captured), std::move(rendered), type, std::move(frame_stats)); } @@ -420,7 +423,8 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( } std::map metadata; - metadata.emplace(kFrameIdMetadataKey, std::to_string(frame_stats.frame_id)); + metadata.emplace(SampleMetadataKey::kFrameIdMetadataKey, + std::to_string(frame_stats.frame_id)); if (psnr > 0) { stats->psnr.AddSample( diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc index 32398ea0a0..085f1b3cf6 100644 --- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc @@ -10,6 +10,9 @@ #include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" +#include +#include + #include "api/stats/rtc_stats.h" #include "api/stats/rtcstats_objects.h" #include "api/test/metrics/metric.h" @@ -17,6 +20,7 @@ #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "rtc_base/checks.h" +#include "test/pc/e2e/metric_metadata_keys.h" namespace webrtc { namespace webrtc_pc_e2e { @@ -24,6 +28,7 @@ namespace { using ::webrtc::test::ImprovementDirection; using ::webrtc::test::Unit; +using ::webrtc::webrtc_pc_e2e::MetricMetadataKey; SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) { return counter * 0.008; @@ -118,30 +123,37 @@ void VideoQualityMetricsReporter::OnStatsReports( void VideoQualityMetricsReporter::StopAndReportResults() { MutexLock video_bwemutex_(&video_bwe_stats_lock_); for (const auto& item : video_bwe_stats_) { - ReportVideoBweResults(GetTestCaseName(item.first), item.second); + ReportVideoBweResults(item.first, item.second); } } std::string VideoQualityMetricsReporter::GetTestCaseName( - const std::string& stream_label) const { - return test_case_name_ + "/" + stream_label; + const std::string& peer_name) const { + return test_case_name_ + "/" + peer_name; } void VideoQualityMetricsReporter::ReportVideoBweResults( - const std::string& test_case_name, + const std::string& peer_name, const VideoBweStats& video_bwe_stats) { + std::string test_case_name = GetTestCaseName(peer_name); + std::map metric_metadata{ + {MetricMetadataKey::kPeerMetadataKey, peer_name}}; + metrics_logger_->LogMetric( "available_send_bandwidth", test_case_name, BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth), - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); metrics_logger_->LogMetric( "transmission_bitrate", test_case_name, BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate), - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); metrics_logger_->LogMetric( "retransmission_bitrate", test_case_name, BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate), - Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); } } // namespace webrtc_pc_e2e diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h index 47e26a5f99..d3d976343b 100644 --- a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h @@ -55,8 +55,8 @@ class VideoQualityMetricsReporter Timestamp sample_time = Timestamp::Zero(); }; - std::string GetTestCaseName(const std::string& stream_label) const; - void ReportVideoBweResults(const std::string& test_case_name, + std::string GetTestCaseName(const std::string& peer_name) const; + void ReportVideoBweResults(const std::string& peer_name, const VideoBweStats& video_bwe_stats); Timestamp Now() const { return clock_->CurrentTime(); } diff --git a/test/pc/e2e/metric_metadata_keys.h b/test/pc/e2e/metric_metadata_keys.h new file mode 100644 index 0000000000..5fe84af88d --- /dev/null +++ b/test/pc/e2e/metric_metadata_keys.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_METRIC_METADATA_KEYS_H_ +#define TEST_PC_E2E_METRIC_METADATA_KEYS_H_ + +namespace webrtc { +namespace webrtc_pc_e2e { + +class MetricMetadataKey { + public: + static constexpr char kPeerMetadataKey[] = "peer"; + static constexpr char kStreamMetadataKey[] = "stream"; + static constexpr char kReceiverMetadataKey[] = "receiver"; + + private: + MetricMetadataKey() = default; +}; + +class SampleMetadataKey { + public: + static constexpr char kFrameIdMetadataKey[] = "frame_id"; + + private: + SampleMetadataKey() = default; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_METRIC_METADATA_KEYS_H_ diff --git a/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc b/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc index b82850b221..fedba275fe 100644 --- a/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc +++ b/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc @@ -8,7 +8,9 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include #include +#include #include "api/test/create_network_emulation_manager.h" #include "api/test/create_peer_connection_quality_test_frame_generator.h" @@ -19,6 +21,7 @@ #include "api/units/time_delta.h" #include "test/gmock.h" #include "test/gtest.h" +#include "test/pc/e2e/metric_metadata_keys.h" #include "test/pc/e2e/peer_connection_quality_test.h" #include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" @@ -71,18 +74,24 @@ struct MetricValidationInfo { std::string name; Unit unit; ImprovementDirection improvement_direction; + std::map metadata; }; bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) { return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit && - a.improvement_direction == b.improvement_direction; + a.improvement_direction == b.improvement_direction && + a.metadata == b.metadata; } std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) { os << "{ test_case=" << m.test_case << "; name=" << m.name << "; unit=" << test::ToString(m.unit) << "; improvement_direction=" << test::ToString(m.improvement_direction) - << " }"; + << "; metadata={ "; + for (const auto& [key, value] : m.metadata) { + os << "{ key=" << key << "; value=" << value << " }"; + } + os << " }}"; return os; } @@ -94,13 +103,14 @@ std::vector ToValidationInfo( MetricValidationInfo{.test_case = m.test_case, .name = m.name, .unit = m.unit, - .improvement_direction = m.improvement_direction}); + .improvement_direction = m.improvement_direction, + .metadata = m.metric_metadata}); } return out; } TEST(PeerConnectionE2EQualityTestMetricNamesTest, - ExportedMetricsHasCorrectName) { + ExportedMetricsHasCorrectNamesAndAnnotation) { std::unique_ptr network_emulation = CreateNetworkEmulationManager(TimeMode::kSimulated); DefaultMetricsLogger metrics_logger( @@ -152,447 +162,634 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest, .test_case = "test_case", .name = "alice_connected", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case", .name = "bob_connected", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {}}, // Metrics from DefaultAudioQualityAnalyzer MetricValidationInfo{ .test_case = "test_case/alice_audio", .name = "expand_rate", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice_audio", .name = "accelerate_rate", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice_audio", .name = "preemptive_rate", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice_audio", .name = "speech_expand_rate", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice_audio", .name = "average_jitter_buffer_delay_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice_audio", .name = "preferred_buffer_size_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob_audio", .name = "expand_rate", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob_audio", .name = "accelerate_rate", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob_audio", .name = "preemptive_rate", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob_audio", .name = "speech_expand_rate", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob_audio", .name = "average_jitter_buffer_delay_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob_audio", .name = "preferred_buffer_size_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, // Metrics from DefaultVideoQualityAnalyzer MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "psnr_dB", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "ssim", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "transport_time", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "total_delay_incl_transport", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "time_between_rendered_frames", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "harmonic_framerate", .unit = Unit::kHertz, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "encode_frame_rate", .unit = Unit::kHertz, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "encode_time", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "time_between_freezes", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "freeze_time_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "pixels_per_frame", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "min_psnr_dB", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "decode_time", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "receive_to_render_time", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "dropped_frames", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "frames_in_flight", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "rendered_frames", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "max_skipped", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "target_encode_bitrate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/alice_video", .name = "actual_encode_bitrate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "psnr_dB", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "ssim", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "transport_time", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "total_delay_incl_transport", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "time_between_rendered_frames", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "harmonic_framerate", .unit = Unit::kHertz, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "encode_frame_rate", .unit = Unit::kHertz, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "encode_time", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "time_between_freezes", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "freeze_time_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "pixels_per_frame", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "min_psnr_dB", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "decode_time", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "receive_to_render_time", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "dropped_frames", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "frames_in_flight", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "rendered_frames", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "max_skipped", .unit = Unit::kCount, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "target_encode_bitrate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob_video", .name = "actual_encode_bitrate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case", .name = "cpu_usage_%", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, // Metrics from StatsBasedNetworkQualityMetricsReporter MetricValidationInfo{ .test_case = "test_case/alice", .name = "bytes_discarded_no_receiver", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "packets_discarded_no_receiver", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "payload_bytes_received", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "payload_bytes_sent", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "bytes_sent", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "packets_sent", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "average_send_rate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "bytes_received", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "packets_received", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "average_receive_rate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "sent_packets_loss", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "bytes_discarded_no_receiver", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "packets_discarded_no_receiver", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "payload_bytes_received", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "payload_bytes_sent", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "bytes_sent", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "packets_sent", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "average_send_rate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "bytes_received", .unit = Unit::kBytes, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "packets_received", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "average_receive_rate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "sent_packets_loss", .unit = Unit::kUnitless, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {}}, // Metrics from VideoQualityMetricsReporter MetricValidationInfo{ .test_case = "test_case/alice", .name = "available_send_bandwidth", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "transmission_bitrate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/alice", .name = "retransmission_bitrate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "available_send_bandwidth", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "transmission_bitrate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}}, MetricValidationInfo{ .test_case = "test_case/bob", .name = "retransmission_bitrate", .unit = Unit::kKilobitsPerSecond, - .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}}, // Metrics from CrossMediaMetricsReporter MetricValidationInfo{ .test_case = "test_case/alice_alice_audio", .name = "audio_ahead_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/alice_alice_video", .name = "video_ahead_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, MetricValidationInfo{ .test_case = "test_case/bob_bob_audio", .name = "audio_ahead_ms", .unit = Unit::kMilliseconds, - .improvement_direction = ImprovementDirection::kSmallerIsBetter}, - MetricValidationInfo{.test_case = "test_case/bob_bob_video", - .name = "video_ahead_ms", - .unit = Unit::kMilliseconds, - .improvement_direction = - ImprovementDirection::kSmallerIsBetter})); + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}}, + MetricValidationInfo{ + .test_case = "test_case/bob_bob_video", + .name = "video_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {}})); } } // namespace