admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[PCLF] Annotate video metrics with peer, stream and receiver

Browse Source 
Bug: b/240540204
Change-Id: I05eddea33a2eb680b59c8247f2acba1e7c7d6a84
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/278260
Reviewed-by: Andrey Logvin <landrey@google.com>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38316}
This commit is contained in:
Artem Titov 2022-10-07 00:54:34 +02:00 committed by WebRTC LUCI CQ
parent 2fb83072db
commit b15faaa264
8 changed files with 442 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
test/pc/e2e/BUILD.gn
View File

@ -524,6 +524,7 @@ if (!build_with_chromium) {
testonly = true
sources = [ "peer_connection_quality_test_metric_names_test.cc" ]
deps = [
":metric_metadata_keys",
":peerconnection_quality_test",
":stats_based_network_quality_metrics_reporter",
"../..:test_support",
@ -741,6 +742,7 @@ if (!build_with_chromium) {
"analyzer/video/video_quality_metrics_reporter.h",
]
deps = [
":metric_metadata_keys",
"../..:perf_test",
"../../../api:peer_connection_quality_test_fixture_api",
"../../../api:rtc_stats_api",
@ -761,6 +763,12 @@ if (!build_with_chromium) {
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
rtc_library("metric_metadata_keys") {
visibility = [ "*" ]
testonly = true
sources = [ "metric_metadata_keys.h" ]
}
rtc_library("default_video_quality_analyzer") {
visibility = [ "*" ]
@ -773,6 +781,7 @@ if (!build_with_chromium) {
deps = [
":default_video_quality_analyzer_internal",
":default_video_quality_analyzer_shared",
":metric_metadata_keys",
"../..:perf_test",
"../../../api:array_view",
"../../../api:video_quality_analyzer_api",
@ -832,6 +841,7 @@ if (!build_with_chromium) {
deps = [
":default_video_quality_analyzer_shared",
":metric_metadata_keys",
":multi_reader_queue",
"../../../api:array_view",
"../../../api:scoped_refptr",

126
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -11,8 +11,10 @@
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
#include <algorithm>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>
@ -34,12 +36,14 @@
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h"
#include "test/pc/e2e/metric_metadata_keys.h"
namespace webrtc {
namespace {
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit;
using ::webrtc::webrtc_pc_e2e::MetricMetadataKey;
constexpr int kBitsInByte = 8;
constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed";
@ -892,7 +896,7 @@ void DefaultVideoQualityAnalyzer::
void DefaultVideoQualityAnalyzer::ReportResults() {
MutexLock lock(&mutex_);
for (auto& item : frames_comparator_.stream_stats()) {
ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second,
ReportResults(item.first, item.second,
stream_frame_counters_.at(item.first));
}
metrics_logger_->LogSingleValueMetric("cpu_usage_%", test_label_,
@ -931,10 +935,15 @@ void DefaultVideoQualityAnalyzer::ReportResults() {
}
void DefaultVideoQualityAnalyzer::ReportResults(
const std::string& test_case_name,
const InternalStatsKey& key,
const StreamStats& stats,
const FrameCounters& frame_counters) {
TimeDelta test_duration = Now() - start_time_;
std::string test_case_name = GetTestCaseName(ToMetricName(key));
std::map<std::string, std::string> metric_metadata{
{MetricMetadataKey::kPeerMetadataKey, peers_->name(key.sender)},
{MetricMetadataKey::kStreamMetadataKey, streams_.name(key.stream)},
{MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)}};
double sum_squared_interframe_delays_secs = 0;
Timestamp video_start_time = Timestamp::PlusInfinity();
@ -962,98 +971,105 @@ void DefaultVideoQualityAnalyzer::ReportResults(
video_duration.seconds<double>() / sum_squared_interframe_delays_secs;
}
metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric(
"psnr_dB", test_case_name, stats.psnr, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"ssim", test_case_name, stats.ssim, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogMetric("transport_time", test_case_name,
stats.transport_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric("harmonic_framerate", test_case_name,
harmonic_framerate_fps, Unit::kHertz,
ImprovementDirection::kBiggerIsBetter);
ImprovementDirection::kSmallerIsBetter,
metric_metadata);
metrics_logger_->LogMetric(
"total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"harmonic_framerate", test_case_name, harmonic_framerate_fps,
Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"encode_frame_rate", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("encode_time", test_case_name,
stats.encode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"encode_time", test_case_name, stats.encode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogMetric("time_between_freezes", test_case_name,
stats.time_between_freezes_ms, Unit::kMilliseconds,
ImprovementDirection::kBiggerIsBetter);
ImprovementDirection::kBiggerIsBetter,
metric_metadata);
metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
stats.freeze_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
ImprovementDirection::kSmallerIsBetter,
metric_metadata);
metrics_logger_->LogMetric(
"pixels_per_frame", test_case_name, stats.resolution_of_rendered_frame,
Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"min_psnr_dB", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("decode_time", test_case_name,
stats.decode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"decode_time", test_case_name, stats.decode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"receive_to_render_time", test_case_name, stats.receive_to_render_time_ms,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric("dropped_frames", test_case_name,
frame_counters.dropped, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter,
metric_metadata);
metrics_logger_->LogSingleValueMetric(
"dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount,
ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"frames_in_flight", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
Unit::kCount, ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric("rendered_frames", test_case_name,
frame_counters.rendered, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("max_skipped", test_case_name,
stats.skipped_between_rendered, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / 1000,
Unit::kKilobitsPerSecond,
ImprovementDirection::kNeitherIsBetter);
Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"rendered_frames", test_case_name, frame_counters.rendered, Unit::kCount,
ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"max_skipped", test_case_name, stats.skipped_between_rendered,
Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / 1000, Unit::kKilobitsPerSecond,
ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"actual_encode_bitrate", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>() * kBitsInByte / 1000,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
metric_metadata);
if (options_.report_detailed_frame_stats) {
metrics_logger_->LogSingleValueMetric(
"num_encoded_frames", test_case_name, frame_counters.encoded,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"num_decoded_frames", test_case_name, frame_counters.decoded,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"num_send_key_frames", test_case_name, stats.num_send_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"num_recv_key_frames", test_case_name, stats.num_recv_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
ImprovementDirection::kBiggerIsBetter,
metric_metadata);
metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
ImprovementDirection::kBiggerIsBetter,
metric_metadata);
}
}

2
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -124,7 +124,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
// Report results for all metrics for all streams.
void ReportResults();
void ReportResults(const std::string& test_case_name,
void ReportResults(const InternalStatsKey& key,
const StreamStats& stats,
const FrameCounters& frame_counters)
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);

16
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
View File

@ -28,13 +28,15 @@
#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
#include "test/pc/e2e/metric_metadata_keys.h"
namespace webrtc {
namespace {
using ::webrtc::webrtc_pc_e2e::SampleMetadataKey;
constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150);
constexpr int kMaxActiveComparisons = 10;
constexpr char kFrameIdMetadataKey[] = "frame_id";
SamplesStatsCounter::StatsSample StatsSample(
double value,
@ -307,10 +309,11 @@ void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
MutexLock lock(&mutex_);
RTC_CHECK_EQ(state_, State::kActive)
<< "Frames comparator has to be started before it will be used";
stream_stats_.at(stats_key).skipped_between_rendered.AddSample(StatsSample(
skipped_between_rendered, Now(),
/*metadata=*/
{{kFrameIdMetadataKey, std::to_string(frame_stats.frame_id)}}));
stream_stats_.at(stats_key).skipped_between_rendered.AddSample(
StatsSample(skipped_between_rendered, Now(),
/*metadata=*/
{{SampleMetadataKey::kFrameIdMetadataKey,
std::to_string(frame_stats.frame_id)}}));
AddComparisonInternal(std::move(stats_key), std::move(captured),
std::move(rendered), type, std::move(frame_stats));
}
@ -420,7 +423,8 @@ void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
}
std::map<std::string, std::string> metadata;
metadata.emplace(kFrameIdMetadataKey, std::to_string(frame_stats.frame_id));
metadata.emplace(SampleMetadataKey::kFrameIdMetadataKey,
std::to_string(frame_stats.frame_id));
if (psnr > 0) {
stats->psnr.AddSample(

26
test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
View File

@ -10,6 +10,9 @@
#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h"
#include <map>
#include <string>
#include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h"
@ -17,6 +20,7 @@
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "rtc_base/checks.h"
#include "test/pc/e2e/metric_metadata_keys.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -24,6 +28,7 @@ namespace {
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit;
using ::webrtc::webrtc_pc_e2e::MetricMetadataKey;
SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) {
return counter * 0.008;
@ -118,30 +123,37 @@ void VideoQualityMetricsReporter::OnStatsReports(
void VideoQualityMetricsReporter::StopAndReportResults() {
MutexLock video_bwemutex_(&video_bwe_stats_lock_);
for (const auto& item : video_bwe_stats_) {
ReportVideoBweResults(GetTestCaseName(item.first), item.second);
ReportVideoBweResults(item.first, item.second);
}
}
std::string VideoQualityMetricsReporter::GetTestCaseName(
const std::string& stream_label) const {
return test_case_name_ + "/" + stream_label;
const std::string& peer_name) const {
return test_case_name_ + "/" + peer_name;
}
void VideoQualityMetricsReporter::ReportVideoBweResults(
const std::string& test_case_name,
const std::string& peer_name,
const VideoBweStats& video_bwe_stats) {
std::string test_case_name = GetTestCaseName(peer_name);
std::map<std::string, std::string> metric_metadata{
{MetricMetadataKey::kPeerMetadataKey, peer_name}};
metrics_logger_->LogMetric(
"available_send_bandwidth", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
metric_metadata);
metrics_logger_->LogMetric(
"transmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
metric_metadata);
metrics_logger_->LogMetric(
"retransmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
metric_metadata);
}
} // namespace webrtc_pc_e2e

4
test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
View File

@ -55,8 +55,8 @@ class VideoQualityMetricsReporter
Timestamp sample_time = Timestamp::Zero();
};
std::string GetTestCaseName(const std::string& stream_label) const;
void ReportVideoBweResults(const std::string& test_case_name,
std::string GetTestCaseName(const std::string& peer_name) const;
void ReportVideoBweResults(const std::string& peer_name,
const VideoBweStats& video_bwe_stats);
Timestamp Now() const { return clock_->CurrentTime(); }

37
test/pc/e2e/metric_metadata_keys.h Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_METRIC_METADATA_KEYS_H_
#define TEST_PC_E2E_METRIC_METADATA_KEYS_H_
namespace webrtc {
namespace webrtc_pc_e2e {
class MetricMetadataKey {
public:
static constexpr char kPeerMetadataKey[] = "peer";
static constexpr char kStreamMetadataKey[] = "stream";
static constexpr char kReceiverMetadataKey[] = "receiver";
private:
MetricMetadataKey() = default;
};
class SampleMetadataKey {
public:
static constexpr char kFrameIdMetadataKey[] = "frame_id";
private:
SampleMetadataKey() = default;
};
} // namespace webrtc_pc_e2e
} // namespace webrtc
#endif // TEST_PC_E2E_METRIC_METADATA_KEYS_H_

387
test/pc/e2e/peer_connection_quality_test_metric_names_test.cc
View File

@ -8,7 +8,9 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include <map>
#include <memory>
#include <string>
#include "api/test/create_network_emulation_manager.h"
#include "api/test/create_peer_connection_quality_test_frame_generator.h"
@ -19,6 +21,7 @@
#include "api/units/time_delta.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/pc/e2e/metric_metadata_keys.h"
#include "test/pc/e2e/peer_connection_quality_test.h"
#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
@ -71,18 +74,24 @@ struct MetricValidationInfo {
std::string name;
Unit unit;
ImprovementDirection improvement_direction;
std::map<std::string, std::string> metadata;
};
bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
a.improvement_direction == b.improvement_direction;
a.improvement_direction == b.improvement_direction &&
a.metadata == b.metadata;
}
std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
os << "{ test_case=" << m.test_case << "; name=" << m.name
<< "; unit=" << test::ToString(m.unit)
<< "; improvement_direction=" << test::ToString(m.improvement_direction)
<< " }";
<< "; metadata={ ";
for (const auto& [key, value] : m.metadata) {
os << "{ key=" << key << "; value=" << value << " }";
}
os << " }}";
return os;
}
@ -94,13 +103,14 @@ std::vector<MetricValidationInfo> ToValidationInfo(
MetricValidationInfo{.test_case = m.test_case,
.name = m.name,
.unit = m.unit,
.improvement_direction = m.improvement_direction});
.improvement_direction = m.improvement_direction,
.metadata = m.metric_metadata});
}
return out;
}
TEST(PeerConnectionE2EQualityTestMetricNamesTest,
ExportedMetricsHasCorrectName) {
ExportedMetricsHasCorrectNamesAndAnnotation) {
std::unique_ptr<NetworkEmulationManager> network_emulation =
CreateNetworkEmulationManager(TimeMode::kSimulated);
DefaultMetricsLogger metrics_logger(
@ -152,447 +162,634 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.test_case = "test_case",
.name = "alice_connected",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case",
.name = "bob_connected",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {}},
// Metrics from DefaultAudioQualityAnalyzer
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "accelerate_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "preemptive_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "speech_expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "average_jitter_buffer_delay_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "preferred_buffer_size_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "accelerate_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "preemptive_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "speech_expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "average_jitter_buffer_delay_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "preferred_buffer_size_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
// Metrics from DefaultVideoQualityAnalyzer
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case",
.name = "cpu_usage_%",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
// Metrics from StatsBasedNetworkQualityMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_discarded_no_receiver",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_discarded_no_receiver",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "payload_bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "payload_bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_sent",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "average_send_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_received",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "average_receive_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "sent_packets_loss",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_discarded_no_receiver",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_discarded_no_receiver",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "payload_bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "payload_bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_sent",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "average_send_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_received",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "average_receive_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "sent_packets_loss",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
// Metrics from VideoQualityMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "available_send_bandwidth",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "transmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "retransmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "available_send_bandwidth",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "transmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "retransmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
// Metrics from CrossMediaMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice_alice_audio",
.name = "audio_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/alice_alice_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob_bob_audio",
.name = "audio_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{.test_case = "test_case/bob_bob_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction =
ImprovementDirection::kSmallerIsBetter}));
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
MetricValidationInfo{
.test_case = "test_case/bob_bob_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}}));
}
} // namespace