diff --git a/api/test/metrics/metrics_logger_and_exporter.cc b/api/test/metrics/metrics_logger_and_exporter.cc index d76003b4a8..9f91eac334 100644 --- a/api/test/metrics/metrics_logger_and_exporter.cc +++ b/api/test/metrics/metrics_logger_and_exporter.cc @@ -24,6 +24,19 @@ namespace webrtc { namespace test { +namespace { + +Metric::Stats ToStats(const SamplesStatsCounter& values) { + if (values.IsEmpty()) { + return Metric::Stats(); + } + return Metric::Stats{.mean = values.GetAverage(), + .stddev = values.GetStandardDeviation(), + .min = values.GetMin(), + .max = values.GetMax()}; +} + +} // namespace MetricsLoggerAndExporter::~MetricsLoggerAndExporter() { bool export_result = Export(); @@ -72,17 +85,13 @@ void MetricsLoggerAndExporter::LogMetric( .sample_metadata = sample.metadata}); } - metrics_.push_back( - Metric{.name = std::string(name), - .unit = unit, - .improvement_direction = improvement_direction, - .test_case = std::string(test_case_name), - .metric_metadata = std::move(metadata), - .time_series = std::move(time_series), - .stats = Metric::Stats{.mean = values.GetAverage(), - .stddev = values.GetStandardDeviation(), - .min = values.GetMin(), - .max = values.GetMax()}}); + metrics_.push_back(Metric{.name = std::string(name), + .unit = unit, + .improvement_direction = improvement_direction, + .test_case = std::string(test_case_name), + .metric_metadata = std::move(metadata), + .time_series = std::move(time_series), + .stats = ToStats(values)}); } void MetricsLoggerAndExporter::LogMetric( diff --git a/api/test/metrics/metrics_logger_and_exporter_test.cc b/api/test/metrics/metrics_logger_and_exporter_test.cc index a19a16203a..65b1d8f68d 100644 --- a/api/test/metrics/metrics_logger_and_exporter_test.cc +++ b/api/test/metrics/metrics_logger_and_exporter_test.cc @@ -145,6 +145,30 @@ TEST(MetricsLoggerAndExporterTest, ASSERT_THAT(metric.stats.max, absl::optional(20.0)); } +TEST(MetricsLoggerAndExporterTest, + LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric) { + TestMetricsExporterFactory exporter_factory; + { + std::vector> exporters; + exporters.push_back(exporter_factory.CreateExporter()); + MetricsLoggerAndExporter logger(Clock::GetRealTimeClock(), + std::move(exporters)); + SamplesStatsCounter values; + logger.LogMetric("metric_name", "test_case_name", values, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, DefaultMetadata()); + } + + std::vector metrics = exporter_factory.exported_metrics; + ASSERT_THAT(metrics.size(), Eq(1lu)); + EXPECT_THAT(metrics[0].name, Eq("metric_name")); + EXPECT_THAT(metrics[0].test_case, Eq("test_case_name")); + EXPECT_THAT(metrics[0].time_series.samples, IsEmpty()); + ASSERT_THAT(metrics[0].stats.mean, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.stddev, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.min, Eq(absl::nullopt)); + ASSERT_THAT(metrics[0].stats.max, Eq(absl::nullopt)); +} + TEST(MetricsLoggerAndExporterTest, LogMetricWithStatsRecordsMetric) { TestMetricsExporterFactory exporter_factory; { diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 49897a3f76..a44041c186 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -35,6 +35,7 @@ if (!build_with_chromium) { deps = [ ":default_video_quality_analyzer_frames_comparator_test", + ":default_video_quality_analyzer_metric_names_test", ":default_video_quality_analyzer_stream_state_test", ":default_video_quality_analyzer_test", ":multi_reader_queue_test", @@ -564,6 +565,29 @@ if (!build_with_chromium) { ] } + rtc_library("default_video_quality_analyzer_metric_names_test") { + testonly = true + sources = [ + "analyzer/video/default_video_quality_analyzer_metric_names_test.cc", + ] + deps = [ + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_shared", + "../..:test_support", + "../../../api:create_frame_generator", + "../../../api:rtp_packet_info", + "../../../api/test/metrics:metric", + "../../../api/test/metrics:metrics_logger_and_exporter", + "../../../api/test/metrics:stdout_metrics_exporter", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../common_video", + "../../../rtc_base:stringutils", + "../../../rtc_tools:video_quality_analysis", + "../../../system_wrappers", + ] + } + rtc_library("default_video_quality_analyzer_frames_comparator_test") { testonly = true sources = [ "analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc" ] @@ -719,6 +743,8 @@ if (!build_with_chromium) { "../../../api:array_view", "../../../api:video_quality_analyzer_api", "../../../api/numerics", + "../../../api/test/metrics:metric", + "../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/units:data_size", "../../../api/units:time_delta", "../../../api/units:timestamp", diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index b758c92d19..46d327081e 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -18,6 +18,7 @@ #include "api/array_view.h" #include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" #include "api/units/time_delta.h" #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" @@ -36,6 +37,9 @@ namespace webrtc { namespace { +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + constexpr int kBitsInByte = 8; constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed"; constexpr absl::string_view kSkipRenderedFrameReasonRendered = "rendered"; @@ -123,9 +127,19 @@ SamplesStatsCounter::StatsSample StatsSample(double value, DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( webrtc::Clock* clock, DefaultVideoQualityAnalyzerOptions options) + : DefaultVideoQualityAnalyzer(clock, + /*metrics_logger=*/nullptr, + std::move(options)) {} + +DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( + webrtc::Clock* clock, + test::MetricsLoggerAndExporter* metrics_logger, + DefaultVideoQualityAnalyzerOptions options) : options_(options), clock_(clock), + metrics_logger_(metrics_logger), frames_comparator_(clock, cpu_measurer_, options) {} + DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { Stop(); } @@ -951,87 +965,186 @@ void DefaultVideoQualityAnalyzer::ReportResults( video_duration.seconds() / sum_squared_interframe_delays_secs; } - ReportResult("psnr", test_case_name, stats.psnr, "dB", - ImproveDirection::kBiggerIsBetter); - ReportResult("ssim", test_case_name, stats.ssim, "unitless", - ImproveDirection::kBiggerIsBetter); - ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("total_delay_incl_transport", test_case_name, - stats.total_delay_incl_transport_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("time_between_rendered_frames", test_case_name, - stats.time_between_rendered_frames_ms, "ms", - ImproveDirection::kSmallerIsBetter); - test::PrintResult("harmonic_framerate", "", test_case_name, - harmonic_framerate_fps, "Hz", /*important=*/false, - ImproveDirection::kBiggerIsBetter); - test::PrintResult("encode_frame_rate", "", test_case_name, - stats.encode_frame_rate.IsEmpty() - ? 0 - : stats.encode_frame_rate.GetEventsPerSecond(), - "Hz", /*important=*/false, - ImproveDirection::kBiggerIsBetter); - ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("time_between_freezes", test_case_name, - stats.time_between_freezes_ms, "ms", - ImproveDirection::kBiggerIsBetter); - ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("pixels_per_frame", test_case_name, - stats.resolution_of_rendered_frame, "count", - ImproveDirection::kBiggerIsBetter); - test::PrintResult("min_psnr", "", test_case_name, - stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - ReportResult("receive_to_render_time", test_case_name, - stats.receive_to_render_time_ms, "ms", - ImproveDirection::kSmallerIsBetter); - test::PrintResult("dropped_frames", "", test_case_name, - frame_counters.dropped, "count", - /*important=*/false, ImproveDirection::kSmallerIsBetter); - test::PrintResult("frames_in_flight", "", test_case_name, - frame_counters.captured - frame_counters.rendered - - frame_counters.dropped, - "count", - /*important=*/false, ImproveDirection::kSmallerIsBetter); - test::PrintResult("rendered_frames", "", test_case_name, - frame_counters.rendered, "count", /*important=*/false, - ImproveDirection::kBiggerIsBetter); - ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered, - "count", ImproveDirection::kSmallerIsBetter); - ReportResult("target_encode_bitrate", test_case_name, - stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond", - ImproveDirection::kNone); - test::PrintResult("actual_encode_bitrate", "", test_case_name, - static_cast(stats.total_encoded_images_payload) / - test_duration.seconds(), - "bytesPerSecond", /*important=*/false, - ImproveDirection::kNone); - - if (options_.report_detailed_frame_stats) { - test::PrintResult("num_encoded_frames", "", test_case_name, - frame_counters.encoded, "count", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - test::PrintResult("num_decoded_frames", "", test_case_name, - frame_counters.decoded, "count", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - test::PrintResult("num_send_key_frames", "", test_case_name, - stats.num_send_key_frames, "count", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - test::PrintResult("num_recv_key_frames", "", test_case_name, - stats.num_recv_key_frames, "count", - /*important=*/false, ImproveDirection::kBiggerIsBetter); - - ReportResult("recv_key_frame_size_bytes", test_case_name, - stats.recv_key_frame_size_bytes, "count", + if (metrics_logger_ == nullptr) { + // Report results through old performance metrics API. + ReportResult("psnr", test_case_name, stats.psnr, "dB", ImproveDirection::kBiggerIsBetter); - ReportResult("recv_delta_frame_size_bytes", test_case_name, - stats.recv_delta_frame_size_bytes, "count", + ReportResult("ssim", test_case_name, stats.ssim, "unitless", ImproveDirection::kBiggerIsBetter); + ReportResult("transport_time", test_case_name, stats.transport_time_ms, + "ms", ImproveDirection::kSmallerIsBetter); + ReportResult("total_delay_incl_transport", test_case_name, + stats.total_delay_incl_transport_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("time_between_rendered_frames", test_case_name, + stats.time_between_rendered_frames_ms, "ms", + ImproveDirection::kSmallerIsBetter); + test::PrintResult("harmonic_framerate", "", test_case_name, + harmonic_framerate_fps, "Hz", /*important=*/false, + ImproveDirection::kBiggerIsBetter); + test::PrintResult("encode_frame_rate", "", test_case_name, + stats.encode_frame_rate.IsEmpty() + ? 0 + : stats.encode_frame_rate.GetEventsPerSecond(), + "Hz", /*important=*/false, + ImproveDirection::kBiggerIsBetter); + ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("time_between_freezes", test_case_name, + stats.time_between_freezes_ms, "ms", + ImproveDirection::kBiggerIsBetter); + ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("pixels_per_frame", test_case_name, + stats.resolution_of_rendered_frame, "count", + ImproveDirection::kBiggerIsBetter); + test::PrintResult("min_psnr", "", test_case_name, + stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("receive_to_render_time", test_case_name, + stats.receive_to_render_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + test::PrintResult("dropped_frames", "", test_case_name, + frame_counters.dropped, "count", + /*important=*/false, ImproveDirection::kSmallerIsBetter); + test::PrintResult("frames_in_flight", "", test_case_name, + frame_counters.captured - frame_counters.rendered - + frame_counters.dropped, + "count", + /*important=*/false, ImproveDirection::kSmallerIsBetter); + test::PrintResult("rendered_frames", "", test_case_name, + frame_counters.rendered, "count", /*important=*/false, + ImproveDirection::kBiggerIsBetter); + ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered, + "count", ImproveDirection::kSmallerIsBetter); + ReportResult("target_encode_bitrate", test_case_name, + stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond", + ImproveDirection::kNone); + test::PrintResult("actual_encode_bitrate", "", test_case_name, + static_cast(stats.total_encoded_images_payload) / + test_duration.seconds(), + "bytesPerSecond", /*important=*/false, + ImproveDirection::kNone); + + if (options_.report_detailed_frame_stats) { + test::PrintResult("num_encoded_frames", "", test_case_name, + frame_counters.encoded, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_decoded_frames", "", test_case_name, + frame_counters.decoded, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_send_key_frames", "", test_case_name, + stats.num_send_key_frames, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_recv_key_frames", "", test_case_name, + stats.num_recv_key_frames, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + + ReportResult("recv_key_frame_size_bytes", test_case_name, + stats.recv_key_frame_size_bytes, "count", + ImproveDirection::kBiggerIsBetter); + ReportResult("recv_delta_frame_size_bytes", test_case_name, + stats.recv_delta_frame_size_bytes, "count", + ImproveDirection::kBiggerIsBetter); + } + } else { + metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr, + Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim, + Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("transport_time", test_case_name, + stats.transport_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name, + stats.total_delay_incl_transport_ms, + Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name, + stats.time_between_rendered_frames_ms, + Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric( + "harmonic_framerate", test_case_name, harmonic_framerate_fps, + Unit::kHertz, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "encode_frame_rate", test_case_name, + stats.encode_frame_rate.IsEmpty() + ? 0 + : stats.encode_frame_rate.GetEventsPerSecond(), + Unit::kHertz, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("encode_time", test_case_name, + stats.encode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "time_between_freezes", test_case_name, stats.time_between_freezes_ms, + Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("freeze_time_ms", test_case_name, + stats.freeze_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("pixels_per_frame", test_case_name, + stats.resolution_of_rendered_frame, Unit::kCount, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "min_psnr_dB", test_case_name, + stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("decode_time", test_case_name, + stats.decode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("receive_to_render_time", test_case_name, + stats.receive_to_render_time_ms, + Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric( + "dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric( + "frames_in_flight", test_case_name, + frame_counters.captured - frame_counters.rendered - + frame_counters.dropped, + Unit::kCount, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric( + "rendered_frames", test_case_name, frame_counters.rendered, + Unit::kCount, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("max_skipped", test_case_name, + stats.skipped_between_rendered, Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("target_encode_bitrate", test_case_name, + stats.target_encode_bitrate / 1000, + Unit::kKilobitsPerSecond, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "actual_encode_bitrate", test_case_name, + static_cast(stats.total_encoded_images_payload) / + test_duration.seconds() * kBitsInByte / 1000, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + + if (options_.report_detailed_frame_stats) { + metrics_logger_->LogSingleValueMetric( + "num_encoded_frames", test_case_name, frame_counters.encoded, + Unit::kCount, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "num_decoded_frames", test_case_name, frame_counters.decoded, + Unit::kCount, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "num_send_key_frames", test_case_name, stats.num_send_key_frames, + Unit::kCount, ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + "num_recv_key_frames", test_case_name, stats.num_recv_key_frames, + Unit::kCount, ImprovementDirection::kBiggerIsBetter); + + metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name, + stats.recv_key_frame_size_bytes, Unit::kCount, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name, + stats.recv_delta_frame_size_bytes, + Unit::kCount, + ImprovementDirection::kBiggerIsBetter); + } } } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 7b56ce985a..c1b7fe870f 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -22,6 +22,7 @@ #include "api/array_view.h" #include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metrics_logger_and_exporter.h" #include "api/test/video_quality_analyzer_interface.h" #include "api/units/data_size.h" #include "api/units/timestamp.h" @@ -49,6 +50,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { explicit DefaultVideoQualityAnalyzer( webrtc::Clock* clock, DefaultVideoQualityAnalyzerOptions options = {}); + DefaultVideoQualityAnalyzer(webrtc::Clock* clock, + test::MetricsLoggerAndExporter* metrics_logger, + DefaultVideoQualityAnalyzerOptions options = {}); ~DefaultVideoQualityAnalyzer() override; void Start(std::string test_case_name, @@ -148,6 +152,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { const DefaultVideoQualityAnalyzerOptions options_; webrtc::Clock* const clock_; + test::MetricsLoggerAndExporter* const metrics_logger_; std::string test_label_; diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc new file mode 100644 index 0000000000..f2ca603d8e --- /dev/null +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc @@ -0,0 +1,649 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "api/test/create_frame_generator.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_logger_and_exporter.h" +#include "api/test/metrics/stdout_metrics_exporter.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "system_wrappers/include/sleep.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +namespace webrtc { +namespace { + +using ::testing::Contains; +using ::testing::SizeIs; +using ::testing::UnorderedElementsAre; + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Metric; +using ::webrtc::test::MetricsExporter; +using ::webrtc::test::MetricsLoggerAndExporter; +using ::webrtc::test::StdoutMetricsExporter; +using ::webrtc::test::Unit; + +constexpr int kAnalyzerMaxThreadsCount = 1; +constexpr int kMaxFramesInFlightPerStream = 10; +constexpr int kFrameWidth = 320; +constexpr int kFrameHeight = 240; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = true; + options.compute_ssim = true; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + options.report_detailed_frame_stats = true; + return options; +} + +VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator, + int64_t timestamp_us) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator->NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .set_timestamp_us(timestamp_us) + .build(); +} + +EncodedImage FakeEncode(const VideoFrame& frame) { + EncodedImage image; + std::vector packet_infos; + packet_infos.push_back(RtpPacketInfo( + /*ssrc=*/1, + /*csrcs=*/{}, + /*rtp_timestamp=*/frame.timestamp(), + /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000))); + image.SetPacketInfos(RtpPacketInfos(packet_infos)); + return image; +} + +VideoFrame DeepCopy(const VideoFrame& frame) { + VideoFrame copy = frame; + copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + return copy; +} + +void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer, + absl::string_view sender, + absl::string_view stream_label, + std::vector receivers, + int frames_count, + test::FrameGeneratorInterface& frame_generator, + int interframe_delay_ms = 0) { + for (int i = 0; i < frames_count; ++i) { + VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, std::string(stream_label), frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + for (absl::string_view receiver : receivers) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(receiver, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(receiver, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(receiver, received_frame); + } + if (i < frames_count - 1 && interframe_delay_ms > 0) { + SleepMs(interframe_delay_ms); + } + } +} + +// Metric fields to assert on +struct MetricValidationInfo { + std::string test_case; + std::string name; + Unit unit; + ImprovementDirection improvement_direction; +}; + +bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) { + return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit && + a.improvement_direction == b.improvement_direction; +} + +std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) { + os << "{ test_case=" << m.test_case << "; name=" << m.name + << "; unit=" << test::ToString(m.unit) + << "; improvement_direction=" << test::ToString(m.improvement_direction) + << " }"; + return os; +} + +std::vector ToValidationInfo( + const std::vector& metrics) { + std::vector out; + for (const Metric& m : metrics) { + out.push_back( + MetricValidationInfo{.test_case = m.test_case, + .name = m.name, + .unit = m.unit, + .improvement_direction = m.improvement_direction}); + } + return out; +} + +std::vector ToTestCases(const std::vector& metrics) { + std::vector out; + for (const Metric& m : metrics) { + out.push_back(m.test_case); + } + return out; +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, MetricNamesForP2PAreCorrect) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + std::vector> exporters; + exporters.push_back(std::make_unique()); + MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(), + std::move(exporters)); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", std::vector{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.Stop(); + + std::vector metrics = + ToValidationInfo(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + UnorderedElementsAre( + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter})); +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, + MetricNamesFor3PeersAreCorrect) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + std::vector> exporters; + exporters.push_back(std::make_unique()); + MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(), + std::move(exporters)); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", + std::vector{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.Stop(); + + std::vector metrics = + ToValidationInfo(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + UnorderedElementsAre( + // Bob + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + + // Charlie + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter})); +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, + TestCaseFor3PeerIsTheSameAfterAllPeersLeft) { + std::unique_ptr frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + std::vector> exporters; + exporters.push_back(std::make_unique()); + MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(), + std::move(exporters)); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", + std::vector{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.UnregisterParticipantInCall("alice"); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.UnregisterParticipantInCall("charlie"); + analyzer.Stop(); + + std::vector metrics = + ToTestCases(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT(metrics, SizeIs(52)); + EXPECT_THAT(metrics, Contains("test_case/alice_video_alice_bob").Times(26)); + EXPECT_THAT(metrics, + Contains("test_case/alice_video_alice_charlie").Times(26)); +} + +} // namespace +} // namespace webrtc