diff --git a/api/BUILD.gn b/api/BUILD.gn index 10337cb55d..9445e3cee3 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -622,6 +622,7 @@ if (rtc_include_tests) { ":time_controller", ":video_quality_analyzer_api", "../test/pc/e2e:peerconnection_quality_test", + "test/metrics:global_metrics_logger_and_exporter", ] } } diff --git a/api/test/create_peerconnection_quality_test_fixture.cc b/api/test/create_peerconnection_quality_test_fixture.cc index 2d9d0821fc..ec38de084d 100644 --- a/api/test/create_peerconnection_quality_test_fixture.cc +++ b/api/test/create_peerconnection_quality_test_fixture.cc @@ -13,6 +13,7 @@ #include #include +#include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/test/time_controller.h" #include "test/pc/e2e/peer_connection_quality_test.h" @@ -27,7 +28,8 @@ CreatePeerConnectionE2EQualityTestFixture( std::unique_ptr video_quality_analyzer) { return std::make_unique( std::move(test_case_name), time_controller, - std::move(audio_quality_analyzer), std::move(video_quality_analyzer)); + std::move(audio_quality_analyzer), std::move(video_quality_analyzer), + test::GetGlobalMetricsLoggerAndExporter()); } } // namespace webrtc_pc_e2e diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn index eb63ce0b12..a7283cd5f6 100644 --- a/api/test/metrics/BUILD.gn +++ b/api/test/metrics/BUILD.gn @@ -13,6 +13,7 @@ if (rtc_enable_protobuf) { group("metrics") { deps = [ + ":global_metrics_logger_and_exporter", ":metric", ":metrics_exporter", ":metrics_logger_and_exporter", @@ -73,28 +74,6 @@ rtc_library("stdout_metrics_exporter") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } -rtc_library("metrics_logger_and_exporter") { - visibility = [ "*" ] - sources = [ - "metrics_logger_and_exporter.cc", - "metrics_logger_and_exporter.h", - ] - deps = [ - ":metric", - ":metrics_exporter", - "../../../rtc_base:checks", - "../../../rtc_base:logging", - "../../../rtc_base/synchronization:mutex", - "../../../system_wrappers", - "../../numerics", - ] - - absl_deps = [ - "//third_party/abseil-cpp/absl/strings", - "//third_party/abseil-cpp/absl/types:optional", - ] -} - rtc_library("chrome_perf_dashboard_metrics_exporter") { visibility = [ "*" ] testonly = true @@ -159,6 +138,42 @@ rtc_library("print_result_proxy_metrics_exporter") { ] } +rtc_library("metrics_logger_and_exporter") { + visibility = [ "*" ] + sources = [ + "metrics_logger_and_exporter.cc", + "metrics_logger_and_exporter.h", + ] + deps = [ + ":metric", + ":metrics_exporter", + "../../../rtc_base:checks", + "../../../rtc_base:logging", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers", + "../../numerics", + ] + + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("global_metrics_logger_and_exporter") { + visibility = [ "*" ] + sources = [ + "global_metrics_logger_and_exporter.cc", + "global_metrics_logger_and_exporter.h", + ] + deps = [ + ":metrics_exporter", + ":metrics_logger_and_exporter", + "../../../rtc_base:checks", + "../../../system_wrappers", + ] +} + if (rtc_include_tests) { rtc_library("stdout_metrics_exporter_test") { testonly = true diff --git a/api/test/metrics/global_metrics_logger_and_exporter.cc b/api/test/metrics/global_metrics_logger_and_exporter.cc new file mode 100644 index 0000000000..d1de0c501e --- /dev/null +++ b/api/test/metrics/global_metrics_logger_and_exporter.cc @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/global_metrics_logger_and_exporter.h" + +#include +#include +#include + +#include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/metrics_logger_and_exporter.h" +#include "rtc_base/checks.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace test { +namespace { + +MetricsLoggerAndExporter* global_metrics_logger_and_exporter = nullptr; + +} // namespace + +MetricsLoggerAndExporter* GetGlobalMetricsLoggerAndExporter() { + return global_metrics_logger_and_exporter; +} + +void SetupGlobalMetricsLoggerAndExporter( + std::vector> exporters) { + RTC_CHECK(global_metrics_logger_and_exporter == nullptr); + global_metrics_logger_and_exporter = new MetricsLoggerAndExporter( + Clock::GetRealTimeClock(), std::move(exporters)); +} + +void ExportAndDestroyGlobalMetricsLoggerAndExporter() { + RTC_CHECK(global_metrics_logger_and_exporter != nullptr); + delete global_metrics_logger_and_exporter; +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/metrics/global_metrics_logger_and_exporter.h b/api/test/metrics/global_metrics_logger_and_exporter.h new file mode 100644 index 0000000000..1029545d47 --- /dev/null +++ b/api/test/metrics/global_metrics_logger_and_exporter.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_ +#define API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_ + +#include +#include + +#include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/metrics_logger_and_exporter.h" + +namespace webrtc { +namespace test { + +// Returns current global `MetricsLoggerAndExporter`. Returns `nullptr` if there +// is global instance wasn't initialized. +MetricsLoggerAndExporter* GetGlobalMetricsLoggerAndExporter(); + +// Initialize global `MetricsLoggerAndExporter` with provided vector of +// exporters. Crashes if there is already initialized global instance. +void SetupGlobalMetricsLoggerAndExporter( + std::vector> exporters); + +// Destroys global `MetricsLoggerAndExporter` forcing it to export all collected +// metrics to the configured exporters. Crashes if there is no initialized +// global instance. +void ExportAndDestroyGlobalMetricsLoggerAndExporter(); + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_ diff --git a/test/BUILD.gn b/test/BUILD.gn index 491691daeb..242a8b21fa 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -509,6 +509,9 @@ if (rtc_include_tests && !build_with_chromium) { ":resources_dir_flag", ":test_flags", ":test_support", + "../api/test/metrics:global_metrics_logger_and_exporter", + "../api/test/metrics:metrics_exporter", + "../api/test/metrics:stdout_metrics_exporter", "../rtc_base", "../rtc_base:checks", "../rtc_base:event_tracer", diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index cc2707ab63..c62e8f7f2c 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -41,6 +41,7 @@ if (!build_with_chromium) { ":multi_reader_queue_test", ":names_collection_test", ":peer_connection_e2e_smoke_test", + ":peer_connection_quality_test_metric_names_test", ":single_process_encoded_image_data_injector_unittest", ":stats_poller_test", ":video_frame_tracking_id_injector_unittest", @@ -407,6 +408,8 @@ if (!build_with_chromium) { "../../../api:video_quality_analyzer_api", "../../../api/rtc_event_log", "../../../api/task_queue", + "../../../api/test/metrics:metric", + "../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/units:time_delta", "../../../api/units:timestamp", "../../../pc:pc_test_utils", @@ -484,6 +487,7 @@ if (!build_with_chromium) { "../../../api:simulated_network_api", "../../../api/audio_codecs:builtin_audio_decoder_factory", "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/test/metrics:global_metrics_logger_and_exporter", "../../../api/video_codecs:builtin_video_decoder_factory", "../../../api/video_codecs:builtin_video_encoder_factory", "../../../call:simulated_network", @@ -510,6 +514,23 @@ if (!build_with_chromium) { } } + rtc_library("peer_connection_quality_test_metric_names_test") { + testonly = true + sources = [ "peer_connection_quality_test_metric_names_test.cc" ] + deps = [ + ":peerconnection_quality_test", + ":stats_based_network_quality_metrics_reporter", + "../..:test_support", + "../../../api:create_network_emulation_manager", + "../../../api:create_peer_connection_quality_test_frame_generator", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/test/metrics:metrics_logger_and_exporter", + "../../../api/test/metrics:stdout_metrics_exporter", + "../../../api/units:time_delta", + ] + } + rtc_library("stats_provider") { visibility = [ "*" ] testonly = true diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index 4ee6590419..8231ade27d 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -15,6 +15,7 @@ #include "api/test/create_network_emulation_manager.h" #include "api/test/create_peer_connection_quality_test_frame_generator.h" #include "api/test/create_peerconnection_quality_test_fixture.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/test/network_emulation_manager.h" #include "api/test/peerconnection_quality_test_fixture.h" #include "call/simulated_network.h" @@ -54,7 +55,8 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { void SetUp() override { network_emulation_ = CreateNetworkEmulationManager(); auto video_quality_analyzer = std::make_unique( - network_emulation_->time_controller()->GetClock()); + network_emulation_->time_controller()->GetClock(), + test::GetGlobalMetricsLoggerAndExporter()); video_quality_analyzer_ = video_quality_analyzer.get(); fixture_ = CreatePeerConnectionE2EQualityTestFixture( testing::UnitTest::GetInstance()->current_test_info()->name(), @@ -181,7 +183,7 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { std::map>( {{"alice", network_links.first->endpoints()}, {"charlie", network_links.second->endpoints()}}), - network_emulation())); + network_emulation(), test::GetGlobalMetricsLoggerAndExporter())); RunParams run_params(TimeDelta::Seconds(2)); run_params.enable_flex_fec_support = true; RunAndCheckEachVideoStreamReceivedFrames(run_params); @@ -233,7 +235,7 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, SmokeH264) { std::map>( {{"alice", network_links.first->endpoints()}, {"charlie", network_links.second->endpoints()}}), - network_emulation())); + network_emulation(), test::GetGlobalMetricsLoggerAndExporter())); RunParams run_params(TimeDelta::Seconds(2)); run_params.enable_flex_fec_support = true; RunAndCheckEachVideoStreamReceivedFrames(run_params); @@ -300,7 +302,7 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) { std::map>( {{"alice", alice_network->endpoints()}, {"bob", bob_network->endpoints()}}), - network_emulation())); + network_emulation(), test::GetGlobalMetricsLoggerAndExporter())); fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) { BuiltInNetworkBehaviorConfig config; diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index 89d5da0d42..8b7e5c6663 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -21,6 +21,7 @@ #include "api/rtc_event_log/rtc_event_log.h" #include "api/rtc_event_log_output_file.h" #include "api/scoped_refptr.h" +#include "api/test/metrics/metric.h" #include "api/test/time_controller.h" #include "api/test/video_quality_analyzer_interface.h" #include "pc/sdp_utils.h" @@ -46,6 +47,8 @@ namespace webrtc { namespace webrtc_pc_e2e { namespace { +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; using VideoCodecConfig = PeerConnectionE2EQualityTestFixture::VideoCodecConfig; @@ -128,17 +131,30 @@ PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( TimeController& time_controller, std::unique_ptr audio_quality_analyzer, std::unique_ptr video_quality_analyzer) + : PeerConnectionE2EQualityTest(std::move(test_case_name), + time_controller, + std::move(audio_quality_analyzer), + std::move(video_quality_analyzer), + /*metrics_logger_=*/nullptr) {} + +PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr audio_quality_analyzer, + std::unique_ptr video_quality_analyzer, + test::MetricsLoggerAndExporter* metrics_logger) : time_controller_(time_controller), task_queue_factory_(time_controller_.CreateTaskQueueFactory()), test_case_name_(std::move(test_case_name)), executor_(std::make_unique( - time_controller_.GetClock())) { + time_controller_.GetClock())), + metrics_logger_(metrics_logger) { // Create default video quality analyzer. We will always create an analyzer, // even if there are no video streams, because it will be installed into video // encoder/decoder factories. if (video_quality_analyzer == nullptr) { video_quality_analyzer = std::make_unique( - time_controller_.GetClock()); + time_controller_.GetClock(), metrics_logger_); } if (field_trial::IsEnabled("WebRTC-VideoFrameTrackingIdAdvertised")) { encoded_image_data_propagator_ = @@ -154,7 +170,8 @@ PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( encoded_image_data_propagator_.get()); if (audio_quality_analyzer == nullptr) { - audio_quality_analyzer = std::make_unique(); + audio_quality_analyzer = + std::make_unique(metrics_logger_); } audio_quality_analyzer_.swap(audio_quality_analyzer); } @@ -289,10 +306,10 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads); RTC_LOG(LS_INFO) << "video_analyzer_threads=" << video_analyzer_threads; quality_metrics_reporters_.push_back( - std::make_unique( - time_controller_.GetClock())); + std::make_unique(time_controller_.GetClock(), + metrics_logger_)); quality_metrics_reporters_.push_back( - std::make_unique()); + std::make_unique(metrics_logger_)); video_quality_analyzer_injection_helper_->Start( test_case_name_, @@ -719,14 +736,24 @@ void PeerConnectionE2EQualityTest::TearDownCall() { } void PeerConnectionE2EQualityTest::ReportGeneralTestResults() { - test::PrintResult(*alice_->params().name + "_connected", "", test_case_name_, - alice_connected_, "unitless", - /*important=*/false, - test::ImproveDirection::kBiggerIsBetter); - test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_, - bob_connected_, "unitless", - /*important=*/false, - test::ImproveDirection::kBiggerIsBetter); + if (metrics_logger_ == nullptr) { + test::PrintResult(*alice_->params().name + "_connected", "", + test_case_name_, alice_connected_, "unitless", + /*important=*/false, + test::ImproveDirection::kBiggerIsBetter); + test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_, + bob_connected_, "unitless", + /*important=*/false, + test::ImproveDirection::kBiggerIsBetter); + } else { + metrics_logger_->LogSingleValueMetric( + *alice_->params().name + "_connected", test_case_name_, + alice_connected_, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter); + metrics_logger_->LogSingleValueMetric( + *bob_->params().name + "_connected", test_case_name_, bob_connected_, + Unit::kUnitless, ImprovementDirection::kBiggerIsBetter); + } } Timestamp PeerConnectionE2EQualityTest::Now() const { diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index 7738a83329..94b2178bbe 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -18,6 +18,7 @@ #include "absl/strings/string_view.h" #include "api/task_queue/task_queue_factory.h" #include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/metrics/metrics_logger_and_exporter.h" #include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/time_controller.h" #include "api/units/time_delta.h" @@ -56,6 +57,12 @@ class PeerConnectionE2EQualityTest TimeController& time_controller, std::unique_ptr audio_quality_analyzer, std::unique_ptr video_quality_analyzer); + PeerConnectionE2EQualityTest( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr audio_quality_analyzer, + std::unique_ptr video_quality_analyzer, + test::MetricsLoggerAndExporter* metrics_logger); ~PeerConnectionE2EQualityTest() override = default; @@ -117,6 +124,7 @@ class PeerConnectionE2EQualityTest std::unique_ptr encoded_image_data_propagator_; std::unique_ptr audio_quality_analyzer_; std::unique_ptr executor_; + test::MetricsLoggerAndExporter* const metrics_logger_; std::vector> peer_configurations_; std::vector peer_handles_; diff --git a/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc b/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc new file mode 100644 index 0000000000..304064a94a --- /dev/null +++ b/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc @@ -0,0 +1,602 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/test/create_network_emulation_manager.h" +#include "api/test/create_peer_connection_quality_test_frame_generator.h" +#include "api/test/metrics/metrics_logger_and_exporter.h" +#include "api/test/metrics/stdout_metrics_exporter.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/time_delta.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/peer_connection_quality_test.h" +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::UnorderedElementsAre; + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Metric; +using ::webrtc::test::MetricsExporter; +using ::webrtc::test::MetricsLoggerAndExporter; +using ::webrtc::test::StdoutMetricsExporter; +using ::webrtc::test::Unit; +using RunParams = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::RunParams; +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; +using AudioConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::AudioConfig; +using PeerConfigurer = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::PeerConfigurer; +using VideoCodecConfig = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoCodecConfig; + +// Adds a peer with some audio and video (the client should not care about +// details about audio and video configs). +void AddDefaultAudioVideoPeer( + absl::string_view peer_name, + absl::string_view audio_stream_label, + absl::string_view video_stream_label, + const PeerNetworkDependencies& network_dependencies, + PeerConnectionE2EQualityTestFixture& fixture) { + fixture.AddPeer(network_dependencies, [&](PeerConfigurer* peer) { + peer->SetName(peer_name); + AudioConfig audio{std::string(audio_stream_label)}; + audio.sync_group = std::string(peer_name); + peer->SetAudioConfig(std::move(audio)); + VideoConfig video(std::string(video_stream_label), 320, 180, 15); + video.sync_group = std::string(peer_name); + peer->AddVideoConfig(std::move(video)); + peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)}); + }); +} + +// Metric fields to assert on +struct MetricValidationInfo { + std::string test_case; + std::string name; + Unit unit; + ImprovementDirection improvement_direction; +}; + +bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) { + return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit && + a.improvement_direction == b.improvement_direction; +} + +std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) { + os << "{ test_case=" << m.test_case << "; name=" << m.name + << "; unit=" << test::ToString(m.unit) + << "; improvement_direction=" << test::ToString(m.improvement_direction) + << " }"; + return os; +} + +std::vector ToValidationInfo( + const std::vector& metrics) { + std::vector out; + for (const Metric& m : metrics) { + out.push_back( + MetricValidationInfo{.test_case = m.test_case, + .name = m.name, + .unit = m.unit, + .improvement_direction = m.improvement_direction}); + } + return out; +} + +TEST(PeerConnectionE2EQualityTestMetricNamesTest, + ExportedMetricsHasCorrectName) { + std::unique_ptr network_emulation = + CreateNetworkEmulationManager(TimeMode::kSimulated); + std::vector> exporters; + exporters.push_back(std::make_unique()); + MetricsLoggerAndExporter metrics_logger( + network_emulation->time_controller()->GetClock(), std::move(exporters)); + PeerConnectionE2EQualityTest fixture( + "test_case", *network_emulation->time_controller(), + /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr, + &metrics_logger); + + EmulatedEndpoint* alice_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation->CreateRoute( + alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()}, + bob_endpoint); + network_emulation->CreateRoute( + bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()}, + alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint}); + + AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video", + alice_network->network_dependencies(), fixture); + AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video", + bob_network->network_dependencies(), fixture); + fixture.AddQualityMetricsReporter( + std::make_unique( + std::map>( + {{"alice", alice_network->endpoints()}, + {"bob", bob_network->endpoints()}}), + network_emulation.get(), &metrics_logger)); + + // Run for at least 7 seconds, so AV-sync metrics will be collected. + fixture.Run(RunParams(TimeDelta::Seconds(7))); + + std::vector metrics = + ToValidationInfo(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + UnorderedElementsAre( + // Metrics from PeerConnectionE2EQualityTest + MetricValidationInfo{ + .test_case = "test_case", + .name = "alice_connected", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case", + .name = "bob_connected", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + + // Metrics from DefaultAudioQualityAnalyzer + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "expand_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "accelerate_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "preemptive_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "speech_expand_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "average_jitter_buffer_delay_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "preferred_buffer_size_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "expand_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "accelerate_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "preemptive_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "speech_expand_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "average_jitter_buffer_delay_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "preferred_buffer_size_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + + // Metrics from DefaultVideoQualityAnalyzer + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case", + .name = "cpu_usage", + .unit = Unit::kPercent, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + + // Metrics from StatsBasedNetworkQualityMetricsReporter + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "bytes_discarded_no_receiver", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "packets_discarded_no_receiver", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "payload_bytes_received", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "payload_bytes_sent", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "bytes_sent", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "packets_sent", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "average_send_rate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "bytes_received", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "packets_received", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "average_receive_rate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "sent_packets_loss", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "bytes_discarded_no_receiver", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "packets_discarded_no_receiver", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "payload_bytes_received", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "payload_bytes_sent", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "bytes_sent", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "packets_sent", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "average_send_rate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "bytes_received", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "packets_received", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "average_receive_rate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "sent_packets_loss", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + + // Metrics from VideoQualityMetricsReporter + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "available_send_bandwidth", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "transmission_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "retransmission_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "available_send_bandwidth", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "transmission_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "retransmission_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + + // Metrics from CrossMediaMetricsReporter + MetricValidationInfo{ + .test_case = "test_case/alice_alice_audio", + .name = "audio_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_alice_video", + .name = "video_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/bob_bob_audio", + .name = "audio_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{.test_case = "test_case/bob_bob_video", + .name = "video_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = + ImprovementDirection::kSmallerIsBetter})); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/test_main_lib.cc b/test/test_main_lib.cc index 204af03ca3..183ff9df21 100644 --- a/test/test_main_lib.cc +++ b/test/test_main_lib.cc @@ -13,11 +13,15 @@ #include #include #include +#include #include "absl/flags/flag.h" #include "absl/memory/memory.h" #include "absl/strings/match.h" #include "absl/types/optional.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/metrics/metrics_exporter.h" +#include "api/test/metrics/stdout_metrics_exporter.h" #include "rtc_base/checks.h" #include "rtc_base/event_tracer.h" #include "rtc_base/logging.h" @@ -65,6 +69,11 @@ ABSL_FLAG(std::string, "", "Path to output an empty JSON file which Chromium infra requires."); +ABSL_FLAG(bool, + export_perf_results_new_api, + false, + "Tells to initialize new API for exporting performance metrics"); + ABSL_FLAG(bool, logs, true, "print logs to stderr"); ABSL_FLAG(bool, verbose, false, "verbose logs to stderr"); @@ -101,6 +110,12 @@ class TestMainImpl : public TestMain { rtc::LogMessage::SetLogToStderr(absl::GetFlag(FLAGS_logs) || absl::GetFlag(FLAGS_verbose)); + if (absl::GetFlag(FLAGS_export_perf_results_new_api)) { + std::vector> exporters; + exporters.push_back(std::make_unique()); + test::SetupGlobalMetricsLoggerAndExporter(std::move(exporters)); + } + // InitFieldTrialsFromString stores the char*, so the char array must // outlive the application. field_trials_ = absl::GetFlag(FLAGS_force_fieldtrials); @@ -158,6 +173,10 @@ class TestMainImpl : public TestMain { webrtc::test::PrintPlottableResults(*metrics_to_plot); } + if (absl::GetFlag(FLAGS_export_perf_results_new_api)) { + test::ExportAndDestroyGlobalMetricsLoggerAndExporter(); + } + std::string result_filename = absl::GetFlag(FLAGS_isolated_script_test_output); if (!result_filename.empty()) {