diff --git a/api/test/peerconnection_quality_test_fixture.h b/api/test/peerconnection_quality_test_fixture.h index 967a17650e..66c056eae8 100644 --- a/api/test/peerconnection_quality_test_fixture.h +++ b/api/test/peerconnection_quality_test_fixture.h @@ -193,6 +193,20 @@ class PeerConnectionE2EQualityTestFixture { double video_encoder_bitrate_multiplier = 1.0; }; + // Represent an entity that will report quality metrics after test. + class QualityMetricsReporter { + public: + virtual ~QualityMetricsReporter() = default; + + // Invoked by framework after peer connection factory and peer connection + // itself will be created but before offer/answer exchange will be started. + virtual void Start(absl::string_view test_case_name) = 0; + + // Invoked by framework after call is ended and peer connection factory and + // peer connection are destroyed. + virtual void StopAndReportResults() = 0; + }; + virtual ~PeerConnectionE2EQualityTestFixture() = default; // Add activity that will be executed on the best effort at least after @@ -209,6 +223,10 @@ class PeerConnectionE2EQualityTestFixture { TimeDelta interval, std::function func) = 0; + // Add stats reporter entity to observe the test. + virtual void AddQualityMetricsReporter( + std::unique_ptr quality_metrics_reporter) = 0; + // Add a new peer to the call and return an object through which caller // can configure peer's behavior. // |network_thread| will be used as network thread for peer's peer connection diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 43d6ede548..1f398f24e5 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -312,6 +312,7 @@ if (rtc_include_tests) { deps = [ ":default_audio_quality_analyzer", ":default_video_quality_analyzer", + ":network_quality_metrics_reporter", "../../../api:callfactory_api", "../../../api:create_network_emulation_manager", "../../../api:create_peerconnection_quality_test_fixture", @@ -436,3 +437,18 @@ rtc_source_set("default_video_quality_analyzer") { "//third_party/abseil-cpp/absl/memory", ] } + +rtc_source_set("network_quality_metrics_reporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "network_quality_metrics_reporter.cc", + "network_quality_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../rtc_base:rtc_event", + ] +} diff --git a/test/pc/e2e/network_quality_metrics_reporter.cc b/test/pc/e2e/network_quality_metrics_reporter.cc new file mode 100644 index 0000000000..70927fe4ea --- /dev/null +++ b/test/pc/e2e/network_quality_metrics_reporter.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/network_quality_metrics_reporter.h" + +#include + +#include "rtc_base/event.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kStatsWaitTimeoutMs = 1000; + +} + +void NetworkQualityMetricsReporter::Start(absl::string_view test_case_name) { + test_case_name_ = std::string(test_case_name); + // Check that network stats are clean before test execution. + EmulatedNetworkStats alice_stats = PopulateStats(alice_network_); + RTC_CHECK_EQ(alice_stats.packets_sent, 0); + RTC_CHECK_EQ(alice_stats.packets_received, 0); + EmulatedNetworkStats bob_stats = PopulateStats(bob_network_); + RTC_CHECK_EQ(bob_stats.packets_sent, 0); + RTC_CHECK_EQ(bob_stats.packets_received, 0); +} + +void NetworkQualityMetricsReporter::StopAndReportResults() { + EmulatedNetworkStats alice_stats = PopulateStats(alice_network_); + EmulatedNetworkStats bob_stats = PopulateStats(bob_network_); + ReportStats("alice", alice_stats, + alice_stats.packets_sent - bob_stats.packets_received); + ReportStats("bob", bob_stats, + bob_stats.packets_sent - alice_stats.packets_received); +} + +EmulatedNetworkStats NetworkQualityMetricsReporter::PopulateStats( + EmulatedNetworkManagerInterface* network) { + rtc::Event wait; + EmulatedNetworkStats stats; + network->GetStats([&](const EmulatedNetworkStats& s) { + stats = s; + wait.Set(); + }); + bool stats_received = wait.Wait(kStatsWaitTimeoutMs); + RTC_CHECK(stats_received); + return stats; +} + +void NetworkQualityMetricsReporter::ReportStats( + const std::string& network_label, + const EmulatedNetworkStats& stats, + int64_t packet_loss) { + ReportResult("bytes_sent", network_label, stats.bytes_sent.bytes(), + "sizeInBytes"); + ReportResult("packets_sent", network_label, stats.packets_sent, "unitless"); + ReportResult("average_send_rate", network_label, + stats.AverageSendRate().bytes_per_sec(), "bytesPerSecond"); + ReportResult("bytes_dropped", network_label, stats.bytes_dropped.bytes(), + "sizeInBytes"); + ReportResult("packets_dropped", network_label, stats.packets_dropped, + "unitless"); + ReportResult("bytes_received", network_label, stats.bytes_received.bytes(), + "sizeInBytes"); + ReportResult("packets_received", network_label, stats.packets_received, + "unitless"); + ReportResult("average_receive_rate", network_label, + stats.AverageReceiveRate().bytes_per_sec(), "bytesPerSecond"); + ReportResult("sent_packets_loss", network_label, packet_loss, "unitless"); +} + +void NetworkQualityMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& network_label, + const double value, + const std::string& unit) const { + test::PrintResult(metric_name, /*modifier=*/"", + GetTestCaseName(network_label), value, unit, + /*important=*/false); +} + +std::string NetworkQualityMetricsReporter::GetTestCaseName( + const std::string& network_label) const { + return test_case_name_ + "/" + network_label; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/network_quality_metrics_reporter.h b/test/pc/e2e/network_quality_metrics_reporter.h new file mode 100644 index 0000000000..52106ef580 --- /dev/null +++ b/test/pc/e2e/network_quality_metrics_reporter.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ + +#include + +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class NetworkQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network, + EmulatedNetworkManagerInterface* bob_network) + : alice_network_(alice_network), bob_network_(bob_network) {} + ~NetworkQualityMetricsReporter() override = default; + + // Network stats must be empty when this method will be invoked. + void Start(absl::string_view test_case_name) override; + void StopAndReportResults() override; + + private: + static EmulatedNetworkStats PopulateStats( + EmulatedNetworkManagerInterface* network); + void ReportStats(const std::string& network_label, + const EmulatedNetworkStats& stats, + int64_t packet_loss); + void ReportResult(const std::string& metric_name, + const std::string& network_label, + const double value, + const std::string& unit) const; + std::string GetTestCaseName(const std::string& network_label) const; + + std::string test_case_name_; + + EmulatedNetworkManagerInterface* alice_network_; + EmulatedNetworkManagerInterface* bob_network_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index 62e0d54c20..33e666b4bb 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -20,6 +20,7 @@ #include "test/gtest.h" #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" +#include "test/pc/e2e/network_quality_metrics_reporter.h" #include "test/testsupport/file_utils.h" namespace webrtc { @@ -113,6 +114,10 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, MAYBE_RunWithEmulatedNetwork) { bob->SetAudioConfig(std::move(audio_config)); }); + fixture->AddQualityMetricsReporter( + absl::make_unique(alice_network, + bob_network)); + RunParams run_params(TimeDelta::seconds(7)); run_params.video_encoder_bitrate_multiplier = 1.1; fixture->Run(run_params); diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index 7f3aafd146..7fa53fac17 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -197,6 +197,11 @@ void PeerConnectionE2EQualityTest::PostTask(ScheduledActivity activity) { remaining_delay.ms()); } +void PeerConnectionE2EQualityTest::AddQualityMetricsReporter( + std::unique_ptr quality_metrics_reporter) { + quality_metrics_reporters_.push_back(std::move(quality_metrics_reporter)); +} + void PeerConnectionE2EQualityTest::AddPeer( rtc::Thread* network_thread, rtc::NetworkManager* network_manager, @@ -292,6 +297,9 @@ void PeerConnectionE2EQualityTest::Run( video_quality_analyzer_injection_helper_->Start(test_case_name_, video_analyzer_threads); audio_quality_analyzer_->Start(test_case_name_, &analyzer_helper_); + for (auto& reporter : quality_metrics_reporters_) { + reporter->Start(test_case_name_); + } // Start RTCEventLog recording if requested. if (alice_->params()->rtc_event_log_path) { @@ -364,6 +372,9 @@ void PeerConnectionE2EQualityTest::Run( audio_quality_analyzer_->Stop(); video_quality_analyzer_injection_helper_->Stop(); + for (auto& reporter : quality_metrics_reporters_) { + reporter->StopAndReportResults(); + } // Ensuring that TestPeers have been destroyed in order to correctly close // Audio dumps. diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index e82792199e..93c75089e9 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -155,6 +155,8 @@ class PeerConnectionE2EQualityTest using RunParams = PeerConnectionE2EQualityTestFixture::RunParams; using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer; + using QualityMetricsReporter = + PeerConnectionE2EQualityTestFixture::QualityMetricsReporter; PeerConnectionE2EQualityTest( std::string test_case_name, @@ -169,6 +171,9 @@ class PeerConnectionE2EQualityTest TimeDelta interval, std::function func) override; + void AddQualityMetricsReporter(std::unique_ptr + quality_metrics_reporter) override; + void AddPeer(rtc::Thread* network_thread, rtc::NetworkManager* network_manager, rtc::FunctionView configurer) override; @@ -232,6 +237,8 @@ class PeerConnectionE2EQualityTest std::unique_ptr alice_; std::unique_ptr bob_; + std::vector> + quality_metrics_reporters_; std::vector> alice_video_sources_;