From 1845922d5a1bf9c27deeffb4a8c8daea124434c1 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Wed, 24 Apr 2019 11:09:35 +0200 Subject: [PATCH] Introduce QualityMetricsReporter and implement network stats gathering QualityMetricsReporter helps to keep network emulation framework and peer connection level test framework separated. Also it provides ability to gather statistics from any component around with correlation with call start and end. Bug: webrtc:10138 Change-Id: Ib3330a8d35481fde77fcf77d2271d6cfcf188fec Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132718 Reviewed-by: Karl Wiberg Reviewed-by: Peter Slatala Reviewed-by: Mirko Bonadei Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Artem Titov Cr-Commit-Position: refs/heads/master@{#27759} --- .../peerconnection_quality_test_fixture.h | 18 ++++ test/pc/e2e/BUILD.gn | 16 ++++ .../e2e/network_quality_metrics_reporter.cc | 96 +++++++++++++++++++ .../pc/e2e/network_quality_metrics_reporter.h | 55 +++++++++++ test/pc/e2e/peer_connection_e2e_smoke_test.cc | 5 + test/pc/e2e/peer_connection_quality_test.cc | 11 +++ test/pc/e2e/peer_connection_quality_test.h | 7 ++ 7 files changed, 208 insertions(+) create mode 100644 test/pc/e2e/network_quality_metrics_reporter.cc create mode 100644 test/pc/e2e/network_quality_metrics_reporter.h diff --git a/api/test/peerconnection_quality_test_fixture.h b/api/test/peerconnection_quality_test_fixture.h index 967a17650e..66c056eae8 100644 --- a/api/test/peerconnection_quality_test_fixture.h +++ b/api/test/peerconnection_quality_test_fixture.h @@ -193,6 +193,20 @@ class PeerConnectionE2EQualityTestFixture { double video_encoder_bitrate_multiplier = 1.0; }; + // Represent an entity that will report quality metrics after test. + class QualityMetricsReporter { + public: + virtual ~QualityMetricsReporter() = default; + + // Invoked by framework after peer connection factory and peer connection + // itself will be created but before offer/answer exchange will be started. + virtual void Start(absl::string_view test_case_name) = 0; + + // Invoked by framework after call is ended and peer connection factory and + // peer connection are destroyed. + virtual void StopAndReportResults() = 0; + }; + virtual ~PeerConnectionE2EQualityTestFixture() = default; // Add activity that will be executed on the best effort at least after @@ -209,6 +223,10 @@ class PeerConnectionE2EQualityTestFixture { TimeDelta interval, std::function func) = 0; + // Add stats reporter entity to observe the test. + virtual void AddQualityMetricsReporter( + std::unique_ptr quality_metrics_reporter) = 0; + // Add a new peer to the call and return an object through which caller // can configure peer's behavior. // |network_thread| will be used as network thread for peer's peer connection diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 43d6ede548..1f398f24e5 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -312,6 +312,7 @@ if (rtc_include_tests) { deps = [ ":default_audio_quality_analyzer", ":default_video_quality_analyzer", + ":network_quality_metrics_reporter", "../../../api:callfactory_api", "../../../api:create_network_emulation_manager", "../../../api:create_peerconnection_quality_test_fixture", @@ -436,3 +437,18 @@ rtc_source_set("default_video_quality_analyzer") { "//third_party/abseil-cpp/absl/memory", ] } + +rtc_source_set("network_quality_metrics_reporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "network_quality_metrics_reporter.cc", + "network_quality_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../rtc_base:rtc_event", + ] +} diff --git a/test/pc/e2e/network_quality_metrics_reporter.cc b/test/pc/e2e/network_quality_metrics_reporter.cc new file mode 100644 index 0000000000..70927fe4ea --- /dev/null +++ b/test/pc/e2e/network_quality_metrics_reporter.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/network_quality_metrics_reporter.h" + +#include + +#include "rtc_base/event.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kStatsWaitTimeoutMs = 1000; + +} + +void NetworkQualityMetricsReporter::Start(absl::string_view test_case_name) { + test_case_name_ = std::string(test_case_name); + // Check that network stats are clean before test execution. + EmulatedNetworkStats alice_stats = PopulateStats(alice_network_); + RTC_CHECK_EQ(alice_stats.packets_sent, 0); + RTC_CHECK_EQ(alice_stats.packets_received, 0); + EmulatedNetworkStats bob_stats = PopulateStats(bob_network_); + RTC_CHECK_EQ(bob_stats.packets_sent, 0); + RTC_CHECK_EQ(bob_stats.packets_received, 0); +} + +void NetworkQualityMetricsReporter::StopAndReportResults() { + EmulatedNetworkStats alice_stats = PopulateStats(alice_network_); + EmulatedNetworkStats bob_stats = PopulateStats(bob_network_); + ReportStats("alice", alice_stats, + alice_stats.packets_sent - bob_stats.packets_received); + ReportStats("bob", bob_stats, + bob_stats.packets_sent - alice_stats.packets_received); +} + +EmulatedNetworkStats NetworkQualityMetricsReporter::PopulateStats( + EmulatedNetworkManagerInterface* network) { + rtc::Event wait; + EmulatedNetworkStats stats; + network->GetStats([&](const EmulatedNetworkStats& s) { + stats = s; + wait.Set(); + }); + bool stats_received = wait.Wait(kStatsWaitTimeoutMs); + RTC_CHECK(stats_received); + return stats; +} + +void NetworkQualityMetricsReporter::ReportStats( + const std::string& network_label, + const EmulatedNetworkStats& stats, + int64_t packet_loss) { + ReportResult("bytes_sent", network_label, stats.bytes_sent.bytes(), + "sizeInBytes"); + ReportResult("packets_sent", network_label, stats.packets_sent, "unitless"); + ReportResult("average_send_rate", network_label, + stats.AverageSendRate().bytes_per_sec(), "bytesPerSecond"); + ReportResult("bytes_dropped", network_label, stats.bytes_dropped.bytes(), + "sizeInBytes"); + ReportResult("packets_dropped", network_label, stats.packets_dropped, + "unitless"); + ReportResult("bytes_received", network_label, stats.bytes_received.bytes(), + "sizeInBytes"); + ReportResult("packets_received", network_label, stats.packets_received, + "unitless"); + ReportResult("average_receive_rate", network_label, + stats.AverageReceiveRate().bytes_per_sec(), "bytesPerSecond"); + ReportResult("sent_packets_loss", network_label, packet_loss, "unitless"); +} + +void NetworkQualityMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& network_label, + const double value, + const std::string& unit) const { + test::PrintResult(metric_name, /*modifier=*/"", + GetTestCaseName(network_label), value, unit, + /*important=*/false); +} + +std::string NetworkQualityMetricsReporter::GetTestCaseName( + const std::string& network_label) const { + return test_case_name_ + "/" + network_label; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/network_quality_metrics_reporter.h b/test/pc/e2e/network_quality_metrics_reporter.h new file mode 100644 index 0000000000..52106ef580 --- /dev/null +++ b/test/pc/e2e/network_quality_metrics_reporter.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ + +#include + +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class NetworkQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network, + EmulatedNetworkManagerInterface* bob_network) + : alice_network_(alice_network), bob_network_(bob_network) {} + ~NetworkQualityMetricsReporter() override = default; + + // Network stats must be empty when this method will be invoked. + void Start(absl::string_view test_case_name) override; + void StopAndReportResults() override; + + private: + static EmulatedNetworkStats PopulateStats( + EmulatedNetworkManagerInterface* network); + void ReportStats(const std::string& network_label, + const EmulatedNetworkStats& stats, + int64_t packet_loss); + void ReportResult(const std::string& metric_name, + const std::string& network_label, + const double value, + const std::string& unit) const; + std::string GetTestCaseName(const std::string& network_label) const; + + std::string test_case_name_; + + EmulatedNetworkManagerInterface* alice_network_; + EmulatedNetworkManagerInterface* bob_network_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index 62e0d54c20..33e666b4bb 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -20,6 +20,7 @@ #include "test/gtest.h" #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" +#include "test/pc/e2e/network_quality_metrics_reporter.h" #include "test/testsupport/file_utils.h" namespace webrtc { @@ -113,6 +114,10 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, MAYBE_RunWithEmulatedNetwork) { bob->SetAudioConfig(std::move(audio_config)); }); + fixture->AddQualityMetricsReporter( + absl::make_unique(alice_network, + bob_network)); + RunParams run_params(TimeDelta::seconds(7)); run_params.video_encoder_bitrate_multiplier = 1.1; fixture->Run(run_params); diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index 7f3aafd146..7fa53fac17 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -197,6 +197,11 @@ void PeerConnectionE2EQualityTest::PostTask(ScheduledActivity activity) { remaining_delay.ms()); } +void PeerConnectionE2EQualityTest::AddQualityMetricsReporter( + std::unique_ptr quality_metrics_reporter) { + quality_metrics_reporters_.push_back(std::move(quality_metrics_reporter)); +} + void PeerConnectionE2EQualityTest::AddPeer( rtc::Thread* network_thread, rtc::NetworkManager* network_manager, @@ -292,6 +297,9 @@ void PeerConnectionE2EQualityTest::Run( video_quality_analyzer_injection_helper_->Start(test_case_name_, video_analyzer_threads); audio_quality_analyzer_->Start(test_case_name_, &analyzer_helper_); + for (auto& reporter : quality_metrics_reporters_) { + reporter->Start(test_case_name_); + } // Start RTCEventLog recording if requested. if (alice_->params()->rtc_event_log_path) { @@ -364,6 +372,9 @@ void PeerConnectionE2EQualityTest::Run( audio_quality_analyzer_->Stop(); video_quality_analyzer_injection_helper_->Stop(); + for (auto& reporter : quality_metrics_reporters_) { + reporter->StopAndReportResults(); + } // Ensuring that TestPeers have been destroyed in order to correctly close // Audio dumps. diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h index e82792199e..93c75089e9 100644 --- a/test/pc/e2e/peer_connection_quality_test.h +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -155,6 +155,8 @@ class PeerConnectionE2EQualityTest using RunParams = PeerConnectionE2EQualityTestFixture::RunParams; using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer; + using QualityMetricsReporter = + PeerConnectionE2EQualityTestFixture::QualityMetricsReporter; PeerConnectionE2EQualityTest( std::string test_case_name, @@ -169,6 +171,9 @@ class PeerConnectionE2EQualityTest TimeDelta interval, std::function func) override; + void AddQualityMetricsReporter(std::unique_ptr + quality_metrics_reporter) override; + void AddPeer(rtc::Thread* network_thread, rtc::NetworkManager* network_manager, rtc::FunctionView configurer) override; @@ -232,6 +237,8 @@ class PeerConnectionE2EQualityTest std::unique_ptr alice_; std::unique_ptr bob_; + std::vector> + quality_metrics_reporters_; std::vector> alice_video_sources_;