diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index b06d2de0a8..62ed0b8a91 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -458,6 +458,7 @@ if (!build_with_chromium) { ":default_audio_quality_analyzer", ":default_video_quality_analyzer", ":network_quality_metrics_reporter", + ":stats_based_network_quality_metrics_reporter", "../../../api:callfactory_api", "../../../api:create_network_emulation_manager", "../../../api:create_peer_connection_quality_test_frame_generator", @@ -677,6 +678,33 @@ if (!build_with_chromium) { absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } + rtc_library("stats_based_network_quality_metrics_reporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "stats_based_network_quality_metrics_reporter.cc", + "stats_based_network_quality_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:array_view", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api:scoped_refptr", + "../../../api/test/network_emulation", + "../../../api/units:data_rate", + "../../../api/units:data_size", + "../../../api/units:timestamp", + "../../../rtc_base", + "../../../rtc_base:rtc_event", + "../../../rtc_base:stringutils", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + rtc_library("cross_media_metrics_reporter") { visibility = [ "*" ] testonly = true diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc index 39dbbf6269..10f62835a9 100644 --- a/test/pc/e2e/peer_connection_e2e_smoke_test.cc +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -23,7 +23,7 @@ #include "test/gtest.h" #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" -#include "test/pc/e2e/network_quality_metrics_reporter.h" +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" #include "test/testsupport/file_utils.h" namespace webrtc { @@ -163,8 +163,11 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { charlie->SetAudioConfig(std::move(audio)); }); fixture()->AddQualityMetricsReporter( - std::make_unique(network_links.first, - network_links.second)); + std::make_unique( + std::map>( + {{"alice", network_links.first->endpoints()}, + {"charlie", network_links.second->endpoints()}}), + network_emulation())); RunParams run_params(TimeDelta::Seconds(2)); run_params.video_codecs = { VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}; @@ -217,8 +220,11 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) { }); AddPeer(bob_network, [](PeerConfigurer* bob) {}); fixture()->AddQualityMetricsReporter( - std::make_unique(alice_network, - bob_network)); + std::make_unique( + std::map>( + {{"alice", alice_network->endpoints()}, + {"bob", bob_network->endpoints()}}), + network_emulation())); fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) { BuiltInNetworkBehaviorConfig config; diff --git a/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc b/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc new file mode 100644 index 0000000000..9f4283dd0a --- /dev/null +++ b/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc @@ -0,0 +1,318 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/network_emulation/network_emulation_interfaces.h" +#include "api/test/network_emulation_manager.h" +#include "api/units/data_rate.h" +#include "api/units/timestamp.h" +#include "rtc_base/event.h" +#include "rtc_base/ip_address.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/field_trial.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kStatsWaitTimeoutMs = 1000; + +// Field trial which controls whether to report standard-compliant bytes +// sent/received per stream. If enabled, padding and headers are not included +// in bytes sent or received. +constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats"; + +std::unique_ptr PopulateStats( + std::vector endpoints, + NetworkEmulationManager* network_emulation) { + rtc::Event stats_loaded; + std::unique_ptr stats; + network_emulation->GetStats(endpoints, + [&](std::unique_ptr s) { + stats = std::move(s); + stats_loaded.Set(); + }); + bool stats_received = stats_loaded.Wait(kStatsWaitTimeoutMs); + RTC_CHECK(stats_received); + return stats; +} + +std::map PopulateIpToPeer( + const std::map>& + peer_endpoints) { + std::map out; + for (const auto& entry : peer_endpoints) { + for (const EmulatedEndpoint* const endpoint : entry.second) { + out.emplace(endpoint->GetPeerLocalAddress(), entry.first); + } + } + return out; +} + +} // namespace + +StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + NetworkLayerStatsCollector( + std::map> peer_endpoints, + NetworkEmulationManager* network_emulation) + : peer_endpoints_(std::move(peer_endpoints)), + ip_to_peer_(PopulateIpToPeer(peer_endpoints_)), + network_emulation_(network_emulation) {} + +void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + Start() { + // Check that network stats are clean before test execution. + for (const auto& entry : peer_endpoints_) { + std::unique_ptr stats = + PopulateStats(entry.second, network_emulation_); + RTC_CHECK_EQ(stats->PacketsSent(), 0); + RTC_CHECK_EQ(stats->PacketsReceived(), 0); + } +} + +std::map +StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + GetStats() { + std::map peer_to_stats; + std::map> sender_to_receivers; + for (const auto& entry : peer_endpoints_) { + NetworkLayerStats stats; + stats.stats = PopulateStats(entry.second, network_emulation_); + const std::string& peer_name = entry.first; + for (const auto& income_stats_entry : + stats.stats->IncomingStatsPerSource()) { + const rtc::IPAddress& source_ip = income_stats_entry.first; + auto it = ip_to_peer_.find(source_ip); + if (it == ip_to_peer_.end()) { + // Source IP is unknown for this collector, so will be skipped. + continue; + } + sender_to_receivers[it->second].push_back(peer_name); + } + peer_to_stats.emplace(peer_name, std::move(stats)); + } + for (auto& entry : peer_to_stats) { + const std::vector& receivers = + sender_to_receivers[entry.first]; + entry.second.receivers = + std::set(receivers.begin(), receivers.end()); + } + return peer_to_stats; +} + +void StatsBasedNetworkQualityMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) { + test_case_name_ = std::string(test_case_name); + collector_.Start(); + start_time_ = clock_->CurrentTime(); +} + +void StatsBasedNetworkQualityMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) { + PCStats cur_stats; + + auto inbound_stats = report->GetStatsOfType(); + for (const auto& stat : inbound_stats) { + cur_stats.payload_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) + + stat->header_bytes_received.ValueOrDefault(0ul)); + } + + auto outbound_stats = report->GetStatsOfType(); + for (const auto& stat : outbound_stats) { + cur_stats.payload_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) + + stat->header_bytes_sent.ValueOrDefault(0ul)); + } + + auto candidate_pairs_stats = report->GetStatsOfType(); + for (const auto& stat : candidate_pairs_stats) { + cur_stats.total_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul)); + cur_stats.total_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul)); + cur_stats.packets_received += stat->packets_received.ValueOrDefault(0ul); + cur_stats.packets_sent += stat->packets_sent.ValueOrDefault(0ul); + } + + MutexLock lock(&mutex_); + pc_stats_[std::string(pc_label)] = cur_stats; +} + +void StatsBasedNetworkQualityMetricsReporter::StopAndReportResults() { + Timestamp end_time = clock_->CurrentTime(); + + if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) { + RTC_LOG(LS_ERROR) + << "Non-standard GetStats; \"payload\" counts include RTP headers"; + } + + std::map stats = collector_.GetStats(); + for (const auto& entry : stats) { + LogNetworkLayerStats(entry.first, entry.second); + } + MutexLock lock(&mutex_); + for (const auto& pair : pc_stats_) { + auto it = stats.find(pair.first); + RTC_CHECK(it != stats.end()) + << "Peer name used for PeerConnection stats collection and peer name " + "used for endpoints naming doesn't match. No endpoints found for " + "peer " + << pair.first; + const NetworkLayerStats& network_layer_stats = it->second; + int64_t total_packets_received = 0; + bool found = false; + for (const auto& dest_peer : network_layer_stats.receivers) { + auto pc_stats_it = pc_stats_.find(dest_peer); + if (pc_stats_it == pc_stats_.end()) { + continue; + } + found = true; + total_packets_received += pc_stats_it->second.packets_received; + } + int64_t packet_loss = -1; + if (found) { + packet_loss = pair.second.packets_sent - total_packets_received; + } + ReportStats(pair.first, pair.second, network_layer_stats, packet_loss, + end_time); + } +} + +void StatsBasedNetworkQualityMetricsReporter::ReportStats( + const std::string& pc_label, + const PCStats& pc_stats, + const NetworkLayerStats& network_layer_stats, + int64_t packet_loss, + const Timestamp& end_time) { + ReportResult("bytes_dropped", pc_label, + network_layer_stats.stats->BytesDropped().bytes(), + "sizeInBytes"); + ReportResult("packets_dropped", pc_label, + network_layer_stats.stats->PacketsDropped(), "unitless"); + + ReportResult("payload_bytes_received", pc_label, + pc_stats.payload_received.bytes(), "sizeInBytes"); + ReportResult("payload_bytes_sent", pc_label, pc_stats.payload_sent.bytes(), + "sizeInBytes"); + + ReportResult("bytes_sent", pc_label, pc_stats.total_sent.bytes(), + "sizeInBytes"); + ReportResult("packets_sent", pc_label, pc_stats.packets_sent, "unitless"); + ReportResult("average_send_rate", pc_label, + (pc_stats.total_sent / (end_time - start_time_)).bytes_per_sec(), + "bytesPerSecond"); + ReportResult("bytes_received", pc_label, pc_stats.total_received.bytes(), + "sizeInBytes"); + ReportResult("packets_received", pc_label, pc_stats.packets_received, + "unitless"); + ReportResult( + "average_receive_rate", pc_label, + (pc_stats.total_received / (end_time - start_time_)).bytes_per_sec(), + "bytesPerSecond"); + ReportResult("sent_packets_loss", pc_label, packet_loss, "unitless"); +} + +void StatsBasedNetworkQualityMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& network_label, + const double value, + const std::string& unit) const { + test::PrintResult(metric_name, /*modifier=*/"", + GetTestCaseName(network_label), value, unit, + /*important=*/false); +} + +std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName( + absl::string_view network_label) const { + rtc::StringBuilder builder; + builder << test_case_name_ << "/" << network_label.data(); + return builder.str(); +} + +void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats( + absl::string_view peer_name, + const NetworkLayerStats& stats) const { + DataRate average_send_rate = stats.stats->PacketsSent() >= 2 + ? stats.stats->AverageSendRate() + : DataRate::Zero(); + DataRate average_receive_rate = stats.stats->PacketsReceived() >= 2 + ? stats.stats->AverageReceiveRate() + : DataRate::Zero(); + rtc::StringBuilder log; + log << "Raw network layer statistic for [" << peer_name << "]:\n" + << "Local IPs:\n"; + std::vector local_ips = stats.stats->LocalAddresses(); + for (size_t i = 0; i < local_ips.size(); ++i) { + log << " " << local_ips[i].ToString() << "\n"; + } + log << "Send statistic:\n" + << " packets: " << stats.stats->PacketsSent() + << " bytes: " << stats.stats->BytesSent().bytes() + << " avg_rate (bytes/sec): " << average_send_rate.bytes_per_sec() + << " avg_rate (bps): " << average_send_rate.bps() << "\n" + << "Send statistic per destination:\n"; + + for (const auto& entry : stats.stats->OutgoingStatsPerDestination()) { + DataRate source_average_send_rate = entry.second->PacketsSent() >= 2 + ? entry.second->AverageSendRate() + : DataRate::Zero(); + log << "(" << entry.first.ToString() << "):\n" + << " packets: " << entry.second->PacketsSent() + << " bytes: " << entry.second->BytesSent().bytes() + << " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec() + << " avg_rate (bps): " << source_average_send_rate.bps() << "\n"; + } + + log << "Receive statistic:\n" + << " packets: " << stats.stats->PacketsReceived() + << " bytes: " << stats.stats->BytesReceived().bytes() + << " avg_rate (bytes/sec): " << average_receive_rate.bytes_per_sec() + << " avg_rate (bps): " << average_receive_rate.bps() << "\n" + << "Receive statistic per source:\n"; + + for (const auto& entry : stats.stats->IncomingStatsPerSource()) { + DataRate source_average_receive_rate = + entry.second->PacketsReceived() >= 2 + ? entry.second->AverageReceiveRate() + : DataRate::Zero(); + log << "(" << entry.first.ToString() << "):\n" + << " packets: " << entry.second->PacketsReceived() + << " bytes: " << entry.second->BytesReceived().bytes() + << " avg_rate (bytes/sec): " + << source_average_receive_rate.bytes_per_sec() + << " avg_rate (bps): " << source_average_receive_rate.bps() << "\n"; + } + + RTC_LOG(INFO) << log.str(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/stats_based_network_quality_metrics_reporter.h b/test/pc/e2e/stats_based_network_quality_metrics_reporter.h new file mode 100644 index 0000000000..f01a49d6da --- /dev/null +++ b/test/pc/e2e/stats_based_network_quality_metrics_reporter.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/test/network_emulation/network_emulation_interfaces.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "rtc_base/ip_address.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class StatsBasedNetworkQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + // |networks| map peer name to network to report network layer stability stats + // and to log network layer metrics. + StatsBasedNetworkQualityMetricsReporter( + std::map> peer_endpoints, + NetworkEmulationManager* network_emulation) + : collector_(std::move(peer_endpoints), network_emulation), + clock_(network_emulation->time_controller()->GetClock()) {} + ~StatsBasedNetworkQualityMetricsReporter() override = default; + + // Network stats must be empty when this method will be invoked. + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr& report) override; + void StopAndReportResults() override; + + private: + struct PCStats { + // TODO(nisse): Separate audio and video counters. Depends on standard stat + // counters, enabled by field trial "WebRTC-UseStandardBytesStats". + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); + + // Total bytes/packets sent/received in all RTCTransport's. + DataSize total_received = DataSize::Zero(); + DataSize total_sent = DataSize::Zero(); + int64_t packets_received = 0; + int64_t packets_sent = 0; + }; + + struct NetworkLayerStats { + std::unique_ptr stats; + std::set receivers; + }; + + class NetworkLayerStatsCollector { + public: + NetworkLayerStatsCollector( + std::map> peer_endpoints, + NetworkEmulationManager* network_emulation); + + void Start(); + + std::map GetStats(); + + private: + const std::map> peer_endpoints_; + const std::map ip_to_peer_; + NetworkEmulationManager* const network_emulation_; + }; + + void ReportStats(const std::string& pc_label, + const PCStats& pc_stats, + const NetworkLayerStats& network_layer_stats, + int64_t packet_loss, + const Timestamp& end_time); + void ReportResult(const std::string& metric_name, + const std::string& network_label, + const double value, + const std::string& unit) const; + std::string GetTestCaseName(absl::string_view network_label) const; + void LogNetworkLayerStats(absl::string_view peer_name, + const NetworkLayerStats& stats) const; + + NetworkLayerStatsCollector collector_; + Clock* const clock_; + + std::string test_case_name_; + Timestamp start_time_ = Timestamp::MinusInfinity(); + + Mutex mutex_; + std::map pc_stats_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_