From dea11f9c4351b8cea0ee351d66376ad6dde123de Mon Sep 17 00:00:00 2001 From: Stefan Holmer Date: Wed, 29 Apr 2015 14:27:35 +0200 Subject: [PATCH] Add per flow throughput and delay metrics. BUG=4548 R=pbos@webrtc.org Review URL: https://webrtc-codereview.appspot.com/48299004 Cr-Commit-Position: refs/heads/master@{#9112} --- .../remote_bitrate_estimators_test.cc | 12 ++-- .../remote_bitrate_estimator/test/bwe_test.cc | 67 ++++++++++++++----- .../remote_bitrate_estimator/test/bwe_test.h | 10 ++- .../test/packet_receiver.cc | 9 ++- .../test/packet_receiver.h | 3 + 5 files changed, 74 insertions(+), 27 deletions(-) diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc index 42f4e1c2d0..227ac4c6af 100644 --- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc +++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc @@ -280,8 +280,8 @@ TEST_P(BweFeedbackTest, Choke1000kbps500kbps1000kbps) { filter.SetCapacity(kHighCapacityKbps); RunFor(60 * 1000); PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0, - counter.GetBitrateStats(), filter.GetDelayStats(), - std::vector>()); + counter.GetBitrateStats(), 0, receiver.GetDelayStats(), + counter.GetBitrateStats()); } TEST_P(BweFeedbackTest, Choke200kbps30kbps200kbps) { @@ -301,8 +301,8 @@ TEST_P(BweFeedbackTest, Choke200kbps30kbps200kbps) { RunFor(60 * 1000); PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0, - counter.GetBitrateStats(), filter.GetDelayStats(), - std::vector>()); + counter.GetBitrateStats(), 0, receiver.GetDelayStats(), + counter.GetBitrateStats()); } TEST_P(BweFeedbackTest, Verizon4gDownlinkTest) { @@ -315,7 +315,7 @@ TEST_P(BweFeedbackTest, Verizon4gDownlinkTest) { ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx"))); RunFor(22 * 60 * 1000); PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(), - filter.GetDelayStats(), std::vector>()); + 0, receiver.GetDelayStats(), counter2.GetBitrateStats()); } // webrtc:3277 @@ -330,7 +330,7 @@ TEST_P(BweFeedbackTest, DISABLED_GoogleWifiTrace3Mbps) { ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx"))); RunFor(300 * 1000); PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(), - filter.GetDelayStats(), std::vector>()); + 0, receiver.GetDelayStats(), counter2.GetBitrateStats()); } TEST_P(BweFeedbackTest, PacedSelfFairness50msTest) { diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc index 83c76a576e..86e8cb0626 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc @@ -177,11 +177,23 @@ string BweTest::GetTestName() const { return string(test_info->name()); } -void BweTest::PrintResults( - double max_throughput_kbps, - Stats throughput_kbps, - Stats delay_ms, - std::vector> flow_throughput_kbps) { +void BweTest::PrintResults(double max_throughput_kbps, + Stats throughput_kbps, + int flow_id, + Stats flow_delay_ms, + Stats flow_throughput_kbps) { + std::map> flow_delays_ms; + flow_delays_ms[flow_id] = flow_delay_ms; + std::map> flow_throughputs_kbps; + flow_throughputs_kbps[flow_id] = flow_throughput_kbps; + PrintResults(max_throughput_kbps, throughput_kbps, flow_delays_ms, + flow_throughputs_kbps); +} + +void BweTest::PrintResults(double max_throughput_kbps, + Stats throughput_kbps, + std::map> flow_delay_ms, + std::map> flow_throughput_kbps) { double utilization = throughput_kbps.GetMean() / max_throughput_kbps; webrtc::test::PrintResult("BwePerformance", GetTestName(), "Utilization", utilization * 100.0, "%", false); @@ -189,15 +201,27 @@ void BweTest::PrintResults( ss << throughput_kbps.GetStdDev() / throughput_kbps.GetMean(); webrtc::test::PrintResult("BwePerformance", GetTestName(), "Utilization var coeff", ss.str(), "", false); - webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(), - "Average delay", delay_ms.AsString(), - "ms", false); + for (auto& kv : flow_throughput_kbps) { + ss.str(""); + ss << "Throughput flow " << kv.first; + webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(), + ss.str(), kv.second.AsString(), + "kbps", false); + } + for (auto& kv : flow_delay_ms) { + ss.str(""); + ss << "Delay flow " << kv.first; + webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(), + ss.str(), kv.second.AsString(), "ms", + false); + } double fairness_index = 1.0; if (!flow_throughput_kbps.empty()) { double squared_bitrate_sum = 0.0; - for (Stats flow : flow_throughput_kbps) { - squared_bitrate_sum += flow.GetMean() * flow.GetMean(); - fairness_index += flow.GetMean(); + fairness_index = 0.0; + for (auto kv : flow_throughput_kbps) { + squared_bitrate_sum += kv.second.GetMean() * kv.second.GetMean(); + fairness_index += kv.second.GetMean(); } fairness_index *= fairness_index; fairness_index /= flow_throughput_kbps.size() * squared_bitrate_sum; @@ -234,13 +258,13 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type, for (int media_flow : media_flow_ids) { // Streams started 20 seconds apart to give them different advantage when // competing for the bandwidth. - const int64_t kFlowStartOffsetMs = i++ * (rand() % 40000); + const int64_t kFlowStartOffsetMs = i++ * (rand() % 10000); sources.push_back(new AdaptiveVideoSource(media_flow, 30, 300, 0, kFlowStartOffsetMs)); senders.push_back(new PacedVideoSender(&uplink_, sources.back(), bwe_type)); } - const int64_t kTcpStartOffsetMs = 20000; + const int64_t kTcpStartOffsetMs = 5000; for (int tcp_flow : tcp_flow_ids) senders.push_back(new TcpSender(&uplink_, tcp_flow, kTcpStartOffsetMs)); @@ -276,11 +300,20 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type, RunFor(run_time_seconds * 1000); - std::vector> flow_throughput_kbps; - for (i = 0; i < all_flow_ids.size(); ++i) - flow_throughput_kbps.push_back(rate_counters[i]->GetBitrateStats()); + std::map> flow_throughput_kbps; + for (RateCounterFilter* rate_counter : rate_counters) { + int flow_id = *rate_counter->flow_ids().begin(); + flow_throughput_kbps[flow_id] = rate_counter->GetBitrateStats(); + } + + std::map> flow_delay_ms; + for (PacketReceiver* receiver : receivers) { + int flow_id = *receiver->flow_ids().begin(); + flow_delay_ms[flow_id] = receiver->GetDelayStats(); + } + PrintResults(capacity_kbps, total_utilization.GetBitrateStats(), - choke.GetDelayStats(), flow_throughput_kbps); + flow_delay_ms, flow_throughput_kbps); for (VideoSource* source : sources) delete source; diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h index 2de5ac6cef..303aca528c 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h @@ -78,8 +78,14 @@ class BweTest { void PrintResults(double max_throughput_kbps, Stats throughput_kbps, - Stats delay_ms, - std::vector> flow_throughput_kbps); + int flow_id, + Stats flow_delay_ms, + Stats flow_throughput_kbps); + + void PrintResults(double max_throughput_kbps, + Stats throughput_kbps, + std::map> flow_delay_ms, + std::map> flow_throughput_kbps); void RunFairnessTest(BandwidthEstimatorType bwe_type, size_t num_media_flows, diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc index 1640137a6a..aa429d03e9 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc @@ -58,8 +58,9 @@ void PacketReceiver::RunFor(int64_t time_ms, Packets* in_out) { // We're treating the send time (from previous filter) as the arrival // time once packet reaches the estimator. int64_t arrival_time_ms = (media_packet->send_time_us() + 500) / 1000; - PlotDelay(arrival_time_ms, - (media_packet->creation_time_us() + 500) / 1000); + int64_t send_time_ms = (media_packet->creation_time_us() + 500) / 1000; + delay_stats_.Push(arrival_time_ms - send_time_ms); + PlotDelay(arrival_time_ms, send_time_ms); bwe_receiver_->ReceivePacket(arrival_time_ms, *media_packet); FeedbackPacket* fb = bwe_receiver_->GetFeedback(arrival_time_ms); @@ -85,6 +86,10 @@ void PacketReceiver::PlotDelay(int64_t arrival_time_ms, int64_t send_time_ms) { last_delay_plot_ms_ = arrival_time_ms; } } + +Stats PacketReceiver::GetDelayStats() const { + return delay_stats_; +} } // namespace bwe } // namespace testing } // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h index 6d94ea7a05..0589d66b3b 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h @@ -36,6 +36,8 @@ class PacketReceiver : public PacketProcessor { void LogStats(); + Stats GetDelayStats() const; + protected: void PlotDelay(int64_t arrival_time_ms, int64_t send_time_ms); @@ -43,6 +45,7 @@ class PacketReceiver : public PacketProcessor { std::string delay_log_prefix_; int64_t last_delay_plot_ms_; bool plot_delay_; + Stats delay_stats_; rtc::scoped_ptr bwe_receiver_; private: