From 09bdc1e5f5a932ef4a20957470c088f0dfc145a8 Mon Sep 17 00:00:00 2001 From: Stefan Holmer Date: Thu, 16 Apr 2015 20:20:36 +0200 Subject: [PATCH] Add a BWE fairness test. Also moves the BWE perf tests to webrtc_perf_tests for tracking. BUG=4548 R=pbos@webrtc.org Review URL: https://webrtc-codereview.appspot.com/45189004 Cr-Commit-Position: refs/heads/master@{#9022} --- webrtc/modules/modules.gyp | 28 +------- .../remote_bitrate_estimator.gypi | 35 ++++++++++ .../remote_bitrate_estimators_test.cc | 69 +++++++++++++++++-- .../test/bwe_test_framework_unittest.cc | 8 +-- webrtc/webrtc_tests.gypi | 5 +- 5 files changed, 107 insertions(+), 38 deletions(-) diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index 1bf8466ecc..dbf01878a5 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -60,6 +60,7 @@ 'audio_device' , 'audio_processing', 'bitrate_controller', + 'bwe_simulator', 'CNG', 'desktop_capture', 'iSACFix', @@ -216,33 +217,6 @@ 'remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc', 'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc', 'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h', - 'remote_bitrate_estimator/remote_bitrate_estimators_test.cc', - 'remote_bitrate_estimator/test/bwe.cc', - 'remote_bitrate_estimator/test/bwe.h', - 'remote_bitrate_estimator/test/bwe_test.cc', - 'remote_bitrate_estimator/test/bwe_test.h', - 'remote_bitrate_estimator/test/bwe_test_baselinefile.cc', - 'remote_bitrate_estimator/test/bwe_test_baselinefile.h', - 'remote_bitrate_estimator/test/bwe_test_fileutils.cc', - 'remote_bitrate_estimator/test/bwe_test_fileutils.h', - 'remote_bitrate_estimator/test/bwe_test_framework.cc', - 'remote_bitrate_estimator/test/bwe_test_framework.h', - 'remote_bitrate_estimator/test/bwe_test_framework_unittest.cc', - 'remote_bitrate_estimator/test/bwe_test_logging.cc', - 'remote_bitrate_estimator/test/bwe_test_logging.h', - 'remote_bitrate_estimator/test/packet_receiver.cc', - 'remote_bitrate_estimator/test/packet_receiver.h', - 'remote_bitrate_estimator/test/packet_sender.cc', - 'remote_bitrate_estimator/test/packet_sender.h', - 'remote_bitrate_estimator/test/packet.h', - 'remote_bitrate_estimator/test/estimators/nada.cc', - 'remote_bitrate_estimator/test/estimators/nada.h', - 'remote_bitrate_estimator/test/estimators/remb.cc', - 'remote_bitrate_estimator/test/estimators/remb.h', - 'remote_bitrate_estimator/test/estimators/send_side.cc', - 'remote_bitrate_estimator/test/estimators/send_side.h', - 'remote_bitrate_estimator/test/estimators/tcp.cc', - 'remote_bitrate_estimator/test/estimators/tcp.h', 'rtp_rtcp/source/mock/mock_rtp_payload_strategy.h', 'rtp_rtcp/source/byte_io_unittest.cc', 'rtp_rtcp/source/fec_receiver_unittest.cc', diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi index 21d3bd584f..c8a48c1a18 100644 --- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi +++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi @@ -55,6 +55,41 @@ 'conditions': [ ['include_tests==1', { 'targets': [ + { + 'target_name': 'bwe_simulator', + 'type': 'static_library', + 'dependencies': [ + '<(DEPTH)/testing/gtest.gyp:gtest', + ], + 'sources': [ + 'test/bwe.cc', + 'test/bwe.h', + 'test/bwe_test.cc', + 'test/bwe_test.h', + 'test/bwe_test_baselinefile.cc', + 'test/bwe_test_baselinefile.h', + 'test/bwe_test_fileutils.cc', + 'test/bwe_test_fileutils.h', + 'test/bwe_test_framework.cc', + 'test/bwe_test_framework.h', + 'test/bwe_test_framework_unittest.cc', + 'test/bwe_test_logging.cc', + 'test/bwe_test_logging.h', + 'test/packet_receiver.cc', + 'test/packet_receiver.h', + 'test/packet_sender.cc', + 'test/packet_sender.h', + 'test/packet.h', + 'test/estimators/nada.cc', + 'test/estimators/nada.h', + 'test/estimators/remb.cc', + 'test/estimators/remb.h', + 'test/estimators/send_side.cc', + 'test/estimators/send_side.h', + 'test/estimators/tcp.cc', + 'test/estimators/tcp.h', + ], + }, { 'target_name': 'bwe_tools_util', 'type': 'static_library', diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc index b64033ee45..837c4dd376 100644 --- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc +++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc @@ -241,8 +241,10 @@ class BweFeedbackTest BweFeedbackTest() : BweTest() {} virtual ~BweFeedbackTest() {} - void PrintResults(double max_throughput_kbps, Stats throughput_kbps, - Stats delay_ms) { + void PrintResults(double max_throughput_kbps, + Stats throughput_kbps, + Stats delay_ms, + std::vector> flow_throughput_kbps) { double utilization = throughput_kbps.GetMean() / max_throughput_kbps; webrtc::test::PrintResult("BwePerformance", GetTestName(), @@ -264,6 +266,16 @@ class BweFeedbackTest delay_ms.AsString(), "ms", false); + double fairness_index = 0.0; + double squared_bitrate_sum = 0.0; + for (Stats flow : flow_throughput_kbps) { + squared_bitrate_sum += flow.GetMean() * flow.GetMean(); + fairness_index += flow.GetMean(); + } + fairness_index *= fairness_index; + fairness_index /= flow_throughput_kbps.size() * squared_bitrate_sum; + webrtc::test::PrintResult("BwePerformance", GetTestName(), "Fairness", + fairness_index * 100, "%", false); } protected: @@ -294,7 +306,8 @@ TEST_P(BweFeedbackTest, Choke1000kbps500kbps1000kbps) { filter.SetCapacity(kHighCapacityKbps); RunFor(60 * 1000); PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0, - counter.GetBitrateStats(), filter.GetDelayStats()); + counter.GetBitrateStats(), filter.GetDelayStats(), + std::vector>()); } TEST_P(BweFeedbackTest, Choke200kbps30kbps200kbps) { @@ -314,7 +327,8 @@ TEST_P(BweFeedbackTest, Choke200kbps30kbps200kbps) { RunFor(60 * 1000); PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0, - counter.GetBitrateStats(), filter.GetDelayStats()); + counter.GetBitrateStats(), filter.GetDelayStats(), + std::vector>()); } TEST_P(BweFeedbackTest, Verizon4gDownlinkTest) { @@ -327,7 +341,7 @@ TEST_P(BweFeedbackTest, Verizon4gDownlinkTest) { ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx"))); RunFor(22 * 60 * 1000); PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(), - filter.GetDelayStats()); + filter.GetDelayStats(), std::vector>()); } // webrtc:3277 @@ -342,7 +356,50 @@ TEST_P(BweFeedbackTest, DISABLED_GoogleWifiTrace3Mbps) { ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx"))); RunFor(300 * 1000); PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(), - filter.GetDelayStats()); + filter.GetDelayStats(), std::vector>()); +} + +TEST_P(BweFeedbackTest, PacedSelfFairnessTest) { + srand(Clock::GetRealTimeClock()->TimeInMicroseconds()); + const int kAllFlowIds[] = {0, 1, 2, 3}; + const size_t kNumFlows = sizeof(kAllFlowIds) / sizeof(kAllFlowIds[0]); + rtc::scoped_ptr sources[kNumFlows]; + rtc::scoped_ptr senders[kNumFlows]; + + for (size_t i = 0; i < kNumFlows; ++i) { + // Streams started woth ramdp, pffsets tp give them different advantage when + // competing for the bandwidth. + sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0, + i * (rand() % 40000))); + senders[i].reset( + new PacedVideoSender(&uplink_, sources[i].get(), GetParam())); + } + + ChokeFilter choke(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows)); + choke.SetCapacity(3000); + choke.SetMaxDelay(1000); + + rtc::scoped_ptr rate_counters[kNumFlows]; + for (size_t i = 0; i < kNumFlows; ++i) { + rate_counters[i].reset(new RateCounterFilter( + &uplink_, CreateFlowIds(&kAllFlowIds[i], 1), "receiver_input")); + } + + RateCounterFilter total_utilization( + &uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "total_utilization"); + + rtc::scoped_ptr receivers[kNumFlows]; + for (size_t i = 0; i < kNumFlows; ++i) { + receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], GetParam(), + i == 0, false)); + } + RunFor(15 * 60 * 1000); + + std::vector> flow_throughput_kbps; + for (size_t i = 0; i < kNumFlows; ++i) + flow_throughput_kbps.push_back(rate_counters[i]->GetBitrateStats()); + PrintResults(3000, total_utilization.GetBitrateStats(), choke.GetDelayStats(), + flow_throughput_kbps); } } // namespace bwe } // namespace testing diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc index 1b1117d4f8..b92b39e84a 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc @@ -615,7 +615,7 @@ class BweTestFramework_ChokeFilterTest : public ::testing::Test { void TestChoke(PacketProcessor* filter, int64_t run_for_ms, uint32_t packets_to_generate, - uint32_t expected_kbit_transmitted) { + size_t expected_kbit_transmitted) { // Generate a bunch of packets, apply choke, verify output is ordered. Packets packets; RTPHeader header; @@ -634,7 +634,7 @@ class BweTestFramework_ChokeFilterTest : public ::testing::Test { ASSERT_TRUE(IsSequenceNumberSorted(output_packets_)); // Sum up the transmitted bytes up until the current time. - uint32_t bytes_transmitted = 0; + size_t bytes_transmitted = 0; while (!output_packets_.empty()) { const Packet* packet = output_packets_.front(); if (packet->send_time_us() > now_ms_ * 1000) { @@ -769,7 +769,7 @@ void TestVideoSender(VideoSender* sender, int64_t run_for_ms, uint32_t expected_packets, uint32_t expected_payload_size, - uint32_t expected_total_payload_size) { + size_t expected_total_payload_size) { assert(sender); Packets packets; sender->RunFor(run_for_ms, &packets); @@ -777,7 +777,7 @@ void TestVideoSender(VideoSender* sender, ASSERT_TRUE(IsSequenceNumberSorted(packets)); EXPECT_EQ(expected_packets, packets.size()); int64_t send_time_us = -1; - uint32_t total_payload_size = 0; + size_t total_payload_size = 0; uint32_t absolute_send_time = 0; uint32_t absolute_send_time_wraps = 0; uint32_t rtp_timestamp = 0; diff --git a/webrtc/webrtc_tests.gypi b/webrtc/webrtc_tests.gypi index c0b3c038f0..08f3ae8ff2 100644 --- a/webrtc/webrtc_tests.gypi +++ b/webrtc/webrtc_tests.gypi @@ -178,6 +178,8 @@ 'type': '<(gtest_target_type)', 'sources': [ 'modules/audio_coding/neteq/test/neteq_performance_unittest.cc', + 'modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc', + 'tools/agc/agc_manager_integrationtest.cc', 'video/call_perf_tests.cc', 'video/full_stack.cc', @@ -190,7 +192,8 @@ '<(webrtc_root)/modules/modules.gyp:video_capture', '<(webrtc_root)/test/test.gyp:channel_transport', '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine', - 'modules/modules.gyp:neteq_test_support', # Needed by neteq_performance_unittest. + 'modules/modules.gyp:neteq_test_support', + 'modules/modules.gyp:bwe_simulator', 'modules/modules.gyp:rtp_rtcp', 'test/test.gyp:test_main', 'test/webrtc_test_common.gyp:webrtc_test_common',