terelius 568c9e72d1 New simulators to test BWE at low bitrates (15-50kbps range).
I've tested both the old and the new delay estimators and they all work ok at 15 kbps. The new estimators are a bit slower to adapt down so the maximum delay is a bit higher, especially at lower bitrates.

None of the estimators work at 10 kbps, but that is likely because the bitrate controller is configured to never go below 10 kbps. This means that it's impossible to empty the queues after a capacity drop to 10kbps (regardless of what the estimators do), so the delay stays high until the capacity increases.

BUG=webrtc:7022

Review-Url: https://codereview.webrtc.org/2644463009
Cr-Commit-Position: refs/heads/master@{#16233}
2017-01-24 10:03:31 +00:00

495 lines
17 KiB
C++

/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include "webrtc/base/constructormagic.h"
#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h"
#include "webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h"
#include "webrtc/modules/remote_bitrate_estimator/test/packet_sender.h"
#include "webrtc/test/gtest.h"
#include "webrtc/test/testsupport/fileutils.h"
namespace webrtc {
namespace testing {
namespace bwe {
// This test fixture is used to instantiate tests running with adaptive video
// senders.
class BweSimulation : public BweTest,
public ::testing::TestWithParam<BandwidthEstimatorType> {
public:
BweSimulation()
: BweTest(), random_(Clock::GetRealTimeClock()->TimeInMicroseconds()) {}
virtual ~BweSimulation() {}
protected:
void SetUp() override {
BweTest::SetUp();
VerboseLogging(true);
}
Random random_;
private:
RTC_DISALLOW_COPY_AND_ASSIGN(BweSimulation);
};
INSTANTIATE_TEST_CASE_P(VideoSendersTest,
BweSimulation,
::testing::Values(kRembEstimator,
kSendSideEstimator,
kNadaEstimator));
TEST_P(BweSimulation, SprintUplinkTest) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
VideoSender sender(&uplink_, &source, GetParam());
RateCounterFilter counter1(&uplink_, 0, "sender_output",
bwe_names[GetParam()]);
TraceBasedDeliveryFilter filter(&uplink_, 0, "link_capacity");
RateCounterFilter counter2(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
ASSERT_TRUE(filter.Init(test::ResourcePath("sprint-uplink", "rx")));
RunFor(60 * 1000);
}
TEST_P(BweSimulation, Verizon4gDownlinkTest) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
VideoSender sender(&downlink_, &source, GetParam());
RateCounterFilter counter1(&downlink_, 0, "sender_output",
bwe_names[GetParam()] + "_up");
TraceBasedDeliveryFilter filter(&downlink_, 0, "link_capacity");
RateCounterFilter counter2(&downlink_, 0, "Receiver",
bwe_names[GetParam()] + "_down");
PacketReceiver receiver(&downlink_, 0, GetParam(), true, true);
ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx")));
RunFor(22 * 60 * 1000);
}
TEST_P(BweSimulation, Choke1000kbps500kbps1000kbpsBiDirectional) {
const int kFlowIds[] = {0, 1};
const size_t kNumFlows = sizeof(kFlowIds) / sizeof(kFlowIds[0]);
AdaptiveVideoSource source(kFlowIds[0], 30, 300, 0, 0);
VideoSender sender(&uplink_, &source, GetParam());
ChokeFilter choke(&uplink_, kFlowIds[0]);
RateCounterFilter counter(&uplink_, kFlowIds[0], "Receiver_0",
bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, kFlowIds[0], GetParam(), true, false);
AdaptiveVideoSource source2(kFlowIds[1], 30, 300, 0, 0);
VideoSender sender2(&downlink_, &source2, GetParam());
ChokeFilter choke2(&downlink_, kFlowIds[1]);
DelayFilter delay(&downlink_, CreateFlowIds(kFlowIds, kNumFlows));
RateCounterFilter counter2(&downlink_, kFlowIds[1], "Receiver_1",
bwe_names[GetParam()]);
PacketReceiver receiver2(&downlink_, kFlowIds[1], GetParam(), true, false);
choke2.set_capacity_kbps(500);
delay.SetOneWayDelayMs(0);
choke.set_capacity_kbps(1000);
choke.set_max_delay_ms(500);
RunFor(60 * 1000);
choke.set_capacity_kbps(500);
RunFor(60 * 1000);
choke.set_capacity_kbps(1000);
RunFor(60 * 1000);
}
TEST_P(BweSimulation, Choke1000kbps500kbps1000kbps) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
VideoSender sender(&uplink_, &source, GetParam());
ChokeFilter choke(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, false);
choke.set_capacity_kbps(1000);
choke.set_max_delay_ms(500);
RunFor(60 * 1000);
choke.set_capacity_kbps(500);
RunFor(60 * 1000);
choke.set_capacity_kbps(1000);
RunFor(60 * 1000);
}
TEST_P(BweSimulation, PacerChoke1000kbps500kbps1000kbps) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
PacedVideoSender sender(&uplink_, &source, GetParam());
ChokeFilter filter(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
filter.set_capacity_kbps(1000);
filter.set_max_delay_ms(500);
RunFor(60 * 1000);
filter.set_capacity_kbps(500);
RunFor(60 * 1000);
filter.set_capacity_kbps(1000);
RunFor(60 * 1000);
}
TEST_P(BweSimulation, PacerChoke10000kbps) {
PeriodicKeyFrameSource source(0, 30, 300, 0, 0, 1000);
PacedVideoSender sender(&uplink_, &source, GetParam());
ChokeFilter filter(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
filter.set_capacity_kbps(10000);
filter.set_max_delay_ms(500);
RunFor(60 * 1000);
}
TEST_P(BweSimulation, PacerChoke200kbps30kbps200kbps) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
PacedVideoSender sender(&uplink_, &source, GetParam());
ChokeFilter filter(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
filter.set_capacity_kbps(200);
filter.set_max_delay_ms(500);
RunFor(60 * 1000);
filter.set_capacity_kbps(30);
RunFor(60 * 1000);
filter.set_capacity_kbps(200);
RunFor(60 * 1000);
}
TEST_P(BweSimulation, Choke200kbps30kbps200kbps) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
VideoSender sender(&uplink_, &source, GetParam());
ChokeFilter filter(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
filter.set_capacity_kbps(200);
filter.set_max_delay_ms(500);
RunFor(60 * 1000);
filter.set_capacity_kbps(30);
RunFor(60 * 1000);
filter.set_capacity_kbps(200);
RunFor(60 * 1000);
}
TEST_P(BweSimulation, PacerChoke50kbps15kbps50kbps) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
PacedVideoSender sender(&uplink_, &source, GetParam());
ChokeFilter filter(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
filter.set_capacity_kbps(50);
filter.set_max_delay_ms(500);
RunFor(60 * 1000);
filter.set_capacity_kbps(15);
RunFor(60 * 1000);
filter.set_capacity_kbps(50);
RunFor(60 * 1000);
}
TEST_P(BweSimulation, Choke50kbps15kbps50kbps) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
VideoSender sender(&uplink_, &source, GetParam());
ChokeFilter filter(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
filter.set_capacity_kbps(50);
filter.set_max_delay_ms(500);
RunFor(60 * 1000);
filter.set_capacity_kbps(15);
RunFor(60 * 1000);
filter.set_capacity_kbps(50);
RunFor(60 * 1000);
}
TEST_P(BweSimulation, GoogleWifiTrace3Mbps) {
AdaptiveVideoSource source(0, 30, 300, 0, 0);
VideoSender sender(&uplink_, &source, GetParam());
RateCounterFilter counter1(&uplink_, 0, "sender_output",
bwe_names[GetParam()]);
TraceBasedDeliveryFilter filter(&uplink_, 0, "link_capacity");
filter.set_max_delay_ms(500);
RateCounterFilter counter2(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx")));
RunFor(300 * 1000);
}
TEST_P(BweSimulation, LinearIncreasingCapacity) {
PeriodicKeyFrameSource source(0, 30, 300, 0, 0, 1000000);
PacedVideoSender sender(&uplink_, &source, GetParam());
ChokeFilter filter(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
filter.set_max_delay_ms(500);
const int kStartingCapacityKbps = 150;
const int kEndingCapacityKbps = 1500;
const int kStepKbps = 5;
const int kStepTimeMs = 1000;
for (int i = kStartingCapacityKbps; i <= kEndingCapacityKbps;
i += kStepKbps) {
filter.set_capacity_kbps(i);
RunFor(kStepTimeMs);
}
}
TEST_P(BweSimulation, LinearDecreasingCapacity) {
PeriodicKeyFrameSource source(0, 30, 300, 0, 0, 1000000);
PacedVideoSender sender(&uplink_, &source, GetParam());
ChokeFilter filter(&uplink_, 0);
RateCounterFilter counter(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
filter.set_max_delay_ms(500);
const int kStartingCapacityKbps = 1500;
const int kEndingCapacityKbps = 150;
const int kStepKbps = -5;
const int kStepTimeMs = 1000;
for (int i = kStartingCapacityKbps; i >= kEndingCapacityKbps;
i += kStepKbps) {
filter.set_capacity_kbps(i);
RunFor(kStepTimeMs);
}
}
TEST_P(BweSimulation, PacerGoogleWifiTrace3Mbps) {
PeriodicKeyFrameSource source(0, 30, 300, 0, 0, 1000);
PacedVideoSender sender(&uplink_, &source, GetParam());
RateCounterFilter counter1(&uplink_, 0, "sender_output",
bwe_names[GetParam()]);
TraceBasedDeliveryFilter filter(&uplink_, 0, "link_capacity");
filter.set_max_delay_ms(500);
RateCounterFilter counter2(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx")));
RunFor(300 * 1000);
}
TEST_P(BweSimulation, PacerGoogleWifiTrace3MbpsLowFramerate) {
PeriodicKeyFrameSource source(0, 5, 300, 0, 0, 1000);
PacedVideoSender sender(&uplink_, &source, GetParam());
RateCounterFilter counter1(&uplink_, 0, "sender_output",
bwe_names[GetParam()]);
TraceBasedDeliveryFilter filter(&uplink_, 0, "link_capacity");
filter.set_max_delay_ms(500);
RateCounterFilter counter2(&uplink_, 0, "Receiver", bwe_names[GetParam()]);
PacketReceiver receiver(&uplink_, 0, GetParam(), true, true);
ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx")));
RunFor(300 * 1000);
}
TEST_P(BweSimulation, SelfFairnessTest) {
Random prng(Clock::GetRealTimeClock()->TimeInMicroseconds());
const int kAllFlowIds[] = {0, 1, 2, 3};
const size_t kNumFlows = sizeof(kAllFlowIds) / sizeof(kAllFlowIds[0]);
std::unique_ptr<VideoSource> sources[kNumFlows];
std::unique_ptr<VideoSender> senders[kNumFlows];
for (size_t i = 0; i < kNumFlows; ++i) {
// Streams started 20 seconds apart to give them different advantage when
// competing for the bandwidth.
sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0,
i * prng.Rand(39999)));
senders[i].reset(new VideoSender(&uplink_, sources[i].get(), GetParam()));
}
ChokeFilter choke(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows));
choke.set_capacity_kbps(1000);
std::unique_ptr<RateCounterFilter> rate_counters[kNumFlows];
for (size_t i = 0; i < kNumFlows; ++i) {
rate_counters[i].reset(
new RateCounterFilter(&uplink_, CreateFlowIds(&kAllFlowIds[i], 1),
"Receiver", bwe_names[GetParam()]));
}
RateCounterFilter total_utilization(
&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "total_utilization",
"Total_link_utilization");
std::unique_ptr<PacketReceiver> receivers[kNumFlows];
for (size_t i = 0; i < kNumFlows; ++i) {
receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], GetParam(),
i == 0, false));
}
RunFor(30 * 60 * 1000);
}
TEST_P(BweSimulation, PacedSelfFairness50msTest) {
const int64_t kAverageOffsetMs = 20 * 1000;
const int kNumRmcatFlows = 4;
int64_t offsets_ms[kNumRmcatFlows];
offsets_ms[0] = random_.Rand(2 * kAverageOffsetMs);
for (int i = 1; i < kNumRmcatFlows; ++i) {
offsets_ms[i] = offsets_ms[i - 1] + random_.Rand(2 * kAverageOffsetMs);
}
RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 1000, 3000, 50, 50, 0,
offsets_ms);
}
TEST_P(BweSimulation, PacedSelfFairness500msTest) {
const int64_t kAverageOffsetMs = 20 * 1000;
const int kNumRmcatFlows = 4;
int64_t offsets_ms[kNumRmcatFlows];
offsets_ms[0] = random_.Rand(2 * kAverageOffsetMs);
for (int i = 1; i < kNumRmcatFlows; ++i) {
offsets_ms[i] = offsets_ms[i - 1] + random_.Rand(2 * kAverageOffsetMs);
}
RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 1000, 3000, 500, 50, 0,
offsets_ms);
}
TEST_P(BweSimulation, PacedSelfFairness1000msTest) {
const int64_t kAverageOffsetMs = 20 * 1000;
const int kNumRmcatFlows = 4;
int64_t offsets_ms[kNumRmcatFlows];
offsets_ms[0] = random_.Rand(2 * kAverageOffsetMs);
for (int i = 1; i < kNumRmcatFlows; ++i) {
offsets_ms[i] = offsets_ms[i - 1] + random_.Rand(2 * kAverageOffsetMs);
}
RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 1000, 50, 0, offsets_ms);
}
TEST_P(BweSimulation, TcpFairness50msTest) {
const int64_t kAverageOffsetMs = 20 * 1000;
int64_t offset_ms[] = {random_.Rand(2 * kAverageOffsetMs), 0};
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 50, 50, 0, offset_ms);
}
TEST_P(BweSimulation, TcpFairness500msTest) {
const int64_t kAverageOffsetMs = 20 * 1000;
int64_t offset_ms[] = {random_.Rand(2 * kAverageOffsetMs), 0};
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 500, 50, 0, offset_ms);
}
TEST_P(BweSimulation, TcpFairness1000msTest) {
const int kAverageOffsetMs = 20 * 1000;
int64_t offset_ms[] = {random_.Rand(2 * kAverageOffsetMs), 0};
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 1000, 50, 0, offset_ms);
}
// The following test cases begin with "Evaluation" as a reference to the
// Internet draft https://tools.ietf.org/html/draft-ietf-rmcat-eval-test-01.
TEST_P(BweSimulation, Evaluation1) {
RunVariableCapacity1SingleFlow(GetParam());
}
TEST_P(BweSimulation, Evaluation2) {
const size_t kNumFlows = 2;
RunVariableCapacity2MultipleFlows(GetParam(), kNumFlows);
}
TEST_P(BweSimulation, Evaluation3) {
RunBidirectionalFlow(GetParam());
}
TEST_P(BweSimulation, Evaluation4) {
RunSelfFairness(GetParam());
}
TEST_P(BweSimulation, Evaluation5) {
RunRoundTripTimeFairness(GetParam());
}
TEST_P(BweSimulation, Evaluation6) {
RunLongTcpFairness(GetParam());
}
// Different calls to the Evaluation7 will create the same FileSizes
// and StartingTimes as long as the seeds remain unchanged. This is essential
// when calling it with multiple estimators for comparison purposes.
TEST_P(BweSimulation, Evaluation7) {
const int kNumTcpFiles = 10;
RunMultipleShortTcpFairness(GetParam(),
BweTest::GetFileSizesBytes(kNumTcpFiles),
BweTest::GetStartingTimesMs(kNumTcpFiles));
}
TEST_P(BweSimulation, Evaluation8) {
RunPauseResumeFlows(GetParam());
}
// Following test cases begin with "GccComparison" run the
// evaluation test cases for both GCC and other calling RMCAT.
TEST_P(BweSimulation, GccComparison1) {
RunVariableCapacity1SingleFlow(GetParam());
BweTest gcc_test(false);
gcc_test.RunVariableCapacity1SingleFlow(kSendSideEstimator);
}
TEST_P(BweSimulation, GccComparison2) {
const size_t kNumFlows = 2;
RunVariableCapacity2MultipleFlows(GetParam(), kNumFlows);
BweTest gcc_test(false);
gcc_test.RunVariableCapacity2MultipleFlows(kSendSideEstimator, kNumFlows);
}
TEST_P(BweSimulation, GccComparison3) {
RunBidirectionalFlow(GetParam());
BweTest gcc_test(false);
gcc_test.RunBidirectionalFlow(kSendSideEstimator);
}
TEST_P(BweSimulation, GccComparison4) {
RunSelfFairness(GetParam());
BweTest gcc_test(false);
gcc_test.RunSelfFairness(GetParam());
}
TEST_P(BweSimulation, GccComparison5) {
RunRoundTripTimeFairness(GetParam());
BweTest gcc_test(false);
gcc_test.RunRoundTripTimeFairness(kSendSideEstimator);
}
TEST_P(BweSimulation, GccComparison6) {
RunLongTcpFairness(GetParam());
BweTest gcc_test(false);
gcc_test.RunLongTcpFairness(kSendSideEstimator);
}
TEST_P(BweSimulation, GccComparison7) {
const int kNumTcpFiles = 10;
std::vector<int> tcp_file_sizes_bytes =
BweTest::GetFileSizesBytes(kNumTcpFiles);
std::vector<int64_t> tcp_starting_times_ms =
BweTest::GetStartingTimesMs(kNumTcpFiles);
RunMultipleShortTcpFairness(GetParam(), tcp_file_sizes_bytes,
tcp_starting_times_ms);
BweTest gcc_test(false);
gcc_test.RunMultipleShortTcpFairness(kSendSideEstimator, tcp_file_sizes_bytes,
tcp_starting_times_ms);
}
TEST_P(BweSimulation, GccComparison8) {
RunPauseResumeFlows(GetParam());
BweTest gcc_test(false);
gcc_test.RunPauseResumeFlows(kSendSideEstimator);
}
TEST_P(BweSimulation, GccComparisonChoke) {
int array[] = {1000, 500, 1000};
std::vector<int> capacities_kbps(array, array + 3);
RunChoke(GetParam(), capacities_kbps);
BweTest gcc_test(false);
gcc_test.RunChoke(kSendSideEstimator, capacities_kbps);
}
} // namespace bwe
} // namespace testing
} // namespace webrtc