Evaluation test cases.
Implemented according to: http://datatracker.ietf.org/doc/draft-ietf-rmcat-eval-test/ Added tests 5.1 - 5.8. Added GccComparison functions. Modified SelfFairness test. R=stefan@webrtc.org Review URL: https://codereview.webrtc.org/1237393002 . Cr-Commit-Position: refs/heads/master@{#9623}
This commit is contained in:
Cesar Magalhaes
parent
66f438f8c3
commit
59adf34d84
@ -16,7 +16,6 @@
|
||||
#include "webrtc/modules/remote_bitrate_estimator/test/packet_sender.h"
|
||||
#include "webrtc/test/testsupport/fileutils.h"
|
||||
|
||||
using std::string;
|
||||
|
||||
namespace webrtc {
|
||||
namespace testing {
|
||||
@ -27,7 +26,8 @@ namespace bwe {
|
||||
class BweSimulation : public BweTest,
|
||||
public ::testing::TestWithParam<BandwidthEstimatorType> {
|
||||
public:
|
||||
BweSimulation() : BweTest() {}
|
||||
BweSimulation()
|
||||
: BweTest(), random_(Clock::GetRealTimeClock()->TimeInMicroseconds()) {}
|
||||
virtual ~BweSimulation() {}
|
||||
|
||||
protected:
|
||||
@ -36,6 +36,8 @@ class BweSimulation : public BweTest,
|
||||
VerboseLogging(true);
|
||||
}
|
||||
|
||||
Random random_;
|
||||
|
||||
private:
|
||||
DISALLOW_COPY_AND_ASSIGN(BweSimulation);
|
||||
};
|
||||
@ -269,36 +271,173 @@ TEST_P(BweSimulation, SelfFairnessTest) {
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, PacedSelfFairness50msTest) {
|
||||
srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
|
||||
RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 50);
|
||||
const int64_t kAverageOffsetMs = 20 * 1000;
|
||||
const int kNumRmcatFlows = 4;
|
||||
int64_t offsets_ms[kNumRmcatFlows];
|
||||
offsets_ms[0] = random_.Rand(0, 2 * kAverageOffsetMs);
|
||||
for (int i = 1; i < kNumRmcatFlows; ++i) {
|
||||
offsets_ms[i] = offsets_ms[i - 1] + random_.Rand(0, 2 * kAverageOffsetMs);
|
||||
}
|
||||
RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 1000, 3000, 50, 50, 0,
|
||||
offsets_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, PacedSelfFairness500msTest) {
|
||||
srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
|
||||
RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 500);
|
||||
const int64_t kAverageOffsetMs = 20 * 1000;
|
||||
const int kNumRmcatFlows = 4;
|
||||
int64_t offsets_ms[kNumRmcatFlows];
|
||||
offsets_ms[0] = random_.Rand(0, 2 * kAverageOffsetMs);
|
||||
for (int i = 1; i < kNumRmcatFlows; ++i) {
|
||||
offsets_ms[i] = offsets_ms[i - 1] + random_.Rand(0, 2 * kAverageOffsetMs);
|
||||
}
|
||||
RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 1000, 3000, 500, 50, 0,
|
||||
offsets_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, PacedSelfFairness1000msTest) {
|
||||
srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
|
||||
RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 1000);
|
||||
const int64_t kAverageOffsetMs = 20 * 1000;
|
||||
const int kNumRmcatFlows = 4;
|
||||
int64_t offsets_ms[kNumRmcatFlows];
|
||||
offsets_ms[0] = random_.Rand(0, 2 * kAverageOffsetMs);
|
||||
for (int i = 1; i < kNumRmcatFlows; ++i) {
|
||||
offsets_ms[i] = offsets_ms[i - 1] + random_.Rand(0, 2 * kAverageOffsetMs);
|
||||
}
|
||||
RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 1000, 50, 0, offsets_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, TcpFairness50msTest) {
|
||||
srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
|
||||
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 50);
|
||||
const int64_t kAverageOffsetMs = 20 * 1000;
|
||||
int64_t offset_ms[] = {random_.Rand(0, 2 * kAverageOffsetMs), 0};
|
||||
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 50, 50, 0, offset_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, TcpFairness500msTest) {
|
||||
srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
|
||||
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 500);
|
||||
const int64_t kAverageOffsetMs = 20 * 1000;
|
||||
int64_t offset_ms[] = {random_.Rand(0, 2 * kAverageOffsetMs), 0};
|
||||
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 500, 50, 0, offset_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, TcpFairness1000msTest) {
|
||||
srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
|
||||
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 1000);
|
||||
const int kAverageOffsetMs = 20 * 1000;
|
||||
int64_t offset_ms[] = {random_.Rand(0, 2 * kAverageOffsetMs), 0};
|
||||
RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 1000, 50, 0, offset_ms);
|
||||
}
|
||||
|
||||
// The following test cases begin with "Evaluation" as a referrence to the
|
||||
// Internet draft https://tools.ietf.org/html/draft-ietf-rmcat-eval-test-01.
|
||||
|
||||
TEST_P(BweSimulation, Evaluation1) {
|
||||
RunVariableCapacity1SingleFlow(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, Evaluation2) {
|
||||
const size_t kNumFlows = 2;
|
||||
RunVariableCapacity2MultipleFlows(GetParam(), kNumFlows);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, Evaluation3) {
|
||||
RunBidirectionalFlow(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, Evaluation4) {
|
||||
RunSelfFairness(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, Evaluation5) {
|
||||
RunRoundTripTimeFairness(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, Evaluation6) {
|
||||
RunLongTcpFairness(GetParam());
|
||||
}
|
||||
|
||||
// Different calls to the Evaluation7 will create the same FileSizes
|
||||
// and StartingTimes as long as the seeds remain unchanged. This is essential
|
||||
// when calling it with multiple estimators for comparison purposes.
|
||||
TEST_P(BweSimulation, Evaluation7) {
|
||||
const int kNumTcpFiles = 10;
|
||||
RunMultipleShortTcpFairness(GetParam(),
|
||||
BweTest::GetFileSizesBytes(kNumTcpFiles),
|
||||
BweTest::GetStartingTimesMs(kNumTcpFiles));
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, Evaluation8) {
|
||||
RunPauseResumeFlows(GetParam());
|
||||
}
|
||||
|
||||
// Following test cases begin with "GccComparison" run the
|
||||
// evaluation test cases for both GCC and other calling RMCAT.
|
||||
|
||||
TEST_P(BweSimulation, GccComparison1) {
|
||||
RunVariableCapacity1SingleFlow(GetParam());
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunVariableCapacity1SingleFlow(kFullSendSideEstimator);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, GccComparison2) {
|
||||
const size_t kNumFlows = 2;
|
||||
RunVariableCapacity2MultipleFlows(GetParam(), kNumFlows);
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunVariableCapacity2MultipleFlows(kFullSendSideEstimator, kNumFlows);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, GccComparison3) {
|
||||
RunBidirectionalFlow(GetParam());
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunBidirectionalFlow(kFullSendSideEstimator);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, GccComparison4) {
|
||||
RunSelfFairness(GetParam());
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunSelfFairness(GetParam());
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, GccComparison5) {
|
||||
RunRoundTripTimeFairness(GetParam());
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunRoundTripTimeFairness(kFullSendSideEstimator);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, GccComparison6) {
|
||||
RunLongTcpFairness(GetParam());
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunLongTcpFairness(kFullSendSideEstimator);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, GccComparison7) {
|
||||
const int kNumTcpFiles = 10;
|
||||
|
||||
std::vector<int> tcp_file_sizes_bytes =
|
||||
BweTest::GetFileSizesBytes(kNumTcpFiles);
|
||||
std::vector<int64_t> tcp_starting_times_ms =
|
||||
BweTest::GetStartingTimesMs(kNumTcpFiles);
|
||||
|
||||
RunMultipleShortTcpFairness(GetParam(), tcp_file_sizes_bytes,
|
||||
tcp_starting_times_ms);
|
||||
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunMultipleShortTcpFairness(
|
||||
kFullSendSideEstimator, tcp_file_sizes_bytes, tcp_starting_times_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, GccComparison8) {
|
||||
RunPauseResumeFlows(GetParam());
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunPauseResumeFlows(kFullSendSideEstimator);
|
||||
}
|
||||
|
||||
TEST_P(BweSimulation, GccComparisonChoke) {
|
||||
int array[] = {1000, 500, 1000};
|
||||
std::vector<int> capacities_kbps(array, array + 3);
|
||||
RunChoke(GetParam(), capacities_kbps);
|
||||
|
||||
BweTest gcc_test(false);
|
||||
gcc_test.RunChoke(kFullSendSideEstimator, capacities_kbps);
|
||||
}
|
||||
|
||||
#endif // BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
|
||||
} // namespace bwe
|
||||
} // namespace testing
|
||||
} // namespace webrtc
|
||||
|
||||
|
||||
@ -348,27 +348,84 @@ TEST_P(BweFeedbackTest, GoogleWifiTrace3Mbps) {
|
||||
}
|
||||
|
||||
TEST_P(BweFeedbackTest, PacedSelfFairness50msTest) {
|
||||
RunFairnessTest(GetParam(), 4, 0, 300, 3000, 50);
|
||||
int64_t kRttMs = 100;
|
||||
int64_t kMaxJitterMs = 15;
|
||||
|
||||
const int kNumRmcatFlows = 4;
|
||||
int64_t offset_ms[kNumRmcatFlows];
|
||||
for (int i = 0; i < kNumRmcatFlows; ++i) {
|
||||
offset_ms[i] = std::max(0, 5000 * i + rand() % 2001 - 1000);
|
||||
}
|
||||
|
||||
RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 300, 3000, 50, kRttMs,
|
||||
kMaxJitterMs, offset_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweFeedbackTest, PacedSelfFairness500msTest) {
|
||||
RunFairnessTest(GetParam(), 4, 0, 300, 3000, 500);
|
||||
int64_t kRttMs = 100;
|
||||
int64_t kMaxJitterMs = 15;
|
||||
|
||||
const int kNumRmcatFlows = 4;
|
||||
int64_t offset_ms[kNumRmcatFlows];
|
||||
for (int i = 0; i < kNumRmcatFlows; ++i) {
|
||||
offset_ms[i] = std::max(0, 5000 * i + rand() % 2001 - 1000);
|
||||
}
|
||||
|
||||
RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 300, 3000, 500, kRttMs,
|
||||
kMaxJitterMs, offset_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweFeedbackTest, PacedSelfFairness1000msTest) {
|
||||
RunFairnessTest(GetParam(), 4, 0, 300, 3000, 1000);
|
||||
int64_t kRttMs = 100;
|
||||
int64_t kMaxJitterMs = 15;
|
||||
|
||||
const int kNumRmcatFlows = 4;
|
||||
int64_t offset_ms[kNumRmcatFlows];
|
||||
for (int i = 0; i < kNumRmcatFlows; ++i) {
|
||||
offset_ms[i] = std::max(0, 5000 * i + rand() % 2001 - 1000);
|
||||
}
|
||||
|
||||
RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 300, 3000, 1000, kRttMs,
|
||||
kMaxJitterMs, offset_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweFeedbackTest, TcpFairness50msTest) {
|
||||
RunFairnessTest(GetParam(), 1, 1, 300, 2000, 50);
|
||||
int64_t kRttMs = 100;
|
||||
int64_t kMaxJitterMs = 15;
|
||||
|
||||
int64_t offset_ms[2]; // One TCP, one RMCAT flow.
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
offset_ms[i] = std::max(0, 5000 * i + rand() % 2001 - 1000);
|
||||
}
|
||||
|
||||
RunFairnessTest(GetParam(), 1, 1, 300, 2000, 50, kRttMs, kMaxJitterMs,
|
||||
offset_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweFeedbackTest, TcpFairness500msTest) {
|
||||
RunFairnessTest(GetParam(), 1, 1, 300, 2000, 500);
|
||||
int64_t kRttMs = 100;
|
||||
int64_t kMaxJitterMs = 15;
|
||||
|
||||
int64_t offset_ms[2]; // One TCP, one RMCAT flow.
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
offset_ms[i] = std::max(0, 5000 * i + rand() % 2001 - 1000);
|
||||
}
|
||||
|
||||
RunFairnessTest(GetParam(), 1, 1, 300, 2000, 500, kRttMs, kMaxJitterMs,
|
||||
offset_ms);
|
||||
}
|
||||
|
||||
TEST_P(BweFeedbackTest, TcpFairness1000msTest) {
|
||||
RunFairnessTest(GetParam(), 1, 1, 300, 2000, 1000);
|
||||
int64_t kRttMs = 100;
|
||||
int64_t kMaxJitterMs = 15;
|
||||
|
||||
int64_t offset_ms[2]; // One TCP, one RMCAT flow.
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
offset_ms[i] = std::max(0, 5000 * i + rand() % 2001 - 1000);
|
||||
}
|
||||
|
||||
RunFairnessTest(GetParam(), 1, 1, 300, 2000, 1000, kRttMs, kMaxJitterMs,
|
||||
offset_ms);
|
||||
}
|
||||
} // namespace bwe
|
||||
} // namespace testing
|
||||
|
||||
@ -10,10 +10,13 @@
|
||||
|
||||
#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h"
|
||||
|
||||
#include <sstream>
|
||||
|
||||
#include "webrtc/base/common.h"
|
||||
#include "webrtc/base/scoped_ptr.h"
|
||||
#include "webrtc/modules/interface/module_common_types.h"
|
||||
#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h"
|
||||
#include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h"
|
||||
#include "webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h"
|
||||
#include "webrtc/modules/remote_bitrate_estimator/test/packet_sender.h"
|
||||
#include "webrtc/system_wrappers/interface/clock.h"
|
||||
@ -89,8 +92,15 @@ void PacketProcessorRunner::QueuePackets(Packets* batch,
|
||||
batch->merge(to_transfer, DereferencingComparator<Packet>);
|
||||
}
|
||||
|
||||
BweTest::BweTest()
|
||||
: run_time_ms_(0), time_now_ms_(-1), simulation_interval_ms_(-1) {
|
||||
// Plot link capacity by default.
|
||||
BweTest::BweTest() : BweTest(true) {
|
||||
}
|
||||
|
||||
BweTest::BweTest(bool plot_capacity)
|
||||
: run_time_ms_(0),
|
||||
time_now_ms_(-1),
|
||||
simulation_interval_ms_(-1),
|
||||
plot_total_available_capacity_(plot_capacity) {
|
||||
links_.push_back(&uplink_);
|
||||
links_.push_back(&downlink_);
|
||||
}
|
||||
@ -234,8 +244,27 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
size_t num_media_flows,
|
||||
size_t num_tcp_flows,
|
||||
int64_t run_time_seconds,
|
||||
int capacity_kbps,
|
||||
int max_delay_ms) {
|
||||
uint32_t capacity_kbps,
|
||||
int64_t max_delay_ms,
|
||||
int64_t rtt_ms,
|
||||
int64_t max_jitter_ms,
|
||||
const int64_t* offsets_ms) {
|
||||
RunFairnessTest(bwe_type, num_media_flows, num_tcp_flows, run_time_seconds,
|
||||
capacity_kbps, max_delay_ms, rtt_ms, max_jitter_ms,
|
||||
offsets_ms, "Fairness_test", bwe_names[bwe_type]);
|
||||
}
|
||||
|
||||
void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
size_t num_media_flows,
|
||||
size_t num_tcp_flows,
|
||||
int64_t run_time_seconds,
|
||||
uint32_t capacity_kbps,
|
||||
int64_t max_delay_ms,
|
||||
int64_t rtt_ms,
|
||||
int64_t max_jitter_ms,
|
||||
const int64_t* offsets_ms,
|
||||
const std::string& title,
|
||||
const std::string& flow_name) {
|
||||
std::set<int> all_flow_ids;
|
||||
std::set<int> media_flow_ids;
|
||||
std::set<int> tcp_flow_ids;
|
||||
@ -253,27 +282,33 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
|
||||
std::vector<VideoSource*> sources;
|
||||
std::vector<PacketSender*> senders;
|
||||
std::vector<MetricRecorder*> metric_recorders;
|
||||
|
||||
int64_t max_offset_ms = 0;
|
||||
|
||||
size_t i = 1;
|
||||
for (int media_flow : media_flow_ids) {
|
||||
// Streams started 20 seconds apart to give them different advantage when
|
||||
// competing for the bandwidth.
|
||||
const int64_t kFlowStartOffsetMs = i++ * (rand() % 10000);
|
||||
sources.push_back(new AdaptiveVideoSource(media_flow, 30, 300, 0,
|
||||
kFlowStartOffsetMs));
|
||||
offsets_ms[media_flow]));
|
||||
senders.push_back(new PacedVideoSender(&uplink_, sources.back(), bwe_type));
|
||||
max_offset_ms = std::max(max_offset_ms, offsets_ms[media_flow]);
|
||||
}
|
||||
|
||||
const int64_t kTcpStartOffsetMs = 5000;
|
||||
for (int tcp_flow : tcp_flow_ids)
|
||||
senders.push_back(new TcpSender(&uplink_, tcp_flow, kTcpStartOffsetMs));
|
||||
for (int tcp_flow : tcp_flow_ids) {
|
||||
senders.push_back(new TcpSender(&uplink_, tcp_flow, offsets_ms[tcp_flow]));
|
||||
max_offset_ms = std::max(max_offset_ms, offsets_ms[tcp_flow]);
|
||||
}
|
||||
|
||||
ChokeFilter choke(&uplink_, all_flow_ids);
|
||||
choke.set_capacity_kbps(capacity_kbps);
|
||||
choke.set_max_delay_ms(max_delay_ms);
|
||||
LinkShare link_share(&choke);
|
||||
|
||||
int64_t one_way_delay_ms = rtt_ms / 2;
|
||||
DelayFilter delay_uplink(&uplink_, all_flow_ids);
|
||||
delay_uplink.SetOneWayDelayMs(25);
|
||||
delay_uplink.SetOneWayDelayMs(one_way_delay_ms);
|
||||
|
||||
JitterFilter jitter(&uplink_, all_flow_ids);
|
||||
jitter.SetMaxJitter(max_jitter_ms);
|
||||
|
||||
std::vector<RateCounterFilter*> rate_counters;
|
||||
for (int flow : all_flow_ids) {
|
||||
@ -285,18 +320,34 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
"total_utilization");
|
||||
|
||||
std::vector<PacketReceiver*> receivers;
|
||||
i = 0;
|
||||
// Delays is being plotted only for the first flow.
|
||||
// To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
|
||||
for (int media_flow : media_flow_ids) {
|
||||
receivers.push_back(
|
||||
new PacketReceiver(&uplink_, media_flow, bwe_type, i++ == 0, false));
|
||||
metric_recorders.push_back(
|
||||
new MetricRecorder(bwe_names[bwe_type], static_cast<int>(media_flow),
|
||||
senders[media_flow], &link_share));
|
||||
receivers.push_back(new PacketReceiver(&uplink_, media_flow, bwe_type,
|
||||
media_flow == 0, false,
|
||||
metric_recorders[media_flow]));
|
||||
metric_recorders[media_flow]->set_plot_available_capacity(
|
||||
media_flow == 0 && plot_total_available_capacity_);
|
||||
metric_recorders[media_flow]->set_start_computing_metrics_ms(max_offset_ms);
|
||||
}
|
||||
// Delays is not being plotted only for TCP flows. To plot all of them,
|
||||
// replace first "false" occurence with "true" on new PacketReceiver().
|
||||
for (int tcp_flow : tcp_flow_ids) {
|
||||
receivers.push_back(
|
||||
new PacketReceiver(&uplink_, tcp_flow, kTcpEstimator, false, false));
|
||||
metric_recorders.push_back(
|
||||
new MetricRecorder(bwe_names[kTcpEstimator], static_cast<int>(tcp_flow),
|
||||
senders[tcp_flow], &link_share));
|
||||
receivers.push_back(new PacketReceiver(&uplink_, tcp_flow, kTcpEstimator,
|
||||
false, false,
|
||||
metric_recorders[tcp_flow]));
|
||||
metric_recorders[tcp_flow]->set_plot_available_capacity(
|
||||
tcp_flow == 0 && plot_total_available_capacity_);
|
||||
}
|
||||
|
||||
DelayFilter delay_downlink(&downlink_, all_flow_ids);
|
||||
delay_downlink.SetOneWayDelayMs(25);
|
||||
delay_downlink.SetOneWayDelayMs(one_way_delay_ms);
|
||||
|
||||
RunFor(run_time_seconds * 1000);
|
||||
|
||||
@ -315,6 +366,23 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
PrintResults(capacity_kbps, total_utilization.GetBitrateStats(),
|
||||
flow_delay_ms, flow_throughput_kbps);
|
||||
|
||||
for (int i : all_flow_ids) {
|
||||
metric_recorders[i]->PlotThroughputHistogram(
|
||||
title, flow_name, static_cast<int>(num_media_flows), 0);
|
||||
|
||||
metric_recorders[i]->PlotLossHistogram(title, flow_name,
|
||||
static_cast<int>(num_media_flows),
|
||||
receivers[i]->GlobalPacketLoss());
|
||||
}
|
||||
|
||||
// Pointless to show delay histogram for TCP flow.
|
||||
for (int i : media_flow_ids) {
|
||||
metric_recorders[i]->PlotDelayHistogram(title, bwe_names[bwe_type],
|
||||
static_cast<int>(num_media_flows),
|
||||
one_way_delay_ms);
|
||||
BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], one_way_delay_ms, i);
|
||||
}
|
||||
|
||||
for (VideoSource* source : sources)
|
||||
delete source;
|
||||
for (PacketSender* sender : senders)
|
||||
@ -323,7 +391,589 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
delete rate_counter;
|
||||
for (PacketReceiver* receiver : receivers)
|
||||
delete receiver;
|
||||
for (MetricRecorder* recorder : metric_recorders)
|
||||
delete recorder;
|
||||
}
|
||||
|
||||
void BweTest::RunChoke(BandwidthEstimatorType bwe_type,
|
||||
std::vector<int> capacities_kbps) {
|
||||
int flow_id = bwe_type;
|
||||
AdaptiveVideoSource source(flow_id, 30, 300, 0, 0);
|
||||
VideoSender sender(&uplink_, &source, bwe_type);
|
||||
ChokeFilter choke(&uplink_, flow_id);
|
||||
LinkShare link_share(&choke);
|
||||
MetricRecorder metric_recorder(bwe_names[bwe_type], flow_id, &sender,
|
||||
&link_share);
|
||||
PacketReceiver receiver(&uplink_, flow_id, bwe_type, true, false,
|
||||
&metric_recorder);
|
||||
metric_recorder.set_plot_available_capacity(plot_total_available_capacity_);
|
||||
|
||||
choke.set_max_delay_ms(500);
|
||||
const int64_t kRunTimeMs = 60 * 1000;
|
||||
|
||||
std::stringstream title("Choke");
|
||||
char delimiter = '_';
|
||||
|
||||
for (auto it = capacities_kbps.begin(); it != capacities_kbps.end(); ++it) {
|
||||
choke.set_capacity_kbps(*it);
|
||||
RunFor(kRunTimeMs);
|
||||
title << delimiter << (*it);
|
||||
delimiter = '-';
|
||||
}
|
||||
|
||||
title << "_kbps,_" << (kRunTimeMs / 1000) << "s_each";
|
||||
metric_recorder.PlotThroughputHistogram(title.str(), bwe_names[bwe_type], 1,
|
||||
0);
|
||||
metric_recorder.PlotDelayHistogram(title.str(), bwe_names[bwe_type], 1, 0);
|
||||
// receiver.PlotLossHistogram(title, bwe_names[bwe_type], 1);
|
||||
// receiver.PlotObjectiveHistogram(title, bwe_names[bwe_type], 1);
|
||||
}
|
||||
|
||||
// 5.1. Single Video and Audio media traffic, forward direction.
|
||||
void BweTest::RunVariableCapacity1SingleFlow(BandwidthEstimatorType bwe_type) {
|
||||
const int kFlowId = 0; // Arbitrary value.
|
||||
AdaptiveVideoSource source(kFlowId, 30, 300, 0, 0);
|
||||
PacedVideoSender sender(&uplink_, &source, bwe_type);
|
||||
|
||||
DefaultEvaluationFilter up_filter(&uplink_, kFlowId);
|
||||
LinkShare link_share(&(up_filter.choke));
|
||||
MetricRecorder metric_recorder(bwe_names[bwe_type], kFlowId, &sender,
|
||||
&link_share);
|
||||
|
||||
PacketReceiver receiver(&uplink_, kFlowId, bwe_type, true, true,
|
||||
&metric_recorder);
|
||||
|
||||
metric_recorder.set_plot_available_capacity(plot_total_available_capacity_);
|
||||
|
||||
DelayFilter down_filter(&downlink_, kFlowId);
|
||||
down_filter.SetOneWayDelayMs(kOneWayDelayMs);
|
||||
|
||||
// Test also with one way propagation delay = 100ms.
|
||||
// up_filter.delay.SetOneWayDelayMs(100);
|
||||
// down_filter.SetOneWayDelayMs(100);
|
||||
|
||||
up_filter.choke.set_capacity_kbps(1000);
|
||||
RunFor(40 * 1000); // 0-40s.
|
||||
up_filter.choke.set_capacity_kbps(2500);
|
||||
RunFor(20 * 1000); // 40-60s.
|
||||
up_filter.choke.set_capacity_kbps(600);
|
||||
RunFor(20 * 1000); // 60-80s.
|
||||
up_filter.choke.set_capacity_kbps(1000);
|
||||
RunFor(20 * 1000); // 80-100s.
|
||||
|
||||
std::string title("5.1_Variable_capacity_single_flow");
|
||||
metric_recorder.PlotThroughputHistogram(title, bwe_names[bwe_type], 1, 0);
|
||||
metric_recorder.PlotDelayHistogram(title, bwe_names[bwe_type], 1,
|
||||
kOneWayDelayMs);
|
||||
metric_recorder.PlotLossHistogram(title, bwe_names[bwe_type], 1,
|
||||
receiver.GlobalPacketLoss());
|
||||
BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, kFlowId);
|
||||
}
|
||||
|
||||
// 5.2. Two forward direction competing flows, variable capacity.
|
||||
void BweTest::RunVariableCapacity2MultipleFlows(BandwidthEstimatorType bwe_type,
|
||||
size_t num_flows) {
|
||||
std::vector<VideoSource*> sources;
|
||||
std::vector<PacketSender*> senders;
|
||||
std::vector<MetricRecorder*> metric_recorders;
|
||||
std::vector<PacketReceiver*> receivers;
|
||||
|
||||
const int64_t kStartingApartMs = 0; // Flows initialized simultaneously.
|
||||
|
||||
for (size_t i = 0; i < num_flows; ++i) {
|
||||
sources.push_back(new AdaptiveVideoSource(static_cast<int>(i), 30, 300, 0,
|
||||
i * kStartingApartMs));
|
||||
senders.push_back(new VideoSender(&uplink_, sources[i], bwe_type));
|
||||
}
|
||||
|
||||
FlowIds flow_ids = CreateFlowIdRange(0, static_cast<int>(num_flows - 1));
|
||||
|
||||
DefaultEvaluationFilter up_filter(&uplink_, flow_ids);
|
||||
LinkShare link_share(&(up_filter.choke));
|
||||
|
||||
RateCounterFilter total_utilization(&uplink_, flow_ids, "Total_utilization");
|
||||
|
||||
// Delays is being plotted only for the first flow.
|
||||
// To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
|
||||
for (size_t i = 0; i < num_flows; ++i) {
|
||||
metric_recorders.push_back(new MetricRecorder(
|
||||
bwe_names[bwe_type], static_cast<int>(i), senders[i], &link_share));
|
||||
|
||||
receivers.push_back(new PacketReceiver(&uplink_, static_cast<int>(i),
|
||||
bwe_type, i == 0, false,
|
||||
metric_recorders[i]));
|
||||
metric_recorders[i]->set_plot_available_capacity(
|
||||
i == 0 && plot_total_available_capacity_);
|
||||
}
|
||||
|
||||
DelayFilter down_filter(&downlink_, flow_ids);
|
||||
down_filter.SetOneWayDelayMs(kOneWayDelayMs);
|
||||
// Test also with one way propagation delay = 100ms.
|
||||
// up_filter.delay.SetOneWayDelayMs(100);
|
||||
// down_filter.SetOneWayDelayMs(100);
|
||||
|
||||
up_filter.choke.set_capacity_kbps(4000);
|
||||
RunFor(25 * 1000); // 0-25s.
|
||||
up_filter.choke.set_capacity_kbps(2000);
|
||||
RunFor(25 * 1000); // 25-50s.
|
||||
up_filter.choke.set_capacity_kbps(3500);
|
||||
RunFor(25 * 1000); // 50-75s.
|
||||
up_filter.choke.set_capacity_kbps(1000);
|
||||
RunFor(25 * 1000); // 75-100s.
|
||||
up_filter.choke.set_capacity_kbps(2000);
|
||||
RunFor(25 * 1000); // 100-125s.
|
||||
|
||||
std::string title("5.2_Variable_capacity_two_flows");
|
||||
for (size_t i = 0; i < num_flows; ++i) {
|
||||
metric_recorders[i]->PlotThroughputHistogram(title, bwe_names[bwe_type],
|
||||
num_flows, 0);
|
||||
metric_recorders[i]->PlotDelayHistogram(title, bwe_names[bwe_type],
|
||||
num_flows, kOneWayDelayMs);
|
||||
metric_recorders[i]->PlotLossHistogram(title, bwe_names[bwe_type],
|
||||
num_flows,
|
||||
receivers[i]->GlobalPacketLoss());
|
||||
BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
|
||||
}
|
||||
|
||||
for (VideoSource* source : sources)
|
||||
delete source;
|
||||
for (PacketSender* sender : senders)
|
||||
delete sender;
|
||||
for (MetricRecorder* recorder : metric_recorders)
|
||||
delete recorder;
|
||||
for (PacketReceiver* receiver : receivers)
|
||||
delete receiver;
|
||||
}
|
||||
|
||||
// 5.3. Bi-directional RMCAT flows.
|
||||
void BweTest::RunBidirectionalFlow(BandwidthEstimatorType bwe_type) {
|
||||
enum direction { kForward = 0, kBackward };
|
||||
const size_t kNumFlows = 2;
|
||||
rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows];
|
||||
rtc::scoped_ptr<VideoSender> senders[kNumFlows];
|
||||
rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows];
|
||||
rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows];
|
||||
|
||||
sources[kForward].reset(new AdaptiveVideoSource(kForward, 30, 300, 0, 0));
|
||||
senders[kForward].reset(
|
||||
new VideoSender(&uplink_, sources[kForward].get(), bwe_type));
|
||||
|
||||
sources[kBackward].reset(new AdaptiveVideoSource(kBackward, 30, 300, 0, 0));
|
||||
senders[kBackward].reset(
|
||||
new VideoSender(&downlink_, sources[kBackward].get(), bwe_type));
|
||||
|
||||
DefaultEvaluationFilter up_filter(&uplink_, kForward);
|
||||
LinkShare up_link_share(&(up_filter.choke));
|
||||
|
||||
metric_recorders[kForward].reset(new MetricRecorder(
|
||||
bwe_names[bwe_type], kForward, senders[kForward].get(), &up_link_share));
|
||||
receivers[kForward].reset(
|
||||
new PacketReceiver(&uplink_, kForward, bwe_type, true, false,
|
||||
metric_recorders[kForward].get()));
|
||||
|
||||
metric_recorders[kForward].get()->set_plot_available_capacity(
|
||||
plot_total_available_capacity_);
|
||||
|
||||
DefaultEvaluationFilter down_filter(&downlink_, kBackward);
|
||||
LinkShare down_link_share(&(down_filter.choke));
|
||||
|
||||
metric_recorders[kBackward].reset(
|
||||
new MetricRecorder(bwe_names[bwe_type], kBackward,
|
||||
senders[kBackward].get(), &down_link_share));
|
||||
receivers[kBackward].reset(
|
||||
new PacketReceiver(&downlink_, kBackward, bwe_type, true, false,
|
||||
metric_recorders[kBackward].get()));
|
||||
|
||||
metric_recorders[kBackward].get()->set_plot_available_capacity(
|
||||
plot_total_available_capacity_);
|
||||
|
||||
// Test also with one way propagation delay = 100ms.
|
||||
// up_filter.delay.SetOneWayDelayMs(100);
|
||||
// down_filter.delay.SetOneWayDelayMs(100);
|
||||
|
||||
up_filter.choke.set_capacity_kbps(2000);
|
||||
down_filter.choke.set_capacity_kbps(2000);
|
||||
RunFor(20 * 1000); // 0-20s.
|
||||
|
||||
up_filter.choke.set_capacity_kbps(1000);
|
||||
RunFor(15 * 1000); // 20-35s.
|
||||
|
||||
down_filter.choke.set_capacity_kbps(800);
|
||||
RunFor(5 * 1000); // 35-40s.
|
||||
|
||||
up_filter.choke.set_capacity_kbps(500);
|
||||
RunFor(20 * 1000); // 40-60s.
|
||||
|
||||
up_filter.choke.set_capacity_kbps(2000);
|
||||
RunFor(10 * 1000); // 60-70s.
|
||||
|
||||
down_filter.choke.set_capacity_kbps(2000);
|
||||
RunFor(30 * 1000); // 70-100s.
|
||||
|
||||
std::string title("5.3_Bidirectional_flows");
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
metric_recorders[i].get()->PlotThroughputHistogram(
|
||||
title, bwe_names[bwe_type], kNumFlows, 0);
|
||||
metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
|
||||
kNumFlows, kOneWayDelayMs);
|
||||
metric_recorders[i].get()->PlotLossHistogram(
|
||||
title, bwe_names[bwe_type], kNumFlows,
|
||||
receivers[i].get()->GlobalPacketLoss());
|
||||
BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
|
||||
}
|
||||
}
|
||||
|
||||
// 5.4. Three forward direction competing flows, constant capacity.
|
||||
void BweTest::RunSelfFairness(BandwidthEstimatorType bwe_type) {
|
||||
const int kNumRmcatFlows = 3;
|
||||
const int kNumTcpFlows = 0;
|
||||
const int64_t kRunTimeS = 120;
|
||||
const int kLinkCapacity = 3500;
|
||||
|
||||
int64_t max_delay_ms = kMaxQueueingDelayMs;
|
||||
int64_t rtt_ms = 2 * kOneWayDelayMs;
|
||||
|
||||
const int64_t kStartingApartMs = 20 * 1000;
|
||||
int64_t offsets_ms[kNumRmcatFlows];
|
||||
for (int i = 0; i < kNumRmcatFlows; ++i) {
|
||||
offsets_ms[i] = kStartingApartMs * i;
|
||||
}
|
||||
|
||||
// Test also with one way propagation delay = 100ms.
|
||||
// rtt_ms = 2 * 100;
|
||||
// Test also with bottleneck queue size = 20ms and 1000ms.
|
||||
// max_delay_ms = 20;
|
||||
// max_delay_ms = 1000;
|
||||
|
||||
std::string title("5.4_Self_fairness_test");
|
||||
|
||||
// Test also with one way propagation delay = 100ms.
|
||||
RunFairnessTest(bwe_type, kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
|
||||
kLinkCapacity, max_delay_ms, rtt_ms, kMaxJitterMs, offsets_ms,
|
||||
title, bwe_names[bwe_type]);
|
||||
}
|
||||
|
||||
// 5.5. Five competing RMCAT flows under different RTTs.
|
||||
void BweTest::RunRoundTripTimeFairness(BandwidthEstimatorType bwe_type) {
|
||||
const int kAllFlowIds[] = {0, 1, 2, 3, 4}; // Five RMCAT flows.
|
||||
const int64_t kAllOneWayDelayMs[] = {10, 25, 50, 100, 150};
|
||||
const size_t kNumFlows = ARRAY_SIZE(kAllFlowIds);
|
||||
rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows];
|
||||
rtc::scoped_ptr<VideoSender> senders[kNumFlows];
|
||||
rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows];
|
||||
|
||||
// Flows initialized 10 seconds apart.
|
||||
const int64_t kStartingApartMs = 10 * 1000;
|
||||
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0,
|
||||
i * kStartingApartMs));
|
||||
senders[i].reset(new VideoSender(&uplink_, sources[i].get(), bwe_type));
|
||||
}
|
||||
|
||||
ChokeFilter choke_filter(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows));
|
||||
LinkShare link_share(&choke_filter);
|
||||
|
||||
JitterFilter jitter_filter(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows));
|
||||
|
||||
rtc::scoped_ptr<DelayFilter> up_delay_filters[kNumFlows];
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
up_delay_filters[i].reset(new DelayFilter(&uplink_, kAllFlowIds[i]));
|
||||
}
|
||||
|
||||
RateCounterFilter total_utilization(
|
||||
&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "Total_utilization");
|
||||
|
||||
// Delays is being plotted only for the first flow.
|
||||
// To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
|
||||
rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows];
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
metric_recorders[i].reset(
|
||||
new MetricRecorder(bwe_names[bwe_type], static_cast<int>(i),
|
||||
senders[i].get(), &link_share));
|
||||
|
||||
receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], bwe_type,
|
||||
i == 0, false,
|
||||
metric_recorders[i].get()));
|
||||
metric_recorders[i].get()->set_start_computing_metrics_ms(kStartingApartMs *
|
||||
(kNumFlows - 1));
|
||||
metric_recorders[i].get()->set_plot_available_capacity(
|
||||
i == 0 && plot_total_available_capacity_);
|
||||
}
|
||||
|
||||
rtc::scoped_ptr<DelayFilter> down_delay_filters[kNumFlows];
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
down_delay_filters[i].reset(new DelayFilter(&downlink_, kAllFlowIds[i]));
|
||||
}
|
||||
|
||||
jitter_filter.SetMaxJitter(kMaxJitterMs);
|
||||
choke_filter.set_max_delay_ms(kMaxQueueingDelayMs);
|
||||
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
up_delay_filters[i]->SetOneWayDelayMs(kAllOneWayDelayMs[i]);
|
||||
down_delay_filters[i]->SetOneWayDelayMs(kAllOneWayDelayMs[i]);
|
||||
}
|
||||
|
||||
choke_filter.set_capacity_kbps(3500);
|
||||
|
||||
RunFor(300 * 1000); // 0-300s.
|
||||
|
||||
std::string title("5.5_Round_Trip_Time_Fairness");
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
metric_recorders[i].get()->PlotThroughputHistogram(
|
||||
title, bwe_names[bwe_type], kNumFlows, 0);
|
||||
metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
|
||||
kNumFlows, kOneWayDelayMs);
|
||||
metric_recorders[i].get()->PlotLossHistogram(
|
||||
title, bwe_names[bwe_type], kNumFlows,
|
||||
receivers[i].get()->GlobalPacketLoss());
|
||||
BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kAllOneWayDelayMs[i],
|
||||
i);
|
||||
}
|
||||
}
|
||||
|
||||
// 5.6. RMCAT Flow competing with a long TCP Flow.
|
||||
void BweTest::RunLongTcpFairness(BandwidthEstimatorType bwe_type) {
|
||||
const size_t kNumRmcatFlows = 1;
|
||||
const size_t kNumTcpFlows = 1;
|
||||
const int64_t kRunTimeS = 120;
|
||||
const int kCapacityKbps = 2000;
|
||||
// Tcp starts at t = 0, media flow at t = 5s.
|
||||
const int64_t kOffSetsMs[] = {5000, 0};
|
||||
|
||||
int64_t max_delay_ms = kMaxQueueingDelayMs;
|
||||
int64_t rtt_ms = 2 * kOneWayDelayMs;
|
||||
|
||||
// Test also with one way propagation delay = 100ms.
|
||||
// rtt_ms = 2 * 100;
|
||||
// Test also with bottleneck queue size = 20ms and 1000ms.
|
||||
// max_delay_ms = 20;
|
||||
// max_delay_ms = 1000;
|
||||
|
||||
std::string title("5.6_Long_TCP_Fairness");
|
||||
std::string flow_name(bwe_names[bwe_type] + 'x' + bwe_names[kTcpEstimator]);
|
||||
|
||||
RunFairnessTest(bwe_type, kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
|
||||
kCapacityKbps, max_delay_ms, rtt_ms, kMaxJitterMs, kOffSetsMs,
|
||||
title, flow_name);
|
||||
}
|
||||
|
||||
// 5.7. RMCAT Flows competing with multiple short TCP Flows.
|
||||
void BweTest::RunMultipleShortTcpFairness(
|
||||
BandwidthEstimatorType bwe_type,
|
||||
std::vector<int> tcp_file_sizes_bytes,
|
||||
std::vector<int64_t> tcp_starting_times_ms) {
|
||||
// Two RMCAT flows and ten TCP flows.
|
||||
const int kAllRmcatFlowIds[] = {0, 1};
|
||||
const int kAllTcpFlowIds[] = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
|
||||
|
||||
assert(tcp_starting_times_ms.size() == tcp_file_sizes_bytes.size() &&
|
||||
tcp_starting_times_ms.size() == ARRAY_SIZE(kAllTcpFlowIds));
|
||||
|
||||
const size_t kNumRmcatFlows = ARRAY_SIZE(kAllRmcatFlowIds);
|
||||
const size_t kNumTotalFlows = kNumRmcatFlows + ARRAY_SIZE(kAllTcpFlowIds);
|
||||
|
||||
rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumRmcatFlows];
|
||||
rtc::scoped_ptr<PacketSender> senders[kNumTotalFlows];
|
||||
rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumTotalFlows];
|
||||
rtc::scoped_ptr<PacketReceiver> receivers[kNumTotalFlows];
|
||||
|
||||
// RMCAT Flows are initialized simultaneosly at t=5 seconds.
|
||||
const int64_t kRmcatStartingTimeMs = 5 * 1000;
|
||||
for (size_t id : kAllRmcatFlowIds) {
|
||||
sources[id].reset(new AdaptiveVideoSource(static_cast<int>(id), 30, 300, 0,
|
||||
kRmcatStartingTimeMs));
|
||||
senders[id].reset(new VideoSender(&uplink_, sources[id].get(), bwe_type));
|
||||
}
|
||||
|
||||
for (size_t id : kAllTcpFlowIds) {
|
||||
senders[id].reset(new TcpSender(&uplink_, static_cast<int>(id),
|
||||
tcp_starting_times_ms[id - kNumRmcatFlows],
|
||||
tcp_file_sizes_bytes[id - kNumRmcatFlows]));
|
||||
}
|
||||
|
||||
FlowIds flow_ids = CreateFlowIdRange(0, static_cast<int>(kNumTotalFlows - 1));
|
||||
DefaultEvaluationFilter up_filter(&uplink_, flow_ids);
|
||||
|
||||
LinkShare link_share(&(up_filter.choke));
|
||||
|
||||
RateCounterFilter total_utilization(&uplink_, flow_ids, "Total_utilization");
|
||||
|
||||
// Delays is being plotted only for the first flow.
|
||||
// To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
|
||||
for (size_t id : kAllRmcatFlowIds) {
|
||||
metric_recorders[id].reset(
|
||||
new MetricRecorder(bwe_names[bwe_type], static_cast<int>(id),
|
||||
senders[id].get(), &link_share));
|
||||
receivers[id].reset(new PacketReceiver(&uplink_, static_cast<int>(id),
|
||||
bwe_type, id == 0, false,
|
||||
metric_recorders[id].get()));
|
||||
metric_recorders[id].get()->set_start_computing_metrics_ms(
|
||||
kRmcatStartingTimeMs);
|
||||
metric_recorders[id].get()->set_plot_available_capacity(
|
||||
id == 0 && plot_total_available_capacity_);
|
||||
}
|
||||
|
||||
// Delays is not being plotted only for TCP flows. To plot all of them,
|
||||
// replace first "false" occurence with "true" on new PacketReceiver().
|
||||
for (size_t id : kAllTcpFlowIds) {
|
||||
metric_recorders[id].reset(
|
||||
new MetricRecorder(bwe_names[kTcpEstimator], static_cast<int>(id),
|
||||
senders[id].get(), &link_share));
|
||||
receivers[id].reset(new PacketReceiver(&uplink_, static_cast<int>(id),
|
||||
kTcpEstimator, false, false,
|
||||
metric_recorders[id].get()));
|
||||
metric_recorders[id].get()->set_plot_available_capacity(
|
||||
id == 0 && plot_total_available_capacity_);
|
||||
}
|
||||
|
||||
DelayFilter down_filter(&downlink_, flow_ids);
|
||||
down_filter.SetOneWayDelayMs(kOneWayDelayMs);
|
||||
|
||||
// Test also with one way propagation delay = 100ms.
|
||||
// up_filter.delay.SetOneWayDelayMs(100);
|
||||
// down_filter.SetOneWayDelayms(100);
|
||||
|
||||
// Test also with bottleneck queue size = 20ms and 1000ms.
|
||||
// up_filter.choke.set_max_delay_ms(20);
|
||||
// up_filter.choke.set_max_delay_ms(1000);
|
||||
|
||||
// Test also with no Jitter:
|
||||
// up_filter.jitter.SetMaxJitter(0);
|
||||
|
||||
up_filter.choke.set_capacity_kbps(2000);
|
||||
|
||||
RunFor(300 * 1000); // 0-300s.
|
||||
|
||||
std::string title("5.7_Multiple_short_TCP_flows");
|
||||
for (size_t id : kAllRmcatFlowIds) {
|
||||
metric_recorders[id].get()->PlotThroughputHistogram(
|
||||
title, bwe_names[bwe_type], kNumRmcatFlows, 0);
|
||||
metric_recorders[id].get()->PlotDelayHistogram(
|
||||
title, bwe_names[bwe_type], kNumRmcatFlows, kOneWayDelayMs);
|
||||
metric_recorders[id].get()->PlotLossHistogram(
|
||||
title, bwe_names[bwe_type], kNumRmcatFlows,
|
||||
receivers[id].get()->GlobalPacketLoss());
|
||||
BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, id);
|
||||
}
|
||||
}
|
||||
|
||||
// 5.8. Three forward direction competing flows, constant capacity.
|
||||
// During the test, one of the flows is paused and later resumed.
|
||||
void BweTest::RunPauseResumeFlows(BandwidthEstimatorType bwe_type) {
|
||||
const int kAllFlowIds[] = {0, 1, 2}; // Three RMCAT flows.
|
||||
const size_t kNumFlows = ARRAY_SIZE(kAllFlowIds);
|
||||
|
||||
rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows];
|
||||
rtc::scoped_ptr<VideoSender> senders[kNumFlows];
|
||||
rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows];
|
||||
rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows];
|
||||
|
||||
// Flows initialized simultaneously.
|
||||
const int64_t kStartingApartMs = 0;
|
||||
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0,
|
||||
i * kStartingApartMs));
|
||||
senders[i].reset(new VideoSender(&uplink_, sources[i].get(), bwe_type));
|
||||
}
|
||||
|
||||
DefaultEvaluationFilter filter(&uplink_,
|
||||
CreateFlowIds(kAllFlowIds, kNumFlows));
|
||||
|
||||
LinkShare link_share(&(filter.choke));
|
||||
|
||||
RateCounterFilter total_utilization(
|
||||
&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "Total_utilization");
|
||||
|
||||
// Delays is being plotted only for the first flow.
|
||||
// To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
metric_recorders[i].reset(
|
||||
new MetricRecorder(bwe_names[bwe_type], static_cast<int>(i),
|
||||
senders[i].get(), &link_share));
|
||||
receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], bwe_type,
|
||||
i == 0, false,
|
||||
metric_recorders[i].get()));
|
||||
metric_recorders[i].get()->set_start_computing_metrics_ms(kStartingApartMs *
|
||||
(kNumFlows - 1));
|
||||
metric_recorders[i].get()->set_plot_available_capacity(
|
||||
i == 0 && plot_total_available_capacity_);
|
||||
}
|
||||
|
||||
// Test also with one way propagation delay = 100ms.
|
||||
// filter.delay.SetOneWayDelayMs(100);
|
||||
filter.choke.set_capacity_kbps(3500);
|
||||
|
||||
RunFor(40 * 1000); // 0-40s.
|
||||
|
||||
senders[0].get()->Pause();
|
||||
metric_recorders[0].get()->PauseFlow();
|
||||
RunFor(20 * 1000); // 40-60s.
|
||||
|
||||
senders[0].get()->Resume();
|
||||
metric_recorders[0].get()->ResumeFlow(20 * 1000);
|
||||
RunFor(60 * 1000); // 60-120s.
|
||||
|
||||
int64_t paused[] = {20 * 1000, 0, 0};
|
||||
|
||||
// First flow is being paused, hence having a different optimum.
|
||||
const std::string optima_lines[] = {"1", "2", "2"};
|
||||
|
||||
std::string title("5.8_Pause_and_resume_media_flow");
|
||||
for (size_t i = 0; i < kNumFlows; ++i) {
|
||||
metric_recorders[i].get()->PlotThroughputHistogram(
|
||||
title, bwe_names[bwe_type], kNumFlows, paused[i], optima_lines[i]);
|
||||
metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
|
||||
kNumFlows, kOneWayDelayMs);
|
||||
metric_recorders[i].get()->PlotLossHistogram(
|
||||
title, bwe_names[bwe_type], kNumFlows,
|
||||
receivers[i].get()->GlobalPacketLoss());
|
||||
BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
|
||||
}
|
||||
}
|
||||
|
||||
// Following functions are used for randomizing TCP file size and
|
||||
// starting time, used on 5.7 RunMultipleShortTcpFairness.
|
||||
// They are pseudo-random generators, creating always the same
|
||||
// value sequence for a given Random seed.
|
||||
|
||||
std::vector<int> BweTest::GetFileSizesBytes(int num_files) {
|
||||
// File size chosen from uniform distribution between [100,1000] kB.
|
||||
const int kMinKbytes = 100;
|
||||
const int kMaxKbytes = 1000;
|
||||
|
||||
Random random(0x12345678);
|
||||
std::vector<int> tcp_file_sizes_bytes;
|
||||
|
||||
while (num_files-- > 0) {
|
||||
tcp_file_sizes_bytes.push_back(random.Rand(kMinKbytes, kMaxKbytes) * 1000);
|
||||
}
|
||||
|
||||
return tcp_file_sizes_bytes;
|
||||
}
|
||||
|
||||
std::vector<int64_t> BweTest::GetStartingTimesMs(int num_files) {
|
||||
// OFF state behaves as an exp. distribution with mean = 10 seconds.
|
||||
const float kMeanMs = 10000.0f;
|
||||
Random random(0x12345678);
|
||||
|
||||
std::vector<int64_t> tcp_starting_times_ms;
|
||||
|
||||
// Two TCP Flows are initialized simultaneosly at t=0 seconds.
|
||||
for (int i = 0; i < 2; ++i, --num_files) {
|
||||
tcp_starting_times_ms.push_back(0);
|
||||
}
|
||||
|
||||
// Other TCP Flows are initialized in an OFF state.
|
||||
while (num_files-- > 0) {
|
||||
tcp_starting_times_ms.push_back(
|
||||
static_cast<int64_t>(random.Exponential(1.0f / kMeanMs)));
|
||||
}
|
||||
|
||||
return tcp_starting_times_ms;
|
||||
}
|
||||
|
||||
} // namespace bwe
|
||||
} // namespace testing
|
||||
} // namespace webrtc
|
||||
|
||||
@ -67,8 +67,49 @@ class Link : public PacketProcessorListener {
|
||||
class BweTest {
|
||||
public:
|
||||
BweTest();
|
||||
explicit BweTest(bool plot_capacity);
|
||||
~BweTest();
|
||||
|
||||
void RunChoke(BandwidthEstimatorType bwe_type,
|
||||
std::vector<int> capacities_kbps);
|
||||
|
||||
void RunVariableCapacity1SingleFlow(BandwidthEstimatorType bwe_type);
|
||||
void RunVariableCapacity2MultipleFlows(BandwidthEstimatorType bwe_type,
|
||||
size_t num_flows);
|
||||
void RunBidirectionalFlow(BandwidthEstimatorType bwe_type);
|
||||
void RunSelfFairness(BandwidthEstimatorType bwe_type);
|
||||
void RunRoundTripTimeFairness(BandwidthEstimatorType bwe_type);
|
||||
void RunLongTcpFairness(BandwidthEstimatorType bwe_type);
|
||||
void RunMultipleShortTcpFairness(BandwidthEstimatorType bwe_type,
|
||||
std::vector<int> tcp_file_sizes_bytes,
|
||||
std::vector<int64_t> tcp_starting_times_ms);
|
||||
void RunPauseResumeFlows(BandwidthEstimatorType bwe_type);
|
||||
|
||||
void RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
size_t num_media_flows,
|
||||
size_t num_tcp_flows,
|
||||
int64_t run_time_seconds,
|
||||
uint32_t capacity_kbps,
|
||||
int64_t max_delay_ms,
|
||||
int64_t rtt_ms,
|
||||
int64_t max_jitter_ms,
|
||||
const int64_t* offsets_ms);
|
||||
|
||||
void RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
size_t num_media_flows,
|
||||
size_t num_tcp_flows,
|
||||
int64_t run_time_seconds,
|
||||
uint32_t capacity_kbps,
|
||||
int64_t max_delay_ms,
|
||||
int64_t rtt_ms,
|
||||
int64_t max_jitter_ms,
|
||||
const int64_t* offsets_ms,
|
||||
const std::string& title,
|
||||
const std::string& flow_name);
|
||||
|
||||
static std::vector<int> GetFileSizesBytes(int num_files);
|
||||
static std::vector<int64_t> GetStartingTimesMs(int num_files);
|
||||
|
||||
protected:
|
||||
void SetUp();
|
||||
|
||||
@ -87,13 +128,6 @@ class BweTest {
|
||||
std::map<int, Stats<double>> flow_delay_ms,
|
||||
std::map<int, Stats<double>> flow_throughput_kbps);
|
||||
|
||||
void RunFairnessTest(BandwidthEstimatorType bwe_type,
|
||||
size_t num_media_flows,
|
||||
size_t num_tcp_flows,
|
||||
int64_t run_time_seconds,
|
||||
int capacity_kbps,
|
||||
int max_delay_ms);
|
||||
|
||||
Link downlink_;
|
||||
Link uplink_;
|
||||
|
||||
@ -107,9 +141,53 @@ class BweTest {
|
||||
int64_t simulation_interval_ms_;
|
||||
std::vector<Link*> links_;
|
||||
Packets packets_;
|
||||
bool plot_total_available_capacity_;
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(BweTest);
|
||||
};
|
||||
|
||||
// Default Evaluation parameters:
|
||||
// Link capacity: 4000ms;
|
||||
// Queueing delay capacity: 300ms.
|
||||
// One-Way propagation delay: 50ms.
|
||||
// Jitter model: Truncated gaussian.
|
||||
// Maximum end-to-end jitter: 30ms = 2*standard_deviation.
|
||||
// Bottleneck queue type: Drop tail.
|
||||
// Path loss ratio: 0%.
|
||||
|
||||
const int kOneWayDelayMs = 50;
|
||||
const int kMaxQueueingDelayMs = 300;
|
||||
const int kMaxCapacityKbps = 4000;
|
||||
const int kMaxJitterMs = 15;
|
||||
|
||||
struct DefaultEvaluationFilter {
|
||||
DefaultEvaluationFilter(PacketProcessorListener* listener, int flow_id)
|
||||
: choke(listener, flow_id),
|
||||
delay(listener, flow_id),
|
||||
jitter(listener, flow_id) {
|
||||
SetDefaultParameters();
|
||||
}
|
||||
|
||||
DefaultEvaluationFilter(PacketProcessorListener* listener,
|
||||
const FlowIds& flow_ids)
|
||||
: choke(listener, flow_ids),
|
||||
delay(listener, flow_ids),
|
||||
jitter(listener, flow_ids) {
|
||||
SetDefaultParameters();
|
||||
}
|
||||
|
||||
void SetDefaultParameters() {
|
||||
delay.SetOneWayDelayMs(kOneWayDelayMs);
|
||||
choke.set_max_delay_ms(kMaxQueueingDelayMs);
|
||||
choke.set_capacity_kbps(kMaxCapacityKbps);
|
||||
jitter.SetMaxJitter(kMaxJitterMs);
|
||||
}
|
||||
|
||||
ChokeFilter choke;
|
||||
DelayFilter delay;
|
||||
JitterFilter jitter;
|
||||
};
|
||||
|
||||
} // namespace bwe
|
||||
} // namespace testing
|
||||
} // namespace webrtc
|
||||
|
||||
@ -302,7 +302,7 @@ double MetricRecorder::ObjectiveFunction() {
|
||||
|
||||
void MetricRecorder::PlotThroughputHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows,
|
||||
size_t num_flows,
|
||||
int64_t extra_offset_ms,
|
||||
const std::string optimum_id) {
|
||||
size_t num_packets_received = delays_ms_.size();
|
||||
@ -347,14 +347,14 @@ void MetricRecorder::PlotThroughputHistogram(const std::string& title,
|
||||
|
||||
void MetricRecorder::PlotThroughputHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows,
|
||||
size_t num_flows,
|
||||
int64_t extra_offset_ms) {
|
||||
PlotThroughputHistogram(title, bwe_name, num_flows, extra_offset_ms, "");
|
||||
}
|
||||
|
||||
void MetricRecorder::PlotDelayHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows,
|
||||
size_t num_flows,
|
||||
int64_t one_way_path_delay_ms) {
|
||||
size_t num_packets_received = delays_ms_.size();
|
||||
double average_delay_ms = Average(delays_ms_, num_packets_received);
|
||||
@ -395,7 +395,7 @@ void MetricRecorder::PlotDelayHistogram(const std::string& title,
|
||||
|
||||
void MetricRecorder::PlotLossHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows,
|
||||
size_t num_flows,
|
||||
float global_loss_ratio) {
|
||||
BWE_TEST_LOGGING_LABEL(6, title, "packet_loss_ratio_(%)", num_flows)
|
||||
BWE_TEST_LOGGING_BAR(6, bwe_name, 100.0f * global_loss_ratio, flow_id_);
|
||||
@ -406,7 +406,7 @@ void MetricRecorder::PlotLossHistogram(const std::string& title,
|
||||
|
||||
void MetricRecorder::PlotObjectiveHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows) {
|
||||
size_t num_flows) {
|
||||
BWE_TEST_LOGGING_LABEL(7, title, "objective_function", num_flows)
|
||||
BWE_TEST_LOGGING_BAR(7, bwe_name, ObjectiveFunction(), flow_id_);
|
||||
}
|
||||
|
||||
@ -84,28 +84,28 @@ class MetricRecorder {
|
||||
|
||||
void PlotThroughputHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows,
|
||||
size_t num_flows,
|
||||
int64_t extra_offset_ms,
|
||||
const std::string optimum_id);
|
||||
|
||||
void PlotThroughputHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows,
|
||||
size_t num_flows,
|
||||
int64_t extra_offset_ms);
|
||||
|
||||
void PlotDelayHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows,
|
||||
size_t num_flows,
|
||||
int64_t one_way_path_delay_ms);
|
||||
|
||||
void PlotLossHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows,
|
||||
size_t num_flows,
|
||||
float global_loss_ratio);
|
||||
|
||||
void PlotObjectiveHistogram(const std::string& title,
|
||||
const std::string& bwe_name,
|
||||
int num_flows);
|
||||
size_t num_flows);
|
||||
|
||||
void set_start_computing_metrics_ms(int64_t start_computing_metrics_ms) {
|
||||
start_computing_metrics_ms_ = start_computing_metrics_ms;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user