admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrates probing end-to-end test to scenario test.

Browse Source 
The previous tests ran in real-time making them flaky, so they were
disabled on a number of platforms.
This CL ports the tests 1:1 (sort of) to use the scenario test
framework which runs with simulated time and much less risk of
flakiness.

Bug: webrtc:10155
Change-Id: I6281f57d73883c8aaa91964e9cfa58d9b47779fa
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/186941
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32333}
This commit is contained in:
Erik Språng 2020-10-06 21:51:21 +02:00 committed by Commit Bot
parent 2a7c57c34f
commit 3e3e16682d
7 changed files with 149 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/scenario/BUILD.gn
View File

@ -168,6 +168,7 @@ if (rtc_include_tests) {
testonly = true
sources = [
"performance_stats_unittest.cc",
"probing_test.cc",
"scenario_unittest.cc",
"stats_collection_unittest.cc",
"video_stream_unittest.cc",

7
test/scenario/call_client.cc
View File

@ -269,6 +269,13 @@ DataRate CallClient::padding_rate() const {
return network_controller_factory_.GetUpdate().pacer_config->pad_rate();
}
void CallClient::UpdateBitrateConstraints(
const BitrateConstraints& constraints) {
SendTask([this, &constraints]() {
call_->GetTransportControllerSend()->SetSdpBitrateParameters(constraints);
});
}
void CallClient::OnPacketReceived(EmulatedIpPacket packet) {
MediaType media_type = MediaType::ANY;
if (!RtpHeaderParser::IsRtcp(packet.cdata(), packet.data.size())) {

1
test/scenario/call_client.h
View File

@ -109,6 +109,7 @@ class CallClient : public EmulatedNetworkReceiverInterface {
DataRate target_rate() const;
DataRate stable_target_rate() const;
DataRate padding_rate() const;
void UpdateBitrateConstraints(const BitrateConstraints& constraints);
void OnPacketReceived(EmulatedIpPacket packet) override;
std::unique_ptr<RtcEventLogOutput> GetLogWriter(std::string name);

133
test/scenario/probing_test.cc Normal file
View File

@ -0,0 +1,133 @@
/*
* Copyright 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/gtest.h"
#include "test/scenario/scenario.h"
namespace webrtc {
namespace test {
TEST(ProbingTest, InitialProbingRampsUpTargetRateWhenNetworkIsGood) {
Scenario s;
NetworkSimulationConfig good_network;
good_network.bandwidth = DataRate::KilobitsPerSec(2000);
VideoStreamConfig video_config;
video_config.encoder.codec =
VideoStreamConfig::Encoder::Codec::kVideoCodecVP8;
CallClientConfig send_config;
auto* caller = s.CreateClient("caller", send_config);
auto* callee = s.CreateClient("callee", CallClientConfig());
auto route =
s.CreateRoutes(caller, {s.CreateSimulationNode(good_network)}, callee,
{s.CreateSimulationNode(NetworkSimulationConfig())});
s.CreateVideoStream(route->forward(), video_config);
s.RunFor(TimeDelta::Seconds(1));
EXPECT_GE(DataRate::BitsPerSec(caller->GetStats().send_bandwidth_bps),
3 * send_config.transport.rates.start_rate);
}
TEST(ProbingTest, MidCallProbingRampupTriggeredByUpdatedBitrateConstraints) {
Scenario s;
const DataRate kStartRate = DataRate::KilobitsPerSec(300);
const DataRate kConstrainedRate = DataRate::KilobitsPerSec(100);
const DataRate kHighRate = DataRate::KilobitsPerSec(2500);
VideoStreamConfig video_config;
video_config.encoder.codec =
VideoStreamConfig::Encoder::Codec::kVideoCodecVP8;
CallClientConfig send_call_config;
send_call_config.transport.rates.start_rate = kStartRate;
send_call_config.transport.rates.max_rate = kHighRate * 2;
auto* caller = s.CreateClient("caller", send_call_config);
auto* callee = s.CreateClient("callee", CallClientConfig());
auto route = s.CreateRoutes(
caller, {s.CreateSimulationNode(NetworkSimulationConfig())}, callee,
{s.CreateSimulationNode(NetworkSimulationConfig())});
s.CreateVideoStream(route->forward(), video_config);
// Wait until initial probing rampup is done and then set a low max bitrate.
s.RunFor(TimeDelta::Seconds(1));
EXPECT_GE(DataRate::BitsPerSec(caller->GetStats().send_bandwidth_bps),
5 * send_call_config.transport.rates.start_rate);
BitrateConstraints bitrate_config;
bitrate_config.max_bitrate_bps = kConstrainedRate.bps();
caller->UpdateBitrateConstraints(bitrate_config);
// Wait until the low send bitrate has taken effect, and then set a much
// higher max bitrate.
s.RunFor(TimeDelta::Seconds(2));
EXPECT_LT(DataRate::BitsPerSec(caller->GetStats().send_bandwidth_bps),
kConstrainedRate * 1.1);
bitrate_config.max_bitrate_bps = 2 * kHighRate.bps();
caller->UpdateBitrateConstraints(bitrate_config);
// Check that the max send bitrate is reached quicker than would be possible
// with simple AIMD rate control.
s.RunFor(TimeDelta::Seconds(1));
EXPECT_GE(DataRate::BitsPerSec(caller->GetStats().send_bandwidth_bps),
kHighRate);
}
TEST(ProbingTest, ProbesRampsUpWhenVideoEncoderConfigChanges) {
Scenario s;
const DataRate kStartRate = DataRate::KilobitsPerSec(50);
const DataRate kHdRate = DataRate::KilobitsPerSec(3250);
// Set up 3-layer simulcast.
VideoStreamConfig video_config;
video_config.encoder.codec =
VideoStreamConfig::Encoder::Codec::kVideoCodecVP8;
video_config.encoder.layers.spatial = 3;
video_config.source.generator.width = 1280;
video_config.source.generator.height = 720;
CallClientConfig send_call_config;
send_call_config.transport.rates.start_rate = kStartRate;
send_call_config.transport.rates.max_rate = kHdRate * 2;
auto* caller = s.CreateClient("caller", send_call_config);
auto* callee = s.CreateClient("callee", CallClientConfig());
auto send_net =
s.CreateMutableSimulationNode([&](NetworkSimulationConfig* c) {
c->bandwidth = DataRate::KilobitsPerSec(200);
});
auto route =
s.CreateRoutes(caller, {send_net->node()}, callee,
{s.CreateSimulationNode(NetworkSimulationConfig())});
auto* video_stream = s.CreateVideoStream(route->forward(), video_config);
// Only QVGA enabled initially. Run until initial probing is done and BWE
// has settled.
video_stream->send()->UpdateActiveLayers({true, false, false});
s.RunFor(TimeDelta::Seconds(2));
// Remove network constraints and run for a while more, BWE should be much
// less than required HD rate.
send_net->UpdateConfig([&](NetworkSimulationConfig* c) {
c->bandwidth = DataRate::PlusInfinity();
});
s.RunFor(TimeDelta::Seconds(2));
DataRate bandwidth =
DataRate::BitsPerSec(caller->GetStats().send_bandwidth_bps);
EXPECT_LT(bandwidth, kHdRate / 4);
// Enable all layers, triggering a probe.
video_stream->send()->UpdateActiveLayers({true, true, true});
// Run for a short while and verify BWE has ramped up fast.
s.RunFor(TimeDelta::Seconds(2));
EXPECT_GT(DataRate::BitsPerSec(caller->GetStats().send_bandwidth_bps),
kHdRate);
}
} // namespace test
} // namespace webrtc

15
test/scenario/video_stream.cc
View File

@ -463,7 +463,8 @@ void SendVideoStream::UpdateConfig(
}
}
// TODO(srte): Add more conditions that should cause reconfiguration.
if (prior_config.encoder.max_framerate != config_.encoder.max_framerate) {
if (prior_config.encoder.max_framerate != config_.encoder.max_framerate ||
prior_config.encoder.max_data_rate != config_.encoder.max_data_rate) {
VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config_);
send_stream_->ReconfigureVideoEncoder(std::move(encoder_config));
}
@ -479,14 +480,12 @@ void SendVideoStream::UpdateActiveLayers(std::vector<bool> active_layers) {
if (config_.encoder.codec ==
VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) {
send_stream_->UpdateActiveSimulcastLayers(active_layers);
} else {
VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config_);
RTC_CHECK_EQ(encoder_config.simulcast_layers.size(),
active_layers.size());
for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i)
encoder_config.simulcast_layers[i].active = active_layers[i];
send_stream_->ReconfigureVideoEncoder(std::move(encoder_config));
}
VideoEncoderConfig encoder_config = CreateVideoEncoderConfig(config_);
RTC_CHECK_EQ(encoder_config.simulcast_layers.size(), active_layers.size());
for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i)
encoder_config.simulcast_layers[i].active = active_layers[i];
send_stream_->ReconfigureVideoEncoder(std::move(encoder_config));
});
}

1
video/BUILD.gn
View File

@ -540,7 +540,6 @@ if (rtc_include_tests) {
"end_to_end_tests/multi_stream_tester.h",
"end_to_end_tests/multi_stream_tests.cc",
"end_to_end_tests/network_state_tests.cc",
"end_to_end_tests/probing_tests.cc",
"end_to_end_tests/retransmission_tests.cc",
"end_to_end_tests/rtp_rtcp_tests.cc",
"end_to_end_tests/ssrc_tests.cc",

326
video/end_to_end_tests/probing_tests.cc
View File

@ -1,326 +0,0 @@
/*
* Copyright 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include "api/task_queue/task_queue_base.h"
#include "api/test/simulated_network.h"
#include "call/fake_network_pipe.h"
#include "call/simulated_network.h"
#include "rtc_base/task_queue_for_test.h"
#include "test/call_test.h"
#include "test/field_trial.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
enum : int { // The first valid value is 1.
kTransportSequenceNumberExtensionId = 1,
};
} // namespace
class ProbingEndToEndTest : public test::CallTest {
public:
ProbingEndToEndTest() {
RegisterRtpExtension(RtpExtension(RtpExtension::kTransportSequenceNumberUri,
kTransportSequenceNumberExtensionId));
}
};
class ProbingTest : public test::EndToEndTest {
public:
explicit ProbingTest(int start_bitrate_bps)
: clock_(Clock::GetRealTimeClock()),
start_bitrate_bps_(start_bitrate_bps),
state_(0),
sender_call_(nullptr) {}
void ModifySenderBitrateConfig(BitrateConstraints* bitrate_config) override {
bitrate_config->start_bitrate_bps = start_bitrate_bps_;
}
void OnCallsCreated(Call* sender_call, Call* receiver_call) override {
sender_call_ = sender_call;
}
protected:
Clock* const clock_;
const int start_bitrate_bps_;
int state_;
Call* sender_call_;
};
// Flaky under MemorySanitizer: bugs.webrtc.org/7419
// Flaky on iOS bots: bugs.webrtc.org/7851
#if defined(MEMORY_SANITIZER)
TEST_F(ProbingEndToEndTest, DISABLED_InitialProbing) {
#elif defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR
TEST_F(ProbingEndToEndTest, DISABLED_InitialProbing) {
#else
TEST_F(ProbingEndToEndTest, InitialProbing) {
#endif
class InitialProbingTest : public ProbingTest {
public:
explicit InitialProbingTest(bool* success, TaskQueueBase* task_queue)
: ProbingTest(300000), success_(success), task_queue_(task_queue) {
*success_ = false;
}
void PerformTest() override {
int64_t start_time_ms = clock_->TimeInMilliseconds();
do {
if (clock_->TimeInMilliseconds() - start_time_ms > kTimeoutMs)
break;
Call::Stats stats;
SendTask(RTC_FROM_HERE, task_queue_,
[this, &stats]() { stats = sender_call_->GetStats(); });
// Initial probing is done with a x3 and x6 multiplier of the start
// bitrate, so a x4 multiplier is a high enough threshold.
if (stats.send_bandwidth_bps > 4 * 300000) {
*success_ = true;
break;
}
} while (!observation_complete_.Wait(20));
}
private:
const int kTimeoutMs = 1000;
bool* const success_;
TaskQueueBase* const task_queue_;
};
bool success = false;
const int kMaxAttempts = 3;
for (int i = 0; i < kMaxAttempts; ++i) {
InitialProbingTest test(&success, task_queue());
RunBaseTest(&test);
if (success)
return;
}
EXPECT_TRUE(success) << "Failed to perform mid initial probing ("
<< kMaxAttempts << " attempts).";
}
// Fails on Linux MSan: bugs.webrtc.org/7428
#if defined(MEMORY_SANITIZER)
TEST_F(ProbingEndToEndTest, DISABLED_TriggerMidCallProbing) {
// Fails on iOS bots: bugs.webrtc.org/7851
#elif defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR
TEST_F(ProbingEndToEndTest, DISABLED_TriggerMidCallProbing) {
#else
TEST_F(ProbingEndToEndTest, TriggerMidCallProbing) {
#endif
class TriggerMidCallProbingTest : public ProbingTest {
public:
TriggerMidCallProbingTest(TaskQueueBase* task_queue, bool* success)
: ProbingTest(300000), success_(success), task_queue_(task_queue) {}
void PerformTest() override {
*success_ = false;
int64_t start_time_ms = clock_->TimeInMilliseconds();
do {
if (clock_->TimeInMilliseconds() - start_time_ms > kTimeoutMs)
break;
Call::Stats stats;
SendTask(RTC_FROM_HERE, task_queue_,
[this, &stats]() { stats = sender_call_->GetStats(); });
switch (state_) {
case 0:
if (stats.send_bandwidth_bps > 5 * 300000) {
BitrateConstraints bitrate_config;
bitrate_config.max_bitrate_bps = 100000;
SendTask(RTC_FROM_HERE, task_queue_, [this, &bitrate_config]() {
sender_call_->GetTransportControllerSend()
->SetSdpBitrateParameters(bitrate_config);
});
++state_;
}
break;
case 1:
if (stats.send_bandwidth_bps < 110000) {
BitrateConstraints bitrate_config;
bitrate_config.max_bitrate_bps = 2500000;
SendTask(RTC_FROM_HERE, task_queue_, [this, &bitrate_config]() {
sender_call_->GetTransportControllerSend()
->SetSdpBitrateParameters(bitrate_config);
});
++state_;
}
break;
case 2:
// During high cpu load the pacer will not be able to pace packets
// at the correct speed, but if we go from 110 to 1250 kbps
// in 5 seconds then it is due to probing.
if (stats.send_bandwidth_bps > 1250000) {
*success_ = true;
observation_complete_.Set();
}
break;
}
} while (!observation_complete_.Wait(20));
}
private:
const int kTimeoutMs = 5000;
bool* const success_;
TaskQueueBase* const task_queue_;
};
bool success = false;
const int kMaxAttempts = 3;
for (int i = 0; i < kMaxAttempts; ++i) {
TriggerMidCallProbingTest test(task_queue(), &success);
RunBaseTest(&test);
if (success)
return;
}
EXPECT_TRUE(success) << "Failed to perform mid call probing (" << kMaxAttempts
<< " attempts).";
}
#if defined(MEMORY_SANITIZER)
TEST_F(ProbingEndToEndTest, DISABLED_ProbeOnVideoEncoderReconfiguration) {
#elif defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR
TEST_F(ProbingEndToEndTest, DISABLED_ProbeOnVideoEncoderReconfiguration) {
#else
TEST_F(ProbingEndToEndTest, ProbeOnVideoEncoderReconfiguration) {
#endif
class ReconfigureTest : public ProbingTest {
public:
ReconfigureTest(TaskQueueBase* task_queue, bool* success)
: ProbingTest(50000), task_queue_(task_queue), success_(success) {}
void ModifyVideoConfigs(
VideoSendStream::Config* send_config,
std::vector<VideoReceiveStream::Config>* receive_configs,
VideoEncoderConfig* encoder_config) override {
encoder_config_ = encoder_config;
}
void OnVideoStreamsCreated(
VideoSendStream* send_stream,
const std::vector<VideoReceiveStream*>& receive_streams) override {
send_stream_ = send_stream;
}
std::unique_ptr<test::PacketTransport> CreateSendTransport(
TaskQueueBase* task_queue,
Call* sender_call) override {
auto network =
std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig());
send_simulated_network_ = network.get();
return std::make_unique<test::PacketTransport>(
task_queue, sender_call, this, test::PacketTransport::kSender,
CallTest::payload_type_map_,
std::make_unique<FakeNetworkPipe>(Clock::GetRealTimeClock(),
std::move(network)));
}
void PerformTest() override {
*success_ = false;
int64_t start_time_ms = clock_->TimeInMilliseconds();
int64_t max_allocation_change_time_ms = -1;
do {
if (clock_->TimeInMilliseconds() - start_time_ms > kTimeoutMs)
break;
Call::Stats stats;
SendTask(RTC_FROM_HERE, task_queue_,
[this, &stats]() { stats = sender_call_->GetStats(); });
switch (state_) {
case 0:
// Wait until initial probing has been completed (6 times start
// bitrate).
if (stats.send_bandwidth_bps >= 250000 &&
stats.send_bandwidth_bps <= 350000) {
BuiltInNetworkBehaviorConfig config;
config.link_capacity_kbps = 200;
send_simulated_network_->SetConfig(config);
// In order to speed up the test we can interrupt exponential
// probing by toggling the network availability. The alternative
// is to wait for it to time out (1000 ms).
sender_call_->GetTransportControllerSend()->OnNetworkAvailability(
false);
sender_call_->GetTransportControllerSend()->OnNetworkAvailability(
true);
++state_;
}
break;
case 1:
if (stats.send_bandwidth_bps <= 200000) {
// Initial probing finished. Increase link capacity and wait
// until BWE ramped up enough to be in ALR. This takes a few
// seconds.
BuiltInNetworkBehaviorConfig config;
config.link_capacity_kbps = 5000;
send_simulated_network_->SetConfig(config);
++state_;
}
break;
case 2:
if (stats.send_bandwidth_bps > 240000) {
// BWE ramped up enough to be in ALR. Setting higher max_bitrate
// should trigger an allocation probe and fast ramp-up.
encoder_config_->max_bitrate_bps = 2000000;
encoder_config_->simulcast_layers[0].max_bitrate_bps = 1200000;
SendTask(RTC_FROM_HERE, task_queue_, [this]() {
send_stream_->ReconfigureVideoEncoder(encoder_config_->Copy());
});
max_allocation_change_time_ms = clock_->TimeInMilliseconds();
++state_;
}
break;
case 3:
if (stats.send_bandwidth_bps >= 1000000) {
EXPECT_LT(
clock_->TimeInMilliseconds() - max_allocation_change_time_ms,
kRampUpMaxDurationMs);
*success_ = true;
observation_complete_.Set();
}
break;
}
} while (!observation_complete_.Wait(20));
}
private:
const int kTimeoutMs = 10000;
const int kRampUpMaxDurationMs = 500;
TaskQueueBase* const task_queue_;
bool* const success_;
SimulatedNetwork* send_simulated_network_;
VideoSendStream* send_stream_;
VideoEncoderConfig* encoder_config_;
};
bool success = false;
const int kMaxAttempts = 3;
for (int i = 0; i < kMaxAttempts; ++i) {
ReconfigureTest test(task_queue(), &success);
RunBaseTest(&test);
if (success) {
return;
}
}
EXPECT_TRUE(success) << "Failed to perform mid call probing (" << kMaxAttempts
<< " attempts).";
}
} // namespace webrtc