Remove the least important WebRTC video tests.

These are considered expandable, and since video tests are very
expensive (45s each), let's remove them.

Bug: webrtc:11426
Change-Id: I4aea18e93d3b3672900650aacf0b5524c52c2900
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/170364
Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
Commit-Queue: Patrik Höglund <phoglund@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30790}
This commit is contained in:
Patrik Höglund 2020-03-13 12:45:32 +01:00 committed by Commit Bot
parent 3453149b28
commit f6767ed71c
2 changed files with 1 additions and 408 deletions

View File

@ -824,9 +824,6 @@ TEST(FullStackTest, ScreenshareSlidesVP8_2TL) {
#if !defined(WEBRTC_MAC) && !defined(WEBRTC_WIN)
// TODO(bugs.webrtc.org/9840): Investigate why is this test flaky on Win/Mac.
const char kScreenshareSimulcastVariableFramerateExperiment[] =
"WebRTC-VP8VariableFramerateScreenshare/"
"Enabled,min_fps:5.0,min_qp:15,undershoot:30/";
TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging screenshare;
@ -855,64 +852,6 @@ TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast) {
fixture->RunWithAnalyzer(screenshare);
}
TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast_Variable_Framerate) {
test::ScopedFieldTrials field_trial(
AppendFieldTrials(kScreenshareSimulcastVariableFramerateExperiment));
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging screenshare;
screenshare.call.send_side_bwe = true;
screenshare.screenshare[0] = {true, false, 10};
screenshare.video[0] = {true, 1850, 1110, 30, 800000, 2500000,
2500000, false, "VP8", 2, 1, 400000,
false, false, false, ""};
screenshare.analyzer = {"screenshare_slides_simulcast_variable_framerate",
0.0, 0.0, kFullStackTestDurationSecs};
ParamsWithLogging screenshare_params_high;
screenshare_params_high.video[0] = {
true, 1850, 1110, 60, 600000, 1250000, 1250000, false,
"VP8", 2, 0, 400000, false, false, false, ""};
VideoQualityTest::Params screenshare_params_low;
screenshare_params_low.video[0] = {true, 1850, 1110, 5, 30000, 200000,
1000000, false, "VP8", 2, 0, 400000,
false, false, false, ""};
std::vector<VideoStream> streams = {
VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0),
VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)};
screenshare.ss[0] = {
streams, 1, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
false};
fixture->RunWithAnalyzer(screenshare);
}
TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Simulcast_low) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging screenshare;
screenshare.call.send_side_bwe = true;
screenshare.screenshare[0] = {true, false, 10};
screenshare.video[0] = {true, 1850, 1110, 30, 800000, 2500000,
2500000, false, "VP8", 2, 1, 400000,
false, false, false, ""};
screenshare.analyzer = {"screenshare_slides_simulcast_low", 0.0, 0.0,
kFullStackTestDurationSecs};
VideoQualityTest::Params screenshare_params_high;
screenshare_params_high.video[0] = {
true, 1850, 1110, 60, 600000, 1250000, 1250000, false,
"VP8", 2, 0, 400000, false, false, false, ""};
VideoQualityTest::Params screenshare_params_low;
screenshare_params_low.video[0] = {true, 1850, 1110, 5, 30000, 200000,
1000000, false, "VP8", 2, 0, 400000,
false, false, false, ""};
std::vector<VideoStream> streams = {
VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0),
VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)};
screenshare.ss[0] = {
streams, 0, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
false};
fixture->RunWithAnalyzer(screenshare);
}
#endif // !defined(WEBRTC_MAC) && !defined(WEBRTC_WIN)
TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) {
@ -1063,66 +1002,7 @@ TEST(FullStackTest, ScreenshareSlidesVP9_3SL_High_Fps) {
fixture->RunWithAnalyzer(screenshare);
}
TEST(FullStackTest, ScreenshareSlidesVP9_3SL_Variable_Fps) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-VP9VariableFramerateScreenshare/"
"Enabled,min_qp:32,min_fps:5.0,undershoot:30,frames_"
"before_steady_state:5/"));
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging screenshare;
screenshare.call.send_side_bwe = true;
screenshare.video[0] = {true, 1850, 1110, 30, 50000, 200000,
2000000, false, "VP9", 1, 0, 400000,
false, false, false, ""};
screenshare.screenshare[0] = {true, false, 10};
screenshare.analyzer = {"screenshare_slides_vp9_3sl_variable_fps", 0.0, 0.0,
kFullStackTestDurationSecs};
screenshare.ss[0] = {
std::vector<VideoStream>(), 0, 3, 2, InterLayerPredMode::kOn,
std::vector<SpatialLayer>(), true};
fixture->RunWithAnalyzer(screenshare);
}
TEST(FullStackTest, VP9SVC_3SL_High) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging simulcast;
simulcast.call.send_side_bwe = true;
simulcast.video[0] = SvcVp9Video();
simulcast.analyzer = {"vp9svc_3sl_high", 0.0, 0.0,
kFullStackTestDurationSecs};
simulcast.ss[0] = {
std::vector<VideoStream>(), 0, 3, 2, InterLayerPredMode::kOn,
std::vector<SpatialLayer>(), false};
fixture->RunWithAnalyzer(simulcast);
}
TEST(FullStackTest, VP9SVC_3SL_Medium) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging simulcast;
simulcast.call.send_side_bwe = true;
simulcast.video[0] = SvcVp9Video();
simulcast.analyzer = {"vp9svc_3sl_medium", 0.0, 0.0,
kFullStackTestDurationSecs};
simulcast.ss[0] = {
std::vector<VideoStream>(), 0, 3, 1, InterLayerPredMode::kOn,
std::vector<SpatialLayer>(), false};
fixture->RunWithAnalyzer(simulcast);
}
TEST(FullStackTest, VP9SVC_3SL_Low) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging simulcast;
simulcast.call.send_side_bwe = true;
simulcast.video[0] = SvcVp9Video();
simulcast.analyzer = {"vp9svc_3sl_low", 0.0, 0.0, kFullStackTestDurationSecs};
simulcast.ss[0] = {
std::vector<VideoStream>(), 0, 3, 0, InterLayerPredMode::kOn,
std::vector<SpatialLayer>(), false};
fixture->RunWithAnalyzer(simulcast);
}
// bugs.webrtc.org/9506
// TODO(http://bugs.webrtc.org/9506): investigate.
#if !defined(WEBRTC_MAC)
TEST(FullStackTest, VP9KSVC_3SL_High) {
@ -1140,21 +1020,6 @@ TEST(FullStackTest, VP9KSVC_3SL_High) {
fixture->RunWithAnalyzer(simulcast);
}
TEST(FullStackTest, VP9KSVC_3SL_Medium) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation/Enabled/"));
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging simulcast;
simulcast.call.send_side_bwe = true;
simulcast.video[0] = SvcVp9Video();
simulcast.analyzer = {"vp9ksvc_3sl_medium", 0.0, 0.0,
kFullStackTestDurationSecs};
simulcast.ss[0] = {
std::vector<VideoStream>(), 0, 3, 1, InterLayerPredMode::kOnKeyPic,
std::vector<SpatialLayer>(), false};
fixture->RunWithAnalyzer(simulcast);
}
TEST(FullStackTest, VP9KSVC_3SL_Low) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation/Enabled/"));
@ -1283,32 +1148,6 @@ TEST(FullStackTest, SimulcastVP8_3SL_High) {
fixture->RunWithAnalyzer(simulcast);
}
TEST(FullStackTest, SimulcastVP8_3SL_Medium) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging simulcast;
simulcast.call.send_side_bwe = true;
simulcast.video[0] = SimulcastVp8VideoHigh();
simulcast.analyzer = {"simulcast_vp8_3sl_medium", 0.0, 0.0,
kFullStackTestDurationSecs};
simulcast.config->loss_percent = 0;
simulcast.config->queue_delay_ms = 100;
ParamsWithLogging video_params_high;
video_params_high.video[0] = SimulcastVp8VideoHigh();
ParamsWithLogging video_params_medium;
video_params_medium.video[0] = SimulcastVp8VideoMedium();
ParamsWithLogging video_params_low;
video_params_low.video[0] = SimulcastVp8VideoLow();
std::vector<VideoStream> streams = {
VideoQualityTest::DefaultVideoStream(video_params_low, 0),
VideoQualityTest::DefaultVideoStream(video_params_medium, 0),
VideoQualityTest::DefaultVideoStream(video_params_high, 0)};
simulcast.ss[0] = {
streams, 1, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
false};
fixture->RunWithAnalyzer(simulcast);
}
TEST(FullStackTest, SimulcastVP8_3SL_Low) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging simulcast;
@ -1372,70 +1211,14 @@ TEST(FullStackTest, MAYBE_HighBitrateWithFakeCodec) {
fixture->RunWithAnalyzer(generator);
}
TEST(FullStackTest, LargeRoomVP8_5thumb) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging large_room;
large_room.call.send_side_bwe = true;
large_room.video[0] = SimulcastVp8VideoHigh();
large_room.analyzer = {"largeroom_5thumb", 0.0, 0.0,
kFullStackTestDurationSecs};
large_room.config->loss_percent = 0;
large_room.config->queue_delay_ms = 100;
ParamsWithLogging video_params_high;
video_params_high.video[0] = SimulcastVp8VideoHigh();
ParamsWithLogging video_params_medium;
video_params_medium.video[0] = SimulcastVp8VideoMedium();
ParamsWithLogging video_params_low;
video_params_low.video[0] = SimulcastVp8VideoLow();
std::vector<VideoStream> streams = {
VideoQualityTest::DefaultVideoStream(video_params_low, 0),
VideoQualityTest::DefaultVideoStream(video_params_medium, 0),
VideoQualityTest::DefaultVideoStream(video_params_high, 0)};
large_room.call.num_thumbnails = 5;
large_room.ss[0] = {
streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
false};
fixture->RunWithAnalyzer(large_room);
}
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
// Fails on mobile devices:
// https://bugs.chromium.org/p/webrtc/issues/detail?id=7301
#define MAYBE_LargeRoomVP8_50thumb DISABLED_LargeRoomVP8_50thumb
#define MAYBE_LargeRoomVP8_15thumb DISABLED_LargeRoomVP8_15thumb
#else
#define MAYBE_LargeRoomVP8_50thumb LargeRoomVP8_50thumb
#define MAYBE_LargeRoomVP8_15thumb LargeRoomVP8_15thumb
#endif
TEST(FullStackTest, MAYBE_LargeRoomVP8_15thumb) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging large_room;
large_room.call.send_side_bwe = true;
large_room.video[0] = SimulcastVp8VideoHigh();
large_room.analyzer = {"largeroom_15thumb", 0.0, 0.0,
kFullStackTestDurationSecs};
large_room.config->loss_percent = 0;
large_room.config->queue_delay_ms = 100;
ParamsWithLogging video_params_high;
video_params_high.video[0] = SimulcastVp8VideoHigh();
ParamsWithLogging video_params_medium;
video_params_medium.video[0] = SimulcastVp8VideoMedium();
ParamsWithLogging video_params_low;
video_params_low.video[0] = SimulcastVp8VideoLow();
std::vector<VideoStream> streams = {
VideoQualityTest::DefaultVideoStream(video_params_low, 0),
VideoQualityTest::DefaultVideoStream(video_params_medium, 0),
VideoQualityTest::DefaultVideoStream(video_params_high, 0)};
large_room.call.num_thumbnails = 15;
large_room.ss[0] = {
streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
false};
fixture->RunWithAnalyzer(large_room);
}
TEST(FullStackTest, MAYBE_LargeRoomVP8_50thumb) {
auto fixture = CreateVideoQualityTestFixture();
ParamsWithLogging large_room;
@ -1469,110 +1252,4 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values("WebRTC-GenericDescriptor/Disabled/",
"WebRTC-GenericDescriptor/Enabled/"));
class DualStreamsTest : public ::testing::TestWithParam<int> {};
// Disable dual video test on mobile device becuase it's too heavy.
// TODO(bugs.webrtc.org/9840): Investigate why is this test flaky on MAC.
#if !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) && !defined(WEBRTC_MAC)
TEST_P(DualStreamsTest,
ModeratelyRestricted_SlidesVp8_2TL_Simulcast_Video_Simulcast_High) {
const int first_stream = GetParam();
ParamsWithLogging dual_streams;
// Screenshare Settings.
dual_streams.screenshare[first_stream] = {true, false, 10};
dual_streams.video[first_stream] = {true, 1850, 1110, 5, 800000, 2500000,
2500000, false, "VP8", 2, 1, 400000,
false, false, false, ""};
ParamsWithLogging screenshare_params_high;
screenshare_params_high.video[0] = {
true, 1850, 1110, 60, 600000, 1250000, 1250000, false,
"VP8", 2, 0, 400000, false, false, false, ""};
VideoQualityTest::Params screenshare_params_low;
screenshare_params_low.video[0] = {true, 1850, 1110, 5, 30000, 200000,
1000000, false, "VP8", 2, 0, 400000,
false, false, false, ""};
std::vector<VideoStream> screenhsare_streams = {
VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0),
VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)};
dual_streams.ss[first_stream] = {
screenhsare_streams, 1, 1, 0, InterLayerPredMode::kOn,
std::vector<SpatialLayer>(), false};
// Video settings.
dual_streams.video[1 - first_stream] = SimulcastVp8VideoHigh();
ParamsWithLogging video_params_high;
video_params_high.video[0] = SimulcastVp8VideoHigh();
ParamsWithLogging video_params_medium;
video_params_medium.video[0] = SimulcastVp8VideoMedium();
ParamsWithLogging video_params_low;
video_params_low.video[0] = SimulcastVp8VideoLow();
std::vector<VideoStream> streams = {
VideoQualityTest::DefaultVideoStream(video_params_low, 0),
VideoQualityTest::DefaultVideoStream(video_params_medium, 0),
VideoQualityTest::DefaultVideoStream(video_params_high, 0)};
dual_streams.ss[1 - first_stream] = {
streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector<SpatialLayer>(),
false};
// Call settings.
dual_streams.call.send_side_bwe = true;
dual_streams.call.dual_video = true;
std::string test_label = "dualstreams_moderately_restricted_screenshare_" +
std::to_string(first_stream);
dual_streams.analyzer = {test_label, 0.0, 0.0, kFullStackTestDurationSecs};
dual_streams.config->loss_percent = 1;
dual_streams.config->link_capacity_kbps = 7500;
dual_streams.config->queue_length_packets = 30;
dual_streams.config->queue_delay_ms = 100;
auto fixture = CreateVideoQualityTestFixture();
fixture->RunWithAnalyzer(dual_streams);
}
#endif // !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) &&
// !defined(WEBRTC_MAC)
TEST_P(DualStreamsTest, Conference_Restricted) {
const int first_stream = GetParam();
ParamsWithLogging dual_streams;
// Screenshare Settings.
dual_streams.screenshare[first_stream] = {true, false, 10};
dual_streams.video[first_stream] = {true, 1850, 1110, 5, 800000, 2500000,
2500000, false, "VP8", 3, 2, 400000,
false, false, false, ""};
// Video settings.
dual_streams.video[1 - first_stream] = {
true, 1280,
720, 30,
150000, 500000,
700000, false,
"VP8", 3,
2, 400000,
false, false,
false, ClipNameToClipPath("ConferenceMotion_1280_720_50")};
// Call settings.
dual_streams.call.send_side_bwe = true;
dual_streams.call.dual_video = true;
std::string test_label = "dualstreams_conference_restricted_screenshare_" +
std::to_string(first_stream);
dual_streams.analyzer = {test_label, 0.0, 0.0, kFullStackTestDurationSecs};
dual_streams.config->loss_percent = 1;
dual_streams.config->link_capacity_kbps = 5000;
dual_streams.config->queue_length_packets = 30;
dual_streams.config->queue_delay_ms = 100;
auto fixture = CreateVideoQualityTestFixture();
fixture->RunWithAnalyzer(dual_streams);
}
INSTANTIATE_TEST_SUITE_P(FullStackTest,
DualStreamsTest,
::testing::Values(0, 1));
} // namespace webrtc

View File

@ -1411,36 +1411,6 @@ TEST(PCFullStackTest, ScreenshareSlidesVP9_3SL_High_Fps) {
fixture->Run(std::move(run_params));
}
TEST(PCFullStackTest, ScreenshareSlidesVP9_3SL_Variable_Fps) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-VP9VariableFramerateScreenshare/"
"Enabled,min_qp:32,min_fps:5.0,undershoot:30,frames_"
"before_steady_state:5/"
"WebRTC-Vp9InterLayerPred/"
"Enabled,inter_layer_pred_mode:on/"));
std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
CreateNetworkEmulationManager();
auto fixture = CreateTestFixture(
"pc_screenshare_slides_vp9_3sl_variable_fps",
CreateTwoNetworkLinks(network_emulation_manager.get(),
BuiltInNetworkBehaviorConfig()),
[](PeerConfigurer* alice) {
VideoConfig video(1850, 1110, 30);
video.stream_label = "alice-video";
video.screen_share_config = ScreenShareConfig(TimeDelta::Seconds(10));
video.simulcast_config = VideoSimulcastConfig(3, 2);
alice->AddVideoConfig(std::move(video));
},
[](PeerConfigurer* bob) {});
RunParams run_params(TimeDelta::Seconds(kTestDurationSec));
run_params.video_codecs = {VideoCodecConfig(
/*name=*/cricket::kVp9CodecName, /*required_params=*/{
{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})};
run_params.use_flex_fec = false;
run_params.use_ulp_fec = false;
fixture->Run(std::move(run_params));
}
TEST(PCFullStackTest, VP9SVC_3SL_High) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-Vp9InterLayerPred/"
@ -1470,35 +1440,6 @@ TEST(PCFullStackTest, VP9SVC_3SL_High) {
fixture->Run(std::move(run_params));
}
TEST(PCFullStackTest, VP9SVC_3SL_Medium) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-Vp9InterLayerPred/"
"Enabled,inter_layer_pred_mode:on/"));
std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
CreateNetworkEmulationManager();
auto fixture = CreateTestFixture(
"pc_vp9svc_3sl_medium",
CreateTwoNetworkLinks(network_emulation_manager.get(),
BuiltInNetworkBehaviorConfig()),
[](PeerConfigurer* alice) {
VideoConfig video(1280, 720, 30);
video.stream_label = "alice-video";
video.input_file_name =
ClipNameToClipPath("ConferenceMotion_1280_720_50");
video.simulcast_config = VideoSimulcastConfig(3, 1);
video.temporal_layers_count = 3;
alice->AddVideoConfig(std::move(video));
},
[](PeerConfigurer* bob) {});
RunParams run_params(TimeDelta::Seconds(kTestDurationSec));
run_params.video_codecs = {VideoCodecConfig(
/*name=*/cricket::kVp9CodecName, /*required_params=*/{
{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})};
run_params.use_flex_fec = false;
run_params.use_ulp_fec = false;
fixture->Run(std::move(run_params));
}
TEST(PCFullStackTest, VP9SVC_3SL_Low) {
webrtc::test::ScopedFieldTrials override_trials(
AppendFieldTrials("WebRTC-Vp9InterLayerPred/"
@ -1680,31 +1621,6 @@ TEST(PCFullStackTest, SimulcastVP8_3SL_High) {
fixture->Run(std::move(run_params));
}
TEST(PCFullStackTest, SimulcastVP8_3SL_Medium) {
std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
CreateNetworkEmulationManager();
BuiltInNetworkBehaviorConfig config;
config.loss_percent = 0;
config.queue_delay_ms = 100;
auto fixture = CreateTestFixture(
"pc_simulcast_vp8_3sl_medium",
CreateTwoNetworkLinks(network_emulation_manager.get(), config),
[](PeerConfigurer* alice) {
VideoConfig video(1280, 720, 30);
video.input_file_name =
ClipNameToClipPath("ConferenceMotion_1280_720_50");
video.simulcast_config = VideoSimulcastConfig(3, 1);
video.stream_label = "alice-video";
alice->AddVideoConfig(std::move(video));
},
[](PeerConfigurer* bob) {});
RunParams run_params(TimeDelta::Seconds(kTestDurationSec));
run_params.video_codecs = {VideoCodecConfig(cricket::kVp8CodecName)};
run_params.use_flex_fec = false;
run_params.use_ulp_fec = false;
fixture->Run(std::move(run_params));
}
TEST(PCFullStackTest, SimulcastVP8_3SL_Low) {
std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
CreateNetworkEmulationManager();