diff --git a/test/DEPS b/test/DEPS index c7ae24f8ad..712dc6b043 100644 --- a/test/DEPS +++ b/test/DEPS @@ -54,4 +54,7 @@ specific_include_rules = { "+pc/test/mock_peer_connection_observers.h", "+p2p/client/basic_port_allocator.h", ], + ".*peer_connection_quality_test\.h": [ + "+pc", + ] } diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 994d61f3f4..11293b82c5 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -22,6 +22,7 @@ group("e2e") { ] if (rtc_include_tests) { deps += [ + ":peerconnection_quality_test", ":test_peer", ":video_quality_analyzer_injection_helper", ] @@ -34,6 +35,7 @@ if (rtc_include_tests) { deps = [ ":default_encoded_image_id_injector_unittest", + ":peer_connection_e2e_smoke_test", ":single_process_encoded_image_id_injector_unittest", ] } @@ -195,6 +197,38 @@ if (rtc_include_tests) { } } + rtc_source_set("peerconnection_quality_test") { + visibility = [ "*" ] + testonly = true + sources = [ + "peer_connection_quality_test.cc", + "peer_connection_quality_test.h", + ] + deps = [ + ":example_video_quality_analyzer", + ":single_process_encoded_image_id_injector", + ":test_peer", + ":video_quality_analyzer_injection_helper", + "../../../api:libjingle_peerconnection_api", + "../../../api:scoped_refptr", + "../../../api/units:time_delta", + "../../../pc:pc_test_utils", + "../../../rtc_base:gunit_helpers", + "../../../rtc_base:rtc_base", + "../../../rtc_base:rtc_base_approved", + "../../../system_wrappers:system_wrappers", + "../../../test:fileutils", + "../../../test:video_test_support", + "api:peer_connection_quality_test_fixture_api", + "api:video_quality_analyzer_api", + "//third_party/abseil-cpp/absl/memory:memory", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + rtc_source_set("single_process_encoded_image_id_injector_unittest") { testonly = true sources = [ @@ -220,6 +254,47 @@ if (rtc_include_tests) { "../../../test:test_support", ] } + + rtc_source_set("peer_connection_e2e_smoke_test") { + testonly = true + sources = [ + "peer_connection_e2e_smoke_test.cc", + ] + deps = [ + ":example_video_quality_analyzer", + "../../../api:callfactory_api", + "../../../api:libjingle_peerconnection_api", + "../../../api:scoped_refptr", + "../../../api:simulated_network_api", + "../../../api/audio_codecs:builtin_audio_decoder_factory", + "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/video_codecs:builtin_video_decoder_factory", + "../../../api/video_codecs:builtin_video_encoder_factory", + "../../../call:simulated_network", + "../../../logging:rtc_event_log_impl_base", + "../../../media:rtc_audio_video", + "../../../modules/audio_device:audio_device_impl", + "../../../p2p:rtc_p2p", + "../../../pc:pc_test_utils", + "../../../pc:peerconnection_wrapper", + "../../../rtc_base:gunit_helpers", + "../../../rtc_base:logging", + "../../../rtc_base:rtc_base", + "../../../rtc_base:rtc_base_tests_utils", + "../../../rtc_base:rtc_event", + "../../../test:fileutils", + "../../../test:test_support", + "../../../test/scenario/network:emulated_network", + "api:create_peerconnection_quality_test_fixture", + "api:peer_connection_quality_test_fixture_api", + "//third_party/abseil-cpp/absl/memory:memory", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } } rtc_source_set("example_video_quality_analyzer") { diff --git a/test/pc/e2e/api/BUILD.gn b/test/pc/e2e/api/BUILD.gn index 6bb1aa3720..8d8c8220d6 100644 --- a/test/pc/e2e/api/BUILD.gn +++ b/test/pc/e2e/api/BUILD.gn @@ -49,6 +49,28 @@ rtc_source_set("peer_connection_quality_test_fixture_api") { "../../../../api/video_codecs:video_codecs_api", "../../../../logging:rtc_event_log_api", "../../../../rtc_base:rtc_base", + "//third_party/abseil-cpp/absl/memory:memory", "//third_party/abseil-cpp/absl/types:optional", ] } + +if (rtc_include_tests) { + rtc_source_set("create_peerconnection_quality_test_fixture") { + visibility = [ "*" ] + testonly = true + sources = [ + "create_peerconnection_quality_test_fixture.cc", + "create_peerconnection_quality_test_fixture.h", + ] + + deps = [ + ":peer_connection_quality_test_fixture_api", + "../:peerconnection_quality_test", + "//third_party/abseil-cpp/absl/memory:memory", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } +} diff --git a/test/pc/e2e/api/create_peerconnection_quality_test_fixture.cc b/test/pc/e2e/api/create_peerconnection_quality_test_fixture.cc new file mode 100644 index 0000000000..0ef36e9b11 --- /dev/null +++ b/test/pc/e2e/api/create_peerconnection_quality_test_fixture.cc @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/api/create_peerconnection_quality_test_fixture.h" + +#include + +#include "absl/memory/memory.h" +#include "test/pc/e2e/peer_connection_quality_test.h" + +namespace webrtc { + +std::unique_ptr +CreatePeerConnectionE2EQualityTestFixture( + std::unique_ptr + alice_components, + std::unique_ptr alice_params, + std::unique_ptr + bob_components, + std::unique_ptr bob_params, + std::unique_ptr analyzers) { + return absl::make_unique( + std::move(alice_components), std::move(alice_params), + std::move(bob_components), std::move(bob_params), std::move(analyzers)); +} + +} // namespace webrtc diff --git a/test/pc/e2e/api/create_peerconnection_quality_test_fixture.h b/test/pc/e2e/api/create_peerconnection_quality_test_fixture.h new file mode 100644 index 0000000000..a9e768f575 --- /dev/null +++ b/test/pc/e2e/api/create_peerconnection_quality_test_fixture.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_API_CREATE_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ +#define TEST_PC_E2E_API_CREATE_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ + +#include + +#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h" + +namespace webrtc { + +// API is in development. Can be changed/removed without notice. +// Create test fixture to establish test call between Alice and Bob. +// During the test Alice will be caller and Bob will answer the call. +std::unique_ptr +CreatePeerConnectionE2EQualityTestFixture( + std::unique_ptr + alice_components, + std::unique_ptr alice_params, + std::unique_ptr + bob_components, + std::unique_ptr bob_params, + std::unique_ptr analyzers); + +} // namespace webrtc + +#endif // TEST_PC_E2E_API_CREATE_PEERCONNECTION_QUALITY_TEST_FIXTURE_H_ diff --git a/test/pc/e2e/api/peerconnection_quality_test_fixture.h b/test/pc/e2e/api/peerconnection_quality_test_fixture.h index 0d85423b4c..adffd0b3d7 100644 --- a/test/pc/e2e/api/peerconnection_quality_test_fixture.h +++ b/test/pc/e2e/api/peerconnection_quality_test_fixture.h @@ -14,6 +14,7 @@ #include #include +#include "absl/memory/memory.h" #include "api/async_resolver_factory.h" #include "api/call/call_factory_interface.h" #include "api/fec_controller.h" @@ -81,7 +82,10 @@ class PeerConnectionE2EQualityTestFixture { // has a network thread, that will be used to communicate with another peers. struct InjectableComponents { explicit InjectableComponents(rtc::Thread* network_thread) - : network_thread(network_thread) { + : network_thread(network_thread), + pcf_dependencies( + absl::make_unique()), + pc_dependencies(absl::make_unique()) { RTC_CHECK(network_thread); } @@ -104,6 +108,8 @@ class PeerConnectionE2EQualityTestFixture { std::vector slides_yuv_file_names; }; + enum VideoGeneratorType { kDefault, kI420A, kI010 }; + // Contains properties of single video stream. struct VideoConfig { size_t width; @@ -113,17 +119,22 @@ class PeerConnectionE2EQualityTestFixture { absl::optional stream_label; // Only single from 3 next fields can be specified. // If specified generator with this name will be used as input. - absl::optional generator_name; - // If specified this file will be used as input. + absl::optional generator; + // If specified this file will be used as input. Input video will be played + // in a circle. absl::optional input_file_name; // If specified screen share video stream will be created as input. absl::optional screen_share_config; // If specified the input stream will be also copied to specified file. + // It is actually one of the test's output file, which contains copy of what + // was captured during the test for this video stream on sender side. + // It is useful when generator is used as input. absl::optional input_dump_file_name; // If specified this file will be used as output on the receiver side for // this stream. If multiple streams will be produced by input stream, - // output files will be appended with indexes. - absl::optional output_file_name; + // output files will be appended with indexes. The produced files contains + // what was rendered for this video stream on receiver side. + absl::optional output_dump_file_name; }; // Contains properties for audio in the call. @@ -138,7 +149,7 @@ class PeerConnectionE2EQualityTestFixture { // If specified the input stream will be also copied to specified file. absl::optional input_dump_file_name; // If specified the output stream will be copied to specified file. - absl::optional output_file_name; + absl::optional output_dump_file_name; // Audio options to use. cricket::AudioOptions audio_options; }; @@ -162,7 +173,16 @@ class PeerConnectionE2EQualityTestFixture { std::unique_ptr video_quality_analyzer; }; - virtual void Run() = 0; + // Contains parameters, that describe how long framework should run quality + // test. + struct RunParams { + // Specifies how long the test should be run. This time shows how long + // the media should flow after connection was established and before + // it will be shut downed. + TimeDelta run_duration; + }; + + virtual void Run(RunParams run_params) = 0; virtual ~PeerConnectionE2EQualityTestFixture() = default; }; diff --git a/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/test/pc/e2e/peer_connection_e2e_smoke_test.cc new file mode 100644 index 0000000000..b922d73fb6 --- /dev/null +++ b/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "absl/memory/memory.h" +#include "call/simulated_network.h" +#include "rtc_base/async_invoker.h" +#include "rtc_base/fake_network.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" +#include "test/pc/e2e/api/create_peerconnection_quality_test_fixture.h" +#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h" +#include "test/scenario/network/network_emulation.h" +#include "test/scenario/network/network_emulation_manager.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { +namespace { + +std::unique_ptr CreateFakeNetworkManager( + std::vector endpoints) { + auto network_manager = absl::make_unique(); + for (auto* endpoint : endpoints) { + network_manager->AddInterface( + rtc::SocketAddress(endpoint->GetPeerLocalAddress(), /*port=*/0)); + } + return network_manager; +} + +} // namespace + +TEST(PeerConnectionE2EQualityTestSmokeTest, RunWithEmulatedNetwork) { + using Params = PeerConnectionE2EQualityTestFixture::Params; + using RunParams = PeerConnectionE2EQualityTestFixture::RunParams; + using VideoGeneratorType = + PeerConnectionE2EQualityTestFixture::VideoGeneratorType; + using Analyzers = PeerConnectionE2EQualityTestFixture::Analyzers; + using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; + using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig; + using InjectableComponents = + PeerConnectionE2EQualityTestFixture::InjectableComponents; + + auto alice_params = absl::make_unique(); + VideoConfig alice_video_config; + alice_video_config.width = 1280; + alice_video_config.height = 720; + alice_video_config.fps = 30; + alice_video_config.stream_label = "alice-video"; + alice_video_config.generator = VideoGeneratorType::kDefault; + + alice_params->video_configs.push_back(alice_video_config); + alice_params->audio_config = AudioConfig{ + AudioConfig::Mode::kGenerated, + /*input_file_name=*/absl::nullopt, + /*input_dump_file_name=*/absl::nullopt, + /*output_dump_file_name=*/absl::nullopt, cricket::AudioOptions()}; + + // Setup emulated network + NetworkEmulationManager network_emulation_manager(Clock::GetRealTimeClock()); + + EmulatedNetworkNode* alice_node = + network_emulation_manager.CreateEmulatedNode( + absl::make_unique(BuiltInNetworkBehaviorConfig())); + EmulatedNetworkNode* bob_node = network_emulation_manager.CreateEmulatedNode( + absl::make_unique(BuiltInNetworkBehaviorConfig())); + EndpointNode* alice_endpoint = + network_emulation_manager.CreateEndpoint(rtc::IPAddress(1)); + EndpointNode* bob_endpoint = + network_emulation_manager.CreateEndpoint(rtc::IPAddress(2)); + network_emulation_manager.CreateRoute(alice_endpoint, {alice_node}, + bob_endpoint); + network_emulation_manager.CreateRoute(bob_endpoint, {bob_node}, + alice_endpoint); + + rtc::Thread* alice_network_thread = + network_emulation_manager.CreateNetworkThread({alice_endpoint}); + rtc::Thread* bob_network_thread = + network_emulation_manager.CreateNetworkThread({bob_endpoint}); + + // Setup components. We need to provide rtc::NetworkManager compatible with + // emulated network layer. + auto alice_components = + absl::make_unique(alice_network_thread); + alice_components->pc_dependencies->network_manager = + CreateFakeNetworkManager({alice_endpoint}); + auto bob_components = + absl::make_unique(bob_network_thread); + bob_components->pc_dependencies->network_manager = + CreateFakeNetworkManager({bob_endpoint}); + + // Create analyzers. + auto analyzers = absl::make_unique(); + analyzers->video_quality_analyzer = + absl::make_unique(); + auto* video_analyzer = static_cast( + analyzers->video_quality_analyzer.get()); + + network_emulation_manager.Start(); + + auto fixture = CreatePeerConnectionE2EQualityTestFixture( + std::move(alice_components), std::move(alice_params), + std::move(bob_components), absl::make_unique(), + std::move(analyzers)); + fixture->Run(RunParams{TimeDelta::seconds(5)}); + + network_emulation_manager.Stop(); + + RTC_LOG(INFO) << "Captured: " << video_analyzer->frames_captured(); + RTC_LOG(INFO) << "Sent : " << video_analyzer->frames_sent(); + RTC_LOG(INFO) << "Received: " << video_analyzer->frames_received(); + RTC_LOG(INFO) << "Rendered: " << video_analyzer->frames_rendered(); + RTC_LOG(INFO) << "Dropped : " << video_analyzer->frames_dropped(); + + // 150 = 30fps * 5s + EXPECT_NEAR(video_analyzer->frames_captured(), 150, 15); + EXPECT_NEAR(video_analyzer->frames_sent(), 150, 15); + EXPECT_NEAR(video_analyzer->frames_received(), 150, 15); + EXPECT_NEAR(video_analyzer->frames_rendered(), 150, 15); +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc new file mode 100644 index 0000000000..e356e59deb --- /dev/null +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/peer_connection_quality_test.h" + +#include +#include + +#include "absl/memory/memory.h" +#include "api/peer_connection_interface.h" +#include "api/scoped_refptr.h" +#include "api/units/time_delta.h" +#include "rtc_base/bind.h" +#include "rtc_base/gunit.h" +#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" +#include "test/pc/e2e/api/video_quality_analyzer_interface.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { +namespace { + +using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; + +constexpr int kDefaultTimeoutMs = 10000; +constexpr char kSignalThreadName[] = "signaling_thread"; + +std::string VideoConfigSourcePresenceToString(const VideoConfig& video_config) { + char buf[1024]; + rtc::SimpleStringBuilder builder(buf); + builder << "video_config.generator=" << video_config.generator.has_value() + << "; video_config.input_file_name=" + << video_config.input_file_name.has_value() + << "; video_config.screen_share_config=" + << video_config.screen_share_config.has_value() << ";"; + return builder.str(); +} + +} // namespace + +PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( + std::unique_ptr alice_components, + std::unique_ptr alice_params, + std::unique_ptr bob_components, + std::unique_ptr bob_params, + std::unique_ptr analyzers) + : clock_(Clock::GetRealTimeClock()), + signaling_thread_(rtc::Thread::Create()) { + RTC_CHECK(alice_components); + RTC_CHECK(alice_params); + RTC_CHECK(bob_components); + RTC_CHECK(bob_params); + RTC_CHECK(analyzers); + + // Print test summary + RTC_LOG(INFO) + << "Media quality test: Alice will make a call to Bob with media video=" + << !alice_params->video_configs.empty() + << "; audio=" << alice_params->audio_config.has_value() + << ". Bob will respond with media video=" + << !bob_params->video_configs.empty() + << "; audio=" << bob_params->audio_config.has_value(); + + // Check that at least Alice or Bob has at least one media stream. + RTC_CHECK(!alice_params->video_configs.empty() || + alice_params->audio_config || !bob_params->video_configs.empty() || + bob_params->audio_config) + << "No media in the call"; + + signaling_thread_->SetName(kSignalThreadName, nullptr); + signaling_thread_->Start(); + + // Create default video quality analyzer. We will always create an analyzer, + // even if there are no video streams, because it will be installed into video + // encoder/decoder factories. + if (analyzers->video_quality_analyzer == nullptr) { + analyzers->video_quality_analyzer = + absl::make_unique(); + } + encoded_image_id_controller_ = + absl::make_unique(); + video_quality_analyzer_injection_helper_ = + absl::make_unique( + std::move(analyzers->video_quality_analyzer), + encoded_image_id_controller_.get(), + encoded_image_id_controller_.get()); + + // Create call participants: Alice and Bob. + // Audio streams are intercepted in AudioDeviceModule, so if it is required to + // catch output of Alice's stream, Alice's output_dump_file_name should be + // passed to Bob's TestPeer setup as audio output file name. + absl::optional alice_audio_output_dump_file_name = + bob_params->audio_config ? bob_params->audio_config->output_dump_file_name + : absl::nullopt; + absl::optional bob_audio_output_dump_file_name = + alice_params->audio_config + ? alice_params->audio_config->output_dump_file_name + : absl::nullopt; + alice_ = TestPeer::CreateTestPeer( + std::move(alice_components), std::move(alice_params), + video_quality_analyzer_injection_helper_.get(), signaling_thread_.get(), + alice_audio_output_dump_file_name); + bob_ = TestPeer::CreateTestPeer( + std::move(bob_components), std::move(bob_params), + video_quality_analyzer_injection_helper_.get(), signaling_thread_.get(), + bob_audio_output_dump_file_name); +} + +void PeerConnectionE2EQualityTest::Run(RunParams run_params) { + SetMissedVideoStreamLabels({alice_->params(), bob_->params()}); + ValidateParams({alice_->params(), bob_->params()}); + signaling_thread_->Invoke( + RTC_FROM_HERE, + rtc::Bind(&PeerConnectionE2EQualityTest::RunOnSignalingThread, this, + run_params)); +} + +void PeerConnectionE2EQualityTest::SetMissedVideoStreamLabels( + std::vector params) { + int counter = 0; + std::set video_labels; + for (auto* p : params) { + for (auto& video_config : p->video_configs) { + if (!video_config.stream_label) { + std::string label; + do { + label = "_auto_video_stream_label_" + std::to_string(counter); + ++counter; + } while (!video_labels.insert(label).second); + video_config.stream_label = label; + } + } + } +} + +void PeerConnectionE2EQualityTest::ValidateParams(std::vector params) { + std::set video_labels; + for (Params* p : params) { + // Validate that each video config has exactly one of |generator|, + // |input_file_name| or |screen_share_config| set. Also validate that all + // video stream labels are unique. + for (auto& video_config : p->video_configs) { + RTC_CHECK(video_config.stream_label); + bool inserted = + video_labels.insert(video_config.stream_label.value()).second; + RTC_CHECK(inserted) << "Duplicate video_config.stream_label=" + << video_config.stream_label.value(); + RTC_CHECK(video_config.generator || video_config.input_file_name || + video_config.screen_share_config) + << VideoConfigSourcePresenceToString(video_config); + RTC_CHECK(!(video_config.input_file_name && video_config.generator)) + << VideoConfigSourcePresenceToString(video_config); + RTC_CHECK( + !(video_config.input_file_name && video_config.screen_share_config)) + << VideoConfigSourcePresenceToString(video_config); + RTC_CHECK(!(video_config.screen_share_config && video_config.generator)) + << VideoConfigSourcePresenceToString(video_config); + } + if (p->audio_config) { + // Check that if mode input file name specified only if mode is kFile. + if (p->audio_config.value().mode == AudioConfig::Mode::kGenerated) { + RTC_CHECK(!p->audio_config.value().input_file_name); + } + if (p->audio_config.value().mode == AudioConfig::Mode::kFile) { + RTC_CHECK(p->audio_config.value().input_file_name); + RTC_CHECK(FileExists(p->audio_config.value().input_file_name.value())); + } + } + } +} + +void PeerConnectionE2EQualityTest::RunOnSignalingThread(RunParams run_params) { + AddMedia(alice_.get()); + AddMedia(bob_.get()); + + SetupCall(alice_.get(), bob_.get()); + + WaitForTransceiversSetup(alice_->params(), bob_.get()); + WaitForTransceiversSetup(bob_->params(), alice_.get()); + SetupVideoSink(alice_->params(), bob_.get()); + SetupVideoSink(bob_->params(), alice_.get()); + + StartVideo(); + + rtc::Event done; + done.Wait(static_cast(run_params.run_duration.ms())); + + TearDownCall(); + video_quality_analyzer_injection_helper_->Stop(); +} + +void PeerConnectionE2EQualityTest::AddMedia(TestPeer* peer) { + AddVideo(peer); + if (peer->params()->audio_config) { + AddAudio(peer); + } +} + +void PeerConnectionE2EQualityTest::AddVideo(TestPeer* peer) { + // Params here valid because of pre-run validation. + Params* params = peer->params(); + for (auto video_config : params->video_configs) { + // Create video generator. + std::unique_ptr frame_generator = + CreateFrameGenerator(video_config); + + // Wrap it to inject video quality analyzer and enable dump of input video + // if required. + VideoFrameWriter* writer = + MaybeCreateVideoWriter(video_config.input_dump_file_name, video_config); + frame_generator = + video_quality_analyzer_injection_helper_->WrapFrameGenerator( + video_config.stream_label.value(), std::move(frame_generator), + writer); + + // Setup FrameGenerator into peer connection. + std::unique_ptr capturer = + absl::WrapUnique(FrameGeneratorCapturer::Create( + std::move(frame_generator), video_config.fps, clock_)); + rtc::scoped_refptr source = + new rtc::RefCountedObject( + move(capturer)); + video_sources_.push_back(source); + RTC_LOG(INFO) << "Adding video with video_config.stream_label=" + << video_config.stream_label.value(); + rtc::scoped_refptr track = + peer->pc_factory()->CreateVideoTrack(video_config.stream_label.value(), + source); + peer->AddTransceiver(track); + } +} + +std::unique_ptr +PeerConnectionE2EQualityTest::CreateFrameGenerator( + const VideoConfig& video_config) { + if (video_config.generator) { + absl::optional frame_generator_type = + absl::nullopt; + if (video_config.generator == VideoGeneratorType::kDefault) { + frame_generator_type = FrameGenerator::OutputType::I420; + } else if (video_config.generator == VideoGeneratorType::kI420A) { + frame_generator_type = FrameGenerator::OutputType::I420A; + } else if (video_config.generator == VideoGeneratorType::kI010) { + frame_generator_type = FrameGenerator::OutputType::I010; + } + return FrameGenerator::CreateSquareGenerator( + static_cast(video_config.width), + static_cast(video_config.height), frame_generator_type, + absl::nullopt); + } + if (video_config.input_file_name) { + return FrameGenerator::CreateFromYuvFile( + std::vector(/*count=*/1, + video_config.input_file_name.value()), + video_config.width, video_config.height, /*frame_repeat_count=*/1); + } + if (video_config.screen_share_config) { + // TODO(titovartem) implement screen share support + // (http://bugs.webrtc.org/10138) + RTC_NOTREACHED() << "Screen share is not implemented"; + return nullptr; + } + RTC_NOTREACHED() << "Unsupported video_config input source"; + return nullptr; +} + +void PeerConnectionE2EQualityTest::AddAudio(TestPeer* peer) { + RTC_CHECK(peer->params()->audio_config); + rtc::scoped_refptr source = + peer->pc_factory()->CreateAudioSource( + peer->params()->audio_config->audio_options); + rtc::scoped_refptr track = + peer->pc_factory()->CreateAudioTrack("audio", source); + peer->AddTransceiver(track); +} + +void PeerConnectionE2EQualityTest::SetupCall(TestPeer* alice, TestPeer* bob) { + // Connect peers. + ASSERT_TRUE(alice->ExchangeOfferAnswerWith(bob)); + // Do the SDP negotiation, and also exchange ice candidates. + ASSERT_EQ_WAIT(alice->signaling_state(), PeerConnectionInterface::kStable, + kDefaultTimeoutMs); + ASSERT_TRUE_WAIT(alice->IsIceGatheringDone(), kDefaultTimeoutMs); + ASSERT_TRUE_WAIT(bob->IsIceGatheringDone(), kDefaultTimeoutMs); + + // Connect an ICE candidate pairs. + ASSERT_TRUE(bob->AddIceCandidates(alice->observer()->GetAllCandidates())); + ASSERT_TRUE(alice->AddIceCandidates(bob->observer()->GetAllCandidates())); + // This means that ICE and DTLS are connected. + ASSERT_TRUE_WAIT(bob->IsIceConnected(), kDefaultTimeoutMs); + ASSERT_TRUE_WAIT(alice->IsIceConnected(), kDefaultTimeoutMs); +} + +void PeerConnectionE2EQualityTest::WaitForTransceiversSetup( + Params* params, + TestPeer* remote_peer) { + uint64_t expected_remote_transceivers = + params->video_configs.size() + (params->audio_config ? 1 : 0); + ASSERT_EQ_WAIT(remote_peer->observer()->on_track_transceivers_.size(), + expected_remote_transceivers, kDefaultTimeoutMs); +} + +void PeerConnectionE2EQualityTest::SetupVideoSink(Params* params, + TestPeer* remote_peer) { + if (params->video_configs.empty()) { + return; + } + std::map video_configs_by_label; + for (auto& video_config : params->video_configs) { + video_configs_by_label.insert(std::pair( + video_config.stream_label.value(), &video_config)); + } + + for (const auto& transceiver : + remote_peer->observer()->on_track_transceivers_) { + const rtc::scoped_refptr& track = + transceiver->receiver()->track(); + if (track->kind() != MediaStreamTrackInterface::kVideoKind) { + continue; + } + + auto it = video_configs_by_label.find(track->id()); + RTC_CHECK(it != video_configs_by_label.end()); + VideoConfig* video_config = it->second; + + VideoFrameWriter* writer = MaybeCreateVideoWriter( + video_config->output_dump_file_name, *video_config); + // It is safe to cast here, because it is checked above that track->kind() + // is kVideoKind. + auto* video_track = static_cast(track.get()); + std::unique_ptr> video_sink = + video_quality_analyzer_injection_helper_->CreateVideoSink(writer); + video_track->AddOrUpdateSink(video_sink.get(), rtc::VideoSinkWants()); + output_video_sinks_.push_back(std::move(video_sink)); + } +} + +void PeerConnectionE2EQualityTest::StartVideo() { + for (const auto& video_source : video_sources_) { + video_source->Start(); + } +} + +void PeerConnectionE2EQualityTest::TearDownCall() { + for (const auto& video_source : video_sources_) { + video_source->Stop(); + } + + alice_->pc()->Close(); + bob_->pc()->Close(); + + for (const auto& video_writer : video_writers_) { + video_writer->Close(); + } +} + +VideoFrameWriter* PeerConnectionE2EQualityTest::MaybeCreateVideoWriter( + absl::optional file_name, + const VideoConfig& config) { + if (!file_name) { + return nullptr; + } + auto video_writer = absl::make_unique( + file_name.value(), config.width, config.height, config.fps); + VideoFrameWriter* out = video_writer.get(); + video_writers_.push_back(std::move(video_writer)); + return out; +} + +} // namespace test +} // namespace webrtc diff --git a/test/pc/e2e/peer_connection_quality_test.h b/test/pc/e2e/peer_connection_quality_test.h new file mode 100644 index 0000000000..a1908af90b --- /dev/null +++ b/test/pc/e2e/peer_connection_quality_test.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ +#define TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ + +#include +#include +#include + +#include "pc/test/frame_generator_capturer_video_track_source.h" +#include "rtc_base/thread.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_id_injector.h" +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" +#include "test/pc/e2e/api/peerconnection_quality_test_fixture.h" +#include "test/pc/e2e/test_peer.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace test { + +class PeerConnectionE2EQualityTest + : public PeerConnectionE2EQualityTestFixture { + public: + using Params = PeerConnectionE2EQualityTestFixture::Params; + using Analyzers = PeerConnectionE2EQualityTestFixture::Analyzers; + using InjectableComponents = + PeerConnectionE2EQualityTestFixture::InjectableComponents; + using VideoGeneratorType = + PeerConnectionE2EQualityTestFixture::VideoGeneratorType; + using RunParams = PeerConnectionE2EQualityTestFixture::RunParams; + using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; + + PeerConnectionE2EQualityTest( + std::unique_ptr alice_components, + std::unique_ptr alice_params, + std::unique_ptr bob_components, + std::unique_ptr bob_params, + std::unique_ptr analyzers); + + ~PeerConnectionE2EQualityTest() override = default; + + void Run(RunParams run_params) override; + + private: + // Sets video stream labels that are not specified in VideoConfigs to unique + // generated values. + void SetMissedVideoStreamLabels(std::vector params); + // Validate peer's parameters, also ensure uniqueness of all video stream + // labels. + void ValidateParams(std::vector params); + // Have to be run on the signaling thread. + void RunOnSignalingThread(RunParams run_params); + void AddMedia(TestPeer* peer); + void AddVideo(TestPeer* peer); + std::unique_ptr CreateFrameGenerator( + const VideoConfig& video_config); + void AddAudio(TestPeer* peer); + void SetupCall(TestPeer* alice, TestPeer* bob); + void WaitForTransceiversSetup(Params* params, TestPeer* remote_peer); + void SetupVideoSink(Params* params, TestPeer* remote_peer); + void StartVideo(); + void TearDownCall(); + VideoFrameWriter* MaybeCreateVideoWriter( + absl::optional file_name, + const VideoConfig& config); + + Clock* const clock_; + std::unique_ptr + video_quality_analyzer_injection_helper_; + std::unique_ptr + encoded_image_id_controller_; + const std::unique_ptr signaling_thread_; + + std::unique_ptr alice_; + std::unique_ptr bob_; + + std::vector> + video_sources_; + std::vector> video_writers_; + std::vector>> + output_video_sinks_; +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ diff --git a/test/pc/e2e/test_peer.cc b/test/pc/e2e/test_peer.cc index 62bbc1d46c..8c46206eeb 100644 --- a/test/pc/e2e/test_peer.cc +++ b/test/pc/e2e/test_peer.cc @@ -77,14 +77,14 @@ std::unique_ptr CreateAudioCapturer( if (audio_config.mode == AudioConfig::Mode::kGenerated) { return TestAudioDeviceModule::CreatePulsedNoiseCapturer( kGeneratedAudioMaxAmplitude, kSamplingFrequencyInHz); - } else if (audio_config.mode == AudioConfig::Mode::kFile) { + } + if (audio_config.mode == AudioConfig::Mode::kFile) { RTC_DCHECK(audio_config.input_file_name); return TestAudioDeviceModule::CreateWavFileReader( audio_config.input_file_name.value()); - } else { - RTC_NOTREACHED() << "Unknown audio_config->mode"; - return nullptr; } + RTC_NOTREACHED() << "Unknown audio_config->mode"; + return nullptr; } rtc::scoped_refptr CreateAudioDeviceModule( @@ -176,12 +176,10 @@ PeerConnectionFactoryDependencies CreatePCFDependencies( VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, rtc::Thread* network_thread, rtc::Thread* signaling_thread, - rtc::Thread* worker_thread, absl::optional audio_output_file_name) { PeerConnectionFactoryDependencies pcf_deps; pcf_deps.network_thread = network_thread; pcf_deps.signaling_thread = signaling_thread; - pcf_deps.worker_thread = worker_thread; pcf_deps.media_engine = CreateMediaEngine( pcf_dependencies.get(), std::move(audio_config), video_analyzer_helper, std::move(audio_output_file_name)); @@ -217,8 +215,7 @@ PeerConnectionDependencies CreatePCDependencies( // until the end of the test. if (pc_dependencies->network_manager == nullptr) { pc_dependencies->network_manager = - // TODO(titovartem) have network manager integrated with emulated - // network layer. + // Use real network (on the loopback interface) absl::make_unique(); } auto port_allocator = absl::make_unique( @@ -250,7 +247,6 @@ std::unique_ptr TestPeer::CreateTestPeer( std::unique_ptr params, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, rtc::Thread* signaling_thread, - rtc::Thread* worker_thread, absl::optional audio_output_file_name) { RTC_DCHECK(components); RTC_DCHECK(params); @@ -264,7 +260,7 @@ std::unique_ptr TestPeer::CreateTestPeer( PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies( std::move(components->pcf_dependencies), params->audio_config, video_analyzer_helper, components->network_thread, signaling_thread, - worker_thread, std::move(audio_output_file_name)); + std::move(audio_output_file_name)); rtc::scoped_refptr pcf = CreateModularPeerConnectionFactory(std::move(pcf_deps)); @@ -284,6 +280,11 @@ bool TestPeer::AddIceCandidates( bool success = true; for (const auto* candidate : candidates) { if (!pc()->AddIceCandidate(candidate)) { + std::string candidate_str; + bool res = candidate->ToString(&candidate_str); + RTC_CHECK(res); + RTC_LOG(LS_ERROR) << "Failed to add ICE candidate, candidate_str=" + << candidate_str; success = false; } } @@ -296,8 +297,8 @@ TestPeer::TestPeer( std::unique_ptr observer, std::unique_ptr params, std::unique_ptr network_manager) - : PeerConnectionWrapper::PeerConnectionWrapper(pc_factory, - pc, + : PeerConnectionWrapper::PeerConnectionWrapper(std::move(pc_factory), + std::move(pc), std::move(observer)), params_(std::move(params)), network_manager_(std::move(network_manager)) {} diff --git a/test/pc/e2e/test_peer.h b/test/pc/e2e/test_peer.h index afd6f0bbbf..f6e878db28 100644 --- a/test/pc/e2e/test_peer.h +++ b/test/pc/e2e/test_peer.h @@ -44,8 +44,6 @@ class TestPeer final : public PeerConnectionWrapper { // also will setup dependencies, that are required for media analyzers // injection. // - // We require |worker_thread| here, because TestPeer can't own worker thread, - // because in such case it will be destroyed before peer connection. // |signaling_thread| will be provided by test fixture implementation. // |params| - describes current peer paramters, like current peer video // streams and audio streams @@ -57,7 +55,6 @@ class TestPeer final : public PeerConnectionWrapper { std::unique_ptr params, VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, rtc::Thread* signaling_thread, - rtc::Thread* worker_thread, absl::optional audio_output_file_name); Params* params() const { return params_.get(); }