diff --git a/BUILD.gn b/BUILD.gn index 7566f2adfc..8cedb5bf22 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -546,13 +546,13 @@ if (rtc_include_tests) { rtc_test("webrtc_perf_tests") { testonly = true configs += [ ":rtc_unittests_config" ] - deps = [ "audio:audio_perf_tests", "call:call_perf_tests", "modules/audio_coding:audio_coding_perf_tests", "modules/audio_processing:audio_processing_perf_tests", "modules/remote_bitrate_estimator:remote_bitrate_estimator_perf_tests", + "pc:peerconnection_perf_tests", "test:test_main", "video:video_full_stack_tests", ] diff --git a/pc/BUILD.gn b/pc/BUILD.gn index a5703b77bb..9d2a05aed3 100644 --- a/pc/BUILD.gn +++ b/pc/BUILD.gn @@ -344,6 +344,39 @@ if (rtc_include_tests) { } } + rtc_source_set("peerconnection_perf_tests") { + testonly = true + sources = [ + "peerconnection_rampup_tests.cc", + "peerconnectionwrapper.cc", + "peerconnectionwrapper.h", + ] + deps = [ + ":pc_test_utils", + "../api:fakemetricsobserver", + "../api:libjingle_peerconnection_api", + "../api:libjingle_peerconnection_test_api", + "../api:rtc_stats_api", + "../api/audio_codecs:builtin_audio_decoder_factory", + "../api/audio_codecs:builtin_audio_encoder_factory", + "../api/video_codecs:builtin_video_decoder_factory", + "../api/video_codecs:builtin_video_encoder_factory", + "../media:rtc_media_tests_utils", + "../p2p:p2p_test_utils", + "../p2p:rtc_p2p", + "../pc:peerconnection", + "../rtc_base:rtc_base", + "../rtc_base:rtc_base_approved", + "../rtc_base:rtc_base_tests_utils", + "../test:perf_test", + "../test:test_support", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + rtc_source_set("pc_test_utils") { testonly = true sources = [ @@ -358,6 +391,7 @@ if (rtc_include_tests) { "test/fakesctptransport.h", "test/fakevideotrackrenderer.h", "test/fakevideotracksource.h", + "test/framegeneratorcapturervideotracksource.h", "test/mock_datachannel.h", "test/mock_peerconnection.h", "test/mock_rtpreceiverinternal.h", @@ -395,6 +429,7 @@ if (rtc_include_tests) { "../rtc_base:rtc_base_tests_utils", "../rtc_base:rtc_task_queue", "../test:test_support", + "../test:video_test_common", ] if (!build_with_chromium && is_clang) { diff --git a/pc/peerconnection_rampup_tests.cc b/pc/peerconnection_rampup_tests.cc new file mode 100644 index 0000000000..74cbb0fdff --- /dev/null +++ b/pc/peerconnection_rampup_tests.cc @@ -0,0 +1,318 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/fakeconstraints.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "p2p/base/testturnserver.h" +#include "p2p/client/basicportallocator.h" +#include "pc/peerconnection.h" +#include "pc/peerconnectionwrapper.h" +#include "pc/test/fakeaudiocapturemodule.h" +#include "pc/test/fakeperiodicvideotracksource.h" +#include "pc/test/fakevideotrackrenderer.h" +#include "pc/test/framegeneratorcapturervideotracksource.h" +#include "rtc_base/fakenetwork.h" +#include "rtc_base/firewallsocketserver.h" +#include "rtc_base/gunit.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/socketaddress.h" +#include "rtc_base/virtualsocketserver.h" +#include "test/gtest.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { + +namespace { +static const int kDefaultTestTimeMs = 15000; +static const int kRampUpTimeMs = 5000; +static const int kPollIntervalTimeMs = 50; +static const int kDefaultTimeoutMs = 10000; +static const rtc::SocketAddress kDefaultLocalAddress("1.1.1.1", 0); +// The video's configured max bitrate in webrtcvideoengine.cc is 1.7 Mbps. +// Setting the network bandwidth to 1 Mbps allows the video's bitrate to push +// the network's limitations. +static const int kNetworkBandwidth = 1000000; +} // namespace + +using RTCConfiguration = PeerConnectionInterface::RTCConfiguration; + +// This is an end to end test to verify that BWE is functioning when setting +// up a one to one call at the PeerConnection level. The intention of the test +// is to catch potential regressions for different ICE path configurations. The +// test uses a VirtualSocketServer for it's underlying simulated network and +// fake audio and video sources. The test is based upon rampup_tests.cc, but +// instead is at the PeerConnection level and uses a different fake network +// (rampup_tests.cc uses SimulatedNetwork). In the future, this test could +// potentially test different network conditions and test video quality as well +// (video_quality_test.cc does this, but at the call level). +// +// The perf test results are printed using the perf test support. If the +// isolated_script_test_perf_output flag is specified in test_main.cc, then +// the results are written to a JSON formatted file for the Chrome perf +// dashboard. Since this test is a webrtc_perf_test, it will be run in the perf +// console every webrtc commit. +class PeerConnectionWrapperForRampUpTest : public PeerConnectionWrapper { + public: + using PeerConnectionWrapper::PeerConnectionWrapper; + + PeerConnectionWrapperForRampUpTest( + rtc::scoped_refptr pc_factory, + rtc::scoped_refptr pc, + std::unique_ptr observer, + rtc::FakeNetworkManager* fake_network_manager) + : PeerConnectionWrapper::PeerConnectionWrapper(pc_factory, + pc, + std::move(observer)), + fake_network_manager_(std::move(fake_network_manager)) {} + + bool AddIceCandidates(std::vector candidates) { + bool success = true; + for (const auto candidate : candidates) { + if (!pc()->AddIceCandidate(candidate)) { + success = false; + } + } + return success; + } + + rtc::scoped_refptr CreateLocalVideoTrack( + FrameGeneratorCapturerVideoTrackSource::Config config, + Clock* clock) { + video_track_sources_.emplace_back( + new rtc::RefCountedObject( + config, clock)); + video_track_sources_.back()->Start(); + return rtc::scoped_refptr( + pc_factory()->CreateVideoTrack(rtc::CreateRandomUuid(), + video_track_sources_.back())); + } + + rtc::scoped_refptr CreateLocalAudioTrack( + const cricket::AudioOptions options) { + rtc::scoped_refptr source = + pc_factory()->CreateAudioSource(options); + return pc_factory()->CreateAudioTrack(rtc::CreateRandomUuid(), source); + } + + private: + // This is owned by the Test, not the Wrapper. It needs to outlive the + // Wrapper, because the port allocator expects its lifetime to be longer than + // the PeerConnection's lifetime. + rtc::FakeNetworkManager* fake_network_manager_; + std::vector> + video_track_sources_; +}; + +// TODO(shampson): Paramaterize the test to run for both Plan B & Unified Plan. +class PeerConnectionRampUpTest : public ::testing::Test { + public: + PeerConnectionRampUpTest() + : clock_(Clock::GetRealTimeClock()), + virtual_socket_server_(new rtc::VirtualSocketServer()), + firewall_socket_server_( + new rtc::FirewallSocketServer(virtual_socket_server_.get())), + network_thread_(new rtc::Thread(firewall_socket_server_.get())), + worker_thread_(rtc::Thread::Create()) { + network_thread_->SetName("PCNetworkThread", this); + worker_thread_->SetName("PCWorkerThread", this); + RTC_CHECK(network_thread_->Start()); + RTC_CHECK(worker_thread_->Start()); + + virtual_socket_server_->set_bandwidth(kNetworkBandwidth / 8); + pc_factory_ = CreatePeerConnectionFactory( + network_thread_.get(), worker_thread_.get(), rtc::Thread::Current(), + rtc::scoped_refptr(FakeAudioCaptureModule::Create()), + CreateBuiltinAudioEncoderFactory(), CreateBuiltinAudioDecoderFactory(), + CreateBuiltinVideoEncoderFactory(), CreateBuiltinVideoDecoderFactory(), + nullptr /* audio_mixer */, nullptr /* audio_processing */); + } + + virtual ~PeerConnectionRampUpTest() { + network_thread()->Invoke(RTC_FROM_HERE, + [this] { turn_servers_.clear(); }); + } + + bool CreatePeerConnectionWrappers(const RTCConfiguration& caller_config, + const RTCConfiguration& callee_config) { + caller_ = CreatePeerConnectionWrapper(caller_config); + callee_ = CreatePeerConnectionWrapper(callee_config); + return caller_ && callee_; + } + + std::unique_ptr + CreatePeerConnectionWrapper(const RTCConfiguration& config) { + auto* fake_network_manager = new rtc::FakeNetworkManager(); + fake_network_manager->AddInterface(kDefaultLocalAddress); + fake_network_managers_.emplace_back(fake_network_manager); + auto port_allocator = + rtc::MakeUnique(fake_network_manager); + + port_allocator->set_step_delay(cricket::kDefaultStepDelay); + auto observer = rtc::MakeUnique(); + auto pc = pc_factory_->CreatePeerConnection( + config, std::move(port_allocator), nullptr, observer.get()); + if (!pc) { + return nullptr; + } + + return rtc::MakeUnique( + pc_factory_, pc, std::move(observer), fake_network_manager); + } + + void SetupOneWayCall() { + ASSERT_TRUE(caller_); + ASSERT_TRUE(callee_); + FrameGeneratorCapturerVideoTrackSource::Config config; + caller_->AddTrack(caller_->CreateLocalVideoTrack(config, clock_)); + // Disable highpass filter so that we can get all the test audio frames. + cricket::AudioOptions options; + options.highpass_filter = false; + caller_->AddTrack(caller_->CreateLocalAudioTrack(options)); + + // Do the SDP negotiation, and also exchange ice candidates. + ASSERT_TRUE(caller_->ExchangeOfferAnswerWith(callee_.get())); + ASSERT_TRUE_WAIT( + caller_->signaling_state() == PeerConnectionInterface::kStable, + kDefaultTimeoutMs); + ASSERT_TRUE_WAIT(caller_->IsIceGatheringDone(), kDefaultTimeoutMs); + ASSERT_TRUE_WAIT(callee_->IsIceGatheringDone(), kDefaultTimeoutMs); + + // Connect an ICE candidate pairs. + ASSERT_TRUE( + callee_->AddIceCandidates(caller_->observer()->GetAllCandidates())); + ASSERT_TRUE( + caller_->AddIceCandidates(callee_->observer()->GetAllCandidates())); + // This means that ICE and DTLS are connected. + ASSERT_TRUE_WAIT(callee_->IsIceConnected(), kDefaultTimeoutMs); + ASSERT_TRUE_WAIT(caller_->IsIceConnected(), kDefaultTimeoutMs); + } + + void CreateTurnServer(cricket::ProtocolType type) { + rtc::Thread* thread = network_thread(); + std::unique_ptr turn_server = + network_thread_->Invoke>( + RTC_FROM_HERE, [thread, type] { + static const rtc::SocketAddress turn_server_internal_address{ + "88.88.88.0", 3478}; + static const rtc::SocketAddress turn_server_external_address{ + "88.88.88.1", 0}; + return rtc::MakeUnique( + thread, turn_server_internal_address, + turn_server_external_address, type); + }); + turn_servers_.push_back(std::move(turn_server)); + } + + // First runs the call for kRampUpTimeMs to ramp up the bandwidth estimate. + // Then runs the test for the remaining test time, grabbing the bandwidth + // estimation stat, every kPollIntervalTimeMs. When finished, averages the + // bandwidth estimations and prints the bandwidth estimation result as a perf + // metric. + void RunTest(const std::string& test_string) { + rtc::Thread::Current()->ProcessMessages(kRampUpTimeMs); + int number_of_polls = + (kDefaultTestTimeMs - kRampUpTimeMs) / kPollIntervalTimeMs; + int total_bwe = 0; + for (int i = 0; i < number_of_polls; ++i) { + rtc::Thread::Current()->ProcessMessages(kPollIntervalTimeMs); + total_bwe += static_cast(GetCallerAvailableBitrateEstimate()); + } + double average_bandwidth_estimate = total_bwe / number_of_polls; + std::string value_description = + "bwe_after_" + std::to_string(kDefaultTestTimeMs / 1000) + "_seconds"; + test::PrintResult("peerconnection_ramp_up_", test_string, value_description, + average_bandwidth_estimate, "bwe", false); + } + + rtc::Thread* network_thread() { return network_thread_.get(); } + + PeerConnectionWrapperForRampUpTest* caller() { return caller_.get(); } + + PeerConnectionWrapperForRampUpTest* callee() { return callee_.get(); } + + private: + // Gets the caller's outgoing available bitrate from the stats. Returns 0 if + // something went wrong. It takes the outgoing bitrate from the current + // selected ICE candidate pair's stats. + double GetCallerAvailableBitrateEstimate() { + auto stats = caller_->GetStats(); + auto transport_stats = stats->GetStatsOfType(); + if (transport_stats.size() == 0u || + !transport_stats[0]->selected_candidate_pair_id.is_defined()) { + return 0; + } + std::string selected_ice_id = + transport_stats[0]->selected_candidate_pair_id.ValueToString(); + // Use the selected ICE candidate pair ID to get the appropriate ICE stats. + const RTCIceCandidatePairStats ice_candidate_pair_stats = + stats->Get(selected_ice_id)->cast_to(); + if (ice_candidate_pair_stats.available_outgoing_bitrate.is_defined()) { + return *ice_candidate_pair_stats.available_outgoing_bitrate; + } + // We couldn't get the |available_outgoing_bitrate| for the active candidate + // pair. + return 0; + } + + Clock* const clock_; + // The turn servers should be accessed & deleted on the network thread to + // avoid a race with the socket read/write which occurs on the network thread. + std::vector> turn_servers_; + // |virtual_socket_server_| is used by |network_thread_| so it must be + // destroyed later. + // TODO(bugs.webrtc.org/7668): We would like to update the virtual network we + // use for this test. VirtualSocketServer isn't ideal because: + // 1) It uses the same queue & network capacity for both directions. + // 2) VirtualSocketServer implements how the network bandwidth affects the + // send delay differently than the SimulatedNetwork, used by the + // FakeNetworkPipe. It would be ideal if all of levels of virtual + // networks used in testing were consistent. + // We would also like to update this test to record the time to ramp up, + // down, and back up (similar to in rampup_tests.cc). This is problematic with + // the VirtualSocketServer. The first ramp down time is very noisy and the + // second ramp up time can take up to 300 seconds, most likely due to a built + // up queue. + std::unique_ptr virtual_socket_server_; + std::unique_ptr firewall_socket_server_; + std::unique_ptr network_thread_; + std::unique_ptr worker_thread_; + // The |pc_factory| uses |network_thread_| & |worker_thread_|, so it must be + // destroyed first. + std::vector> fake_network_managers_; + rtc::scoped_refptr pc_factory_; + std::unique_ptr caller_; + std::unique_ptr callee_; +}; + +TEST_F(PeerConnectionRampUpTest, TurnOverTCP) { + CreateTurnServer(cricket::ProtocolType::PROTO_TCP); + PeerConnectionInterface::IceServer ice_server; + ice_server.urls.push_back("turn:88.88.88.0:3478?transport=tcp"); + ice_server.username = "test"; + ice_server.password = "test"; + PeerConnectionInterface::RTCConfiguration client_1_config; + client_1_config.servers.push_back(ice_server); + client_1_config.type = PeerConnectionInterface::kRelay; + PeerConnectionInterface::RTCConfiguration client_2_config; + client_2_config.servers.push_back(ice_server); + client_2_config.type = PeerConnectionInterface::kRelay; + ASSERT_TRUE(CreatePeerConnectionWrappers(client_1_config, client_2_config)); + + SetupOneWayCall(); + RunTest("turn_over_tcp"); +} + +// TODO(bugs.webrtc.org/7668): Test other ICE configurations. + +} // namespace webrtc diff --git a/pc/test/framegeneratorcapturervideotracksource.h b/pc/test/framegeneratorcapturervideotracksource.h new file mode 100644 index 0000000000..b406e3bfba --- /dev/null +++ b/pc/test/framegeneratorcapturervideotracksource.h @@ -0,0 +1,72 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef PC_TEST_FRAMEGENERATORCAPTURERVIDEOTRACKSOURCE_H_ +#define PC_TEST_FRAMEGENERATORCAPTURERVIDEOTRACKSOURCE_H_ + +#include + +#include "pc/videotracksource.h" +#include "test/frame_generator_capturer.h" + +namespace webrtc { + +// Implements a VideoTrackSourceInterface to be used for creating VideoTracks. +// The video source is generated using a FrameGeneratorCapturer, specifically +// a SquareGenerator that generates frames with randomly sized and colored +// squares. +class FrameGeneratorCapturerVideoTrackSource : public VideoTrackSource { + public: + static const int kDefaultFramesPerSecond = 30; + static const int kDefaultWidth = 640; + static const int kDefaultHeight = 480; + static const int kNumSquaresGenerated = 50; + + struct Config { + int frames_per_second = kDefaultFramesPerSecond; + int width = kDefaultWidth; + int height = kDefaultHeight; + int num_squares_generated = 50; + }; + + explicit FrameGeneratorCapturerVideoTrackSource(Clock* clock) + : FrameGeneratorCapturerVideoTrackSource(Config(), clock) {} + + FrameGeneratorCapturerVideoTrackSource(Config config, Clock* clock) + : VideoTrackSource(false /* remote */) { + video_capturer_.reset(test::FrameGeneratorCapturer::Create( + config.width, config.height, rtc::nullopt, config.num_squares_generated, + config.frames_per_second, clock)); + } + + ~FrameGeneratorCapturerVideoTrackSource() = default; + + void Start() { + video_capturer_->Start(); + SetState(kLive); + } + + void Stop() { + video_capturer_->Stop(); + SetState(kMuted); + } + + protected: + rtc::VideoSourceInterface* source() override { + return video_capturer_.get(); + } + + private: + std::unique_ptr video_capturer_; +}; + +} // namespace webrtc + +#endif // PC_TEST_FRAMEGENERATORCAPTURERVIDEOTRACKSOURCE_H_ diff --git a/pc/test/mockpeerconnectionobservers.h b/pc/test/mockpeerconnectionobservers.h index e864706fcb..147b674918 100644 --- a/pc/test/mockpeerconnectionobservers.h +++ b/pc/test/mockpeerconnectionobservers.h @@ -93,8 +93,13 @@ class MockPeerConnectionObserver : public PeerConnectionObserver { void OnIceConnectionChange( PeerConnectionInterface::IceConnectionState new_state) override { RTC_DCHECK(pc_->ice_connection_state() == new_state); + // When ICE is finished, the caller will get to a kIceConnectionCompleted + // state, because it has the ICE controlling role, while the callee + // will get to a kIceConnectionConnected state. This means that both ICE + // and DTLS are connected. ice_connected_ = - (new_state == PeerConnectionInterface::kIceConnectionConnected); + (new_state == PeerConnectionInterface::kIceConnectionConnected) || + (new_state == PeerConnectionInterface::kIceConnectionCompleted); callback_triggered_ = true; } void OnIceGatheringChange( @@ -188,6 +193,14 @@ class MockPeerConnectionObserver : public PeerConnectionObserver { } } + std::vector GetAllCandidates() { + std::vector candidates; + for (const auto& candidate : candidates_) { + candidates.push_back(candidate.get()); + } + return candidates; + } + std::vector GetCandidatesByMline(int mline_index) { std::vector candidates; for (const auto& candidate : candidates_) {