diff --git a/api/BUILD.gn b/api/BUILD.gn index 2121744ba8..30e414cddb 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -173,6 +173,7 @@ rtc_library("libjingle_peerconnection_api") { ":rtp_parameters", ":rtp_transceiver_direction", ":scoped_refptr", + "adaptation:resource_adaptation_api", "audio:audio_mixer_api", "audio_codecs:audio_codecs_api", "crypto:frame_decryptor_interface", diff --git a/api/peer_connection_interface.h b/api/peer_connection_interface.h index 0ae47b2a2f..3293910bfe 100644 --- a/api/peer_connection_interface.h +++ b/api/peer_connection_interface.h @@ -73,6 +73,7 @@ #include #include +#include "api/adaptation/resource.h" #include "api/async_resolver_factory.h" #include "api/audio/audio_mixer.h" #include "api/audio_codecs/audio_decoder_factory.h" @@ -1116,6 +1117,14 @@ class RTC_EXPORT PeerConnectionInterface : public rtc::RefCountInterface { return absl::nullopt; } + // When a resource is overused, the PeerConnection will try to reduce the load + // on the sysem, for example by reducing the resolution or frame rate of + // encoded streams. The Resource API allows injecting platform-specific usage + // measurements. The conditions to trigger kOveruse or kUnderuse are up to the + // implementation. + // TODO(hbos): Make pure virtual when implemented by downstream projects. + virtual void AddAdaptationResource(rtc::scoped_refptr resource) {} + // Start RtcEventLog using an existing output-sink. Takes ownership of // |output| and passes it on to Call, which will take the ownership. If the // operation fails the output will be closed and deallocated. The event log diff --git a/api/peer_connection_proxy.h b/api/peer_connection_proxy.h index c278308ccb..23887e53da 100644 --- a/api/peer_connection_proxy.h +++ b/api/peer_connection_proxy.h @@ -132,6 +132,7 @@ PROXY_METHOD0(IceConnectionState, standardized_ice_connection_state) PROXY_METHOD0(PeerConnectionState, peer_connection_state) PROXY_METHOD0(IceGatheringState, ice_gathering_state) PROXY_METHOD0(absl::optional, can_trickle_ice_candidates) +PROXY_METHOD1(void, AddAdaptationResource, rtc::scoped_refptr) PROXY_METHOD2(bool, StartRtcEventLog, std::unique_ptr, diff --git a/pc/BUILD.gn b/pc/BUILD.gn index 1e832734f1..12a7fcc19f 100644 --- a/pc/BUILD.gn +++ b/pc/BUILD.gn @@ -522,6 +522,7 @@ if (rtc_include_tests) { "jsep_session_description_unittest.cc", "local_audio_source_unittest.cc", "media_stream_unittest.cc", + "peer_connection_adaptation_integrationtest.cc", "peer_connection_bundle_unittest.cc", "peer_connection_crypto_unittest.cc", "peer_connection_data_channel_unittest.cc", @@ -589,6 +590,7 @@ if (rtc_include_tests) { "../api/transport/rtp:rtp_source", "../api/units:time_delta", "../api/video:builtin_video_bitrate_allocator_factory", + "../call/adaptation:resource_adaptation_test_utilities", "../logging:fake_rtc_event_log", "../media:rtc_media_config", "../media:rtc_media_engine_defaults", diff --git a/pc/peer_connection.cc b/pc/peer_connection.cc index 059d5dd9e0..c46eaa2b9e 100644 --- a/pc/peer_connection.cc +++ b/pc/peer_connection.cc @@ -4369,6 +4369,21 @@ PeerConnection::GetFirstAudioTransceiver() const { return nullptr; } +void PeerConnection::AddAdaptationResource( + rtc::scoped_refptr resource) { + if (!worker_thread()->IsCurrent()) { + return worker_thread()->Invoke(RTC_FROM_HERE, [this, resource]() { + return AddAdaptationResource(resource); + }); + } + RTC_DCHECK_RUN_ON(worker_thread()); + if (!call_) { + // The PeerConnection has been closed. + return; + } + call_->AddAdaptationResource(resource); +} + bool PeerConnection::StartRtcEventLog(std::unique_ptr output, int64_t output_period_ms) { return worker_thread()->Invoke( diff --git a/pc/peer_connection.h b/pc/peer_connection.h index f3102572fb..3bb962bb1d 100644 --- a/pc/peer_connection.h +++ b/pc/peer_connection.h @@ -237,6 +237,8 @@ class PeerConnection : public PeerConnectionInternal, rtc::scoped_refptr GetSctpTransport() const override; + void AddAdaptationResource(rtc::scoped_refptr resource) override; + bool StartRtcEventLog(std::unique_ptr output, int64_t output_period_ms) override; bool StartRtcEventLog(std::unique_ptr output) override; diff --git a/pc/peer_connection_adaptation_integrationtest.cc b/pc/peer_connection_adaptation_integrationtest.cc new file mode 100644 index 0000000000..71d054eb90 --- /dev/null +++ b/pc/peer_connection_adaptation_integrationtest.cc @@ -0,0 +1,161 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" +#include "call/adaptation/test/fake_resource.h" +#include "pc/test/fake_periodic_video_source.h" +#include "pc/test/fake_periodic_video_track_source.h" +#include "pc/test/peer_connection_test_wrapper.h" +#include "rtc_base/checks.h" +#include "rtc_base/gunit.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/thread.h" +#include "rtc_base/virtual_socket_server.h" +#include "test/gtest.h" + +namespace webrtc { + +const int64_t kDefaultTimeoutMs = 5000; + +struct TrackWithPeriodicSource { + rtc::scoped_refptr track; + rtc::scoped_refptr periodic_track_source; +}; + +// Performs an O/A exchange and waits until the signaling state is stable again. +void Negotiate(rtc::scoped_refptr caller, + rtc::scoped_refptr callee) { + // Wire up callbacks and listeners such that a full O/A is performed in + // response to CreateOffer(). + PeerConnectionTestWrapper::Connect(caller.get(), callee.get()); + caller->CreateOffer(PeerConnectionInterface::RTCOfferAnswerOptions()); + caller->WaitForNegotiation(); +} + +TrackWithPeriodicSource CreateTrackWithPeriodicSource( + rtc::scoped_refptr factory) { + FakePeriodicVideoSource::Config periodic_track_source_config; + periodic_track_source_config.frame_interval_ms = 100; + periodic_track_source_config.timestamp_offset_ms = rtc::TimeMillis(); + rtc::scoped_refptr periodic_track_source = + new rtc::RefCountedObject( + periodic_track_source_config, /* remote */ false); + TrackWithPeriodicSource track_with_source; + track_with_source.track = + factory->CreateVideoTrack("PeriodicTrack", periodic_track_source); + track_with_source.periodic_track_source = periodic_track_source; + return track_with_source; +} + +// Triggers overuse and obtains VideoSinkWants. Adaptation processing happens in +// parallel and this function makes no guarantee that the returnd VideoSinkWants +// have yet to reflect the overuse signal. Used together with EXPECT_TRUE_WAIT +// to "spam overuse until a change is observed". +rtc::VideoSinkWants TriggerOveruseAndGetSinkWants( + rtc::scoped_refptr fake_resource, + const FakePeriodicVideoSource& source) { + fake_resource->SetUsageState(ResourceUsageState::kOveruse); + return source.wants(); +} + +class PeerConnectionAdaptationIntegrationTest : public ::testing::Test { + public: + PeerConnectionAdaptationIntegrationTest() + : virtual_socket_server_(), + network_thread_(new rtc::Thread(&virtual_socket_server_)), + worker_thread_(rtc::Thread::Create()) { + RTC_CHECK(network_thread_->Start()); + RTC_CHECK(worker_thread_->Start()); + } + + rtc::scoped_refptr CreatePcWrapper( + const char* name) { + rtc::scoped_refptr pc_wrapper = + new rtc::RefCountedObject( + name, network_thread_.get(), worker_thread_.get()); + PeerConnectionInterface::RTCConfiguration config; + config.sdp_semantics = SdpSemantics::kUnifiedPlan; + EXPECT_TRUE(pc_wrapper->CreatePc(config, CreateBuiltinAudioEncoderFactory(), + CreateBuiltinAudioDecoderFactory())); + return pc_wrapper; + } + + protected: + rtc::VirtualSocketServer virtual_socket_server_; + std::unique_ptr network_thread_; + std::unique_ptr worker_thread_; +}; + +TEST_F(PeerConnectionAdaptationIntegrationTest, + ResouceInjectedAfterNegotiationCausesReductionInResolution) { + auto caller_wrapper = CreatePcWrapper("caller"); + auto caller = caller_wrapper->pc(); + auto callee_wrapper = CreatePcWrapper("callee"); + + // Adding a track and negotiating ensures that a VideoSendStream exists. + TrackWithPeriodicSource track_with_source = + CreateTrackWithPeriodicSource(caller_wrapper->pc_factory()); + auto sender = caller->AddTrack(track_with_source.track, {}).value(); + Negotiate(caller_wrapper, callee_wrapper); + // Prefer degrading resolution. + auto parameters = sender->GetParameters(); + parameters.degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE; + sender->SetParameters(parameters); + + const auto& source = + track_with_source.periodic_track_source->fake_periodic_source(); + int pixel_count_before_overuse = source.wants().max_pixel_count; + + // Inject a fake resource and spam kOveruse until resolution becomes limited. + auto fake_resource = FakeResource::Create("FakeResource"); + caller->AddAdaptationResource(fake_resource); + EXPECT_TRUE_WAIT( + TriggerOveruseAndGetSinkWants(fake_resource, source).max_pixel_count < + pixel_count_before_overuse, + kDefaultTimeoutMs); +} + +TEST_F(PeerConnectionAdaptationIntegrationTest, + ResouceInjectedBeforeNegotiationCausesReductionInResolution) { + auto caller_wrapper = CreatePcWrapper("caller"); + auto caller = caller_wrapper->pc(); + auto callee_wrapper = CreatePcWrapper("callee"); + + // Inject a fake resource before adding any tracks or negotiating. + auto fake_resource = FakeResource::Create("FakeResource"); + caller->AddAdaptationResource(fake_resource); + + // Adding a track and negotiating ensures that a VideoSendStream exists. + TrackWithPeriodicSource track_with_source = + CreateTrackWithPeriodicSource(caller_wrapper->pc_factory()); + auto sender = caller->AddTrack(track_with_source.track, {}).value(); + Negotiate(caller_wrapper, callee_wrapper); + // Prefer degrading resolution. + auto parameters = sender->GetParameters(); + parameters.degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE; + sender->SetParameters(parameters); + + const auto& source = + track_with_source.periodic_track_source->fake_periodic_source(); + int pixel_count_before_overuse = source.wants().max_pixel_count; + + // Spam kOveruse until resolution becomes limited. + EXPECT_TRUE_WAIT( + TriggerOveruseAndGetSinkWants(fake_resource, source).max_pixel_count < + pixel_count_before_overuse, + kDefaultTimeoutMs); +} + +} // namespace webrtc diff --git a/pc/test/fake_periodic_video_source.h b/pc/test/fake_periodic_video_source.h index 1684ca4adb..b1cff4e5ed 100644 --- a/pc/test/fake_periodic_video_source.h +++ b/pc/test/fake_periodic_video_source.h @@ -16,6 +16,7 @@ #include "api/video/video_source_interface.h" #include "media/base/fake_frame_source.h" #include "media/base/video_broadcaster.h" +#include "rtc_base/critical_section.h" #include "rtc_base/task_queue_for_test.h" #include "rtc_base/task_utils/repeating_task.h" @@ -59,6 +60,11 @@ class FakePeriodicVideoSource final }); } + rtc::VideoSinkWants wants() const { + rtc::CritScope cs(&crit_); + return wants_; + } + void RemoveSink(rtc::VideoSinkInterface* sink) override { RTC_DCHECK(thread_checker_.IsCurrent()); broadcaster_.RemoveSink(sink); @@ -67,6 +73,10 @@ class FakePeriodicVideoSource final void AddOrUpdateSink(rtc::VideoSinkInterface* sink, const rtc::VideoSinkWants& wants) override { RTC_DCHECK(thread_checker_.IsCurrent()); + { + rtc::CritScope cs(&crit_); + wants_ = wants; + } broadcaster_.AddOrUpdateSink(sink, wants); } @@ -80,6 +90,8 @@ class FakePeriodicVideoSource final rtc::VideoBroadcaster broadcaster_; cricket::FakeFrameSource frame_source_; + rtc::CriticalSection crit_; + rtc::VideoSinkWants wants_ RTC_GUARDED_BY(&crit_); std::unique_ptr task_queue_; }; diff --git a/pc/test/fake_periodic_video_track_source.h b/pc/test/fake_periodic_video_track_source.h index cc406d6d3f..98a456f232 100644 --- a/pc/test/fake_periodic_video_track_source.h +++ b/pc/test/fake_periodic_video_track_source.h @@ -29,6 +29,10 @@ class FakePeriodicVideoTrackSource : public VideoTrackSource { ~FakePeriodicVideoTrackSource() = default; + const FakePeriodicVideoSource& fake_periodic_source() const { + return source_; + } + protected: rtc::VideoSourceInterface* source() override { return &source_; } diff --git a/pc/test/peer_connection_test_wrapper.cc b/pc/test/peer_connection_test_wrapper.cc index 4f0d72e667..946f459f3b 100644 --- a/pc/test/peer_connection_test_wrapper.cc +++ b/pc/test/peer_connection_test_wrapper.cc @@ -80,7 +80,8 @@ PeerConnectionTestWrapper::PeerConnectionTestWrapper( rtc::Thread* worker_thread) : name_(name), network_thread_(network_thread), - worker_thread_(worker_thread) { + worker_thread_(worker_thread), + pending_negotiation_(false) { pc_thread_checker_.Detach(); } @@ -135,6 +136,17 @@ PeerConnectionTestWrapper::CreateDataChannel( return peer_connection_->CreateDataChannel(label, &init); } +void PeerConnectionTestWrapper::WaitForNegotiation() { + EXPECT_TRUE_WAIT(!pending_negotiation_, kMaxWait); +} + +void PeerConnectionTestWrapper::OnSignalingChange( + webrtc::PeerConnectionInterface::SignalingState new_state) { + if (new_state == webrtc::PeerConnectionInterface::SignalingState::kStable) { + pending_negotiation_ = false; + } +} + void PeerConnectionTestWrapper::OnAddTrack( rtc::scoped_refptr receiver, const std::vector>& streams) { @@ -182,6 +194,7 @@ void PeerConnectionTestWrapper::OnSuccess(SessionDescriptionInterface* desc) { void PeerConnectionTestWrapper::CreateOffer( const webrtc::PeerConnectionInterface::RTCOfferAnswerOptions& options) { RTC_LOG(LS_INFO) << "PeerConnectionTestWrapper " << name_ << ": CreateOffer."; + pending_negotiation_ = true; peer_connection_->CreateOffer(this, options); } @@ -189,6 +202,7 @@ void PeerConnectionTestWrapper::CreateAnswer( const webrtc::PeerConnectionInterface::RTCOfferAnswerOptions& options) { RTC_LOG(LS_INFO) << "PeerConnectionTestWrapper " << name_ << ": CreateAnswer."; + pending_negotiation_ = true; peer_connection_->CreateAnswer(this, options); } diff --git a/pc/test/peer_connection_test_wrapper.h b/pc/test/peer_connection_test_wrapper.h index 2dc88e9309..92599b78ab 100644 --- a/pc/test/peer_connection_test_wrapper.h +++ b/pc/test/peer_connection_test_wrapper.h @@ -49,15 +49,21 @@ class PeerConnectionTestWrapper rtc::scoped_refptr audio_encoder_factory, rtc::scoped_refptr audio_decoder_factory); + rtc::scoped_refptr pc_factory() + const { + return peer_connection_factory_; + } webrtc::PeerConnectionInterface* pc() { return peer_connection_.get(); } rtc::scoped_refptr CreateDataChannel( const std::string& label, const webrtc::DataChannelInit& init); + void WaitForNegotiation(); + // Implements PeerConnectionObserver. void OnSignalingChange( - webrtc::PeerConnectionInterface::SignalingState new_state) override {} + webrtc::PeerConnectionInterface::SignalingState new_state) override; void OnAddTrack( rtc::scoped_refptr receiver, const std::vector>& @@ -121,6 +127,7 @@ class PeerConnectionTestWrapper rtc::scoped_refptr fake_audio_capture_module_; std::unique_ptr renderer_; int num_get_user_media_calls_ = 0; + bool pending_negotiation_; }; #endif // PC_TEST_PEER_CONNECTION_TEST_WRAPPER_H_