[Adaptation] Add ability to inject resources on the PeerConnection.

This unblocks injecting platform-specific resources, such as power
usage signals in Chrome.

This CL adds AddAdaptationResource to PeerConnectionInterface and
integration tests verifying that if an injected resource is overusing,
resolution will soon be reduced.

To aid testing, some testing-only classes have been updated.

Bug: webrtc:11525
Change-Id: I820099e79f18d910fd641ee1412ad064b99ebce9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/177003
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31505}

api/BUILD.gn

@@ -173,6 +173,7 @@ rtc_library("libjingle_peerconnection_api") {
     ":rtp_parameters",
     ":rtp_transceiver_direction",
     ":scoped_refptr",
+    "adaptation:resource_adaptation_api",
     "audio:audio_mixer_api",
     "audio_codecs:audio_codecs_api",
     "crypto:frame_decryptor_interface",

api/peer_connection_interface.h

@@ -73,6 +73,7 @@
 #include <string>
 #include <vector>
 
+#include "api/adaptation/resource.h"
 #include "api/async_resolver_factory.h"
 #include "api/audio/audio_mixer.h"
 #include "api/audio_codecs/audio_decoder_factory.h"
@@ -1116,6 +1117,14 @@ class RTC_EXPORT PeerConnectionInterface : public rtc::RefCountInterface {
     return absl::nullopt;
   }
 
+  // When a resource is overused, the PeerConnection will try to reduce the load
+  // on the sysem, for example by reducing the resolution or frame rate of
+  // encoded streams. The Resource API allows injecting platform-specific usage
+  // measurements. The conditions to trigger kOveruse or kUnderuse are up to the
+  // implementation.
+  // TODO(hbos): Make pure virtual when implemented by downstream projects.
+  virtual void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) {}
+
   // Start RtcEventLog using an existing output-sink. Takes ownership of
   // |output| and passes it on to Call, which will take the ownership. If the
   // operation fails the output will be closed and deallocated. The event log

api/peer_connection_proxy.h

@@ -132,6 +132,7 @@ PROXY_METHOD0(IceConnectionState, standardized_ice_connection_state)
 PROXY_METHOD0(PeerConnectionState, peer_connection_state)
 PROXY_METHOD0(IceGatheringState, ice_gathering_state)
 PROXY_METHOD0(absl::optional<bool>, can_trickle_ice_candidates)
+PROXY_METHOD1(void, AddAdaptationResource, rtc::scoped_refptr<Resource>)
 PROXY_METHOD2(bool,
               StartRtcEventLog,
               std::unique_ptr<RtcEventLogOutput>,

pc/BUILD.gn

@@ -522,6 +522,7 @@ if (rtc_include_tests) {
       "jsep_session_description_unittest.cc",
       "local_audio_source_unittest.cc",
       "media_stream_unittest.cc",
+      "peer_connection_adaptation_integrationtest.cc",
       "peer_connection_bundle_unittest.cc",
       "peer_connection_crypto_unittest.cc",
       "peer_connection_data_channel_unittest.cc",
@@ -589,6 +590,7 @@ if (rtc_include_tests) {
       "../api/transport/rtp:rtp_source",
       "../api/units:time_delta",
       "../api/video:builtin_video_bitrate_allocator_factory",
+      "../call/adaptation:resource_adaptation_test_utilities",
       "../logging:fake_rtc_event_log",
       "../media:rtc_media_config",
       "../media:rtc_media_engine_defaults",

pc/peer_connection.cc

@@ -4369,6 +4369,21 @@ PeerConnection::GetFirstAudioTransceiver() const {
   return nullptr;
 }
 
+void PeerConnection::AddAdaptationResource(
+    rtc::scoped_refptr<Resource> resource) {
+  if (!worker_thread()->IsCurrent()) {
+    return worker_thread()->Invoke<void>(RTC_FROM_HERE, [this, resource]() {
+      return AddAdaptationResource(resource);
+    });
+  }
+  RTC_DCHECK_RUN_ON(worker_thread());
+  if (!call_) {
+    // The PeerConnection has been closed.
+    return;
+  }
+  call_->AddAdaptationResource(resource);
+}
+
 bool PeerConnection::StartRtcEventLog(std::unique_ptr<RtcEventLogOutput> output,
                                       int64_t output_period_ms) {
   return worker_thread()->Invoke<bool>(

pc/peer_connection.h

@@ -237,6 +237,8 @@ class PeerConnection : public PeerConnectionInternal,
 
   rtc::scoped_refptr<SctpTransportInterface> GetSctpTransport() const override;
 
+  void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override;
+
   bool StartRtcEventLog(std::unique_ptr<RtcEventLogOutput> output,
                         int64_t output_period_ms) override;
   bool StartRtcEventLog(std::unique_ptr<RtcEventLogOutput> output) override;

pc/peer_connection_adaptation_integrationtest.cc

@@ -0,0 +1,161 @@
+/*
+ *  Copyright 2020 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/rtp_parameters.h"
+#include "api/scoped_refptr.h"
+#include "call/adaptation/test/fake_resource.h"
+#include "pc/test/fake_periodic_video_source.h"
+#include "pc/test/fake_periodic_video_track_source.h"
+#include "pc/test/peer_connection_test_wrapper.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/gunit.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/thread.h"
+#include "rtc_base/virtual_socket_server.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+const int64_t kDefaultTimeoutMs = 5000;
+
+struct TrackWithPeriodicSource {
+  rtc::scoped_refptr<VideoTrackInterface> track;
+  rtc::scoped_refptr<FakePeriodicVideoTrackSource> periodic_track_source;
+};
+
+// Performs an O/A exchange and waits until the signaling state is stable again.
+void Negotiate(rtc::scoped_refptr<PeerConnectionTestWrapper> caller,
+               rtc::scoped_refptr<PeerConnectionTestWrapper> callee) {
+  // Wire up callbacks and listeners such that a full O/A is performed in
+  // response to CreateOffer().
+  PeerConnectionTestWrapper::Connect(caller.get(), callee.get());
+  caller->CreateOffer(PeerConnectionInterface::RTCOfferAnswerOptions());
+  caller->WaitForNegotiation();
+}
+
+TrackWithPeriodicSource CreateTrackWithPeriodicSource(
+    rtc::scoped_refptr<PeerConnectionFactoryInterface> factory) {
+  FakePeriodicVideoSource::Config periodic_track_source_config;
+  periodic_track_source_config.frame_interval_ms = 100;
+  periodic_track_source_config.timestamp_offset_ms = rtc::TimeMillis();
+  rtc::scoped_refptr<FakePeriodicVideoTrackSource> periodic_track_source =
+      new rtc::RefCountedObject<FakePeriodicVideoTrackSource>(
+          periodic_track_source_config, /* remote */ false);
+  TrackWithPeriodicSource track_with_source;
+  track_with_source.track =
+      factory->CreateVideoTrack("PeriodicTrack", periodic_track_source);
+  track_with_source.periodic_track_source = periodic_track_source;
+  return track_with_source;
+}
+
+// Triggers overuse and obtains VideoSinkWants. Adaptation processing happens in
+// parallel and this function makes no guarantee that the returnd VideoSinkWants
+// have yet to reflect the overuse signal. Used together with EXPECT_TRUE_WAIT
+// to "spam overuse until a change is observed".
+rtc::VideoSinkWants TriggerOveruseAndGetSinkWants(
+    rtc::scoped_refptr<FakeResource> fake_resource,
+    const FakePeriodicVideoSource& source) {
+  fake_resource->SetUsageState(ResourceUsageState::kOveruse);
+  return source.wants();
+}
+
+class PeerConnectionAdaptationIntegrationTest : public ::testing::Test {
+ public:
+  PeerConnectionAdaptationIntegrationTest()
+      : virtual_socket_server_(),
+        network_thread_(new rtc::Thread(&virtual_socket_server_)),
+        worker_thread_(rtc::Thread::Create()) {
+    RTC_CHECK(network_thread_->Start());
+    RTC_CHECK(worker_thread_->Start());
+  }
+
+  rtc::scoped_refptr<PeerConnectionTestWrapper> CreatePcWrapper(
+      const char* name) {
+    rtc::scoped_refptr<PeerConnectionTestWrapper> pc_wrapper =
+        new rtc::RefCountedObject<PeerConnectionTestWrapper>(
+            name, network_thread_.get(), worker_thread_.get());
+    PeerConnectionInterface::RTCConfiguration config;
+    config.sdp_semantics = SdpSemantics::kUnifiedPlan;
+    EXPECT_TRUE(pc_wrapper->CreatePc(config, CreateBuiltinAudioEncoderFactory(),
+                                     CreateBuiltinAudioDecoderFactory()));
+    return pc_wrapper;
+  }
+
+ protected:
+  rtc::VirtualSocketServer virtual_socket_server_;
+  std::unique_ptr<rtc::Thread> network_thread_;
+  std::unique_ptr<rtc::Thread> worker_thread_;
+};
+
+TEST_F(PeerConnectionAdaptationIntegrationTest,
+       ResouceInjectedAfterNegotiationCausesReductionInResolution) {
+  auto caller_wrapper = CreatePcWrapper("caller");
+  auto caller = caller_wrapper->pc();
+  auto callee_wrapper = CreatePcWrapper("callee");
+
+  // Adding a track and negotiating ensures that a VideoSendStream exists.
+  TrackWithPeriodicSource track_with_source =
+      CreateTrackWithPeriodicSource(caller_wrapper->pc_factory());
+  auto sender = caller->AddTrack(track_with_source.track, {}).value();
+  Negotiate(caller_wrapper, callee_wrapper);
+  // Prefer degrading resolution.
+  auto parameters = sender->GetParameters();
+  parameters.degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE;
+  sender->SetParameters(parameters);
+
+  const auto& source =
+      track_with_source.periodic_track_source->fake_periodic_source();
+  int pixel_count_before_overuse = source.wants().max_pixel_count;
+
+  // Inject a fake resource and spam kOveruse until resolution becomes limited.
+  auto fake_resource = FakeResource::Create("FakeResource");
+  caller->AddAdaptationResource(fake_resource);
+  EXPECT_TRUE_WAIT(
+      TriggerOveruseAndGetSinkWants(fake_resource, source).max_pixel_count <
+          pixel_count_before_overuse,
+      kDefaultTimeoutMs);
+}
+
+TEST_F(PeerConnectionAdaptationIntegrationTest,
+       ResouceInjectedBeforeNegotiationCausesReductionInResolution) {
+  auto caller_wrapper = CreatePcWrapper("caller");
+  auto caller = caller_wrapper->pc();
+  auto callee_wrapper = CreatePcWrapper("callee");
+
+  // Inject a fake resource before adding any tracks or negotiating.
+  auto fake_resource = FakeResource::Create("FakeResource");
+  caller->AddAdaptationResource(fake_resource);
+
+  // Adding a track and negotiating ensures that a VideoSendStream exists.
+  TrackWithPeriodicSource track_with_source =
+      CreateTrackWithPeriodicSource(caller_wrapper->pc_factory());
+  auto sender = caller->AddTrack(track_with_source.track, {}).value();
+  Negotiate(caller_wrapper, callee_wrapper);
+  // Prefer degrading resolution.
+  auto parameters = sender->GetParameters();
+  parameters.degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE;
+  sender->SetParameters(parameters);
+
+  const auto& source =
+      track_with_source.periodic_track_source->fake_periodic_source();
+  int pixel_count_before_overuse = source.wants().max_pixel_count;
+
+  // Spam kOveruse until resolution becomes limited.
+  EXPECT_TRUE_WAIT(
+      TriggerOveruseAndGetSinkWants(fake_resource, source).max_pixel_count <
+          pixel_count_before_overuse,
+      kDefaultTimeoutMs);
+}
+
+}  // namespace webrtc

pc/test/fake_periodic_video_source.h

@@ -16,6 +16,7 @@
 #include "api/video/video_source_interface.h"
 #include "media/base/fake_frame_source.h"
 #include "media/base/video_broadcaster.h"
+#include "rtc_base/critical_section.h"
 #include "rtc_base/task_queue_for_test.h"
 #include "rtc_base/task_utils/repeating_task.h"
 
@@ -59,6 +60,11 @@ class FakePeriodicVideoSource final
     });
   }
 
+  rtc::VideoSinkWants wants() const {
+    rtc::CritScope cs(&crit_);
+    return wants_;
+  }
+
   void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override {
     RTC_DCHECK(thread_checker_.IsCurrent());
     broadcaster_.RemoveSink(sink);
@@ -67,6 +73,10 @@ class FakePeriodicVideoSource final
   void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
                        const rtc::VideoSinkWants& wants) override {
     RTC_DCHECK(thread_checker_.IsCurrent());
+    {
+      rtc::CritScope cs(&crit_);
+      wants_ = wants;
+    }
     broadcaster_.AddOrUpdateSink(sink, wants);
   }
 
@@ -80,6 +90,8 @@ class FakePeriodicVideoSource final
 
   rtc::VideoBroadcaster broadcaster_;
   cricket::FakeFrameSource frame_source_;
+  rtc::CriticalSection crit_;
+  rtc::VideoSinkWants wants_ RTC_GUARDED_BY(&crit_);
 
   std::unique_ptr<TaskQueueForTest> task_queue_;
 };

pc/test/fake_periodic_video_track_source.h

@@ -29,6 +29,10 @@ class FakePeriodicVideoTrackSource : public VideoTrackSource {
 
   ~FakePeriodicVideoTrackSource() = default;
 
+  const FakePeriodicVideoSource& fake_periodic_source() const {
+    return source_;
+  }
+
  protected:
   rtc::VideoSourceInterface<VideoFrame>* source() override { return &source_; }
 

pc/test/peer_connection_test_wrapper.cc

@@ -80,7 +80,8 @@ PeerConnectionTestWrapper::PeerConnectionTestWrapper(
     rtc::Thread* worker_thread)
     : name_(name),
       network_thread_(network_thread),
-      worker_thread_(worker_thread) {
+      worker_thread_(worker_thread),
+      pending_negotiation_(false) {
   pc_thread_checker_.Detach();
 }
 
@@ -135,6 +136,17 @@ PeerConnectionTestWrapper::CreateDataChannel(
   return peer_connection_->CreateDataChannel(label, &init);
 }
 
+void PeerConnectionTestWrapper::WaitForNegotiation() {
+  EXPECT_TRUE_WAIT(!pending_negotiation_, kMaxWait);
+}
+
+void PeerConnectionTestWrapper::OnSignalingChange(
+    webrtc::PeerConnectionInterface::SignalingState new_state) {
+  if (new_state == webrtc::PeerConnectionInterface::SignalingState::kStable) {
+    pending_negotiation_ = false;
+  }
+}
+
 void PeerConnectionTestWrapper::OnAddTrack(
     rtc::scoped_refptr<RtpReceiverInterface> receiver,
     const std::vector<rtc::scoped_refptr<MediaStreamInterface>>& streams) {
@@ -182,6 +194,7 @@ void PeerConnectionTestWrapper::OnSuccess(SessionDescriptionInterface* desc) {
 void PeerConnectionTestWrapper::CreateOffer(
     const webrtc::PeerConnectionInterface::RTCOfferAnswerOptions& options) {
   RTC_LOG(LS_INFO) << "PeerConnectionTestWrapper " << name_ << ": CreateOffer.";
+  pending_negotiation_ = true;
   peer_connection_->CreateOffer(this, options);
 }
 
@@ -189,6 +202,7 @@ void PeerConnectionTestWrapper::CreateAnswer(
     const webrtc::PeerConnectionInterface::RTCOfferAnswerOptions& options) {
   RTC_LOG(LS_INFO) << "PeerConnectionTestWrapper " << name_
                    << ": CreateAnswer.";
+  pending_negotiation_ = true;
   peer_connection_->CreateAnswer(this, options);
 }
 

pc/test/peer_connection_test_wrapper.h

@@ -49,15 +49,21 @@ class PeerConnectionTestWrapper
       rtc::scoped_refptr<webrtc::AudioEncoderFactory> audio_encoder_factory,
       rtc::scoped_refptr<webrtc::AudioDecoderFactory> audio_decoder_factory);
 
+  rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> pc_factory()
+      const {
+    return peer_connection_factory_;
+  }
   webrtc::PeerConnectionInterface* pc() { return peer_connection_.get(); }
 
   rtc::scoped_refptr<webrtc::DataChannelInterface> CreateDataChannel(
       const std::string& label,
       const webrtc::DataChannelInit& init);
 
+  void WaitForNegotiation();
+
   // Implements PeerConnectionObserver.
   void OnSignalingChange(
-      webrtc::PeerConnectionInterface::SignalingState new_state) override {}
+      webrtc::PeerConnectionInterface::SignalingState new_state) override;
   void OnAddTrack(
       rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
       const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>>&
@@ -121,6 +127,7 @@ class PeerConnectionTestWrapper
   rtc::scoped_refptr<FakeAudioCaptureModule> fake_audio_capture_module_;
   std::unique_ptr<webrtc::FakeVideoTrackRenderer> renderer_;
   int num_get_user_media_calls_ = 0;
+  bool pending_negotiation_;
 };
 
 #endif  // PC_TEST_PEER_CONNECTION_TEST_WRAPPER_H_