[Adaptation] Adding adaptation resources from Call.

This CL adds AddAdaptationResource to Call and
AddAdaptationResource/GetAdaptationResources method to relevant
VideoSendStream and VideoStreamEncoder interfaces and implementations.

Unittests are added to ensure that resources can be added to the Call
both before and after the creation of a VideoSendStream and that the
resources always gets added to the streams.

In a follow-up CL, we will continue to plumb the resources all the way
to PeerConnectionInterface, and an integration test will then be added
to ensure that injected resources are capable of triggering adaptation.

Bug: webrtc:11525
Change-Id: I499e9c23c3e359df943414d420b2e0ce2e9b2d56
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/177002
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31499}

api/video/BUILD.gn

@@ -259,8 +259,10 @@ rtc_source_set("video_stream_encoder") {
     ":video_codec_constants",
     ":video_frame",
     "..:rtp_parameters",
+    "..:scoped_refptr",
     "../:fec_controller_api",
     "../:rtp_parameters",
+    "../adaptation:resource_adaptation_api",
     "../units:data_rate",
     "../video_codecs:video_codecs_api",
   ]

api/video/video_stream_encoder_interface.h

@@ -13,8 +13,10 @@
 
 #include <vector>
 
+#include "api/adaptation/resource.h"
 #include "api/fec_controller_override.h"
 #include "api/rtp_parameters.h"  // For DegradationPreference.
+#include "api/scoped_refptr.h"
 #include "api/units/data_rate.h"
 #include "api/video/video_bitrate_allocator.h"
 #include "api/video/video_sink_interface.h"
@@ -49,6 +51,15 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface<VideoFrame> {
         int min_transmit_bitrate_bps) = 0;
   };
 
+  // If the resource is overusing, the VideoStreamEncoder will try to reduce
+  // resolution or frame rate until no resource is overusing.
+  // TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor
+  // is moved to Call this method could be deleted altogether in favor of
+  // Call-level APIs only.
+  virtual void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) = 0;
+  virtual std::vector<rtc::scoped_refptr<Resource>>
+  GetAdaptationResources() = 0;
+
   // Sets the source that will provide video frames to the VideoStreamEncoder's
   // OnFrame method. |degradation_preference| control whether or not resolution
   // or frame rate may be reduced. The VideoStreamEncoder registers itself with

call/BUILD.gn

@@ -39,6 +39,7 @@ rtc_library("call_interfaces") {
     "../api:rtp_parameters",
     "../api:scoped_refptr",
     "../api:transport_api",
+    "../api/adaptation:resource_adaptation_api",
     "../api/audio:audio_mixer_api",
     "../api/audio_codecs:audio_codecs_api",
     "../api/crypto:frame_decryptor_interface",
@@ -306,7 +307,9 @@ rtc_library("video_stream_api") {
     "../api:frame_transformer_interface",
     "../api:rtp_headers",
     "../api:rtp_parameters",
+    "../api:scoped_refptr",
     "../api:transport_api",
+    "../api/adaptation:resource_adaptation_api",
     "../api/crypto:frame_decryptor_interface",
     "../api/crypto:frame_encryptor_interface",
     "../api/crypto:options",
@@ -407,7 +410,9 @@ if (rtc_include_tests) {
       "../api/audio_codecs:builtin_audio_decoder_factory",
       "../api/rtc_event_log",
       "../api/task_queue:default_task_queue_factory",
+      "../api/test/video:function_video_factory",
       "../api/transport:field_trial_based_config",
+      "../api/video:builtin_video_bitrate_allocator_factory",
       "../api/video:video_frame",
       "../api/video:video_rtp_headers",
       "../audio",
@@ -441,6 +446,7 @@ if (rtc_include_tests) {
       "../test:video_test_common",
       "../test/time_controller:time_controller",
       "../video",
+      "adaptation:resource_adaptation_test_utilities",
       "//test/scenario:scenario",
       "//testing/gmock",
       "//testing/gtest",

call/adaptation/resource_adaptation_processor.cc

@@ -172,6 +172,12 @@ void ResourceAdaptationProcessor::AddResource(
   resources_.push_back(resource);
 }
 
+std::vector<rtc::scoped_refptr<Resource>>
+ResourceAdaptationProcessor::GetResources() const {
+  RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
+  return resources_;
+}
+
 void ResourceAdaptationProcessor::RemoveResource(
     rtc::scoped_refptr<Resource> resource) {
   RTC_DCHECK_RUN_ON(resource_adaptation_queue_);

call/adaptation/resource_adaptation_processor.h

@@ -72,6 +72,7 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
   void RemoveRestrictionsListener(
       VideoSourceRestrictionsListener* restrictions_listener) override;
   void AddResource(rtc::scoped_refptr<Resource> resource) override;
+  std::vector<rtc::scoped_refptr<Resource>> GetResources() const override;
   void RemoveResource(rtc::scoped_refptr<Resource> resource) override;
   void AddAdaptationConstraint(
       AdaptationConstraint* adaptation_constraint) override;

call/adaptation/resource_adaptation_processor_interface.h

@@ -12,6 +12,7 @@
 #define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_
 
 #include <map>
+#include <vector>
 
 #include "absl/types/optional.h"
 #include "api/adaptation/resource.h"
@@ -80,6 +81,7 @@ class ResourceAdaptationProcessorInterface {
   virtual void RemoveRestrictionsListener(
       VideoSourceRestrictionsListener* restrictions_listener) = 0;
   virtual void AddResource(rtc::scoped_refptr<Resource> resource) = 0;
+  virtual std::vector<rtc::scoped_refptr<Resource>> GetResources() const = 0;
   virtual void RemoveResource(rtc::scoped_refptr<Resource> resource) = 0;
   virtual void AddAdaptationConstraint(
       AdaptationConstraint* adaptation_constraint) = 0;

call/call.cc

@@ -212,6 +212,8 @@ class Call final : public webrtc::Call,
   void DestroyFlexfecReceiveStream(
       FlexfecReceiveStream* receive_stream) override;
 
+  void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override;
+
   RtpTransportControllerSendInterface* GetTransportControllerSend() override;
 
   Stats GetStats() const override;
@@ -333,6 +335,9 @@ class Call final : public webrtc::Call,
       RTC_GUARDED_BY(worker_thread_);
   std::set<VideoSendStream*> video_send_streams_ RTC_GUARDED_BY(worker_thread_);
 
+  std::vector<rtc::scoped_refptr<Resource>> adaptation_resources_
+      RTC_GUARDED_BY(worker_thread_);
+
   using RtpStateMap = std::map<uint32_t, RtpState>;
   RtpStateMap suspended_audio_send_ssrcs_ RTC_GUARDED_BY(worker_thread_);
   RtpStateMap suspended_video_send_ssrcs_ RTC_GUARDED_BY(worker_thread_);
@@ -860,6 +865,10 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream(
     video_send_ssrcs_[ssrc] = send_stream;
   }
   video_send_streams_.insert(send_stream);
+  // Add resources that were previously added to the call to the new stream.
+  for (const auto& adaptation_resource : adaptation_resources_) {
+    send_stream->AddAdaptationResource(adaptation_resource);
+  }
 
   UpdateAggregateNetworkState();
 
@@ -1024,6 +1033,14 @@ void Call::DestroyFlexfecReceiveStream(FlexfecReceiveStream* receive_stream) {
   delete receive_stream;
 }
 
+void Call::AddAdaptationResource(rtc::scoped_refptr<Resource> resource) {
+  RTC_DCHECK_RUN_ON(worker_thread_);
+  adaptation_resources_.push_back(resource);
+  for (VideoSendStream* stream : video_send_streams_) {
+    stream->AddAdaptationResource(resource);
+  }
+}
+
 RtpTransportControllerSendInterface* Call::GetTransportControllerSend() {
   return transport_send_ptr_;
 }

call/call.h

@@ -15,6 +15,7 @@
 #include <string>
 #include <vector>
 
+#include "api/adaptation/resource.h"
 #include "api/media_types.h"
 #include "call/audio_receive_stream.h"
 #include "call/audio_send_stream.h"
@@ -125,6 +126,11 @@ class Call {
   virtual void DestroyFlexfecReceiveStream(
       FlexfecReceiveStream* receive_stream) = 0;
 
+  // When a resource is overused, the Call will try to reduce the load on the
+  // sysem, for example by reducing the resolution or frame rate of encoded
+  // streams.
+  virtual void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) = 0;
+
   // All received RTP and RTCP packets for the call should be inserted to this
   // PacketReceiver. The PacketReceiver pointer is valid as long as the
   // Call instance exists.

call/call_unittest.cc

@@ -20,9 +20,12 @@
 #include "api/rtc_event_log/rtc_event_log.h"
 #include "api/task_queue/default_task_queue_factory.h"
 #include "api/test/mock_audio_mixer.h"
+#include "api/test/video/function_video_encoder_factory.h"
 #include "api/transport/field_trial_based_config.h"
+#include "api/video/builtin_video_bitrate_allocator_factory.h"
 #include "audio/audio_receive_stream.h"
 #include "audio/audio_send_stream.h"
+#include "call/adaptation/test/fake_resource.h"
 #include "call/audio_state.h"
 #include "modules/audio_device/include/mock_audio_device.h"
 #include "modules/audio_processing/include/mock_audio_processing.h"
@@ -35,6 +38,8 @@
 
 namespace {
 
+using ::testing::Contains;
+
 struct CallHelper {
   explicit CallHelper(bool use_null_audio_processing) {
     task_queue_factory_ = webrtc::CreateDefaultTaskQueueFactory();
@@ -325,6 +330,59 @@ TEST(CallTest, RecreatingAudioStreamWithSameSsrcReusesRtpState) {
   }
 }
 
+TEST(CallTest, AddAdaptationResourceAfterCreatingVideoSendStream) {
+  CallHelper call(true);
+  // Create a VideoSendStream.
+  test::FunctionVideoEncoderFactory fake_encoder_factory([]() {
+    return std::make_unique<test::FakeEncoder>(Clock::GetRealTimeClock());
+  });
+  auto bitrate_allocator_factory = CreateBuiltinVideoBitrateAllocatorFactory();
+  MockTransport send_transport;
+  VideoSendStream::Config config(&send_transport);
+  config.rtp.payload_type = 110;
+  config.rtp.ssrcs = {42};
+  config.encoder_settings.encoder_factory = &fake_encoder_factory;
+  config.encoder_settings.bitrate_allocator_factory =
+      bitrate_allocator_factory.get();
+  VideoEncoderConfig encoder_config;
+  encoder_config.max_bitrate_bps = 1337;
+  VideoSendStream* stream =
+      call->CreateVideoSendStream(std::move(config), std::move(encoder_config));
+  EXPECT_NE(stream, nullptr);
+  // Add a fake resource. It should get added to the stream.
+  auto fake_resource = FakeResource::Create("FakeResource");
+  call->AddAdaptationResource(fake_resource);
+  EXPECT_THAT(stream->GetAdaptationResources(), Contains(fake_resource));
+  call->DestroyVideoSendStream(stream);
+}
+
+TEST(CallTest, AddAdaptationResourceBeforeCreatingVideoSendStream) {
+  CallHelper call(true);
+  // Add a fake resource.
+  auto fake_resource = FakeResource::Create("FakeResource");
+  call->AddAdaptationResource(fake_resource);
+  // Create a VideoSendStream.
+  test::FunctionVideoEncoderFactory fake_encoder_factory([]() {
+    return std::make_unique<test::FakeEncoder>(Clock::GetRealTimeClock());
+  });
+  auto bitrate_allocator_factory = CreateBuiltinVideoBitrateAllocatorFactory();
+  MockTransport send_transport;
+  VideoSendStream::Config config(&send_transport);
+  config.rtp.payload_type = 110;
+  config.rtp.ssrcs = {42};
+  config.encoder_settings.encoder_factory = &fake_encoder_factory;
+  config.encoder_settings.bitrate_allocator_factory =
+      bitrate_allocator_factory.get();
+  VideoEncoderConfig encoder_config;
+  encoder_config.max_bitrate_bps = 1337;
+  VideoSendStream* stream =
+      call->CreateVideoSendStream(std::move(config), std::move(encoder_config));
+  EXPECT_NE(stream, nullptr);
+  // The fake resource should automatically get added to the stream.
+  EXPECT_THAT(stream->GetAdaptationResources(), Contains(fake_resource));
+  call->DestroyVideoSendStream(stream);
+}
+
 TEST(CallTest, SharedModuleThread) {
   class SharedModuleThreadUser : public Module {
    public:

call/degraded_call.cc

@@ -245,6 +245,11 @@ void DegradedCall::DestroyFlexfecReceiveStream(
   call_->DestroyFlexfecReceiveStream(receive_stream);
 }
 
+void DegradedCall::AddAdaptationResource(
+    rtc::scoped_refptr<Resource> resource) {
+  call_->AddAdaptationResource(std::move(resource));
+}
+
 PacketReceiver* DegradedCall::Receiver() {
   if (receive_config_) {
     return this;

call/degraded_call.h

@@ -77,6 +77,8 @@ class DegradedCall : public Call, private PacketReceiver {
   void DestroyFlexfecReceiveStream(
       FlexfecReceiveStream* receive_stream) override;
 
+  void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override;
+
   PacketReceiver* Receiver() override;
 
   RtpTransportControllerSendInterface* GetTransportControllerSend() override;

call/video_send_stream.h

@@ -18,10 +18,12 @@
 #include <vector>
 
 #include "absl/types/optional.h"
+#include "api/adaptation/resource.h"
 #include "api/call/transport.h"
 #include "api/crypto/crypto_options.h"
 #include "api/frame_transformer_interface.h"
 #include "api/rtp_parameters.h"
+#include "api/scoped_refptr.h"
 #include "api/video/video_content_type.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_sink_interface.h"
@@ -215,6 +217,15 @@ class VideoSendStream {
   // When a stream is stopped, it can't receive, process or deliver packets.
   virtual void Stop() = 0;
 
+  // If the resource is overusing, the VideoSendStream will try to reduce
+  // resolution or frame rate until no resource is overusing.
+  // TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor
+  // is moved to Call this method could be deleted altogether in favor of
+  // Call-level APIs only.
+  virtual void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) = 0;
+  virtual std::vector<rtc::scoped_refptr<Resource>>
+  GetAdaptationResources() = 0;
+
   virtual void SetSource(
       rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
       const DegradationPreference& degradation_preference) = 0;

media/engine/fake_webrtc_call.cc

@@ -279,6 +279,14 @@ void FakeVideoSendStream::Stop() {
   sending_ = false;
 }
 
+void FakeVideoSendStream::AddAdaptationResource(
+    rtc::scoped_refptr<webrtc::Resource> resource) {}
+
+std::vector<rtc::scoped_refptr<webrtc::Resource>>
+FakeVideoSendStream::GetAdaptationResources() {
+  return {};
+}
+
 void FakeVideoSendStream::SetSource(
     rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
     const webrtc::DegradationPreference& degradation_preference) {
@@ -570,6 +578,9 @@ void FakeCall::DestroyFlexfecReceiveStream(
   }
 }
 
+void FakeCall::AddAdaptationResource(
+    rtc::scoped_refptr<webrtc::Resource> resource) {}
+
 webrtc::PacketReceiver* FakeCall::Receiver() {
   return this;
 }

media/engine/fake_webrtc_call.h

@@ -173,6 +173,10 @@ class FakeVideoSendStream final
       const std::vector<bool> active_layers) override;
   void Start() override;
   void Stop() override;
+  void AddAdaptationResource(
+      rtc::scoped_refptr<webrtc::Resource> resource) override;
+  std::vector<rtc::scoped_refptr<webrtc::Resource>> GetAdaptationResources()
+      override;
   void SetSource(
       rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
       const webrtc::DegradationPreference& degradation_preference) override;
@@ -341,6 +345,9 @@ class FakeCall final : public webrtc::Call, public webrtc::PacketReceiver {
   void DestroyFlexfecReceiveStream(
       webrtc::FlexfecReceiveStream* receive_stream) override;
 
+  void AddAdaptationResource(
+      rtc::scoped_refptr<webrtc::Resource> resource) override;
+
   webrtc::PacketReceiver* Receiver() override;
 
   DeliveryStatus DeliverPacket(webrtc::MediaType media_type,

video/test/mock_video_stream_encoder.h

@@ -10,6 +10,8 @@
 #ifndef VIDEO_TEST_MOCK_VIDEO_STREAM_ENCODER_H_
 #define VIDEO_TEST_MOCK_VIDEO_STREAM_ENCODER_H_
 
+#include <vector>
+
 #include "api/video/video_stream_encoder_interface.h"
 #include "test/gmock.h"
 
@@ -17,6 +19,14 @@ namespace webrtc {
 
 class MockVideoStreamEncoder : public VideoStreamEncoderInterface {
  public:
+  MOCK_METHOD(void,
+              AddAdaptationResource,
+              (rtc::scoped_refptr<Resource>),
+              (override));
+  MOCK_METHOD(std::vector<rtc::scoped_refptr<Resource>>,
+              GetAdaptationResources,
+              (),
+              (override));
   MOCK_METHOD(void,
               SetSource,
               (rtc::VideoSourceInterface<VideoFrame>*,

video/video_send_stream.cc

@@ -179,6 +179,18 @@ void VideoSendStream::Stop() {
   worker_queue_->PostTask([send_stream] { send_stream->Stop(); });
 }
 
+void VideoSendStream::AddAdaptationResource(
+    rtc::scoped_refptr<Resource> resource) {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  video_stream_encoder_->AddAdaptationResource(resource);
+}
+
+std::vector<rtc::scoped_refptr<Resource>>
+VideoSendStream::GetAdaptationResources() {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  return video_stream_encoder_->GetAdaptationResources();
+}
+
 void VideoSendStream::SetSource(
     rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
     const DegradationPreference& degradation_preference) {

video/video_send_stream.h

@@ -79,6 +79,9 @@ class VideoSendStream : public webrtc::VideoSendStream {
   void Start() override;
   void Stop() override;
 
+  void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override;
+  std::vector<rtc::scoped_refptr<Resource>> GetAdaptationResources() override;
+
   void SetSource(rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
                  const DegradationPreference& degradation_preference) override;
 

video/video_stream_encoder.cc

@@ -376,6 +376,57 @@ void VideoStreamEncoder::SetFecControllerOverride(
   });
 }
 
+void VideoStreamEncoder::AddAdaptationResource(
+    rtc::scoped_refptr<Resource> resource) {
+  // Map any externally added resources as kCpu for the sake of stats reporting.
+  // TODO(hbos): Make the manager map any unknown resources to kCpu and get rid
+  // of this MapResourceToReason() call.
+  rtc::Event map_resource_event;
+  encoder_queue_.PostTask([this, resource, &map_resource_event] {
+    RTC_DCHECK_RUN_ON(&encoder_queue_);
+    stream_resource_manager_.MapResourceToReason(resource,
+                                                 VideoAdaptationReason::kCpu);
+    map_resource_event.Set();
+  });
+  map_resource_event.Wait(rtc::Event::kForever);
+
+  // Add the resource to the processor.
+  rtc::Event add_resource_event;
+  resource_adaptation_queue_.PostTask([this, resource, &add_resource_event] {
+    RTC_DCHECK_RUN_ON(&resource_adaptation_queue_);
+    if (!resource_adaptation_processor_) {
+      // The VideoStreamEncoder was stopped and the processor destroyed before
+      // this task had a chance to execute. No action needed.
+      return;
+    }
+    // TODO(hbos): When https://webrtc-review.googlesource.com/c/src/+/176406
+    // has landed, there is no need to Stop+Start when adding a resource.
+    resource_adaptation_processor_->StopResourceAdaptation();
+    resource_adaptation_processor_->AddResource(resource);
+    resource_adaptation_processor_->StartResourceAdaptation();
+    add_resource_event.Set();
+  });
+  add_resource_event.Wait(rtc::Event::kForever);
+}
+
+std::vector<rtc::scoped_refptr<Resource>>
+VideoStreamEncoder::GetAdaptationResources() {
+  std::vector<rtc::scoped_refptr<Resource>> resources;
+  rtc::Event event;
+  resource_adaptation_queue_.PostTask([this, &resources, &event] {
+    RTC_DCHECK_RUN_ON(&resource_adaptation_queue_);
+    if (!resource_adaptation_processor_) {
+      // The VideoStreamEncoder was stopped and the processor destroyed before
+      // this task had a chance to execute. No action needed.
+      return;
+    }
+    resources = resource_adaptation_processor_->GetResources();
+    event.Set();
+  });
+  event.Wait(rtc::Event::kForever);
+  return resources;
+}
+
 void VideoStreamEncoder::SetSource(
     rtc::VideoSourceInterface<VideoFrame>* source,
     const DegradationPreference& degradation_preference) {

video/video_stream_encoder.h

@@ -70,6 +70,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
                      TaskQueueFactory* task_queue_factory);
   ~VideoStreamEncoder() override;
 
+  void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override;
+  std::vector<rtc::scoped_refptr<Resource>> GetAdaptationResources() override;
+
   void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
                  const DegradationPreference& degradation_preference) override;