Route min/max FPS constraints to VideoStreamEncoder.

This change
- adds new type VideoTrackSourceConstraints expressing min/max FPS
  constraints.
- adds new method VideoTrackSourceInterface::ProcessConstraints.
- adds new method VideoSinkInterface<>::OnConstraintsChanged.
- updates AdaptedVideoTrackSource and VideoBroadcaster to forward
  the constraints to sinks.
- adds several unit tests for the added functionality.
- and finally, implements OnConstraintsChanged in VideoStreamEncoder.

Chromium will be updated in coming CLs to supply constraints set
through the MediaStream module.

go/rtc-0hz-present

Bug: chromium:1255737
No-Try: true
Change-Id: Iffef239217269c332a1aaa902ddeae2440929e22
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/235040
Commit-Queue: Markus Handell <handellm@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Henrik Andreassson <henrika@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35197}

api/BUILD.gn

@@ -98,6 +98,12 @@ rtc_library("rtp_packet_info") {
   absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
 }
 
+rtc_source_set("video_track_source_constraints") {
+  visibility = [ "*" ]
+  sources = [ "video_track_source_constraints.h" ]
+  absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
 rtc_library("media_stream_interface") {
   visibility = [ "*" ]
   sources = [
@@ -110,6 +116,7 @@ rtc_library("media_stream_interface") {
     ":audio_options_api",
     ":rtp_parameters",
     ":scoped_refptr",
+    ":video_track_source_constraints",
     "../modules/audio_processing:audio_processing_statistics",
     "../rtc_base:checks",
     "../rtc_base:refcount",

api/media_stream_interface.h

@@ -28,6 +28,7 @@
 #include "api/video/video_frame.h"
 #include "api/video/video_sink_interface.h"
 #include "api/video/video_source_interface.h"
+#include "api/video_track_source_constraints.h"
 #include "modules/audio_processing/include/audio_processing_statistics.h"
 #include "rtc_base/ref_count.h"
 #include "rtc_base/system/rtc_export.h"
@@ -146,8 +147,6 @@ class VideoTrackSourceInterface : public MediaSourceInterface,
   // Add an encoded video sink to the source and additionally cause
   // a key frame to be generated from the source. The sink will be
   // invoked from a decoder queue.
-  // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project
-  // adapts.
   virtual void AddEncodedSink(
       rtc::VideoSinkInterface<RecordableEncodedFrame>* sink) = 0;
 
@@ -155,6 +154,13 @@ class VideoTrackSourceInterface : public MediaSourceInterface,
   virtual void RemoveEncodedSink(
       rtc::VideoSinkInterface<RecordableEncodedFrame>* sink) = 0;
 
+  // Notify about constraints set on the source. The information eventually gets
+  // routed to attached sinks via VideoSinkInterface<>::OnConstraintsChanged.
+  // The call is expected to happen on the network thread.
+  // TODO(crbug/1255737): make pure virtual once downstream project adapts.
+  virtual void ProcessConstraints(
+      const webrtc::VideoTrackSourceConstraints& constraints) {}
+
  protected:
   ~VideoTrackSourceInterface() override = default;
 };

api/video/BUILD.gn

@@ -60,6 +60,7 @@ rtc_library("video_frame") {
     "..:array_view",
     "..:rtp_packet_info",
     "..:scoped_refptr",
+    "..:video_track_source_constraints",
     "../../rtc_base:checks",
     "../../rtc_base:rtc_base_approved",
     "../../rtc_base/memory:aligned_malloc",

api/video/video_sink_interface.h

@@ -11,6 +11,8 @@
 #ifndef API_VIDEO_VIDEO_SINK_INTERFACE_H_
 #define API_VIDEO_VIDEO_SINK_INTERFACE_H_
 
+#include "absl/types/optional.h"
+#include "api/video_track_source_constraints.h"
 #include "rtc_base/checks.h"
 
 namespace rtc {
@@ -25,6 +27,11 @@ class VideoSinkInterface {
   // Should be called by the source when it discards the frame due to rate
   // limiting.
   virtual void OnDiscardedFrame() {}
+
+  // Called on the network thread when video constraints change.
+  // TODO(crbug/1255737): make pure virtual once downstream project adapts.
+  virtual void OnConstraintsChanged(
+      const webrtc::VideoTrackSourceConstraints& constraints) {}
 };
 
 }  // namespace rtc

api/video_track_source_constraints.h

@@ -0,0 +1,32 @@
+/*
+ *  Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+// This file contains interfaces for MediaStream, MediaTrack and MediaSource.
+// These interfaces are used for implementing MediaStream and MediaTrack as
+// defined in http://dev.w3.org/2011/webrtc/editor/webrtc.html#stream-api. These
+// interfaces must be used only with PeerConnection.
+
+#ifndef API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_
+#define API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_
+
+#include "absl/types/optional.h"
+
+namespace webrtc {
+
+// This struct definition describes constraints on the video source that may be
+// set with VideoTrackSourceInterface::ProcessConstraints.
+struct VideoTrackSourceConstraints {
+  absl::optional<double> min_fps;
+  absl::optional<double> max_fps;
+};
+
+}  // namespace webrtc
+
+#endif  // API_VIDEO_TRACK_SOURCE_CONSTRAINTS_H_

media/base/adapted_video_track_source.cc

@@ -114,4 +114,9 @@ bool AdaptedVideoTrackSource::AdaptFrame(int width,
   return true;
 }
 
+void AdaptedVideoTrackSource::ProcessConstraints(
+    const webrtc::VideoTrackSourceConstraints& constraints) {
+  broadcaster_.ProcessConstraints(constraints);
+}
+
 }  // namespace rtc

media/base/adapted_video_track_source.h

@@ -86,6 +86,8 @@ class RTC_EXPORT AdaptedVideoTrackSource
       rtc::VideoSinkInterface<webrtc::RecordableEncodedFrame>* sink) override {}
   void RemoveEncodedSink(
       rtc::VideoSinkInterface<webrtc::RecordableEncodedFrame>* sink) override {}
+  void ProcessConstraints(
+      const webrtc::VideoTrackSourceConstraints& constraints) override;
 
   cricket::VideoAdapter video_adapter_;
 

media/base/video_broadcaster.cc

@@ -32,6 +32,13 @@ void VideoBroadcaster::AddOrUpdateSink(
   if (!FindSinkPair(sink)) {
     // `Sink` is a new sink, which didn't receive previous frame.
     previous_frame_sent_to_all_sinks_ = false;
+
+    if (last_constraints_.has_value()) {
+      RTC_LOG(LS_INFO) << __func__ << " forwarding stored constraints min_fps "
+                       << last_constraints_->min_fps.value_or(-1) << " max_fps "
+                       << last_constraints_->max_fps.value_or(-1);
+      sink->OnConstraintsChanged(*last_constraints_);
+    }
   }
   VideoSourceBase::AddOrUpdateSink(sink, wants);
   UpdateWants();
@@ -100,6 +107,18 @@ void VideoBroadcaster::OnDiscardedFrame() {
   }
 }
 
+void VideoBroadcaster::ProcessConstraints(
+    const webrtc::VideoTrackSourceConstraints& constraints) {
+  webrtc::MutexLock lock(&sinks_and_wants_lock_);
+  RTC_LOG(LS_INFO) << __func__ << " min_fps "
+                   << constraints.min_fps.value_or(-1) << " max_fps "
+                   << constraints.max_fps.value_or(-1) << " broadcasting to "
+                   << sink_pairs().size() << " sinks.";
+  last_constraints_ = constraints;
+  for (auto& sink_pair : sink_pairs())
+    sink_pair.sink->OnConstraintsChanged(constraints);
+}
+
 void VideoBroadcaster::UpdateWants() {
   VideoSinkWants wants;
   wants.rotation_applied = false;

media/base/video_broadcaster.h

@@ -11,6 +11,7 @@
 #ifndef MEDIA_BASE_VIDEO_BROADCASTER_H_
 #define MEDIA_BASE_VIDEO_BROADCASTER_H_
 
+#include "api/media_stream_interface.h"
 #include "api/scoped_refptr.h"
 #include "api/sequence_checker.h"
 #include "api/video/video_frame_buffer.h"
@@ -31,6 +32,11 @@ class VideoBroadcaster : public VideoSourceBase,
  public:
   VideoBroadcaster();
   ~VideoBroadcaster() override;
+
+  // Adds a new, or updates an already existing sink. If the sink is new and
+  // ProcessConstraints has been called previously, the new sink's
+  // OnConstraintsCalled method will be invoked with the most recent
+  // constraints.
   void AddOrUpdateSink(VideoSinkInterface<webrtc::VideoFrame>* sink,
                        const VideoSinkWants& wants) override;
   void RemoveSink(VideoSinkInterface<webrtc::VideoFrame>* sink) override;
@@ -50,6 +56,11 @@ class VideoBroadcaster : public VideoSourceBase,
 
   void OnDiscardedFrame() override;
 
+  // Called on the network thread when constraints change. Forwards the
+  // constraints to sinks added with AddOrUpdateSink via OnConstraintsChanged.
+  void ProcessConstraints(
+      const webrtc::VideoTrackSourceConstraints& constraints);
+
  protected:
   void UpdateWants() RTC_EXCLUSIVE_LOCKS_REQUIRED(sinks_and_wants_lock_);
   const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& GetBlackFrameBuffer(
@@ -62,6 +73,8 @@ class VideoBroadcaster : public VideoSourceBase,
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> black_frame_buffer_;
   bool previous_frame_sent_to_all_sinks_ RTC_GUARDED_BY(sinks_and_wants_lock_) =
       true;
+  absl::optional<webrtc::VideoTrackSourceConstraints> last_constraints_
+      RTC_GUARDED_BY(sinks_and_wants_lock_);
 };
 
 }  // namespace rtc

media/base/video_broadcaster_unittest.cc

@@ -16,13 +16,31 @@
 #include "api/video/i420_buffer.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_rotation.h"
+#include "api/video/video_source_interface.h"
 #include "media/base/fake_video_renderer.h"
+#include "test/gmock.h"
 #include "test/gtest.h"
 
 using cricket::FakeVideoRenderer;
 using rtc::VideoBroadcaster;
 using rtc::VideoSinkWants;
 
+using ::testing::AllOf;
+using ::testing::Eq;
+using ::testing::Field;
+using ::testing::Mock;
+using ::testing::Optional;
+
+class MockSink : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
+ public:
+  void OnFrame(const webrtc::VideoFrame&) override {}
+
+  MOCK_METHOD(void,
+              OnConstraintsChanged,
+              (const webrtc::VideoTrackSourceConstraints& constraints),
+              (override));
+};
+
 TEST(VideoBroadcasterTest, frame_wanted) {
   VideoBroadcaster broadcaster;
   EXPECT_FALSE(broadcaster.frame_wanted());
@@ -232,3 +250,83 @@ TEST(VideoBroadcasterTest, SinkWantsBlackFrames) {
   EXPECT_TRUE(sink2.black_frame());
   EXPECT_EQ(30, sink2.timestamp_us());
 }
+
+TEST(VideoBroadcasterTest, ConstraintsChangedNotCalledOnSinkAddition) {
+  MockSink sink;
+  VideoBroadcaster broadcaster;
+  EXPECT_CALL(sink, OnConstraintsChanged).Times(0);
+  broadcaster.AddOrUpdateSink(&sink, VideoSinkWants());
+}
+
+TEST(VideoBroadcasterTest, ForwardsLastConstraintsOnAdd) {
+  MockSink sink;
+  VideoBroadcaster broadcaster;
+  broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{2, 3});
+  broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{1, 4});
+  EXPECT_CALL(
+      sink,
+      OnConstraintsChanged(AllOf(
+          Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(1)),
+          Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(4)))));
+  broadcaster.AddOrUpdateSink(&sink, VideoSinkWants());
+}
+
+TEST(VideoBroadcasterTest, UpdatesOnlyNewSinksWithConstraints) {
+  MockSink sink1;
+  VideoBroadcaster broadcaster;
+  broadcaster.AddOrUpdateSink(&sink1, VideoSinkWants());
+  broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{1, 4});
+  Mock::VerifyAndClearExpectations(&sink1);
+  EXPECT_CALL(sink1, OnConstraintsChanged).Times(0);
+  MockSink sink2;
+  EXPECT_CALL(
+      sink2,
+      OnConstraintsChanged(AllOf(
+          Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(1)),
+          Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(4)))));
+  broadcaster.AddOrUpdateSink(&sink2, VideoSinkWants());
+}
+
+TEST(VideoBroadcasterTest, ForwardsConstraintsToSink) {
+  MockSink sink;
+  VideoBroadcaster broadcaster;
+  EXPECT_CALL(sink, OnConstraintsChanged).Times(0);
+  broadcaster.AddOrUpdateSink(&sink, VideoSinkWants());
+  Mock::VerifyAndClearExpectations(&sink);
+
+  EXPECT_CALL(sink, OnConstraintsChanged(AllOf(
+                        Field(&webrtc::VideoTrackSourceConstraints::min_fps,
+                              Eq(absl::nullopt)),
+                        Field(&webrtc::VideoTrackSourceConstraints::max_fps,
+                              Eq(absl::nullopt)))));
+  broadcaster.ProcessConstraints(
+      webrtc::VideoTrackSourceConstraints{absl::nullopt, absl::nullopt});
+  Mock::VerifyAndClearExpectations(&sink);
+
+  EXPECT_CALL(
+      sink,
+      OnConstraintsChanged(AllOf(
+          Field(&webrtc::VideoTrackSourceConstraints::min_fps,
+                Eq(absl::nullopt)),
+          Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(3)))));
+  broadcaster.ProcessConstraints(
+      webrtc::VideoTrackSourceConstraints{absl::nullopt, 3});
+  Mock::VerifyAndClearExpectations(&sink);
+
+  EXPECT_CALL(
+      sink,
+      OnConstraintsChanged(AllOf(
+          Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(2)),
+          Field(&webrtc::VideoTrackSourceConstraints::max_fps,
+                Eq(absl::nullopt)))));
+  broadcaster.ProcessConstraints(
+      webrtc::VideoTrackSourceConstraints{2, absl::nullopt});
+  Mock::VerifyAndClearExpectations(&sink);
+
+  EXPECT_CALL(
+      sink,
+      OnConstraintsChanged(AllOf(
+          Field(&webrtc::VideoTrackSourceConstraints::min_fps, Optional(2)),
+          Field(&webrtc::VideoTrackSourceConstraints::max_fps, Optional(3)))));
+  broadcaster.ProcessConstraints(webrtc::VideoTrackSourceConstraints{2, 3});
+}

pc/video_track_source_proxy.h

@@ -42,6 +42,9 @@ PROXY_SECONDARY_METHOD1(void,
 PROXY_SECONDARY_METHOD1(void,
                         RemoveEncodedSink,
                         rtc::VideoSinkInterface<RecordableEncodedFrame>*)
+PROXY_SECONDARY_METHOD1(void,
+                        ProcessConstraints,
+                        const webrtc::VideoTrackSourceConstraints&)
 END_PROXY_MAP(VideoTrackSource)
 
 }  // namespace webrtc

video/video_stream_encoder.cc

@@ -1372,6 +1372,18 @@ void VideoStreamEncoder::OnDiscardedFrame() {
       VideoStreamEncoderObserver::DropReason::kSource);
 }
 
+void VideoStreamEncoder::OnConstraintsChanged(
+    const webrtc::VideoTrackSourceConstraints& constraints) {
+  // This method is called on the network thread.
+  RTC_LOG(LS_INFO) << __func__ << " min_fps "
+                   << constraints.min_fps.value_or(-1) << " max_fps "
+                   << constraints.max_fps.value_or(-1);
+  main_queue_->PostTask(ToQueuedTask(task_safety_, [this, constraints] {
+    RTC_DCHECK_RUN_ON(main_queue_);
+    source_constraints_ = constraints;
+  }));
+}
+
 bool VideoStreamEncoder::EncoderPaused() const {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   // Pause video if paused by caller or as long as the network is down or the

video/video_stream_encoder.h

@@ -176,6 +176,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
   // Implements VideoSinkInterface.
   void OnFrame(const VideoFrame& video_frame) override;
   void OnDiscardedFrame() override;
+  void OnConstraintsChanged(
+      const webrtc::VideoTrackSourceConstraints& constraints) override;
 
   void MaybeEncodeVideoFrame(const VideoFrame& frame,
                              int64_t time_when_posted_in_ms);
@@ -239,6 +241,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
       encoder_selector_;
   VideoStreamEncoderObserver* const encoder_stats_observer_;
 
+  // The source's constraints.
+  absl::optional<VideoTrackSourceConstraints> source_constraints_
+      RTC_GUARDED_BY(main_queue_);
+
   VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_);
   std::unique_ptr<VideoEncoder> encoder_ RTC_GUARDED_BY(&encoder_queue_)
       RTC_PT_GUARDED_BY(&encoder_queue_);