RtpEncodingParameters::request_resolution patch 3

This cl/ adds resource adapation to the requested_resolution
feature. The restrictions that are sent to the video source
are also saved inside video_stream_encoder and used when
determining layer resolution.

Anticipated further patches
4) Let VideoSource do adaption if possible

Bug: webrtc:14451
Change-Id: Ia9b990a6b92b76af7ff6665a562f84585f79c35b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/277580
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Jonas Oreland <jonaso@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38306}

call/adaptation/video_source_restrictions.cc

@@ -10,6 +10,7 @@
 
 #include "call/adaptation/video_source_restrictions.h"
 
+#include <algorithm>
 #include <limits>
 
 #include "rtc_base/checks.h"
@@ -79,6 +80,30 @@ void VideoSourceRestrictions::set_max_frame_rate(
   max_frame_rate_ = std::move(max_frame_rate);
 }
 
+void VideoSourceRestrictions::UpdateMin(const VideoSourceRestrictions& other) {
+  if (max_pixels_per_frame_.has_value()) {
+    max_pixels_per_frame_ = std::min(*max_pixels_per_frame_,
+                                     other.max_pixels_per_frame().value_or(
+                                         std::numeric_limits<size_t>::max()));
+  } else {
+    max_pixels_per_frame_ = other.max_pixels_per_frame();
+  }
+  if (target_pixels_per_frame_.has_value()) {
+    target_pixels_per_frame_ = std::min(
+        *target_pixels_per_frame_, other.target_pixels_per_frame().value_or(
+                                       std::numeric_limits<size_t>::max()));
+  } else {
+    target_pixels_per_frame_ = other.target_pixels_per_frame();
+  }
+  if (max_frame_rate_.has_value()) {
+    max_frame_rate_ = std::min(
+        *max_frame_rate_,
+        other.max_frame_rate().value_or(std::numeric_limits<double>::max()));
+  } else {
+    max_frame_rate_ = other.max_frame_rate();
+  }
+}
+
 bool DidRestrictionsIncrease(VideoSourceRestrictions before,
                              VideoSourceRestrictions after) {
   bool decreased_resolution = DidDecreaseResolution(before, after);

call/adaptation/video_source_restrictions.h

@@ -60,6 +60,9 @@ class VideoSourceRestrictions {
       absl::optional<size_t> target_pixels_per_frame);
   void set_max_frame_rate(absl::optional<double> max_frame_rate);
 
+  // Update `this` with min(`this`, `other`).
+  void UpdateMin(const VideoSourceRestrictions& other);
+
  private:
   // These map to rtc::VideoSinkWants's `max_pixel_count` and
   // `target_pixel_count`.

call/adaptation/video_source_restrictions_unittest.cc

@@ -126,4 +126,21 @@ TEST(VideoSourceRestrictions,
   EXPECT_FALSE(DidRestrictionsDecrease(kHd, k15fps));
 }
 
+TEST(VideoSourceRestrictions, UpdateMin) {
+  VideoSourceRestrictions one(kHdPixels / 2, kHdPixels, 7.0);
+  VideoSourceRestrictions two(kHdPixels, kHdPixels / 3, 15.0);
+
+  one.UpdateMin(two);
+
+  EXPECT_EQ(one.max_pixels_per_frame(), kHdPixels / 2);
+  EXPECT_EQ(one.target_pixels_per_frame(), kHdPixels / 3);
+  EXPECT_EQ(one.max_frame_rate(), 7.0);
+
+  two.UpdateMin(one);
+
+  EXPECT_EQ(two.max_pixels_per_frame(), kHdPixels / 2);
+  EXPECT_EQ(two.target_pixels_per_frame(), kHdPixels / 3);
+  EXPECT_EQ(two.max_frame_rate(), 7.0);
+}
+
 }  // namespace webrtc

video/BUILD.gn

@@ -938,6 +938,7 @@ if (rtc_include_tests) {
       "adaptation:video_adaptation",
       "config:encoder_config",
       "config:streams_config",
+      "config:video_config_tests",
     ]
     absl_deps = [
       "//third_party/abseil-cpp/absl/algorithm:container",

video/config/BUILD.gn

@@ -23,6 +23,7 @@ rtc_library("streams_config") {
     "../../api/units:data_rate",
     "../../api/video:video_codec_constants",
     "../../api/video_codecs:video_codecs_api",
+    "../../call/adaptation:resource_adaptation",
     "../../media:rtc_media_base",
     "../../modules/video_coding:video_coding_utility",
     "../../modules/video_coding:webrtc_vp9_helpers",
@@ -69,10 +70,14 @@ if (rtc_include_tests) {
     testonly = true
 
     defines = []
-    sources = [ "simulcast_unittest.cc" ]
+    sources = [
+      "encoder_stream_factory_unittest.cc",
+      "simulcast_unittest.cc",
+    ]
     deps = [
       ":streams_config",
       "../../api/transport:field_trial_based_config",
+      "../../call/adaptation:resource_adaptation",
       "../../test:field_trial",
       "../../test:test_support",
     ]

video/config/encoder_stream_factory.cc

@@ -10,6 +10,7 @@
 #include "video/config/encoder_stream_factory.h"
 
 #include <algorithm>
+#include <limits>
 #include <set>
 #include <string>
 #include <utility>
@@ -115,6 +116,7 @@ EncoderStreamFactory::EncoderStreamFactory(
     bool is_screenshare,
     bool conference_mode,
     const webrtc::VideoEncoder::EncoderInfo& encoder_info,
+    absl::optional<webrtc::VideoSourceRestrictions> restrictions,
     const webrtc::FieldTrialsView* trials)
     : codec_name_(codec_name),
       max_qp_(max_qp),
@@ -122,7 +124,8 @@ EncoderStreamFactory::EncoderStreamFactory(
       conference_mode_(conference_mode),
       trials_(trials ? *trials : fallback_trials_),
       encoder_info_requested_resolution_alignment_(
-          encoder_info.requested_resolution_alignment) {}
+          encoder_info.requested_resolution_alignment),
+      restrictions_(restrictions) {}
 
 std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
     int frame_width,
@@ -434,6 +437,16 @@ EncoderStreamFactory::GetLayerResolutionFromRequestedResolution(
   adapter.OnOutputFormatRequest(requested_resolution.ToPair(),
                                 requested_resolution.PixelCount(),
                                 absl::nullopt);
+  if (restrictions_) {
+    rtc::VideoSinkWants wants;
+    wants.is_active = true;
+    wants.target_pixel_count = restrictions_->target_pixels_per_frame();
+    wants.max_pixel_count =
+        rtc::dchecked_cast<int>(restrictions_->max_pixels_per_frame().value_or(
+            std::numeric_limits<int>::max()));
+    wants.resolution_alignment = encoder_info_requested_resolution_alignment_;
+    adapter.OnSinkWants(wants);
+  }
   int cropped_width, cropped_height;
   int out_width = 0, out_height = 0;
   if (!adapter.AdaptFrameResolution(frame_width, frame_height, 0,

video/config/encoder_stream_factory.h

@@ -16,6 +16,7 @@
 #include "api/transport/field_trial_based_config.h"
 #include "api/units/data_rate.h"
 #include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/video_source_restrictions.h"
 #include "video/config/video_encoder_config.h"
 
 namespace cricket {
@@ -34,14 +35,16 @@ class EncoderStreamFactory
                        bool is_screenshare,
                        bool conference_mode,
                        const webrtc::VideoEncoder::EncoderInfo& encoder_info,
+                       absl::optional<webrtc::VideoSourceRestrictions>
+                           restrictions = absl::nullopt,
                        const webrtc::FieldTrialsView* trials = nullptr);
 
- private:
   std::vector<webrtc::VideoStream> CreateEncoderStreams(
       int width,
       int height,
       const webrtc::VideoEncoderConfig& encoder_config) override;
 
+ private:
   std::vector<webrtc::VideoStream> CreateDefaultVideoStreams(
       int width,
       int height,
@@ -69,6 +72,7 @@ class EncoderStreamFactory
   const webrtc::FieldTrialBasedConfig fallback_trials_;
   const webrtc::FieldTrialsView& trials_;
   const int encoder_info_requested_resolution_alignment_;
+  const absl::optional<webrtc::VideoSourceRestrictions> restrictions_;
 };
 
 }  // namespace cricket

video/config/encoder_stream_factory_unittest.cc

@@ -0,0 +1,79 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/config/encoder_stream_factory.h"
+
+#include "call/adaptation/video_source_restrictions.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+using cricket::EncoderStreamFactory;
+constexpr int kMaxQp = 48;
+
+namespace {
+
+std::vector<Resolution> GetStreamResolutions(
+    const std::vector<VideoStream>& streams) {
+  std::vector<Resolution> res;
+  for (const auto& s : streams) {
+    if (s.active) {
+      res.push_back(
+          {rtc::checked_cast<int>(s.width), rtc::checked_cast<int>(s.height)});
+    }
+  }
+  return res;
+}
+
+VideoStream LayerWithRequestedResolution(Resolution res) {
+  VideoStream s;
+  s.requested_resolution = res;
+  return s;
+}
+
+}  // namespace
+
+TEST(EncoderStreamFactory, SinglecastRequestedResolution) {
+  VideoEncoder::EncoderInfo encoder_info;
+  auto factory = rtc::make_ref_counted<EncoderStreamFactory>(
+      "VP8", kMaxQp,
+      /* is_screenshare= */ false,
+      /* conference_mode= */ false, encoder_info);
+  VideoEncoderConfig encoder_config;
+  encoder_config.number_of_streams = 1;
+  encoder_config.simulcast_layers.push_back(
+      LayerWithRequestedResolution({.width = 640, .height = 360}));
+  auto streams = factory->CreateEncoderStreams(1280, 720, encoder_config);
+  EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{
+                                               {.width = 640, .height = 360},
+                                           }));
+}
+
+TEST(EncoderStreamFactory, SinglecastRequestedResolutionWithAdaptation) {
+  VideoSourceRestrictions restrictions(
+      /* max_pixels_per_frame= */ (320 * 320),
+      /* target_pixels_per_frame= */ absl::nullopt,
+      /* max_frame_rate= */ absl::nullopt);
+  VideoEncoder::EncoderInfo encoder_info;
+  auto factory = rtc::make_ref_counted<EncoderStreamFactory>(
+      "VP8", kMaxQp,
+      /* is_screenshare= */ false,
+      /* conference_mode= */ false, encoder_info, restrictions);
+  VideoEncoderConfig encoder_config;
+  encoder_config.number_of_streams = 1;
+  encoder_config.simulcast_layers.push_back(
+      LayerWithRequestedResolution({.width = 640, .height = 360}));
+  auto streams = factory->CreateEncoderStreams(1280, 720, encoder_config);
+  EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{
+                                               {.width = 320, .height = 180},
+                                           }));
+}
+
+}  // namespace webrtc

video/video_stream_encoder.cc

@@ -33,6 +33,7 @@
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_encoder.h"
 #include "call/adaptation/resource_adaptation_processor.h"
+#include "call/adaptation/video_source_restrictions.h"
 #include "call/adaptation/video_stream_adapter.h"
 #include "modules/video_coding/include/video_codec_initializer.h"
 #include "modules/video_coding/svc/svc_rate_allocator.h"
@@ -508,7 +509,7 @@ void ApplyEncoderBitrateLimitsIfSingleActiveStream(
 }
 
 absl::optional<int> ParseVp9LowTierCoreCountThreshold(
-    const webrtc::FieldTrialsView& trials) {
+    const FieldTrialsView& trials) {
   FieldTrialFlag disable_low_tier("Disabled");
   FieldTrialParameter<int> max_core_count("max_core_count", 2);
   ParseFieldTrial({&disable_low_tier, &max_core_count},
@@ -519,6 +520,22 @@ absl::optional<int> ParseVp9LowTierCoreCountThreshold(
   return max_core_count.Get();
 }
 
+absl::optional<VideoSourceRestrictions> MergeRestrictions(
+    const std::vector<absl::optional<VideoSourceRestrictions>>& list) {
+  absl::optional<VideoSourceRestrictions> return_value;
+  for (const auto& res : list) {
+    if (!res) {
+      continue;
+    }
+    if (!return_value) {
+      return_value = *res;
+      continue;
+    }
+    return_value->UpdateMin(*res);
+  }
+  return return_value;
+}
+
 }  //  namespace
 
 VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
@@ -949,7 +966,9 @@ void VideoStreamEncoder::ReconfigureEncoder() {
             encoder_config_.video_format.name, encoder_config_.max_qp,
             encoder_config_.content_type ==
                 webrtc::VideoEncoderConfig::ContentType::kScreen,
-            encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo());
+            encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo(),
+            MergeRestrictions({latest_restrictions_, animate_restrictions_}),
+            &field_trials_);
 
     streams = factory->CreateEncoderStreams(
         last_frame_info_->width, last_frame_info_->height, encoder_config_);
@@ -2313,6 +2332,11 @@ void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
   RTC_LOG(LS_INFO) << "Updating sink restrictions from "
                    << (reason ? reason->Name() : std::string("<null>"))
                    << " to " << restrictions.ToString();
+
+  // TODO(webrtc:14451) Split video_source_sink_controller_
+  // so that ownership on restrictions/wants is kept on &encoder_queue_
+  latest_restrictions_ = restrictions;
+
   worker_queue_->PostTask(SafeTask(
       task_safety_.flag(), [this, restrictions = std::move(restrictions)]() {
         RTC_DCHECK_RUN_ON(worker_queue_);
@@ -2454,6 +2478,17 @@ void VideoStreamEncoder::CheckForAnimatedContent(
       RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent "
                           "animation detection.";
     }
+    // TODO(webrtc:14451) Split video_source_sink_controller_
+    // so that ownership on restrictions/wants is kept on &encoder_queue_
+    if (should_cap_resolution) {
+      animate_restrictions_ =
+          VideoSourceRestrictions(kMaxAnimationPixels,
+                                  /* target_pixels_per_frame= */ absl::nullopt,
+                                  /* max_frame_rate= */ absl::nullopt);
+    } else {
+      animate_restrictions_.reset();
+    }
+
     worker_queue_->PostTask(
         SafeTask(task_safety_.flag(), [this, should_cap_resolution]() {
           RTC_DCHECK_RUN_ON(worker_queue_);

video/video_stream_encoder.h

@@ -457,6 +457,20 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
 
   const absl::optional<int> vp9_low_tier_core_threshold_;
 
+  // These are copies of restrictions (glorified max_pixel_count) set by
+  // a) OnVideoSourceRestrictionsUpdated
+  // b) CheckForAnimatedContent
+  // They are used to scale down encoding resolution if needed when using
+  // requested_resolution.
+  //
+  // TODO(webrtc:14451) Split video_source_sink_controller_
+  // so that ownership on restrictions/wants is kept on &encoder_queue_, that
+  // these extra copies would not be needed.
+  absl::optional<VideoSourceRestrictions> latest_restrictions_
+      RTC_GUARDED_BY(&encoder_queue_);
+  absl::optional<VideoSourceRestrictions> animate_restrictions_
+      RTC_GUARDED_BY(&encoder_queue_);
+
   // Used to cancel any potentially pending tasks to the worker thread.
   // Refrenced by tasks running on `encoder_queue_` so need to be destroyed
   // after stopping that queue. Must be created and destroyed on