Add option to disable quality scaling for AV1.

The main goal of this change is to disable the quality scaler when multiple spatial layers are used.

Bug: b/295129711
Change-Id: I25e0b7440a8c2adee3e97720a1e0ee5e0a914334
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/319181
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#40709}

api/video_codecs/video_codec.cc

@@ -102,6 +102,16 @@ const VideoCodecH264& VideoCodec::H264() const {
   return codec_specific_.H264;
 }
 
+VideoCodecAV1* VideoCodec::AV1() {
+  RTC_DCHECK_EQ(codecType, kVideoCodecAV1);
+  return &codec_specific_.AV1;
+}
+
+const VideoCodecAV1& VideoCodec::AV1() const {
+  RTC_DCHECK_EQ(codecType, kVideoCodecAV1);
+  return codec_specific_.AV1;
+}
+
 const char* CodecTypeToPayloadString(VideoCodecType type) {
   switch (type) {
     case kVideoCodecVP8:

api/video_codecs/video_codec.h

@@ -97,6 +97,16 @@ struct VideoCodecH264 {
   uint8_t numberOfTemporalLayers;
 };
 
+struct VideoCodecAV1 {
+  bool operator==(const VideoCodecAV1& other) const {
+    return automatic_resize_on == other.automatic_resize_on;
+  }
+  bool operator!=(const VideoCodecAV1& other) const {
+    return !(*this == other);
+  }
+  bool automatic_resize_on;
+};
+
 // Translates from name of codec to codec type and vice versa.
 RTC_EXPORT const char* CodecTypeToPayloadString(VideoCodecType type);
 RTC_EXPORT VideoCodecType PayloadStringToCodecType(const std::string& name);
@@ -105,6 +115,7 @@ union VideoCodecUnion {
   VideoCodecVP8 VP8;
   VideoCodecVP9 VP9;
   VideoCodecH264 H264;
+  VideoCodecAV1 AV1;
 };
 
 enum class VideoCodecMode { kRealtimeVideo, kScreensharing };
@@ -193,6 +204,8 @@ class RTC_EXPORT VideoCodec {
   const VideoCodecVP9& VP9() const;
   VideoCodecH264* H264();
   const VideoCodecH264& H264() const;
+  VideoCodecAV1* AV1();
+  const VideoCodecAV1& AV1() const;
 
  private:
   // TODO(hta): Consider replacing the union with a pointer type.

media/engine/fake_webrtc_call.cc

@@ -211,6 +211,16 @@ bool FakeVideoSendStream::GetH264Settings(
   return true;
 }
 
+bool FakeVideoSendStream::GetAv1Settings(
+    webrtc::VideoCodecAV1* settings) const {
+  if (!codec_settings_set_) {
+    return false;
+  }
+
+  *settings = codec_specific_settings_.av1;
+  return true;
+}
+
 int FakeVideoSendStream::GetNumberOfSwappedFrames() const {
   return num_swapped_frames_;
 }
@@ -315,6 +325,9 @@ void FakeVideoSendStream::ReconfigureVideoEncoder(
     } else if (config_.rtp.payload_name == "H264") {
       codec_specific_settings_.h264.numberOfTemporalLayers =
           num_temporal_layers;
+    } else if (config_.rtp.payload_name == "AV1") {
+      config.encoder_specific_settings->FillVideoCodecAv1(
+          &codec_specific_settings_.av1);
     } else {
       ADD_FAILURE() << "Unsupported encoder payload: "
                     << config_.rtp.payload_name;

media/engine/fake_webrtc_call.h

@@ -169,6 +169,7 @@ class FakeVideoSendStream final
   bool GetVp8Settings(webrtc::VideoCodecVP8* settings) const;
   bool GetVp9Settings(webrtc::VideoCodecVP9* settings) const;
   bool GetH264Settings(webrtc::VideoCodecH264* settings) const;
+  bool GetAv1Settings(webrtc::VideoCodecAV1* settings) const;
 
   int GetNumberOfSwappedFrames() const;
   int GetLastWidth() const;
@@ -226,6 +227,7 @@ class FakeVideoSendStream final
     webrtc::VideoCodecVP8 vp8;
     webrtc::VideoCodecVP9 vp9;
     webrtc::VideoCodecH264 h264;
+    webrtc::VideoCodecAV1 av1;
   } codec_specific_settings_;
   bool resolution_scaling_enabled_;
   bool framerate_scaling_enabled_;

media/engine/fake_webrtc_video_engine.cc

@@ -281,11 +281,13 @@ void FakeWebRtcVideoEncoderFactory::AddSupportedVideoCodec(
 }
 
 void FakeWebRtcVideoEncoderFactory::AddSupportedVideoCodecType(
-    const std::string& name) {
+    const std::string& name,
+    const std::vector<webrtc::ScalabilityMode>& scalability_modes) {
   // This is to match the default H264 params of cricket::VideoCodec.
   cricket::VideoCodec video_codec = cricket::CreateVideoCodec(name);
-  formats_.push_back(
-      webrtc::SdpVideoFormat(video_codec.name, video_codec.params));
+  formats_.push_back(webrtc::SdpVideoFormat(
+      video_codec.name, video_codec.params,
+      {scalability_modes.begin(), scalability_modes.end()}));
 }
 
 int FakeWebRtcVideoEncoderFactory::GetNumCreatedEncoders() {

media/engine/fake_webrtc_video_engine.h

@@ -124,7 +124,9 @@ class FakeWebRtcVideoEncoderFactory : public webrtc::VideoEncoderFactory {
   void EncoderDestroyed(FakeWebRtcVideoEncoder* encoder);
   void set_encoders_have_internal_sources(bool internal_source);
   void AddSupportedVideoCodec(const webrtc::SdpVideoFormat& format);
-  void AddSupportedVideoCodecType(const std::string& name);
+  void AddSupportedVideoCodecType(
+      const std::string& name,
+      const std::vector<webrtc::ScalabilityMode>& scalability_modes = {});
   int GetNumCreatedEncoders();
   const std::vector<FakeWebRtcVideoEncoder*> encoders();
 

media/engine/webrtc_video_engine.cc

@@ -963,6 +963,16 @@ WebRtcVideoSendChannel::WebRtcVideoSendStream::ConfigureVideoEncoderSettings(
     return rtc::make_ref_counted<
         webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
   }
+  if (absl::EqualsIgnoreCase(codec.name, kAv1CodecName)) {
+    webrtc::VideoCodecAV1 av1_settings = {.automatic_resize_on =
+                                              automatic_resize};
+    if (NumSpatialLayersFromEncoding(rtp_parameters_, /*idx=*/0) > 1 ||
+        rtp_parameters_.encodings.size() > 1) {
+      av1_settings.automatic_resize_on = false;
+    }
+    return rtc::make_ref_counted<
+        webrtc::VideoEncoderConfig::Av1EncoderSpecificSettings>(av1_settings);
+  }
   return nullptr;
 }
 std::vector<VideoCodecSettings> WebRtcVideoSendChannel::SelectSendVideoCodecs(

media/engine/webrtc_video_engine_unittest.cc

@@ -382,7 +382,9 @@ class WebRtcVideoEngineTest : public ::testing::Test {
   // Find the codec in the engine with the given name. The codec must be
   // present.
   cricket::VideoCodec GetEngineCodec(const std::string& name) const;
-  void AddSupportedVideoCodecType(const std::string& name);
+  void AddSupportedVideoCodecType(
+      const std::string& name,
+      const std::vector<webrtc::ScalabilityMode>& scalability_modes = {});
   std::unique_ptr<VideoMediaSendChannelInterface>
   SetSendParamsWithAllSupportedCodecs();
 
@@ -855,8 +857,9 @@ cricket::VideoCodec WebRtcVideoEngineTest::GetEngineCodec(
 }
 
 void WebRtcVideoEngineTest::AddSupportedVideoCodecType(
-    const std::string& name) {
-  encoder_factory_->AddSupportedVideoCodecType(name);
+    const std::string& name,
+    const std::vector<webrtc::ScalabilityMode>& scalability_modes) {
+  encoder_factory_->AddSupportedVideoCodecType(name, scalability_modes);
   decoder_factory_->AddSupportedVideoCodecType(name);
 }
 
@@ -2645,6 +2648,8 @@ class WebRtcVideoChannelTest : public WebRtcVideoEngineTest {
   void SetUp() override {
     AddSupportedVideoCodecType("VP8");
     AddSupportedVideoCodecType("VP9");
+    AddSupportedVideoCodecType(
+        "AV1", {ScalabilityMode::kL1T3, ScalabilityMode::kL2T3});
 #if defined(WEBRTC_USE_H264)
     AddSupportedVideoCodecType("H264");
 #endif
@@ -3664,6 +3669,52 @@ TEST_F(WebRtcVideoChannelTest, VerifyVp8SpecificSettings) {
   EXPECT_TRUE(send_channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
 }
 
+TEST_F(WebRtcVideoChannelTest, VerifyAv1SpecificSettings) {
+  cricket::VideoSenderParameters parameters;
+  parameters.codecs.push_back(GetEngineCodec("AV1"));
+  ASSERT_TRUE(send_channel_->SetSenderParameters(parameters));
+  webrtc::test::FrameForwarder frame_forwarder;
+  webrtc::VideoCodecAV1 settings;
+
+  // Single-stream settings should apply with RTX as well (verifies that we
+  // check number of regular SSRCs and not StreamParams::ssrcs which contains
+  // both RTX and regular SSRCs).
+  FakeVideoSendStream* stream = SetUpSimulcast(false, /*with_rtx=*/true);
+  EXPECT_TRUE(
+      send_channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder));
+  send_channel_->SetSend(true);
+  frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame());
+
+  ASSERT_TRUE(stream->GetAv1Settings(&settings)) << "No AV1 config set.";
+  EXPECT_TRUE(settings.automatic_resize_on);
+
+  webrtc::RtpParameters rtp_parameters =
+      send_channel_->GetRtpSendParameters(last_ssrc_);
+  EXPECT_THAT(
+      rtp_parameters.encodings,
+      ElementsAre(Field(&webrtc::RtpEncodingParameters::scalability_mode,
+                        absl::nullopt)));
+  rtp_parameters.encodings[0].scalability_mode = "L2T3";
+  EXPECT_TRUE(
+      send_channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters).ok());
+  frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame());
+
+  ASSERT_TRUE(stream->GetAv1Settings(&settings)) << "No AV1 config set.";
+  EXPECT_FALSE(settings.automatic_resize_on);
+
+  EXPECT_TRUE(send_channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
+  stream = SetUpSimulcast(true, /*with_rtx=*/false);
+  EXPECT_TRUE(
+      send_channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder));
+  send_channel_->SetSend(true);
+  frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame());
+
+  ASSERT_TRUE(stream->GetAv1Settings(&settings)) << "No AV1 config set.";
+  EXPECT_FALSE(settings.automatic_resize_on);
+
+  EXPECT_TRUE(send_channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
+}
+
 // Test that setting the same options doesn't result in the encoder being
 // reconfigured.
 TEST_F(WebRtcVideoChannelTest, SetIdenticalOptionsDoesntReconfigureEncoder) {

modules/video_coding/codecs/av1/libaom_av1_encoder.cc

@@ -824,7 +824,10 @@ VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const {
   info.implementation_name = "libaom";
   info.has_trusted_rate_controller = true;
   info.is_hardware_accelerated = false;
-  info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
+  info.scaling_settings =
+      (inited_ && !encoder_settings_.AV1().automatic_resize_on)
+          ? VideoEncoder::ScalingSettings::kOff
+          : VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
   info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420,
                                   VideoFrameBuffer::Type::kNV12};
   if (SvcEnabled()) {

modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc

@@ -36,6 +36,7 @@ using ::testing::SizeIs;
 
 VideoCodec DefaultCodecSettings() {
   VideoCodec codec_settings;
+  codec_settings.codecType = kVideoCodecAV1;
   codec_settings.width = 320;
   codec_settings.height = 180;
   codec_settings.maxFramerate = 30;
@@ -398,5 +399,16 @@ TEST(LibaomAv1EncoderTest, AdheresToTargetBitrateDespiteUnevenFrameTiming) {
       kTargetBitrateBps, kTargetBitrateBps / 10);
 }
 
+TEST(LibaomAv1EncoderTest, DisableAutomaticResize) {
+  std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
+  ASSERT_TRUE(encoder);
+  VideoCodec codec_settings = DefaultCodecSettings();
+  codec_settings.AV1()->automatic_resize_on = false;
+  EXPECT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
+            WEBRTC_VIDEO_CODEC_OK);
+  EXPECT_EQ(encoder->GetEncoderInfo().scaling_settings.thresholds,
+            absl::nullopt);
+}
+
 }  // namespace
 }  // namespace webrtc

video/config/video_encoder_config.cc

@@ -96,6 +96,8 @@ void VideoEncoderConfig::EncoderSpecificSettings::FillEncoderSpecificSettings(
     FillVideoCodecVp8(codec->VP8());
   } else if (codec->codecType == kVideoCodecVP9) {
     FillVideoCodecVp9(codec->VP9());
+  } else if (codec->codecType == kVideoCodecAV1) {
+    FillVideoCodecAv1(codec->AV1());
   } else {
     RTC_DCHECK_NOTREACHED()
         << "Encoder specifics set/used for unknown codec type.";
@@ -112,6 +114,11 @@ void VideoEncoderConfig::EncoderSpecificSettings::FillVideoCodecVp9(
   RTC_DCHECK_NOTREACHED();
 }
 
+void VideoEncoderConfig::EncoderSpecificSettings::FillVideoCodecAv1(
+    VideoCodecAV1* av1_settings) const {
+  RTC_DCHECK_NOTREACHED();
+}
+
 VideoEncoderConfig::Vp8EncoderSpecificSettings::Vp8EncoderSpecificSettings(
     const VideoCodecVP8& specifics)
     : specifics_(specifics) {}
@@ -130,4 +137,13 @@ void VideoEncoderConfig::Vp9EncoderSpecificSettings::FillVideoCodecVp9(
   *vp9_settings = specifics_;
 }
 
+VideoEncoderConfig::Av1EncoderSpecificSettings::Av1EncoderSpecificSettings(
+    const VideoCodecAV1& specifics)
+    : specifics_(specifics) {}
+
+void VideoEncoderConfig::Av1EncoderSpecificSettings::FillVideoCodecAv1(
+    VideoCodecAV1* av1_settings) const {
+  *av1_settings = specifics_;
+}
+
 }  // namespace webrtc

video/config/video_encoder_config.h

@@ -99,6 +99,7 @@ class VideoEncoderConfig {
 
     virtual void FillVideoCodecVp8(VideoCodecVP8* vp8_settings) const;
     virtual void FillVideoCodecVp9(VideoCodecVP9* vp9_settings) const;
+    virtual void FillVideoCodecAv1(VideoCodecAV1* av1_settings) const;
 
    private:
     ~EncoderSpecificSettings() override {}
@@ -123,6 +124,15 @@ class VideoEncoderConfig {
     VideoCodecVP9 specifics_;
   };
 
+  class Av1EncoderSpecificSettings : public EncoderSpecificSettings {
+   public:
+    explicit Av1EncoderSpecificSettings(const VideoCodecAV1& specifics);
+    void FillVideoCodecAv1(VideoCodecAV1* av1_settings) const override;
+
+   private:
+    VideoCodecAV1 specifics_;
+  };
+
   enum class ContentType {
     kRealtimeVideo,
     kScreen,