Add av1 svc configuration for target bitrates

This configuration mostly copies vp9 configuration for regular video,
but is done separately to allow tune av1 svc bitrates independently of vp9.

Bug: webrtc:12148
Change-Id: Icd11817ada8f9b6135ee2da57204eadb50de3954
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195329
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32713}

modules/video_coding/BUILD.gn

@@ -99,29 +99,6 @@ rtc_library("nack_module") {
 
 rtc_library("video_coding") {
   visibility = [ "*" ]
-  deps = [
-    "..:module_fec_api",
-    "../../api:array_view",
-    "../../api:scoped_refptr",
-    "../../api/video:encoded_image",
-    "../../api/video:video_adaptation",
-    "../../api/video:video_bitrate_allocation",
-    "../../api/video:video_bitrate_allocator_factory",
-    "../../rtc_base:deprecation",
-    "../../rtc_base/system:no_unique_address",
-    "../../rtc_base/task_utils:to_queued_task",
-    "../../system_wrappers:field_trial",
-    "../../system_wrappers:metrics",
-    "../rtp_rtcp:rtp_video_header",
-  ]
-  absl_deps = [
-    "//third_party/abseil-cpp/absl/base:core_headers",
-    "//third_party/abseil-cpp/absl/container:inlined_vector",
-    "//third_party/abseil-cpp/absl/memory",
-    "//third_party/abseil-cpp/absl/types:optional",
-    "//third_party/abseil-cpp/absl/types:variant",
-  ]
-
   sources = [
     "codec_timer.cc",
     "codec_timer.h",
@@ -167,7 +144,7 @@ rtc_library("video_coding") {
     "video_receiver2.h",
   ]
 
-  deps += [
+  deps = [
     ":codec_globals_headers",
     ":encoded_frame",
     ":video_codec_interface",
@@ -175,15 +152,22 @@ rtc_library("video_coding") {
     ":webrtc_vp9_helpers",
     "..:module_api",
     "..:module_api_public",
+    "..:module_fec_api",
+    "../../api:array_view",
     "../../api:fec_controller_api",
     "../../api:rtp_headers",
     "../../api:rtp_packet_info",
+    "../../api:scoped_refptr",
     "../../api/units:data_rate",
     "../../api/units:time_delta",
     "../../api/video:builtin_video_bitrate_allocator_factory",
     "../../api/video:encoded_frame",
+    "../../api/video:encoded_image",
     "../../api/video:video_adaptation",
+    "../../api/video:video_adaptation",
+    "../../api/video:video_bitrate_allocation",
     "../../api/video:video_bitrate_allocator",
+    "../../api/video:video_bitrate_allocator_factory",
     "../../api/video:video_frame",
     "../../api/video:video_frame_type",
     "../../api/video:video_rtp_headers",
@@ -191,6 +175,7 @@ rtc_library("video_coding") {
     "../../common_video",
     "../../rtc_base",
     "../../rtc_base:checks",
+    "../../rtc_base:deprecation",
     "../../rtc_base:rtc_base_approved",
     "../../rtc_base:rtc_numerics",
     "../../rtc_base:rtc_task_queue",
@@ -202,12 +187,25 @@ rtc_library("video_coding") {
     "../../rtc_base/experiments:rtt_mult_experiment",
     "../../rtc_base/synchronization:mutex",
     "../../rtc_base/synchronization:sequence_checker",
+    "../../rtc_base/system:no_unique_address",
     "../../rtc_base/task_utils:repeating_task",
+    "../../rtc_base/task_utils:to_queued_task",
     "../../rtc_base/third_party/base64",
     "../../rtc_base/time:timestamp_extrapolator",
     "../../system_wrappers",
+    "../../system_wrappers:field_trial",
+    "../../system_wrappers:metrics",
     "../rtp_rtcp",
     "../rtp_rtcp:rtp_rtcp_format",
+    "../rtp_rtcp:rtp_video_header",
+    "codecs/av1:av1_svc_config",
+  ]
+  absl_deps = [
+    "//third_party/abseil-cpp/absl/base:core_headers",
+    "//third_party/abseil-cpp/absl/container:inlined_vector",
+    "//third_party/abseil-cpp/absl/memory",
+    "//third_party/abseil-cpp/absl/types:optional",
+    "//third_party/abseil-cpp/absl/types:variant",
   ]
 }
 

modules/video_coding/codecs/av1/BUILD.gn

@@ -9,6 +9,20 @@
 import("//third_party/libaom/options.gni")
 import("../../../../webrtc.gni")
 
+rtc_library("av1_svc_config") {
+  sources = [
+    "av1_svc_config.cc",
+    "av1_svc_config.h",
+  ]
+  deps = [
+    "../../../../api/video_codecs:video_codecs_api",
+    "../../../../rtc_base:checks",
+    "../../../../rtc_base:logging",
+    "../../svc:scalability_structures",
+    "../../svc:scalable_video_controller",
+  ]
+}
+
 rtc_library("libaom_av1_decoder") {
   visibility = [ "*" ]
   poisonous = [ "software_video_codecs" ]
@@ -70,12 +84,18 @@ if (rtc_include_tests) {
   rtc_library("video_coding_codecs_av1_tests") {
     testonly = true
 
+    sources = [ "av1_svc_config_unittest.cc" ]
+    deps = [
+      ":av1_svc_config",
+      "../../../../api/video_codecs:video_codecs_api",
+    ]
+
     if (enable_libaom) {
-      sources = [
+      sources += [
         "libaom_av1_encoder_unittest.cc",
         "libaom_av1_unittest.cc",
       ]
-      deps = [
+      deps += [
         ":libaom_av1_decoder",
         ":libaom_av1_encoder",
         "../..:encoded_video_frame_producer",
@@ -84,7 +104,6 @@ if (rtc_include_tests) {
         "../../../../api/units:data_size",
         "../../../../api/units:time_delta",
         "../../../../api/video:video_frame",
-        "../../../../api/video_codecs:video_codecs_api",
         "../../../../test:test_support",
         "../../svc:scalability_structures",
         "../../svc:scalable_video_controller",

modules/video_coding/codecs/av1/av1_svc_config.cc

@@ -0,0 +1,73 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+
+#include "modules/video_coding/svc/create_scalability_structure.h"
+#include "modules/video_coding/svc/scalable_video_controller.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+bool SetAv1SvcConfig(VideoCodec& video_codec) {
+  RTC_DCHECK_EQ(video_codec.codecType, kVideoCodecAV1);
+
+  if (video_codec.ScalabilityMode().empty()) {
+    RTC_LOG(LS_INFO) << "No scalability mode set.";
+    return false;
+  }
+  std::unique_ptr<ScalableVideoController> structure =
+      CreateScalabilityStructure(video_codec.ScalabilityMode());
+  if (structure == nullptr) {
+    RTC_LOG(LS_INFO) << "Failed to create structure "
+                     << video_codec.ScalabilityMode();
+    return false;
+  }
+  ScalableVideoController::StreamLayersConfig info = structure->StreamConfig();
+  for (int sl_idx = 0; sl_idx < info.num_spatial_layers; ++sl_idx) {
+    SpatialLayer& spatial_layer = video_codec.spatialLayers[sl_idx];
+    spatial_layer.width = video_codec.width * info.scaling_factor_num[sl_idx] /
+                          info.scaling_factor_den[sl_idx];
+    spatial_layer.height = video_codec.height *
+                           info.scaling_factor_num[sl_idx] /
+                           info.scaling_factor_den[sl_idx];
+    spatial_layer.maxFramerate = video_codec.maxFramerate;
+    spatial_layer.numberOfTemporalLayers = info.num_temporal_layers;
+    spatial_layer.active = true;
+  }
+
+  if (info.num_spatial_layers == 1) {
+    SpatialLayer& spatial_layer = video_codec.spatialLayers[0];
+    spatial_layer.minBitrate = video_codec.minBitrate;
+    spatial_layer.targetBitrate = video_codec.startBitrate;
+    spatial_layer.maxBitrate = video_codec.maxBitrate;
+    return true;
+  }
+
+  for (int sl_idx = 0; sl_idx < info.num_spatial_layers; ++sl_idx) {
+    SpatialLayer& spatial_layer = video_codec.spatialLayers[sl_idx];
+    // minBitrate and maxBitrate formulas are copied from vp9 settings and
+    // are not yet tuned for av1.
+    const int num_pixels = spatial_layer.width * spatial_layer.height;
+    int min_bitrate_kbps = (600.0 * std::sqrt(num_pixels) - 95'000.0) / 1000.0;
+    spatial_layer.minBitrate = std::max(min_bitrate_kbps, 20);
+    spatial_layer.maxBitrate = 50 + static_cast<int>(1.6 * num_pixels / 1000.0);
+    spatial_layer.targetBitrate =
+        (spatial_layer.minBitrate + spatial_layer.maxBitrate) / 2;
+  }
+  return true;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/av1_svc_config.h

@@ -0,0 +1,22 @@
+/* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_
+
+#include "api/video_codecs/video_codec.h"
+
+namespace webrtc {
+
+// Fills `video_codec.spatialLayers` using other members.
+bool SetAv1SvcConfig(VideoCodec& video_codec);
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_

modules/video_coding/codecs/av1/av1_svc_config_unittest.cc

@@ -0,0 +1,142 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
+
+#include "api/video_codecs/video_codec.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+TEST(Av1SvcConfigTest, RequireScalabilityMode) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+
+  video_codec.SetScalabilityMode("");
+  EXPECT_FALSE(SetAv1SvcConfig(video_codec));
+
+  video_codec.SetScalabilityMode("Unknown");
+  EXPECT_FALSE(SetAv1SvcConfig(video_codec));
+
+  video_codec.SetScalabilityMode("NONE");
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+}
+
+TEST(Av1SvcConfigTest, SetsActiveSpatialLayersFromScalabilityMode) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L2T1");
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_TRUE(video_codec.spatialLayers[0].active);
+  EXPECT_TRUE(video_codec.spatialLayers[1].active);
+  EXPECT_FALSE(video_codec.spatialLayers[2].active);
+}
+
+TEST(Av1SvcConfigTest, ConfiguresDobuleResolutionRatioFromScalabilityMode) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L2T1");
+  video_codec.width = 1200;
+  video_codec.height = 800;
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].width, 600);
+  EXPECT_EQ(video_codec.spatialLayers[0].height, 400);
+  EXPECT_EQ(video_codec.spatialLayers[1].width, 1200);
+  EXPECT_EQ(video_codec.spatialLayers[1].height, 800);
+}
+
+TEST(Av1SvcConfigTest, ConfiguresSmallResolutionRatioFromScalabilityMode) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  // h mode uses 1.5:1 ratio
+  video_codec.SetScalabilityMode("L2T1h");
+  video_codec.width = 1500;
+  video_codec.height = 900;
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].width, 1000);
+  EXPECT_EQ(video_codec.spatialLayers[0].height, 600);
+  EXPECT_EQ(video_codec.spatialLayers[1].width, 1500);
+  EXPECT_EQ(video_codec.spatialLayers[1].height, 900);
+}
+
+TEST(Av1SvcConfigTest, CopiesFramrate) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  // h mode uses 1.5:1 ratio
+  video_codec.SetScalabilityMode("L2T1");
+  video_codec.maxFramerate = 27;
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].maxFramerate, 27);
+  EXPECT_EQ(video_codec.spatialLayers[1].maxFramerate, 27);
+}
+
+TEST(Av1SvcConfigTest, SetsNumberOfTemporalLayers) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L1T3");
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].numberOfTemporalLayers, 3);
+}
+
+TEST(Av1SvcConfigTest, CopiesBitrateForSingleSpatialLayer) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L1T3");
+  video_codec.minBitrate = 100;
+  video_codec.startBitrate = 200;
+  video_codec.maxBitrate = 500;
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_EQ(video_codec.spatialLayers[0].minBitrate, 100u);
+  EXPECT_EQ(video_codec.spatialLayers[0].targetBitrate, 200u);
+  EXPECT_EQ(video_codec.spatialLayers[0].maxBitrate, 500u);
+}
+
+TEST(Av1SvcConfigTest, SetsBitratesForMultipleSpatialLayers) {
+  VideoCodec video_codec;
+  video_codec.codecType = kVideoCodecAV1;
+  video_codec.SetScalabilityMode("L3T3");
+
+  EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+  EXPECT_GT(video_codec.spatialLayers[0].minBitrate, 0u);
+  EXPECT_LE(video_codec.spatialLayers[0].minBitrate,
+            video_codec.spatialLayers[0].targetBitrate);
+  EXPECT_LE(video_codec.spatialLayers[0].targetBitrate,
+            video_codec.spatialLayers[0].maxBitrate);
+
+  EXPECT_GT(video_codec.spatialLayers[1].minBitrate, 0u);
+  EXPECT_LE(video_codec.spatialLayers[1].minBitrate,
+            video_codec.spatialLayers[1].targetBitrate);
+  EXPECT_LE(video_codec.spatialLayers[1].targetBitrate,
+            video_codec.spatialLayers[1].maxBitrate);
+
+  EXPECT_GT(video_codec.spatialLayers[2].minBitrate, 0u);
+  EXPECT_LE(video_codec.spatialLayers[2].minBitrate,
+            video_codec.spatialLayers[2].targetBitrate);
+  EXPECT_LE(video_codec.spatialLayers[2].targetBitrate,
+            video_codec.spatialLayers[2].maxBitrate);
+}
+
+}  // namespace
+}  // namespace webrtc

modules/video_coding/video_codec_initializer.cc

@@ -20,6 +20,7 @@
 #include "api/units/data_rate.h"
 #include "api/video/video_bitrate_allocation.h"
 #include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
 #include "modules/video_coding/codecs/vp9/svc_config.h"
 #include "modules/video_coding/include/video_coding_defines.h"
 #include "rtc_base/checks.h"
@@ -56,7 +57,6 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
   RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
 
   VideoCodec video_codec;
-  memset(&video_codec, 0, sizeof(video_codec));
   video_codec.codecType = config.codec_type;
 
   switch (config.content_type) {
@@ -255,6 +255,11 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
 
       break;
     }
+    case kVideoCodecAV1:
+      if (!SetAv1SvcConfig(video_codec)) {
+        RTC_LOG(LS_WARNING) << "Failed to configure svc bitrates for av1.";
+      }
+      break;
     case kVideoCodecH264: {
       if (!config.encoder_specific_settings)
         *video_codec.H264() = VideoEncoder::GetDefaultH264Settings();