From 4005e5abb8bd160f237d89dd207c357cf56195cf Mon Sep 17 00:00:00 2001 From: Danil Chapovalov Date: Wed, 25 Nov 2020 17:01:22 +0100 Subject: [PATCH] Add av1 svc configuration for target bitrates This configuration mostly copies vp9 configuration for regular video, but is done separately to allow tune av1 svc bitrates independently of vp9. Bug: webrtc:12148 Change-Id: Icd11817ada8f9b6135ee2da57204eadb50de3954 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195329 Reviewed-by: Philip Eliasson Commit-Queue: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#32713} --- modules/video_coding/BUILD.gn | 46 +++--- modules/video_coding/codecs/av1/BUILD.gn | 25 ++- .../video_coding/codecs/av1/av1_svc_config.cc | 73 +++++++++ .../video_coding/codecs/av1/av1_svc_config.h | 22 +++ .../codecs/av1/av1_svc_config_unittest.cc | 142 ++++++++++++++++++ .../video_coding/video_codec_initializer.cc | 7 +- 6 files changed, 287 insertions(+), 28 deletions(-) create mode 100644 modules/video_coding/codecs/av1/av1_svc_config.cc create mode 100644 modules/video_coding/codecs/av1/av1_svc_config.h create mode 100644 modules/video_coding/codecs/av1/av1_svc_config_unittest.cc diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 742a3056ff..2844d52ed1 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -99,29 +99,6 @@ rtc_library("nack_module") { rtc_library("video_coding") { visibility = [ "*" ] - deps = [ - "..:module_fec_api", - "../../api:array_view", - "../../api:scoped_refptr", - "../../api/video:encoded_image", - "../../api/video:video_adaptation", - "../../api/video:video_bitrate_allocation", - "../../api/video:video_bitrate_allocator_factory", - "../../rtc_base:deprecation", - "../../rtc_base/system:no_unique_address", - "../../rtc_base/task_utils:to_queued_task", - "../../system_wrappers:field_trial", - "../../system_wrappers:metrics", - "../rtp_rtcp:rtp_video_header", - ] - absl_deps = [ - "//third_party/abseil-cpp/absl/base:core_headers", - "//third_party/abseil-cpp/absl/container:inlined_vector", - "//third_party/abseil-cpp/absl/memory", - "//third_party/abseil-cpp/absl/types:optional", - "//third_party/abseil-cpp/absl/types:variant", - ] - sources = [ "codec_timer.cc", "codec_timer.h", @@ -167,7 +144,7 @@ rtc_library("video_coding") { "video_receiver2.h", ] - deps += [ + deps = [ ":codec_globals_headers", ":encoded_frame", ":video_codec_interface", @@ -175,15 +152,22 @@ rtc_library("video_coding") { ":webrtc_vp9_helpers", "..:module_api", "..:module_api_public", + "..:module_fec_api", + "../../api:array_view", "../../api:fec_controller_api", "../../api:rtp_headers", "../../api:rtp_packet_info", + "../../api:scoped_refptr", "../../api/units:data_rate", "../../api/units:time_delta", "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_frame", + "../../api/video:encoded_image", "../../api/video:video_adaptation", + "../../api/video:video_adaptation", + "../../api/video:video_bitrate_allocation", "../../api/video:video_bitrate_allocator", + "../../api/video:video_bitrate_allocator_factory", "../../api/video:video_frame", "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", @@ -191,6 +175,7 @@ rtc_library("video_coding") { "../../common_video", "../../rtc_base", "../../rtc_base:checks", + "../../rtc_base:deprecation", "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_numerics", "../../rtc_base:rtc_task_queue", @@ -202,12 +187,25 @@ rtc_library("video_coding") { "../../rtc_base/experiments:rtt_mult_experiment", "../../rtc_base/synchronization:mutex", "../../rtc_base/synchronization:sequence_checker", + "../../rtc_base/system:no_unique_address", "../../rtc_base/task_utils:repeating_task", + "../../rtc_base/task_utils:to_queued_task", "../../rtc_base/third_party/base64", "../../rtc_base/time:timestamp_extrapolator", "../../system_wrappers", + "../../system_wrappers:field_trial", + "../../system_wrappers:metrics", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", + "../rtp_rtcp:rtp_video_header", + "codecs/av1:av1_svc_config", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/container:inlined_vector", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/types:optional", + "//third_party/abseil-cpp/absl/types:variant", ] } diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn index 27b22a0a59..95b5ad1274 100644 --- a/modules/video_coding/codecs/av1/BUILD.gn +++ b/modules/video_coding/codecs/av1/BUILD.gn @@ -9,6 +9,20 @@ import("//third_party/libaom/options.gni") import("../../../../webrtc.gni") +rtc_library("av1_svc_config") { + sources = [ + "av1_svc_config.cc", + "av1_svc_config.h", + ] + deps = [ + "../../../../api/video_codecs:video_codecs_api", + "../../../../rtc_base:checks", + "../../../../rtc_base:logging", + "../../svc:scalability_structures", + "../../svc:scalable_video_controller", + ] +} + rtc_library("libaom_av1_decoder") { visibility = [ "*" ] poisonous = [ "software_video_codecs" ] @@ -70,12 +84,18 @@ if (rtc_include_tests) { rtc_library("video_coding_codecs_av1_tests") { testonly = true + sources = [ "av1_svc_config_unittest.cc" ] + deps = [ + ":av1_svc_config", + "../../../../api/video_codecs:video_codecs_api", + ] + if (enable_libaom) { - sources = [ + sources += [ "libaom_av1_encoder_unittest.cc", "libaom_av1_unittest.cc", ] - deps = [ + deps += [ ":libaom_av1_decoder", ":libaom_av1_encoder", "../..:encoded_video_frame_producer", @@ -84,7 +104,6 @@ if (rtc_include_tests) { "../../../../api/units:data_size", "../../../../api/units:time_delta", "../../../../api/video:video_frame", - "../../../../api/video_codecs:video_codecs_api", "../../../../test:test_support", "../../svc:scalability_structures", "../../svc:scalable_video_controller", diff --git a/modules/video_coding/codecs/av1/av1_svc_config.cc b/modules/video_coding/codecs/av1/av1_svc_config.cc new file mode 100644 index 0000000000..1e61477b78 --- /dev/null +++ b/modules/video_coding/codecs/av1/av1_svc_config.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/av1/av1_svc_config.h" + +#include +#include +#include + +#include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/svc/scalable_video_controller.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +bool SetAv1SvcConfig(VideoCodec& video_codec) { + RTC_DCHECK_EQ(video_codec.codecType, kVideoCodecAV1); + + if (video_codec.ScalabilityMode().empty()) { + RTC_LOG(LS_INFO) << "No scalability mode set."; + return false; + } + std::unique_ptr structure = + CreateScalabilityStructure(video_codec.ScalabilityMode()); + if (structure == nullptr) { + RTC_LOG(LS_INFO) << "Failed to create structure " + << video_codec.ScalabilityMode(); + return false; + } + ScalableVideoController::StreamLayersConfig info = structure->StreamConfig(); + for (int sl_idx = 0; sl_idx < info.num_spatial_layers; ++sl_idx) { + SpatialLayer& spatial_layer = video_codec.spatialLayers[sl_idx]; + spatial_layer.width = video_codec.width * info.scaling_factor_num[sl_idx] / + info.scaling_factor_den[sl_idx]; + spatial_layer.height = video_codec.height * + info.scaling_factor_num[sl_idx] / + info.scaling_factor_den[sl_idx]; + spatial_layer.maxFramerate = video_codec.maxFramerate; + spatial_layer.numberOfTemporalLayers = info.num_temporal_layers; + spatial_layer.active = true; + } + + if (info.num_spatial_layers == 1) { + SpatialLayer& spatial_layer = video_codec.spatialLayers[0]; + spatial_layer.minBitrate = video_codec.minBitrate; + spatial_layer.targetBitrate = video_codec.startBitrate; + spatial_layer.maxBitrate = video_codec.maxBitrate; + return true; + } + + for (int sl_idx = 0; sl_idx < info.num_spatial_layers; ++sl_idx) { + SpatialLayer& spatial_layer = video_codec.spatialLayers[sl_idx]; + // minBitrate and maxBitrate formulas are copied from vp9 settings and + // are not yet tuned for av1. + const int num_pixels = spatial_layer.width * spatial_layer.height; + int min_bitrate_kbps = (600.0 * std::sqrt(num_pixels) - 95'000.0) / 1000.0; + spatial_layer.minBitrate = std::max(min_bitrate_kbps, 20); + spatial_layer.maxBitrate = 50 + static_cast(1.6 * num_pixels / 1000.0); + spatial_layer.targetBitrate = + (spatial_layer.minBitrate + spatial_layer.maxBitrate) / 2; + } + return true; +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/av1/av1_svc_config.h b/modules/video_coding/codecs/av1/av1_svc_config.h new file mode 100644 index 0000000000..15d94e03a9 --- /dev/null +++ b/modules/video_coding/codecs/av1/av1_svc_config.h @@ -0,0 +1,22 @@ +/* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_ + +#include "api/video_codecs/video_codec.h" + +namespace webrtc { + +// Fills `video_codec.spatialLayers` using other members. +bool SetAv1SvcConfig(VideoCodec& video_codec); + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_ diff --git a/modules/video_coding/codecs/av1/av1_svc_config_unittest.cc b/modules/video_coding/codecs/av1/av1_svc_config_unittest.cc new file mode 100644 index 0000000000..02ded1c70d --- /dev/null +++ b/modules/video_coding/codecs/av1/av1_svc_config_unittest.cc @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/av1/av1_svc_config.h" + +#include "api/video_codecs/video_codec.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(Av1SvcConfigTest, RequireScalabilityMode) { + VideoCodec video_codec; + video_codec.codecType = kVideoCodecAV1; + + video_codec.SetScalabilityMode(""); + EXPECT_FALSE(SetAv1SvcConfig(video_codec)); + + video_codec.SetScalabilityMode("Unknown"); + EXPECT_FALSE(SetAv1SvcConfig(video_codec)); + + video_codec.SetScalabilityMode("NONE"); + EXPECT_TRUE(SetAv1SvcConfig(video_codec)); +} + +TEST(Av1SvcConfigTest, SetsActiveSpatialLayersFromScalabilityMode) { + VideoCodec video_codec; + video_codec.codecType = kVideoCodecAV1; + video_codec.SetScalabilityMode("L2T1"); + + EXPECT_TRUE(SetAv1SvcConfig(video_codec)); + + EXPECT_TRUE(video_codec.spatialLayers[0].active); + EXPECT_TRUE(video_codec.spatialLayers[1].active); + EXPECT_FALSE(video_codec.spatialLayers[2].active); +} + +TEST(Av1SvcConfigTest, ConfiguresDobuleResolutionRatioFromScalabilityMode) { + VideoCodec video_codec; + video_codec.codecType = kVideoCodecAV1; + video_codec.SetScalabilityMode("L2T1"); + video_codec.width = 1200; + video_codec.height = 800; + + EXPECT_TRUE(SetAv1SvcConfig(video_codec)); + + EXPECT_EQ(video_codec.spatialLayers[0].width, 600); + EXPECT_EQ(video_codec.spatialLayers[0].height, 400); + EXPECT_EQ(video_codec.spatialLayers[1].width, 1200); + EXPECT_EQ(video_codec.spatialLayers[1].height, 800); +} + +TEST(Av1SvcConfigTest, ConfiguresSmallResolutionRatioFromScalabilityMode) { + VideoCodec video_codec; + video_codec.codecType = kVideoCodecAV1; + // h mode uses 1.5:1 ratio + video_codec.SetScalabilityMode("L2T1h"); + video_codec.width = 1500; + video_codec.height = 900; + + EXPECT_TRUE(SetAv1SvcConfig(video_codec)); + + EXPECT_EQ(video_codec.spatialLayers[0].width, 1000); + EXPECT_EQ(video_codec.spatialLayers[0].height, 600); + EXPECT_EQ(video_codec.spatialLayers[1].width, 1500); + EXPECT_EQ(video_codec.spatialLayers[1].height, 900); +} + +TEST(Av1SvcConfigTest, CopiesFramrate) { + VideoCodec video_codec; + video_codec.codecType = kVideoCodecAV1; + // h mode uses 1.5:1 ratio + video_codec.SetScalabilityMode("L2T1"); + video_codec.maxFramerate = 27; + + EXPECT_TRUE(SetAv1SvcConfig(video_codec)); + + EXPECT_EQ(video_codec.spatialLayers[0].maxFramerate, 27); + EXPECT_EQ(video_codec.spatialLayers[1].maxFramerate, 27); +} + +TEST(Av1SvcConfigTest, SetsNumberOfTemporalLayers) { + VideoCodec video_codec; + video_codec.codecType = kVideoCodecAV1; + video_codec.SetScalabilityMode("L1T3"); + + EXPECT_TRUE(SetAv1SvcConfig(video_codec)); + + EXPECT_EQ(video_codec.spatialLayers[0].numberOfTemporalLayers, 3); +} + +TEST(Av1SvcConfigTest, CopiesBitrateForSingleSpatialLayer) { + VideoCodec video_codec; + video_codec.codecType = kVideoCodecAV1; + video_codec.SetScalabilityMode("L1T3"); + video_codec.minBitrate = 100; + video_codec.startBitrate = 200; + video_codec.maxBitrate = 500; + + EXPECT_TRUE(SetAv1SvcConfig(video_codec)); + + EXPECT_EQ(video_codec.spatialLayers[0].minBitrate, 100u); + EXPECT_EQ(video_codec.spatialLayers[0].targetBitrate, 200u); + EXPECT_EQ(video_codec.spatialLayers[0].maxBitrate, 500u); +} + +TEST(Av1SvcConfigTest, SetsBitratesForMultipleSpatialLayers) { + VideoCodec video_codec; + video_codec.codecType = kVideoCodecAV1; + video_codec.SetScalabilityMode("L3T3"); + + EXPECT_TRUE(SetAv1SvcConfig(video_codec)); + + EXPECT_GT(video_codec.spatialLayers[0].minBitrate, 0u); + EXPECT_LE(video_codec.spatialLayers[0].minBitrate, + video_codec.spatialLayers[0].targetBitrate); + EXPECT_LE(video_codec.spatialLayers[0].targetBitrate, + video_codec.spatialLayers[0].maxBitrate); + + EXPECT_GT(video_codec.spatialLayers[1].minBitrate, 0u); + EXPECT_LE(video_codec.spatialLayers[1].minBitrate, + video_codec.spatialLayers[1].targetBitrate); + EXPECT_LE(video_codec.spatialLayers[1].targetBitrate, + video_codec.spatialLayers[1].maxBitrate); + + EXPECT_GT(video_codec.spatialLayers[2].minBitrate, 0u); + EXPECT_LE(video_codec.spatialLayers[2].minBitrate, + video_codec.spatialLayers[2].targetBitrate); + EXPECT_LE(video_codec.spatialLayers[2].targetBitrate, + video_codec.spatialLayers[2].maxBitrate); +} + +} // namespace +} // namespace webrtc diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc index 4777fe51c4..983e2a07a8 100644 --- a/modules/video_coding/video_codec_initializer.cc +++ b/modules/video_coding/video_codec_initializer.cc @@ -20,6 +20,7 @@ #include "api/units/data_rate.h" #include "api/video/video_bitrate_allocation.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" #include "modules/video_coding/codecs/vp9/svc_config.h" #include "modules/video_coding/include/video_coding_defines.h" #include "rtc_base/checks.h" @@ -56,7 +57,6 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); VideoCodec video_codec; - memset(&video_codec, 0, sizeof(video_codec)); video_codec.codecType = config.codec_type; switch (config.content_type) { @@ -255,6 +255,11 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( break; } + case kVideoCodecAV1: + if (!SetAv1SvcConfig(video_codec)) { + RTC_LOG(LS_WARNING) << "Failed to configure svc bitrates for av1."; + } + break; case kVideoCodecH264: { if (!config.encoder_specific_settings) *video_codec.H264() = VideoEncoder::GetDefaultH264Settings();