admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move SvcRateAllocator out of codecs/vp9 and make it not vp9 specific

Browse Source 
to reuse it for scalable av1

Bug: webrtc:12148
Change-Id: I0d413c5466a2d66df3eea9cdf13687f4ddd1a843
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/191765
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32560}
This commit is contained in:
Danil Chapovalov 2020-11-05 18:06:00 +01:00 committed by Commit Bot
parent 85fae2acc5
commit 649648e722
6 changed files with 263 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/video_coding/BUILD.gn
View File

@ -512,7 +512,6 @@ rtc_library("webrtc_vp9_helpers") {
sources = [
"codecs/vp9/svc_config.cc",
"codecs/vp9/svc_config.h",
"codecs/vp9/svc_rate_allocator.cc",
"codecs/vp9/svc_rate_allocator.h",
]
@ -527,6 +526,7 @@ rtc_library("webrtc_vp9_helpers") {
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../rtc_base/experiments:stable_target_rate_experiment",
"svc:svc_rate_allocator",
]
absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ]
}
@ -900,7 +900,6 @@ if (rtc_include_tests) {
"codecs/vp8/libvpx_vp8_simulcast_test.cc",
"codecs/vp8/screenshare_layers_unittest.cc",
"codecs/vp9/svc_config_unittest.cc",
"codecs/vp9/svc_rate_allocator_unittest.cc",
"decoding_state_unittest.cc",
"fec_controller_unittest.cc",
"frame_buffer2_unittest.cc",
@ -1009,6 +1008,7 @@ if (rtc_include_tests) {
"codecs/av1:video_coding_codecs_av1_tests",
"deprecated:nack_module",
"svc:scalability_structure_tests",
"svc:svc_rate_allocator_tests",
]
absl_deps = [
"//third_party/abseil-cpp/absl/memory",

49
modules/video_coding/codecs/vp9/svc_rate_allocator.h
View File

@ -11,52 +11,7 @@
#ifndef MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_
#define MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_
#include <stddef.h>
#include <stdint.h>
#include "absl/container/inlined_vector.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/video_codec.h"
#include "rtc_base/experiments/stable_target_rate_experiment.h"
namespace webrtc {
class SvcRateAllocator : public VideoBitrateAllocator {
public:
explicit SvcRateAllocator(const VideoCodec& codec);
VideoBitrateAllocation Allocate(
VideoBitrateAllocationParameters parameters) override;
static DataRate GetMaxBitrate(const VideoCodec& codec);
static DataRate GetPaddingBitrate(const VideoCodec& codec);
static absl::InlinedVector<DataRate, kMaxSpatialLayers> GetLayerStartBitrates(
const VideoCodec& codec);
private:
VideoBitrateAllocation GetAllocationNormalVideo(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const;
VideoBitrateAllocation GetAllocationScreenSharing(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const;
// Returns the number of layers that are active and have enough bitrate to
// actually be enabled.
size_t FindNumEnabledLayers(DataRate target_rate) const;
const VideoCodec codec_;
const StableTargetRateExperiment experiment_settings_;
const absl::InlinedVector<DataRate, kMaxSpatialLayers>
cumulative_layer_start_bitrates_;
size_t last_active_layer_count_;
};
} // namespace webrtc
// TODO(danilchap): Update dependent includes and remove this forwarding header.
#include "modules/video_coding/svc/svc_rate_allocator.h"
#endif // MODULES_VIDEO_CODING_CODECS_VP9_SVC_RATE_ALLOCATOR_H_

31
modules/video_coding/svc/BUILD.gn
View File

@ -68,8 +68,25 @@ rtc_source_set("scalability_structures") {
]
}
rtc_source_set("svc_rate_allocator") {
sources = [
"svc_rate_allocator.cc",
"svc_rate_allocator.h",
]
deps = [
":scalability_structures",
"../../../api/video:video_bitrate_allocation",
"../../../api/video:video_bitrate_allocator",
"../../../api/video:video_codec_constants",
"../../../api/video_codecs:video_codecs_api",
"../../../rtc_base:checks",
"../../../rtc_base/experiments:stable_target_rate_experiment",
]
absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ]
}
if (rtc_include_tests) {
rtc_library("scalability_structure_tests") {
rtc_source_set("scalability_structure_tests") {
testonly = true
sources = [
"scalability_structure_key_svc_unittest.cc",
@ -93,4 +110,16 @@ if (rtc_include_tests) {
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_source_set("svc_rate_allocator_tests") {
testonly = true
sources = [ "svc_rate_allocator_unittest.cc" ]
deps = [
":svc_rate_allocator",
"..:webrtc_vp9_helpers",
"../../../rtc_base:checks",
"../../../test:field_trial",
"../../../test:test_support",
]
}
}

114
modules/video_coding/codecs/vp9/svc_rate_allocator.cc → modules/video_coding/svc/svc_rate_allocator.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
#include "modules/video_coding/svc/svc_rate_allocator.h"
#include <algorithm>
#include <cmath>
@ -17,40 +17,38 @@
#include <vector>
#include "absl/container/inlined_vector.h"
#include "modules/video_coding/svc/create_scalability_structure.h"
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
const float kSpatialLayeringRateScalingFactor = 0.55f;
const float kTemporalLayeringRateScalingFactor = 0.55f;
constexpr float kSpatialLayeringRateScalingFactor = 0.55f;
constexpr float kTemporalLayeringRateScalingFactor = 0.55f;
// Returns numberOfSpatialLayers if no layers are active.
size_t GetFirstActiveLayer(const VideoCodec& codec) {
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
size_t layer = 0;
for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) {
if (codec.spatialLayers[layer].active) {
struct ActiveSpatialLayers {
size_t first = 0;
size_t num = 0;
};
ActiveSpatialLayers GetActiveSpatialLayers(const VideoCodec& codec,
size_t num_spatial_layers) {
ActiveSpatialLayers active;
for (active.first = 0; active.first < num_spatial_layers; ++active.first) {
if (codec.spatialLayers[active.first].active) {
break;
}
}
return layer;
}
static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) {
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
const size_t first_active_layer = GetFirstActiveLayer(codec);
size_t last_active_layer = first_active_layer;
for (; last_active_layer < codec.VP9().numberOfSpatialLayers;
++last_active_layer) {
size_t last_active_layer = active.first;
for (; last_active_layer < num_spatial_layers; ++last_active_layer) {
if (!codec.spatialLayers[last_active_layer].active) {
break;
}
}
return last_active_layer - first_active_layer;
active.num = last_active_layer - active.first;
return active;
}
std::vector<DataRate> AdjustAndVerify(
@ -173,16 +171,39 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
} // namespace
SvcRateAllocator::NumLayers SvcRateAllocator::GetNumLayers(
const VideoCodec& codec) {
NumLayers layers;
if (!codec.ScalabilityMode().empty()) {
if (auto structure = CreateScalabilityStructure(codec.ScalabilityMode())) {
ScalableVideoController::StreamLayersConfig config =
structure->StreamConfig();
layers.spatial = config.num_spatial_layers;
layers.temporal = config.num_temporal_layers;
return layers;
}
}
if (codec.codecType == kVideoCodecVP9) {
layers.spatial = codec.VP9().numberOfSpatialLayers;
layers.temporal = codec.VP9().numberOfTemporalLayers;
return layers;
}
layers.spatial = 1;
layers.temporal = 1;
return layers;
}
SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec)
: codec_(codec),
num_layers_(GetNumLayers(codec)),
experiment_settings_(StableTargetRateExperiment::ParseFromFieldTrials()),
cumulative_layer_start_bitrates_(GetLayerStartBitrates(codec)),
last_active_layer_count_(0) {
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
RTC_DCHECK_GT(codec.VP9().numberOfTemporalLayers, 0u);
for (size_t layer_idx = 0; layer_idx < codec.VP9().numberOfSpatialLayers;
++layer_idx) {
RTC_DCHECK_GT(num_layers_.spatial, 0);
RTC_DCHECK_LE(num_layers_.spatial, kMaxSpatialLayers);
RTC_DCHECK_GT(num_layers_.temporal, 0);
RTC_DCHECK_LE(num_layers_.temporal, 3);
for (size_t layer_idx = 0; layer_idx < num_layers_.spatial; ++layer_idx) {
// Verify min <= target <= max.
if (codec.spatialLayers[layer_idx].active) {
RTC_DCHECK_GT(codec.spatialLayers[layer_idx].maxBitrate, 0);
@ -205,16 +226,16 @@ VideoBitrateAllocation SvcRateAllocator::Allocate(
}
if (codec_.spatialLayers[0].targetBitrate == 0) {
// Delegate rate distribution to VP9 encoder wrapper if bitrate thresholds
// Delegate rate distribution to encoder wrapper if bitrate thresholds
// are not set.
VideoBitrateAllocation bitrate_allocation;
bitrate_allocation.SetBitrate(0, 0, total_bitrate.bps());
return bitrate_allocation;
}
const size_t first_active_layer = GetFirstActiveLayer(codec_);
const size_t num_active_layers = GetNumActiveSpatialLayers(codec_);
size_t num_spatial_layers = num_active_layers;
const ActiveSpatialLayers active_layers =
GetActiveSpatialLayers(codec_, num_layers_.spatial);
size_t num_spatial_layers = active_layers.num;
if (num_spatial_layers == 0) {
return VideoBitrateAllocation(); // All layers are deactivated.
@ -249,13 +270,13 @@ VideoBitrateAllocation SvcRateAllocator::Allocate(
VideoBitrateAllocation allocation;
if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
allocation = GetAllocationNormalVideo(total_bitrate, first_active_layer,
allocation = GetAllocationNormalVideo(total_bitrate, active_layers.first,
num_spatial_layers);
} else {
allocation = GetAllocationScreenSharing(total_bitrate, first_active_layer,
allocation = GetAllocationScreenSharing(total_bitrate, active_layers.first,
num_spatial_layers);
}
allocation.set_bw_limited(num_spatial_layers < num_active_layers);
allocation.set_bw_limited(num_spatial_layers < active_layers.num);
return allocation;
}
@ -279,25 +300,24 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
VideoBitrateAllocation bitrate_allocation;
const size_t num_temporal_layers = codec_.VP9().numberOfTemporalLayers;
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
std::vector<DataRate> temporal_layer_rates =
SplitBitrate(num_temporal_layers, spatial_layer_rates[sl_idx],
SplitBitrate(num_layers_.temporal, spatial_layer_rates[sl_idx],
kTemporalLayeringRateScalingFactor);
// Distribute rate across temporal layers. Allocate more bits to lower
// layers since they are used for prediction of higher layers and their
// references are far apart.
if (num_temporal_layers == 1) {
if (num_layers_.temporal == 1) {
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
temporal_layer_rates[0].bps());
} else if (num_temporal_layers == 2) {
} else if (num_layers_.temporal == 2) {
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
temporal_layer_rates[1].bps());
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
temporal_layer_rates[0].bps());
} else {
RTC_CHECK_EQ(num_temporal_layers, 3);
RTC_CHECK_EQ(num_layers_.temporal, 3);
// In case of three temporal layers the high layer has two frames and the
// middle layer has one frame within GOP (in between two consecutive low
// layer frames). Thus high layer requires more bits (comparing pure
@ -383,13 +403,14 @@ size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const {
}
DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) {
const size_t first_active_layer = GetFirstActiveLayer(codec);
const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec);
const NumLayers num_layers = GetNumLayers(codec);
const ActiveSpatialLayers active_layers =
GetActiveSpatialLayers(codec, num_layers.spatial);
DataRate max_bitrate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
for (size_t sl_idx = 0; sl_idx < active_layers.num; ++sl_idx) {
max_bitrate += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
codec.spatialLayers[active_layers.first + sl_idx].maxBitrate);
}
if (codec.maxBitrate != 0) {
@ -412,12 +433,13 @@ DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) {
absl::InlinedVector<DataRate, kMaxSpatialLayers>
SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) {
absl::InlinedVector<DataRate, kMaxSpatialLayers> start_bitrates;
const size_t first_active_layer = GetFirstActiveLayer(codec);
const size_t num_layers = GetNumActiveSpatialLayers(codec);
const NumLayers num_layers = GetNumLayers(codec);
const ActiveSpatialLayers active_layers =
GetActiveSpatialLayers(codec, num_layers.spatial);
DataRate last_rate = DataRate::Zero();
for (size_t i = 1; i <= num_layers; ++i) {
for (size_t i = 1; i <= active_layers.num; ++i) {
DataRate layer_toggling_rate =
FindLayerTogglingThreshold(codec, first_active_layer, i);
FindLayerTogglingThreshold(codec, active_layers.first, i);
start_bitrates.push_back(layer_toggling_rate);
RTC_DCHECK_LE(last_rate, layer_toggling_rate);
last_rate = layer_toggling_rate;

69
modules/video_coding/svc/svc_rate_allocator.h Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_SVC_SVC_RATE_ALLOCATOR_H_
#define MODULES_VIDEO_CODING_SVC_SVC_RATE_ALLOCATOR_H_
#include <stddef.h>
#include <stdint.h>
#include "absl/container/inlined_vector.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_bitrate_allocator.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/video_codec.h"
#include "rtc_base/experiments/stable_target_rate_experiment.h"
namespace webrtc {
class SvcRateAllocator : public VideoBitrateAllocator {
public:
explicit SvcRateAllocator(const VideoCodec& codec);
VideoBitrateAllocation Allocate(
VideoBitrateAllocationParameters parameters) override;
static DataRate GetMaxBitrate(const VideoCodec& codec);
static DataRate GetPaddingBitrate(const VideoCodec& codec);
static absl::InlinedVector<DataRate, kMaxSpatialLayers> GetLayerStartBitrates(
const VideoCodec& codec);
private:
struct NumLayers {
size_t spatial = 1;
size_t temporal = 1;
};
static NumLayers GetNumLayers(const VideoCodec& codec);
VideoBitrateAllocation GetAllocationNormalVideo(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const;
VideoBitrateAllocation GetAllocationScreenSharing(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const;
// Returns the number of layers that are active and have enough bitrate to
// actually be enabled.
size_t FindNumEnabledLayers(DataRate target_rate) const;
const VideoCodec codec_;
const NumLayers num_layers_;
const StableTargetRateExperiment experiment_settings_;
const absl::InlinedVector<DataRate, kMaxSpatialLayers>
cumulative_layer_start_bitrates_;
size_t last_active_layer_count_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_SVC_SVC_RATE_ALLOCATOR_H_

93
modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc → modules/video_coding/svc/svc_rate_allocator_unittest.cc
View File

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
#include "modules/video_coding/svc/svc_rate_allocator.h"
#include <algorithm>
#include <vector>
@ -270,6 +270,97 @@ TEST(SvcRateAllocatorTest, FindLayerTogglingThreshold) {
EXPECT_EQ(layer_start_bitrates[2], kThreeLayerMinRate);
}
TEST(SvcRateAllocatorTest, SupportsAv1) {
VideoCodec codec;
codec.width = 640;
codec.height = 360;
codec.codecType = kVideoCodecAV1;
codec.SetScalabilityMode("L3T3");
codec.spatialLayers[0].active = true;
codec.spatialLayers[0].minBitrate = 30;
codec.spatialLayers[0].targetBitrate = 51;
codec.spatialLayers[0].maxBitrate = 73;
codec.spatialLayers[1].active = true;
codec.spatialLayers[1].minBitrate = 49;
codec.spatialLayers[1].targetBitrate = 64;
codec.spatialLayers[1].maxBitrate = 97;
codec.spatialLayers[2].active = true;
codec.spatialLayers[2].minBitrate = 193;
codec.spatialLayers[2].targetBitrate = 305;
codec.spatialLayers[2].maxBitrate = 418;
SvcRateAllocator allocator(codec);
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(1'000'000, 30));
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0u);
EXPECT_GT(allocation.GetSpatialLayerSum(1), 0u);
EXPECT_GT(allocation.GetSpatialLayerSum(2), 0u);
}
TEST(SvcRateAllocatorTest, SupportsAv1WithSkippedLayer) {
VideoCodec codec;
codec.width = 640;
codec.height = 360;
codec.codecType = kVideoCodecAV1;
codec.SetScalabilityMode("L3T3");
codec.spatialLayers[0].active = false;
codec.spatialLayers[0].minBitrate = 30;
codec.spatialLayers[0].targetBitrate = 51;
codec.spatialLayers[0].maxBitrate = 73;
codec.spatialLayers[1].active = true;
codec.spatialLayers[1].minBitrate = 49;
codec.spatialLayers[1].targetBitrate = 64;
codec.spatialLayers[1].maxBitrate = 97;
codec.spatialLayers[2].active = true;
codec.spatialLayers[2].minBitrate = 193;
codec.spatialLayers[2].targetBitrate = 305;
codec.spatialLayers[2].maxBitrate = 418;
SvcRateAllocator allocator(codec);
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(1'000'000, 30));
EXPECT_EQ(allocation.GetSpatialLayerSum(0), 0u);
EXPECT_GT(allocation.GetSpatialLayerSum(1), 0u);
EXPECT_GT(allocation.GetSpatialLayerSum(2), 0u);
}
TEST(SvcRateAllocatorTest, UsesScalabilityModeToGetNumberOfLayers) {
VideoCodec codec;
codec.width = 640;
codec.height = 360;
codec.codecType = kVideoCodecAV1;
codec.SetScalabilityMode("L2T2");
codec.spatialLayers[0].active = true;
codec.spatialLayers[0].minBitrate = 30;
codec.spatialLayers[0].targetBitrate = 51;
codec.spatialLayers[0].maxBitrate = 73;
codec.spatialLayers[1].active = true;
codec.spatialLayers[1].minBitrate = 49;
codec.spatialLayers[1].targetBitrate = 64;
codec.spatialLayers[1].maxBitrate = 97;
codec.spatialLayers[2].active = true;
codec.spatialLayers[2].minBitrate = 193;
codec.spatialLayers[2].targetBitrate = 305;
codec.spatialLayers[2].maxBitrate = 418;
SvcRateAllocator allocator(codec);
VideoBitrateAllocation allocation =
allocator.Allocate(VideoBitrateAllocationParameters(1'000'000, 30));
// Expect bitrates for 2 temporal layers.
EXPECT_TRUE(allocation.HasBitrate(1, /*temporal_index=*/0));
EXPECT_TRUE(allocation.HasBitrate(1, /*temporal_index=*/1));
EXPECT_FALSE(allocation.HasBitrate(1, /*temporal_index=*/2));
// expect codec.spatialLayers[2].active is ignored because scability mode uses
// just 2 spatial layers.
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0u);
}
class SvcRateAllocatorTestParametrizedContentType
: public ::testing::Test,
public ::testing::WithParamInterface<bool> {