diff --git a/modules/video_coding/codecs/vp9/svc_config.cc b/modules/video_coding/codecs/vp9/svc_config.cc index 77eee3dbf5..3a32a43622 100644 --- a/modules/video_coding/codecs/vp9/svc_config.cc +++ b/modules/video_coding/codecs/vp9/svc_config.cc @@ -172,6 +172,21 @@ std::vector GetVp9SvcConfig(VideoCodec& codec) { absl::optional scalability_mode = codec.GetScalabilityMode(); RTC_DCHECK(scalability_mode.has_value()); + // Limit number of spatial layers for given resolution. + int limited_num_spatial_layers = + GetLimitedNumSpatialLayers(codec.width, codec.height); + if (limited_num_spatial_layers < + ScalabilityModeToNumSpatialLayers(*scalability_mode)) { + ScalabilityMode limited_scalability_mode = + LimitNumSpatialLayers(*scalability_mode, limited_num_spatial_layers); + RTC_LOG(LS_WARNING) + << "Reducing number of spatial layers due to low input resolution: " + << ScalabilityModeToString(*scalability_mode) << " to " + << ScalabilityModeToString(limited_scalability_mode); + scalability_mode = limited_scalability_mode; + codec.SetScalabilityMode(limited_scalability_mode); + } + absl::optional info = ScalabilityStructureConfig(*scalability_mode); if (!info.has_value()) { @@ -180,16 +195,6 @@ std::vector GetVp9SvcConfig(VideoCodec& codec) { return {}; } - if (static_cast(GetLimitedNumSpatialLayers(codec.width, codec.height)) < - info->num_spatial_layers) { - // Layers will be reduced, do not use scalability mode for now. - // TODO(bugs.webrtc.org/11607): Use a lower scalability mode once all lower - // modes are supported. - codec.UnsetScalabilityMode(); - codec.VP9()->interLayerPred = - ScalabilityModeToInterLayerPredMode(*scalability_mode); - } - // TODO(bugs.webrtc.org/11607): Add support for screensharing. std::vector spatial_layers = GetSvcConfig(codec.width, codec.height, codec.maxFramerate, diff --git a/modules/video_coding/codecs/vp9/svc_config_unittest.cc b/modules/video_coding/codecs/vp9/svc_config_unittest.cc index 4de3c5b2a6..762fd39287 100644 --- a/modules/video_coding/codecs/vp9/svc_config_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_config_unittest.cc @@ -72,15 +72,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityMode) { codec.height = 270; codec.SetScalabilityMode(ScalabilityMode::kL3T3_KEY); - // Scalability mode reset, configuration should be in accordance to L2T3_KEY. + // Scalability mode updated. std::vector spatial_layers = GetVp9SvcConfig(codec); EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::height, 135), Field(&SpatialLayer::height, 270))); EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 3), Field(&SpatialLayer::numberOfTemporalLayers, 3))); - EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOnKeyPic); - EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt); + EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL2T3_KEY); } TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModePortrait) { @@ -90,15 +89,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModePortrait) { codec.height = 480; codec.SetScalabilityMode(ScalabilityMode::kL3T1); - // Scalability mode reset, configuration should be in accordance to L2T1. + // Scalability mode updated. std::vector spatial_layers = GetVp9SvcConfig(codec); EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::width, 135), Field(&SpatialLayer::width, 270))); EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 1), Field(&SpatialLayer::numberOfTemporalLayers, 1))); - EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOn); - EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt); + EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL2T1); } TEST(SvcConfig, NumSpatialLayersWithScalabilityModeResolutionRatio1_5) { @@ -122,15 +120,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModeResolutionRatio1_5) { codec.codecType = kVideoCodecVP9; codec.width = 320; codec.height = 180; - codec.SetScalabilityMode(ScalabilityMode::kL2T1h); // 1.5:1 + codec.SetScalabilityMode(ScalabilityMode::kL3T1h); // 1.5:1 - // Scalability mode reset, configuration should be in accordance to L1T1. + // Scalability mode updated. std::vector spatial_layers = GetVp9SvcConfig(codec); EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::width, 320))); EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 1))); - EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOn); - EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt); + EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL1T1); } TEST(SvcConfig, AlwaysSendsAtLeastOneLayer) { diff --git a/modules/video_coding/svc/scalability_mode_util.cc b/modules/video_coding/svc/scalability_mode_util.cc index 39a4f1fd1e..35d66df203 100644 --- a/modules/video_coding/svc/scalability_mode_util.cc +++ b/modules/video_coding/svc/scalability_mode_util.cc @@ -292,4 +292,99 @@ absl::optional ScalabilityModeToResolutionRatio( RTC_CHECK_NOTREACHED(); } +ScalabilityMode LimitNumSpatialLayers(ScalabilityMode scalability_mode, + int max_spatial_layers) { + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + if (max_spatial_layers >= num_spatial_layers) { + return scalability_mode; + } + + switch (scalability_mode) { + case ScalabilityMode::kL1T1: + return ScalabilityMode::kL1T1; + case ScalabilityMode::kL1T2: + return ScalabilityMode::kL1T2; + case ScalabilityMode::kL1T3: + return ScalabilityMode::kL1T3; + case ScalabilityMode::kL2T1: + return ScalabilityMode::kL1T1; + case ScalabilityMode::kL2T1h: + return ScalabilityMode::kL1T1; + case ScalabilityMode::kL2T1_KEY: + return ScalabilityMode::kL1T1; + case ScalabilityMode::kL2T2: + return ScalabilityMode::kL1T2; + case ScalabilityMode::kL2T2h: + return ScalabilityMode::kL1T2; + case ScalabilityMode::kL2T2_KEY: + return ScalabilityMode::kL1T2; + case ScalabilityMode::kL2T2_KEY_SHIFT: + return ScalabilityMode::kL1T2; + case ScalabilityMode::kL2T3: + return ScalabilityMode::kL1T3; + case ScalabilityMode::kL2T3h: + return ScalabilityMode::kL1T3; + case ScalabilityMode::kL2T3_KEY: + return ScalabilityMode::kL1T3; + case ScalabilityMode::kL3T1: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T1 + : ScalabilityMode::kL1T1; + case ScalabilityMode::kL3T1h: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T1h + : ScalabilityMode::kL1T1; + case ScalabilityMode::kL3T1_KEY: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T1_KEY + : ScalabilityMode::kL1T1; + case ScalabilityMode::kL3T2: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T2 + : ScalabilityMode::kL1T2; + case ScalabilityMode::kL3T2h: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T2h + : ScalabilityMode::kL1T2; + case ScalabilityMode::kL3T2_KEY: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T2_KEY + : ScalabilityMode::kL1T2; + case ScalabilityMode::kL3T3: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T3 + : ScalabilityMode::kL1T3; + case ScalabilityMode::kL3T3h: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T3h + : ScalabilityMode::kL1T3; + case ScalabilityMode::kL3T3_KEY: + return max_spatial_layers == 2 ? ScalabilityMode::kL2T3_KEY + : ScalabilityMode::kL1T3; + case ScalabilityMode::kS2T1: + return ScalabilityMode::kL1T1; + case ScalabilityMode::kS2T1h: + return ScalabilityMode::kL1T1; + case ScalabilityMode::kS2T2: + return ScalabilityMode::kL1T2; + case ScalabilityMode::kS2T2h: + return ScalabilityMode::kL1T2; + case ScalabilityMode::kS2T3: + return ScalabilityMode::kL1T3; + case ScalabilityMode::kS2T3h: + return ScalabilityMode::kL1T3; + case ScalabilityMode::kS3T1: + return max_spatial_layers == 2 ? ScalabilityMode::kS2T1 + : ScalabilityMode::kL1T1; + case ScalabilityMode::kS3T1h: + return max_spatial_layers == 2 ? ScalabilityMode::kS2T1h + : ScalabilityMode::kL1T1; + case ScalabilityMode::kS3T2: + return max_spatial_layers == 2 ? ScalabilityMode::kS2T2 + : ScalabilityMode::kL1T2; + case ScalabilityMode::kS3T2h: + return max_spatial_layers == 2 ? ScalabilityMode::kS2T2h + : ScalabilityMode::kL1T2; + case ScalabilityMode::kS3T3: + return max_spatial_layers == 2 ? ScalabilityMode::kS2T3 + : ScalabilityMode::kL1T3; + case ScalabilityMode::kS3T3h: + return max_spatial_layers == 2 ? ScalabilityMode::kS2T3h + : ScalabilityMode::kL1T3; + } + RTC_CHECK_NOTREACHED(); +} + } // namespace webrtc diff --git a/modules/video_coding/svc/scalability_mode_util.h b/modules/video_coding/svc/scalability_mode_util.h index fa0b730e52..aef955a9a5 100644 --- a/modules/video_coding/svc/scalability_mode_util.h +++ b/modules/video_coding/svc/scalability_mode_util.h @@ -36,6 +36,9 @@ int ScalabilityModeToNumTemporalLayers(ScalabilityMode scalability_mode); absl::optional ScalabilityModeToResolutionRatio( ScalabilityMode scalability_mode); +ScalabilityMode LimitNumSpatialLayers(ScalabilityMode scalability_mode, + int max_spatial_layers); + } // namespace webrtc #endif // MODULES_VIDEO_CODING_SVC_SCALABILITY_MODE_UTIL_H_ diff --git a/modules/video_coding/svc/scalability_mode_util_unittest.cc b/modules/video_coding/svc/scalability_mode_util_unittest.cc index 7fb103631f..448494ffcc 100644 --- a/modules/video_coding/svc/scalability_mode_util_unittest.cc +++ b/modules/video_coding/svc/scalability_mode_util_unittest.cc @@ -10,6 +10,10 @@ #include "modules/video_coding/svc/scalability_mode_util.h" +#include +#include +#include + #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/video_codecs/scalability_mode.h" @@ -30,7 +34,7 @@ TEST(ScalabilityModeUtil, RejectsUnknownString) { // Check roundtrip conversion of all enum values. TEST(ScalabilityModeUtil, ConvertsAllToAndFromString) { - const ScalabilityMode kLastEnum = ScalabilityMode::kS3T3; + const ScalabilityMode kLastEnum = ScalabilityMode::kS3T3h; for (int numerical_enum = 0; numerical_enum <= static_cast(kLastEnum); numerical_enum++) { ScalabilityMode scalability_mode = @@ -43,5 +47,70 @@ TEST(ScalabilityModeUtil, ConvertsAllToAndFromString) { } } +struct TestParams { + std::string scalability_mode; + std::vector, std::string>> + limited_scalability_mode; +}; + +class NumSpatialLayersTest : public ::testing::TestWithParam {}; + +INSTANTIATE_TEST_SUITE_P( + MaxLayers, + NumSpatialLayersTest, + ::testing::ValuesIn( + {{"L1T1", {{{0, 1}, "L1T1"}, {{2}, "L1T1"}, {{3}, "L1T1"}}}, + {"L1T2", {{{0, 1}, "L1T2"}, {{2}, "L1T2"}, {{3}, "L1T2"}}}, + {"L1T3", {{{0, 1}, "L1T3"}, {{2}, "L1T3"}, {{3}, "L1T3"}}}, + {"L2T1", {{{0, 1}, "L1T1"}, {{2}, "L2T1"}, {{3}, "L2T1"}}}, + {"L2T1h", {{{0, 1}, "L1T1"}, {{2}, "L2T1h"}, {{3}, "L2T1h"}}}, + {"L2T1_KEY", {{{0, 1}, "L1T1"}, {{2}, "L2T1_KEY"}, {{3}, "L2T1_KEY"}}}, + {"L2T2", {{{0, 1}, "L1T2"}, {{2}, "L2T2"}, {{3}, "L2T2"}}}, + {"L2T2h", {{{0, 1}, "L1T2"}, {{2}, "L2T2h"}, {{3}, "L2T2h"}}}, + {"L2T2_KEY", {{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY"}, {{3}, "L2T2_KEY"}}}, + {"L2T2_KEY_SHIFT", + {{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY_SHIFT"}, {{3}, "L2T2_KEY_SHIFT"}}}, + {"L2T3", {{{0, 1}, "L1T3"}, {{2}, "L2T3"}, {{3}, "L2T3"}}}, + {"L2T3h", {{{0, 1}, "L1T3"}, {{2}, "L2T3h"}, {{3}, "L2T3h"}}}, + {"L2T3_KEY", {{{0, 1}, "L1T3"}, {{2}, "L2T3_KEY"}, {{3}, "L2T3_KEY"}}}, + {"L3T1", {{{0, 1}, "L1T1"}, {{2}, "L2T1"}, {{3}, "L3T1"}}}, + {"L3T1h", {{{0, 1}, "L1T1"}, {{2}, "L2T1h"}, {{3}, "L3T1h"}}}, + {"L3T1_KEY", {{{0, 1}, "L1T1"}, {{2}, "L2T1_KEY"}, {{3}, "L3T1_KEY"}}}, + {"L3T2", {{{0, 1}, "L1T2"}, {{2}, "L2T2"}, {{3}, "L3T2"}}}, + {"L3T2h", {{{0, 1}, "L1T2"}, {{2}, "L2T2h"}, {{3}, "L3T2h"}}}, + {"L3T2_KEY", {{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY"}, {{3}, "L3T2_KEY"}}}, + {"L3T3", {{{0, 1}, "L1T3"}, {{2}, "L2T3"}, {{3}, "L3T3"}}}, + {"L3T3h", {{{0, 1}, "L1T3"}, {{2}, "L2T3h"}, {{3}, "L3T3h"}}}, + {"L3T3_KEY", {{{0, 1}, "L1T3"}, {{2}, "L2T3_KEY"}, {{3}, "L3T3_KEY"}}}, + {"S2T1", {{{0, 1}, "L1T1"}, {{2}, "S2T1"}, {{3}, "S2T1"}}}, + {"S2T1h", {{{0, 1}, "L1T1"}, {{2}, "S2T1h"}, {{3}, "S2T1h"}}}, + {"S2T2", {{{0, 1}, "L1T2"}, {{2}, "S2T2"}, {{3}, "S2T2"}}}, + {"S2T2h", {{{0, 1}, "L1T2"}, {{2}, "S2T2h"}, {{3}, "S2T2h"}}}, + {"S2T3", {{{0, 1}, "L1T3"}, {{2}, "S2T3"}, {{3}, "S2T3"}}}, + {"S2T3h", {{{0, 1}, "L1T3"}, {{2}, "S2T3h"}, {{3}, "S2T3h"}}}, + {"S3T1", {{{0, 1}, "L1T1"}, {{2}, "S2T1"}, {{3}, "S3T1"}}}, + {"S3T1h", {{{0, 1}, "L1T1"}, {{2}, "S2T1h"}, {{3}, "S3T1h"}}}, + {"S3T2", {{{0, 1}, "L1T2"}, {{2}, "S2T2"}, {{3}, "S3T2"}}}, + {"S3T2h", {{{0, 1}, "L1T2"}, {{2}, "S2T2h"}, {{3}, "S3T2h"}}}, + {"S3T3", {{{0, 1}, "L1T3"}, {{2}, "S2T3"}, {{3}, "S3T3"}}}, + {"S3T3h", {{{0, 1}, "L1T3"}, {{2}, "S2T3h"}, {{3}, "S3T3h"}}}}), + [](const ::testing::TestParamInfo& info) { + return info.param.scalability_mode; + }); + +TEST_P(NumSpatialLayersTest, LimitsSpatialLayers) { + const ScalabilityMode mode = + *ScalabilityModeFromString(GetParam().scalability_mode); + for (const auto& param : GetParam().limited_scalability_mode) { + const std::vector max_num_spatial_layers = + std::get>(param); + const ScalabilityMode expected_mode = + *ScalabilityModeFromString(std::get(param)); + for (const auto& max_layers : max_num_spatial_layers) { + EXPECT_EQ(expected_mode, LimitNumSpatialLayers(mode, max_layers)); + } + } +} + } // namespace } // namespace webrtc