diff --git a/api/video_codecs/scalability_mode.h b/api/video_codecs/scalability_mode.h index 262e7ced7b..a33b9e420d 100644 --- a/api/video_codecs/scalability_mode.h +++ b/api/video_codecs/scalability_mode.h @@ -44,6 +44,7 @@ enum class ScalabilityMode { kL3T3h, kL3T3_KEY, kS2T1, + kS2T3, kS3T3, }; diff --git a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc index b252fb708a..35675f5cbd 100644 --- a/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc +++ b/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc @@ -466,7 +466,7 @@ void LibvpxVp9Encoder::EnableSpatialLayer(int sid) { } for (int tid = 0; tid < num_temporal_layers_; ++tid) { config_->layer_target_bitrate[sid * num_temporal_layers_ + tid] = - current_bitrate_allocation_.GetBitrate(sid, tid) / 1000; + current_bitrate_allocation_.GetTemporalLayerSum(sid, tid) / 1000; } config_->ss_target_bitrate[sid] = current_bitrate_allocation_.GetSpatialLayerSum(sid) / 1000; diff --git a/modules/video_coding/svc/BUILD.gn b/modules/video_coding/svc/BUILD.gn index d82f3164a5..b8ce91d99a 100644 --- a/modules/video_coding/svc/BUILD.gn +++ b/modules/video_coding/svc/BUILD.gn @@ -112,6 +112,7 @@ if (rtc_include_tests) { "../../../api/video:video_frame_type", "../../../api/video_codecs:scalability_mode", "../../../common_video/generic_frame_descriptor", + "../../../rtc_base:stringutils", "../../../test:test_support", ] absl_deps = [ diff --git a/modules/video_coding/svc/create_scalability_structure.cc b/modules/video_coding/svc/create_scalability_structure.cc index dfbfb14968..d5a85e0e3c 100644 --- a/modules/video_coding/svc/create_scalability_structure.cc +++ b/modules/video_coding/svc/create_scalability_structure.cc @@ -106,6 +106,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS2T1 = { {1, 1}, {2, 1}}; +constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3 = { + /*num_spatial_layers=*/2, + /*num_temporal_layers=*/3, + /*uses_reference_scaling=*/false, + {1, 1}, + {2, 1}}; + constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3 = { /*num_spatial_layers=*/3, /*num_temporal_layers=*/3, @@ -127,6 +134,7 @@ constexpr NamedStructureFactory kFactories[] = { kConfigL2T2}, {ScalabilityMode::kL2T2_KEY_SHIFT, Create, kConfigL2T2}, + {ScalabilityMode::kL2T3, Create, kConfigL2T3}, {ScalabilityMode::kL2T3_KEY, Create, kConfigL2T3}, {ScalabilityMode::kL3T1, Create, kConfigL3T1}, @@ -134,6 +142,7 @@ constexpr NamedStructureFactory kFactories[] = { {ScalabilityMode::kL3T3_KEY, Create, kConfigL3T3}, {ScalabilityMode::kS2T1, Create, kConfigS2T1}, + {ScalabilityMode::kS2T3, Create, kConfigS2T3}, {ScalabilityMode::kS3T3, Create, kConfigS3T3}, }; diff --git a/modules/video_coding/svc/scalability_mode_util.cc b/modules/video_coding/svc/scalability_mode_util.cc index d0a56afebc..510c9fd32c 100644 --- a/modules/video_coding/svc/scalability_mode_util.cc +++ b/modules/video_coding/svc/scalability_mode_util.cc @@ -75,6 +75,8 @@ absl::optional ScalabilityModeFromString( if (mode_string == "S2T1") return ScalabilityMode::kS2T1; + if (mode_string == "S2T3") + return ScalabilityMode::kS2T3; if (mode_string == "S3T3") return ScalabilityMode::kS3T3; @@ -133,6 +135,8 @@ absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode) { return "L3T3_KEY"; case ScalabilityMode::kS2T1: return "S2T1"; + case ScalabilityMode::kS2T3: + return "S2T3"; case ScalabilityMode::kS3T3: return "S3T3"; } @@ -179,6 +183,7 @@ InterLayerPredMode ScalabilityModeToInterLayerPredMode( case ScalabilityMode::kL3T3_KEY: return InterLayerPredMode::kOnKeyPic; case ScalabilityMode::kS2T1: + case ScalabilityMode::kS2T3: case ScalabilityMode::kS3T3: return InterLayerPredMode::kOff; } @@ -215,6 +220,7 @@ int ScalabilityModeToNumSpatialLayers(ScalabilityMode scalability_mode) { case ScalabilityMode::kL3T3_KEY: return 3; case ScalabilityMode::kS2T1: + case ScalabilityMode::kS2T3: return 2; case ScalabilityMode::kS3T3: return 3; @@ -259,6 +265,7 @@ int ScalabilityModeToNumTemporalLayers(ScalabilityMode scalability_mode) { return 3; case ScalabilityMode::kS2T1: return 1; + case ScalabilityMode::kS2T3: case ScalabilityMode::kS3T3: return 3; } diff --git a/modules/video_coding/svc/scalability_structure_full_svc.cc b/modules/video_coding/svc/scalability_structure_full_svc.cc index 892059297c..d73f167231 100644 --- a/modules/video_coding/svc/scalability_structure_full_svc.cc +++ b/modules/video_coding/svc/scalability_structure_full_svc.cc @@ -353,6 +353,26 @@ FrameDependencyStructure ScalabilityStructureL2T2::DependencyStructure() const { return structure; } +FrameDependencyStructure ScalabilityStructureL2T3::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 6; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1}; + auto& t = structure.templates; + t.resize(10); + t[1].S(0).T(0).Dtis("SSSSSS").ChainDiffs({0, 0}); + t[6].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 1}).FrameDiffs({1}); + t[3].S(0).T(2).Dtis("--D--R").ChainDiffs({2, 1}).FrameDiffs({2}); + t[8].S(1).T(2).Dtis("-----D").ChainDiffs({3, 2}).FrameDiffs({2, 1}); + t[2].S(0).T(1).Dtis("-DS-RR").ChainDiffs({4, 3}).FrameDiffs({4}); + t[7].S(1).T(1).Dtis("----DS").ChainDiffs({5, 4}).FrameDiffs({4, 1}); + t[4].S(0).T(2).Dtis("--D--R").ChainDiffs({6, 5}).FrameDiffs({2}); + t[9].S(1).T(2).Dtis("-----D").ChainDiffs({7, 6}).FrameDiffs({2, 1}); + t[0].S(0).T(0).Dtis("SSSRRR").ChainDiffs({8, 7}).FrameDiffs({8}); + t[5].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 1}).FrameDiffs({8, 1}); + return structure; +} + FrameDependencyStructure ScalabilityStructureL3T1::DependencyStructure() const { FrameDependencyStructure structure; structure.num_decode_targets = 3; diff --git a/modules/video_coding/svc/scalability_structure_full_svc.h b/modules/video_coding/svc/scalability_structure_full_svc.h index a3cad0af8a..03141ffb10 100644 --- a/modules/video_coding/svc/scalability_structure_full_svc.h +++ b/modules/video_coding/svc/scalability_structure_full_svc.h @@ -133,6 +133,23 @@ class ScalabilityStructureL2T2 : public ScalabilityStructureFullSvc { FrameDependencyStructure DependencyStructure() const override; }; +// S1T2 4 ,8 +// S1T1 / | 6' | +// S1T0 2--+-'+--+-... +// | | | | +// S0T2 | 3 | ,7 +// S0T1 | / 5' +// S0T0 1----'-----... +// Time-> 0 1 2 3 +class ScalabilityStructureL2T3 : public ScalabilityStructureFullSvc { + public: + explicit ScalabilityStructureL2T3(ScalingFactor resolution_factor = {}) + : ScalabilityStructureFullSvc(2, 3, resolution_factor) {} + ~ScalabilityStructureL2T3() override = default; + + FrameDependencyStructure DependencyStructure() const override; +}; + // S2 0-0-0- // | | | // S1 0-0-0-... diff --git a/modules/video_coding/svc/scalability_structure_simulcast.cc b/modules/video_coding/svc/scalability_structure_simulcast.cc index e5fa4c4368..0a06e9ee88 100644 --- a/modules/video_coding/svc/scalability_structure_simulcast.cc +++ b/modules/video_coding/svc/scalability_structure_simulcast.cc @@ -242,6 +242,26 @@ FrameDependencyStructure ScalabilityStructureS2T1::DependencyStructure() const { return structure; } +FrameDependencyStructure ScalabilityStructureS2T3::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 6; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1}; + auto& t = structure.templates; + t.resize(10); + t[1].S(0).T(0).Dtis("SSS---").ChainDiffs({0, 0}); + t[6].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 0}); + t[3].S(0).T(2).Dtis("--D---").ChainDiffs({2, 1}).FrameDiffs({2}); + t[8].S(1).T(2).Dtis("-----D").ChainDiffs({3, 2}).FrameDiffs({2}); + t[2].S(0).T(1).Dtis("-DS---").ChainDiffs({4, 3}).FrameDiffs({4}); + t[7].S(1).T(1).Dtis("----DS").ChainDiffs({5, 4}).FrameDiffs({4}); + t[4].S(0).T(2).Dtis("--D---").ChainDiffs({6, 5}).FrameDiffs({2}); + t[9].S(1).T(2).Dtis("-----D").ChainDiffs({7, 6}).FrameDiffs({2}); + t[0].S(0).T(0).Dtis("SSS---").ChainDiffs({8, 7}).FrameDiffs({8}); + t[5].S(1).T(0).Dtis("---SSS").ChainDiffs({1, 8}).FrameDiffs({8}); + return structure; +} + FrameDependencyStructure ScalabilityStructureS3T3::DependencyStructure() const { FrameDependencyStructure structure; structure.num_decode_targets = 9; diff --git a/modules/video_coding/svc/scalability_structure_simulcast.h b/modules/video_coding/svc/scalability_structure_simulcast.h index 7b57df2985..53f491c2b6 100644 --- a/modules/video_coding/svc/scalability_structure_simulcast.h +++ b/modules/video_coding/svc/scalability_structure_simulcast.h @@ -76,6 +76,26 @@ class ScalabilityStructureS2T1 : public ScalabilityStructureSimulcast { FrameDependencyStructure DependencyStructure() const override; }; +// S1T2 3 7 +// | / +// S1T1 / 5 +// |_/ +// S1T0 1-------9... +// +// S0T2 2 6 +// | / +// S0T1 / 4 +// |_/ +// S0T0 0-------8... +// Time-> 0 1 2 3 4 +class ScalabilityStructureS2T3 : public ScalabilityStructureSimulcast { + public: + ScalabilityStructureS2T3() : ScalabilityStructureSimulcast(2, 3) {} + ~ScalabilityStructureS2T3() override = default; + + FrameDependencyStructure DependencyStructure() const override; +}; + class ScalabilityStructureS3T3 : public ScalabilityStructureSimulcast { public: ScalabilityStructureS3T3() : ScalabilityStructureSimulcast(3, 3) {} diff --git a/modules/video_coding/svc/scalability_structure_unittest.cc b/modules/video_coding/svc/scalability_structure_unittest.cc index ffc085da04..86d7cc0fcf 100644 --- a/modules/video_coding/svc/scalability_structure_unittest.cc +++ b/modules/video_coding/svc/scalability_structure_unittest.cc @@ -22,6 +22,7 @@ #include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/scalability_structure_test_helpers.h" #include "modules/video_coding/svc/scalable_video_controller.h" +#include "rtc_base/strings/string_builder.h" #include "test/gmock.h" #include "test/gtest.h" @@ -43,6 +44,41 @@ using ::testing::SizeIs; using ::testing::TestWithParam; using ::testing::Values; +std::string FrameDependencyTemplateToString(const FrameDependencyTemplate& t) { + rtc::StringBuilder sb; + sb << "S" << t.spatial_id << "T" << t.temporal_id; + sb << ": dtis = "; + for (const auto dtis : t.decode_target_indications) { + switch (dtis) { + case DecodeTargetIndication::kNotPresent: + sb << "-"; + break; + case DecodeTargetIndication::kDiscardable: + sb << "D"; + break; + case DecodeTargetIndication::kSwitch: + sb << "S"; + break; + case DecodeTargetIndication::kRequired: + sb << "R"; + break; + default: + sb << "?"; + break; + } + } + sb << ", frame diffs = { "; + for (int d : t.frame_diffs) { + sb << d << ", "; + } + sb << "}, chain diffs = { "; + for (int d : t.chain_diffs) { + sb << d << ", "; + } + sb << "}"; + return sb.Release(); +} + struct SvcTestParam { friend std::ostream& operator<<(std::ostream& os, const SvcTestParam& param) { return os << param.name; @@ -176,7 +212,8 @@ TEST_P(ScalabilityStructureTest, ThereIsAPerfectTemplateForEachFrame) { .GenerateFrames(GetParam().num_temporal_units); for (size_t frame_id = 0; frame_id < frame_infos.size(); ++frame_id) { EXPECT_THAT(structure.templates, Contains(frame_infos[frame_id])) - << " for frame " << frame_id; + << " for frame " << frame_id << ", Expected " + << FrameDependencyTemplateToString(frame_infos[frame_id]); } } @@ -336,10 +373,12 @@ INSTANTIATE_TEST_SUITE_P( SvcTestParam{"L3T1", /*num_temporal_units=*/3}, SvcTestParam{"L3T3", /*num_temporal_units=*/8}, SvcTestParam{"S2T1", /*num_temporal_units=*/3}, + SvcTestParam{"S2T3", /*num_temporal_units=*/8}, SvcTestParam{"S3T3", /*num_temporal_units=*/8}, SvcTestParam{"L2T2", /*num_temporal_units=*/4}, SvcTestParam{"L2T2_KEY", /*num_temporal_units=*/4}, SvcTestParam{"L2T2_KEY_SHIFT", /*num_temporal_units=*/4}, + SvcTestParam{"L2T3", /*num_temporal_units=*/8}, SvcTestParam{"L2T3_KEY", /*num_temporal_units=*/8}, SvcTestParam{"L3T3_KEY", /*num_temporal_units=*/8}), [](const testing::TestParamInfo& info) { diff --git a/pc/test/svc_e2e_tests.cc b/pc/test/svc_e2e_tests.cc index 30a0ca4475..993d18bbfc 100644 --- a/pc/test/svc_e2e_tests.cc +++ b/pc/test/svc_e2e_tests.cc @@ -295,6 +295,7 @@ INSTANTIATE_TEST_SUITE_P( SvcTestParameters{cricket::kVp9CodecName, "L2T2", 2, 2}, SvcTestParameters{cricket::kVp9CodecName, "L2T2_KEY", 2, 2}, SvcTestParameters{cricket::kVp9CodecName, "L2T2_KEY_SHIFT", 2, 2}, + SvcTestParameters{cricket::kVp9CodecName, "L2T3", 2, 3}, SvcTestParameters{cricket::kVp9CodecName, "L2T3_KEY", 2, 3}, SvcTestParameters{cricket::kVp9CodecName, "L3T1", 3, 1}, SvcTestParameters{cricket::kVp9CodecName, "L3T3", 3, 3}),