diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn index df1c7da71e..4c61be72dc 100644 --- a/modules/video_coding/codecs/av1/BUILD.gn +++ b/modules/video_coding/codecs/av1/BUILD.gn @@ -55,6 +55,8 @@ rtc_source_set("scalability_structures") { sources = [ "scalability_structure_l1t2.cc", "scalability_structure_l1t2.h", + "scalability_structure_l1t3.cc", + "scalability_structure_l1t3.h", "scalability_structure_l2t1.cc", "scalability_structure_l2t1.h", "scalability_structure_l2t1_key.cc", @@ -65,6 +67,10 @@ rtc_source_set("scalability_structures") { "scalability_structure_l2t2_key.h", "scalability_structure_l2t2_key_shift.cc", "scalability_structure_l2t2_key_shift.h", + "scalability_structure_l3t1.cc", + "scalability_structure_l3t1.h", + "scalability_structure_l3t3.cc", + "scalability_structure_l3t3.h", "scalability_structure_s2t1.cc", "scalability_structure_s2t1.h", ] diff --git a/modules/video_coding/codecs/av1/libaom_av1_unittest.cc b/modules/video_coding/codecs/av1/libaom_av1_unittest.cc index 90c9e67dd6..4bca6b28b9 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_unittest.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_unittest.cc @@ -22,11 +22,14 @@ #include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" #include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t2.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l3t1.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l3t3.h" #include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h" #include "modules/video_coding/codecs/av1/scalable_video_controller.h" #include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h" @@ -283,10 +286,16 @@ INSTANTIATE_TEST_SUITE_P( /*num_frames_to_generate=*/4}, SvcTestParam{std::make_unique, /*num_frames_to_generate=*/4}, + SvcTestParam{std::make_unique, + /*num_frames_to_generate=*/8}, SvcTestParam{std::make_unique, /*num_frames_to_generate=*/3}, SvcTestParam{std::make_unique, /*num_frames_to_generate=*/3}, + SvcTestParam{std::make_unique, + /*num_frames_to_generate=*/3}, + SvcTestParam{std::make_unique, + /*num_frames_to_generate=*/8}, SvcTestParam{std::make_unique, /*num_frames_to_generate=*/3}, SvcTestParam{std::make_unique, diff --git a/modules/video_coding/codecs/av1/scalability_structure_l1t3.cc b/modules/video_coding/codecs/av1/scalability_structure_l1t3.cc new file mode 100644 index 0000000000..4ef5628fa8 --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l1t3.cc @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h" + +#include +#include + +#include "absl/base/macros.h" +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +constexpr DecodeTargetIndication kDtis[3][3] = { + {kSwitch, kSwitch, kSwitch}, // T0 + {kNotPresent, kDiscardable, kSwitch}, // T1 + {kNotPresent, kNotPresent, kDiscardable}, // T2 +}; + +} // namespace + +ScalabilityStructureL1T3::~ScalabilityStructureL1T3() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL1T3::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 1; + result.num_temporal_layers = 3; + return result; +} + +FrameDependencyStructure ScalabilityStructureL1T3::DependencyStructure() const { + using Builder = GenericFrameInfo::Builder; + FrameDependencyStructure structure; + structure.num_decode_targets = 3; + structure.num_chains = 1; + structure.decode_target_protected_by_chain = {0, 0, 0}; + structure.templates = { + Builder().T(0).Dtis("SSS").ChainDiffs({0}).Build(), + Builder().T(0).Dtis("SSS").ChainDiffs({4}).Fdiffs({4}).Build(), + Builder().T(1).Dtis("-DS").ChainDiffs({2}).Fdiffs({2}).Build(), + Builder().T(2).Dtis("--D").ChainDiffs({1}).Fdiffs({1}).Build(), + Builder().T(2).Dtis("--D").ChainDiffs({3}).Fdiffs({1}).Build(), + }; + return structure; +} + +std::vector +ScalabilityStructureL1T3::NextFrameConfig(bool restart) { + if (restart) { + next_pattern_ = kKeyFrame; + } + std::vector config(1); + + switch (next_pattern_) { + case kKeyFrame: + config[0].T(0).Keyframe().Update(0); + next_pattern_ = kDeltaFrameT2A; + break; + case kDeltaFrameT2A: + config[0].T(2).Reference(0); + next_pattern_ = kDeltaFrameT1; + break; + case kDeltaFrameT1: + config[0].T(1).Reference(0).Update(1); + next_pattern_ = kDeltaFrameT2B; + break; + case kDeltaFrameT2B: + config[0].T(2).Reference(1); + next_pattern_ = kDeltaFrameT0; + break; + case kDeltaFrameT0: + config[0].T(0).ReferenceAndUpdate(0); + next_pattern_ = kDeltaFrameT2A; + break; + } + return config; +} + +absl::optional ScalabilityStructureL1T3::OnEncodeDone( + LayerFrameConfig config) { + absl::optional frame_info; + if (config.TemporalId() < 0 || + config.TemporalId() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected temporal id " << config.TemporalId(); + return frame_info; + } + frame_info.emplace(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign( + std::begin(kDtis[config.TemporalId()]), + std::end(kDtis[config.TemporalId()])); + frame_info->part_of_chain = {config.TemporalId() == 0}; + return frame_info; +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_l1t3.h b/modules/video_coding/codecs/av1/scalability_structure_l1t3.h new file mode 100644 index 0000000000..562d0f2a50 --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l1t3.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_ + +#include + +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// T2 0 0 0 0 +// | / | / +// T1 / 0 / 0 ... +// |_/ |_/ +// T0 0-------0------ +// Time-> 0 1 2 3 4 5 6 7 +class ScalabilityStructureL1T3 : public ScalableVideoController { + public: + ~ScalabilityStructureL1T3() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector NextFrameConfig(bool restart) override; + absl::optional OnEncodeDone( + LayerFrameConfig config) override; + + private: + enum FramePattern { + kKeyFrame, + kDeltaFrameT2A, + kDeltaFrameT1, + kDeltaFrameT2B, + kDeltaFrameT0, + }; + + FramePattern next_pattern_ = kKeyFrame; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_ diff --git a/modules/video_coding/codecs/av1/scalability_structure_l3t1.cc b/modules/video_coding/codecs/av1/scalability_structure_l3t1.cc new file mode 100644 index 0000000000..b151641c4a --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l3t1.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l3t1.h" + +#include +#include + +#include "absl/base/macros.h" +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; +constexpr auto kRequired = DecodeTargetIndication::kRequired; + +constexpr DecodeTargetIndication kDtis[5][3] = { + {kSwitch, kSwitch, kSwitch}, // Key, S0 + {kNotPresent, kSwitch, kSwitch}, // Key, S1 + {kNotPresent, kNotPresent, kSwitch}, // Key and Delta, S2 + {kSwitch, kRequired, kRequired}, // Delta, S0 + {kNotPresent, kSwitch, kRequired}, // Delta, S1 +}; + +} // namespace + +ScalabilityStructureL3T1::~ScalabilityStructureL3T1() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL3T1::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 3; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure ScalabilityStructureL3T1::DependencyStructure() const { + using Builder = GenericFrameInfo::Builder; + FrameDependencyStructure structure; + structure.num_decode_targets = 3; + structure.num_chains = 3; + structure.decode_target_protected_by_chain = {0, 1, 2}; + structure.templates = { + Builder().S(0).Dtis("SRR").ChainDiffs({3, 2, 1}).Fdiffs({3}).Build(), + Builder().S(0).Dtis("SSS").ChainDiffs({0, 0, 0}).Build(), + Builder().S(1).Dtis("-SR").ChainDiffs({1, 1, 1}).Fdiffs({3, 1}).Build(), + Builder().S(1).Dtis("-SS").ChainDiffs({1, 1, 1}).Fdiffs({1}).Build(), + Builder().S(2).Dtis("--S").ChainDiffs({2, 1, 1}).Fdiffs({3, 1}).Build(), + Builder().S(2).Dtis("--S").ChainDiffs({2, 1, 1}).Fdiffs({1}).Build(), + }; + return structure; +} + +std::vector +ScalabilityStructureL3T1::NextFrameConfig(bool restart) { + std::vector config(3); + + // Buffer i keeps latest frame for spatial layer i + if (restart || keyframe_) { + config[0].Id(0).S(0).Keyframe().Update(0); + config[1].Id(1).S(1).Update(1).Reference(0); + config[2].Id(2).S(2).Update(2).Reference(1); + keyframe_ = false; + } else { + config[0].Id(3).S(0).ReferenceAndUpdate(0); + config[1].Id(4).S(1).ReferenceAndUpdate(1).Reference(0); + config[2].Id(2).S(2).ReferenceAndUpdate(2).Reference(1); + } + return config; +} + +absl::optional ScalabilityStructureL3T1::OnEncodeDone( + LayerFrameConfig config) { + absl::optional frame_info; + if (config.IsKeyframe() && config.Id() != 0) { + // Encoder generated a key frame without asking to. + if (config.SpatialId() > 0) { + RTC_LOG(LS_WARNING) << "Unexpected spatial id " << config.SpatialId() + << " for key frame."; + } + config = LayerFrameConfig().Id(0).S(0).Keyframe().Update(0); + } + + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + frame_info->part_of_chain = {config.SpatialId() == 0, config.SpatialId() <= 1, + true}; + return frame_info; +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_l3t1.h b/modules/video_coding/codecs/av1/scalability_structure_l3t1.h new file mode 100644 index 0000000000..404860d08f --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l3t1.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_ + +#include + +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S2 0-0-0- +// | | | +// S1 0-0-0-... +// | | | +// S0 0-0-0- +// Time-> 0 1 2 +class ScalabilityStructureL3T1 : public ScalableVideoController { + public: + ~ScalabilityStructureL3T1() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector NextFrameConfig(bool restart) override; + absl::optional OnEncodeDone( + LayerFrameConfig config) override; + + private: + bool keyframe_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_ diff --git a/modules/video_coding/codecs/av1/scalability_structure_l3t3.cc b/modules/video_coding/codecs/av1/scalability_structure_l3t3.cc new file mode 100644 index 0000000000..62c52fcd1e --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l3t3.cc @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l3t3.h" + +#include +#include + +#include "absl/base/macros.h" +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; +constexpr auto kRequired = DecodeTargetIndication::kRequired; + +constexpr DecodeTargetIndication kDtis[12][9] = { + // Key, S0 + {kSwitch, kSwitch, kSwitch, // S0 + kSwitch, kSwitch, kSwitch, // S1 + kSwitch, kSwitch, kSwitch}, // S2 + // Key, S1 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kSwitch, kSwitch, kSwitch, // S1 + kSwitch, kSwitch, kSwitch}, // S2 + // Key, S2 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kNotPresent, // S1 + kSwitch, kSwitch, kSwitch}, // S2 + // Delta, S0T2 + {kNotPresent, kNotPresent, kDiscardable, // S0 + kNotPresent, kNotPresent, kRequired, // S1 + kNotPresent, kNotPresent, kRequired}, // S2 + // Delta, S1T2 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kDiscardable, // S1 + kNotPresent, kNotPresent, kRequired}, // S2 + // Delta, S2T2 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kNotPresent, // S1 + kNotPresent, kNotPresent, kDiscardable}, // S2 + // Delta, S0T1 + {kNotPresent, kDiscardable, kSwitch, // S0 + kNotPresent, kRequired, kRequired, // S1 + kNotPresent, kRequired, kRequired}, // S2 + // Delta, S1T1 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kDiscardable, kSwitch, // S1 + kNotPresent, kRequired, kRequired}, // S2 + // Delta, S2T1 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kNotPresent, // S1 + kNotPresent, kDiscardable, kSwitch}, // S2 + // Delta, S0T0 + {kSwitch, kSwitch, kSwitch, // S0 + kRequired, kRequired, kRequired, // S1 + kRequired, kRequired, kRequired}, // S2 + // Delta, S1T0 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kSwitch, kSwitch, kSwitch, // S1 + kRequired, kRequired, kRequired}, // S2 + // Delta, S2T0 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kNotPresent, // S1 + kSwitch, kSwitch, kSwitch}, // S2 +}; + +} // namespace + +ScalabilityStructureL3T3::~ScalabilityStructureL3T3() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL3T3::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 3; + result.num_temporal_layers = 3; + return result; +} + +FrameDependencyStructure ScalabilityStructureL3T3::DependencyStructure() const { + using Builder = GenericFrameInfo::Builder; + FrameDependencyStructure structure; + structure.num_decode_targets = 9; + structure.num_chains = 3; + structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1, 2, 2, 2}; + structure.templates = { + Builder().S(0).T(0).Dtis("SSSSSSSSS").ChainDiffs({0, 0, 0}).Build(), + Builder() + .S(0) + .T(0) + .Dtis("SSSRRRRRR") + .Fdiffs({12}) + .ChainDiffs({12, 11, 10}) + .Build(), + Builder() + .S(0) + .T(1) + .Dtis("-DS-RR-RR") + .Fdiffs({6}) + .ChainDiffs({6, 5, 4}) + .Build(), + Builder() + .S(0) + .T(2) + .Dtis("--D--R--R") + .Fdiffs({3}) + .ChainDiffs({3, 2, 1}) + .Build(), + Builder() + .S(0) + .T(2) + .Dtis("--D--R--R") + .Fdiffs({3}) + .ChainDiffs({9, 8, 7}) + .Build(), + Builder() + .S(1) + .T(0) + .Dtis("---SSSSSS") + .Fdiffs({1}) + .ChainDiffs({1, 1, 1}) + .Build(), + Builder() + .S(1) + .T(0) + .Dtis("---SSSRRR") + .Fdiffs({12, 1}) + .ChainDiffs({1, 1, 1}) + .Build(), + Builder() + .S(1) + .T(1) + .Dtis("----DS-RR") + .Fdiffs({6, 1}) + .ChainDiffs({7, 6, 5}) + .Build(), + Builder() + .S(1) + .T(2) + .Dtis("-----D--R") + .Fdiffs({3, 1}) + .ChainDiffs({4, 3, 2}) + .Build(), + Builder() + .S(1) + .T(2) + .Dtis("-----D--R") + .Fdiffs({3, 1}) + .ChainDiffs({10, 9, 8}) + .Build(), + Builder() + .S(2) + .T(0) + .Dtis("------SSS") + .Fdiffs({1}) + .ChainDiffs({2, 1, 1}) + .Build(), + Builder() + .S(2) + .T(0) + .Dtis("------SSS") + .Fdiffs({12, 1}) + .ChainDiffs({2, 1, 1}) + .Build(), + Builder() + .S(2) + .T(1) + .Dtis("-------DS") + .Fdiffs({6, 1}) + .ChainDiffs({8, 7, 6}) + .Build(), + Builder() + .S(2) + .T(2) + .Dtis("--------D") + .Fdiffs({3, 1}) + .ChainDiffs({5, 4, 3}) + .Build(), + Builder() + .S(2) + .T(2) + .Dtis("--------D") + .Fdiffs({3, 1}) + .ChainDiffs({11, 10, 9}) + .Build(), + }; + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL3T3::KeyFrameConfig() const { + return LayerFrameConfig().Id(0).S(0).T(0).Keyframe().Update(0); +} + +std::vector +ScalabilityStructureL3T3::NextFrameConfig(bool restart) { + if (restart) { + next_pattern_ = kKeyFrame; + } + std::vector config(3); + + // For this structure name each of 8 buffers after the layer of the frame that + // buffer keeps. + static constexpr int kS0T0 = 0; + static constexpr int kS1T0 = 1; + static constexpr int kS2T0 = 2; + static constexpr int kS0T1 = 3; + static constexpr int kS1T1 = 4; + static constexpr int kS2T1 = 5; + static constexpr int kS0T2 = 6; + static constexpr int kS1T2 = 7; + switch (next_pattern_) { + case kKeyFrame: + config[0].Id(0).S(0).T(0).Keyframe().Update(kS0T0); + config[1].Id(1).S(1).T(0).Update(kS1T0).Reference(kS0T0); + config[2].Id(2).S(2).T(0).Update(kS2T0).Reference(kS1T0); + next_pattern_ = kDeltaFrameT2A; + break; + case kDeltaFrameT2A: + config[0].Id(3).S(0).T(2).Reference(kS0T0).Update(kS0T2); + config[1].Id(4).S(1).T(2).Reference(kS1T0).Reference(kS0T2).Update(kS1T2); + config[2].Id(5).S(2).T(2).Reference(kS2T0).Reference(kS1T2); + next_pattern_ = kDeltaFrameT1; + break; + case kDeltaFrameT1: + config[0].Id(6).S(0).T(1).Reference(kS0T0).Update(kS0T1); + config[1].Id(7).S(1).T(1).Reference(kS1T0).Reference(kS0T1).Update(kS1T1); + config[2].Id(8).S(2).T(1).Reference(kS2T0).Reference(kS1T1).Update(kS2T1); + next_pattern_ = kDeltaFrameT2B; + break; + case kDeltaFrameT2B: + config[0].Id(3).S(0).T(2).Reference(kS0T1).Update(kS0T2); + config[1].Id(4).S(1).T(2).Reference(kS1T1).Reference(kS0T2).Update(kS1T2); + config[2].Id(5).S(2).T(2).Reference(kS2T1).Reference(kS1T2); + next_pattern_ = kDeltaFrameT0; + break; + case kDeltaFrameT0: + config[0].Id(9).S(0).T(0).ReferenceAndUpdate(kS0T0); + config[1].Id(10).S(1).T(0).ReferenceAndUpdate(kS1T0).Reference(kS0T0); + config[2].Id(11).S(2).T(0).ReferenceAndUpdate(kS2T0).Reference(kS1T0); + next_pattern_ = kDeltaFrameT2A; + break; + } + return config; +} + +absl::optional ScalabilityStructureL3T3::OnEncodeDone( + LayerFrameConfig config) { + if (config.IsKeyframe() && config.Id() != 0) { + // Encoder generated a key frame without asking to. + if (config.SpatialId() > 0) { + RTC_LOG(LS_WARNING) << "Unexpected spatial id " << config.SpatialId() + << " for key frame."; + } + config = LayerFrameConfig() + .Keyframe() + .Id(0) + .S(0) + .T(0) + .Update(0) + .Update(1) + .Update(2) + .Update(3) + .Update(4) + .Update(5) + .Update(6) + .Update(7); + } + + absl::optional frame_info; + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + if (config.TemporalId() == 0) { + frame_info->part_of_chain = {config.SpatialId() == 0, + config.SpatialId() <= 1, true}; + } else { + frame_info->part_of_chain = {false, false, false}; + } + return frame_info; +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_l3t3.h b/modules/video_coding/codecs/av1/scalability_structure_l3t3.h new file mode 100644 index 0000000000..363f07e015 --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l3t3.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_ + +#include + +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// https://aomediacodec.github.io/av1-rtp-spec/#a63-l3t3-full-svc +class ScalabilityStructureL3T3 : public ScalableVideoController { + public: + ~ScalabilityStructureL3T3() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector NextFrameConfig(bool restart) override; + absl::optional OnEncodeDone( + LayerFrameConfig config) override; + + private: + enum FramePattern { + kKeyFrame, + kDeltaFrameT2A, + kDeltaFrameT1, + kDeltaFrameT2B, + kDeltaFrameT0, + }; + LayerFrameConfig KeyFrameConfig() const; + + FramePattern next_pattern_ = kKeyFrame; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_ diff --git a/modules/video_coding/codecs/av1/scalability_structure_unittest.cc b/modules/video_coding/codecs/av1/scalability_structure_unittest.cc index 9b47cfa16a..7d19b1b21e 100644 --- a/modules/video_coding/codecs/av1/scalability_structure_unittest.cc +++ b/modules/video_coding/codecs/av1/scalability_structure_unittest.cc @@ -20,11 +20,14 @@ #include "api/video/video_frame_type.h" #include "modules/video_coding/chain_diff_calculator.h" #include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t2.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h" #include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l3t1.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l3t3.h" #include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h" #include "modules/video_coding/codecs/av1/scalable_video_controller.h" #include "modules/video_coding/frame_dependencies_calculator.h" @@ -284,11 +287,17 @@ INSTANTIATE_TEST_SUITE_P( ScalabilityStructureTest, Values(SvcTestParam{"L1T2", std::make_unique, /*num_temporal_units=*/4}, + SvcTestParam{"L1T3", std::make_unique, + /*num_temporal_units=*/8}, SvcTestParam{"L2T1", std::make_unique, /*num_temporal_units=*/3}, SvcTestParam{"L2T1Key", std::make_unique, /*num_temporal_units=*/3}, + SvcTestParam{"L3T1", std::make_unique, + /*num_temporal_units=*/3}, + SvcTestParam{"L3T3", std::make_unique, + /*num_temporal_units=*/8}, SvcTestParam{"S2T1", std::make_unique, /*num_temporal_units=*/3}, SvcTestParam{"L2T2", std::make_unique,