diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn index b06fc9e8bf..81559db38b 100644 --- a/modules/video_coding/codecs/av1/BUILD.gn +++ b/modules/video_coding/codecs/av1/BUILD.gn @@ -55,6 +55,12 @@ rtc_source_set("scalability_structures") { sources = [ "scalability_structure_l1t2.cc", "scalability_structure_l1t2.h", + "scalability_structure_l2t1.cc", + "scalability_structure_l2t1.h", + "scalability_structure_l2t1_key.cc", + "scalability_structure_l2t1_key.h", + "scalability_structure_s2t1.cc", + "scalability_structure_s2t1.h", ] deps = [ ":scalable_video_controller", diff --git a/modules/video_coding/codecs/av1/libaom_av1_unittest.cc b/modules/video_coding/codecs/av1/libaom_av1_unittest.cc index a2752e6377..3c224f7df9 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_unittest.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_unittest.cc @@ -22,6 +22,9 @@ #include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" #include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h" +#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h" #include "modules/video_coding/codecs/av1/scalable_video_controller.h" #include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -275,7 +278,13 @@ INSTANTIATE_TEST_SUITE_P( Values(SvcTestParam{std::make_unique, /*num_frames_to_generate=*/4}, SvcTestParam{std::make_unique, - /*num_frames_to_generate=*/4})); + /*num_frames_to_generate=*/4}, + SvcTestParam{std::make_unique, + /*num_frames_to_generate=*/3}, + SvcTestParam{std::make_unique, + /*num_frames_to_generate=*/3}, + SvcTestParam{std::make_unique, + /*num_frames_to_generate=*/3})); } // namespace } // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_l2t1.cc b/modules/video_coding/codecs/av1/scalability_structure_l2t1.cc new file mode 100644 index 0000000000..0a397d963b --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l2t1.cc @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h" + +#include +#include + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; +constexpr auto kRequired = DecodeTargetIndication::kRequired; + +constexpr DecodeTargetIndication kDtis[4][2] = { + {kSwitch, kSwitch}, // Key, S0 + {kNotPresent, kSwitch}, // Key, S1 + {kSwitch, kRequired}, // Delta, S0 + {kNotPresent, kRequired}, // Delta, S1 +}; + +} // namespace + +ScalabilityStructureL2T1::~ScalabilityStructureL2T1() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL2T1::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure ScalabilityStructureL2T1::DependencyStructure() const { + using Builder = GenericFrameInfo::Builder; + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 1}; + structure.templates = { + Builder().S(0).Dtis("SR").Fdiffs({2}).ChainDiffs({2, 1}).Build(), + Builder().S(0).Dtis("SS").ChainDiffs({0, 0}).Build(), + Builder().S(1).Dtis("-R").Fdiffs({1, 2}).ChainDiffs({1, 2}).Build(), + Builder().S(1).Dtis("-S").Fdiffs({1}).ChainDiffs({1, 1}).Build(), + }; + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL2T1::KeyFrameConfig() const { + LayerFrameConfig result; + result.id = 0; + result.spatial_id = 0; + result.is_keyframe = true; + result.buffers = {{/*id=*/0, /*references=*/false, /*updates=*/true}}; + return result; +} + +std::vector +ScalabilityStructureL2T1::NextFrameConfig(bool restart) { + std::vector result(2); + + // Buffer0 keeps latest S0 frame, Buffer1 keeps latest S1 frame. + if (restart || keyframe_) { + result[0] = KeyFrameConfig(); + + result[1].id = 1; + result[1].spatial_id = 1; + result[1].is_keyframe = false; + result[1].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/false}, + {/*id=*/1, /*references=*/false, /*updates=*/true}}; + + keyframe_ = false; + } else { + result[0].id = 2; + result[0].spatial_id = 0; + result[0].is_keyframe = false; + result[0].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/true}}; + + result[1].id = 3; + result[1].spatial_id = 1; + result[1].is_keyframe = false; + result[1].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/false}, + {/*id=*/1, /*references=*/true, /*updates=*/true}}; + } + return result; +} + +absl::optional ScalabilityStructureL2T1::OnEncodeDone( + LayerFrameConfig config) { + absl::optional frame_info; + if (config.is_keyframe) { + config = KeyFrameConfig(); + } + + if (config.id < 0 || config.id >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.id; + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.spatial_id; + frame_info->temporal_id = config.temporal_id; + frame_info->encoder_buffers = std::move(config.buffers); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.id]), + std::end(kDtis[config.id])); + frame_info->part_of_chain = {config.spatial_id == 0, true}; + return frame_info; +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_l2t1.h b/modules/video_coding/codecs/av1/scalability_structure_l2t1.h new file mode 100644 index 0000000000..0f53602604 --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l2t1.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_ + +#include + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1 0--0--0- +// | | | ... +// S0 0--0--0- +class ScalabilityStructureL2T1 : public ScalableVideoController { + public: + ~ScalabilityStructureL2T1() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector NextFrameConfig(bool restart) override; + absl::optional OnEncodeDone( + LayerFrameConfig config) override; + + private: + LayerFrameConfig KeyFrameConfig() const; + + bool keyframe_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_ diff --git a/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc b/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc new file mode 100644 index 0000000000..fa2a943ffc --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h" + +#include +#include + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +constexpr DecodeTargetIndication kDtis[3][2] = { + {kSwitch, kSwitch}, // Key, S0 + {kSwitch, kNotPresent}, // Delta, S0 + {kNotPresent, kSwitch}, // Key and Delta, S1 +}; + +} // namespace + +ScalabilityStructureL2T1Key::~ScalabilityStructureL2T1Key() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL2T1Key::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure ScalabilityStructureL2T1Key::DependencyStructure() + const { + using Builder = GenericFrameInfo::Builder; + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 1}; + structure.templates = { + Builder().S(0).Dtis("S-").Fdiffs({2}).ChainDiffs({2, 1}).Build(), + Builder().S(0).Dtis("SS").ChainDiffs({0, 0}).Build(), + Builder().S(1).Dtis("-S").Fdiffs({2}).ChainDiffs({1, 2}).Build(), + Builder().S(1).Dtis("-S").Fdiffs({1}).ChainDiffs({1, 1}).Build(), + }; + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL2T1Key::KeyFrameConfig() const { + LayerFrameConfig result; + result.id = 0; + result.spatial_id = 0; + result.is_keyframe = true; + result.buffers = {{/*id=*/0, /*references=*/false, /*updates=*/true}}; + return result; +} + +std::vector +ScalabilityStructureL2T1Key::NextFrameConfig(bool restart) { + std::vector result(2); + + // Buffer0 keeps latest S0T0 frame, Buffer1 keeps latest S1T0 frame. + if (restart || keyframe_) { + result[0] = KeyFrameConfig(); + + result[1].id = 2; + result[1].spatial_id = 1; + result[1].is_keyframe = false; + result[1].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/false}, + {/*id=*/1, /*references=*/false, /*updates=*/true}}; + + keyframe_ = false; + } else { + result[0].id = 1; + result[0].spatial_id = 0; + result[0].is_keyframe = false; + result[0].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/true}}; + + result[1].id = 2; + result[1].spatial_id = 1; + result[1].is_keyframe = false; + result[1].buffers = {{/*id=*/0, /*references=*/false, /*updates=*/false}, + {/*id=*/1, /*references=*/true, /*updates=*/true}}; + } + return result; +} + +absl::optional ScalabilityStructureL2T1Key::OnEncodeDone( + LayerFrameConfig config) { + absl::optional frame_info; + if (config.is_keyframe) { + config = KeyFrameConfig(); + } + + if (config.id < 0 || config.id >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.id; + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.spatial_id; + frame_info->temporal_id = config.temporal_id; + frame_info->encoder_buffers = std::move(config.buffers); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.id]), + std::end(kDtis[config.id])); + if (config.is_keyframe) { + frame_info->part_of_chain = {true, true}; + } else { + frame_info->part_of_chain = {config.spatial_id == 0, + config.spatial_id == 1}; + } + return frame_info; +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h b/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h new file mode 100644 index 0000000000..c1d8c8947f --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_ + +#include + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1 0--0--0- +// | ... +// S0 0--0--0- +class ScalabilityStructureL2T1Key : public ScalableVideoController { + public: + ~ScalabilityStructureL2T1Key() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector NextFrameConfig(bool restart) override; + absl::optional OnEncodeDone( + LayerFrameConfig config) override; + + private: + LayerFrameConfig KeyFrameConfig() const; + + bool keyframe_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_ diff --git a/modules/video_coding/codecs/av1/scalability_structure_s2t1.cc b/modules/video_coding/codecs/av1/scalability_structure_s2t1.cc new file mode 100644 index 0000000000..d1d257d586 --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_s2t1.cc @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h" + +#include +#include + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +constexpr DecodeTargetIndication kDtis[2][2] = { + {kSwitch, kNotPresent}, // S0 + {kNotPresent, kSwitch}, // S1 +}; + +} // namespace + +ScalabilityStructureS2T1::~ScalabilityStructureS2T1() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureS2T1::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure ScalabilityStructureS2T1::DependencyStructure() const { + using Builder = GenericFrameInfo::Builder; + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 1}; + structure.templates = { + Builder().S(0).Dtis("S-").Fdiffs({2}).ChainDiffs({2, 1}).Build(), + Builder().S(0).Dtis("S-").ChainDiffs({0, 0}).Build(), + Builder().S(1).Dtis("-S").Fdiffs({2}).ChainDiffs({1, 2}).Build(), + Builder().S(1).Dtis("-S").ChainDiffs({1, 0}).Build(), + }; + return structure; +} + +std::vector +ScalabilityStructureS2T1::NextFrameConfig(bool restart) { + if (restart) { + keyframe_ = true; + } + std::vector result(2); + + // Buffer0 keeps latest S0T0 frame, Buffer1 keeps latest S1T0 frame. + result[0].spatial_id = 0; + result[0].is_keyframe = keyframe_; + result[0].buffers = {{/*id=*/0, /*references=*/!keyframe_, /*updates=*/true}}; + + result[1].spatial_id = 1; + result[1].is_keyframe = keyframe_; + result[1].buffers = {{/*id=*/1, /*references=*/!keyframe_, /*updates=*/true}}; + + keyframe_ = false; + return result; +} + +absl::optional ScalabilityStructureS2T1::OnEncodeDone( + LayerFrameConfig config) { + absl::optional frame_info; + if (config.id != 0) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.id; + } + if (config.spatial_id < 0 || + config.spatial_id >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected spatial id " << config.spatial_id; + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.spatial_id; + frame_info->temporal_id = config.temporal_id; + frame_info->encoder_buffers = std::move(config.buffers); + frame_info->decode_target_indications.assign( + std::begin(kDtis[config.spatial_id]), std::end(kDtis[config.spatial_id])); + frame_info->part_of_chain = {config.spatial_id == 0, config.spatial_id == 1}; + return frame_info; +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_s2t1.h b/modules/video_coding/codecs/av1/scalability_structure_s2t1.h new file mode 100644 index 0000000000..06a99775c4 --- /dev/null +++ b/modules/video_coding/codecs/av1/scalability_structure_s2t1.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_ + +#include + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1 0--0--0- +// ... +// S0 0--0--0- +class ScalabilityStructureS2T1 : public ScalableVideoController { + public: + ~ScalabilityStructureS2T1() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector NextFrameConfig(bool restart) override; + absl::optional OnEncodeDone( + LayerFrameConfig config) override; + + private: + bool keyframe_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_