From 9df335374a00381f0f763d02838c2a115197df41 Mon Sep 17 00:00:00 2001 From: philipel Date: Mon, 4 Mar 2019 16:37:50 +0100 Subject: [PATCH] Generic Frame Descriptor (GFD) VP8 templates. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In this CL: - Updated Vp8TemporalLayers::OnEncodeDone to take a CodecSpecificInfo instead of a CodecSpecificInfoVP8, so that both the VP8 specific and generic information can be populated. - Added structs to represent the GFD template structure. - Added code to generate templates for video/screensharing. Bug: webrtc:10342 Change-Id: I978f9d708597a6f86bbdc494e62acf7a7b400db3 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/123422 Commit-Queue: Philip Eliasson Reviewed-by: Erik Språng Reviewed-by: Stefan Holmer Reviewed-by: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#26987} --- api/video_codecs/vp8_temporal_layers.h | 8 +- .../generic_frame_descriptor/BUILD.gn | 23 ++++ common_video/generic_frame_descriptor/OWNERS | 7 ++ .../generic_frame_info.cc | 70 ++++++++++++ .../generic_frame_info.h | 70 ++++++++++++ modules/video_coding/BUILD.gn | 2 + .../codecs/vp8/default_temporal_layers.cc | 94 +++++++++++++--- .../codecs/vp8/default_temporal_layers.h | 3 +- .../vp8/default_temporal_layers_unittest.cc | 105 +++++++++--------- .../codecs/vp8/libvpx_vp8_encoder.cc | 9 +- .../codecs/vp8/screenshare_layers.cc | 63 ++++++++--- .../codecs/vp8/screenshare_layers.h | 6 +- .../codecs/vp8/screenshare_layers_unittest.cc | 90 +++++++-------- .../include/video_codec_interface.cc | 20 ++++ .../include/video_codec_interface.h | 12 +- .../simulcast_rate_allocator_unittest.cc | 2 +- test/fake_vp8_encoder.cc | 7 +- 17 files changed, 441 insertions(+), 150 deletions(-) create mode 100644 common_video/generic_frame_descriptor/BUILD.gn create mode 100644 common_video/generic_frame_descriptor/OWNERS create mode 100644 common_video/generic_frame_descriptor/generic_frame_info.cc create mode 100644 common_video/generic_frame_descriptor/generic_frame_info.h create mode 100644 modules/video_coding/include/video_codec_interface.cc diff --git a/api/video_codecs/vp8_temporal_layers.h b/api/video_codecs/vp8_temporal_layers.h index 67e9ebda89..e5dc14eb5d 100644 --- a/api/video_codecs/vp8_temporal_layers.h +++ b/api/video_codecs/vp8_temporal_layers.h @@ -44,7 +44,7 @@ namespace webrtc { // the bitrate produced. enum class Vp8TemporalLayersType { kFixedPattern, kBitrateDynamic }; -struct CodecSpecificInfoVP8; +struct CodecSpecificInfo; struct Vp8EncoderConfig { static constexpr size_t kMaxPeriodicity = 16; @@ -115,8 +115,8 @@ class Vp8TemporalLayers { // a keyframe. // If the encoder decided to drop this frame, |size_bytes| must be set to 0, // otherwise it should indicate the size in bytes of the encoded frame. - // If |size_bytes| > 0, and |vp8_info| is not null, the TemporalLayers - // instance my update |vp8_info| with codec specific data such as temporal id. + // If |size_bytes| > 0, and |info| is not null, the TemporalLayers + // instance my update |info| with codec specific data such as temporal id. // Some fields of this struct may have already been populated by the encoder, // check before overwriting. // If |size_bytes| > 0, |qp| should indicate the frame-level QP this frame was @@ -126,7 +126,7 @@ class Vp8TemporalLayers { size_t size_bytes, bool is_keyframe, int qp, - CodecSpecificInfoVP8* vp8_info) = 0; + CodecSpecificInfo* info) = 0; }; } // namespace webrtc diff --git a/common_video/generic_frame_descriptor/BUILD.gn b/common_video/generic_frame_descriptor/BUILD.gn new file mode 100644 index 0000000000..d9bd494ab2 --- /dev/null +++ b/common_video/generic_frame_descriptor/BUILD.gn @@ -0,0 +1,23 @@ +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +rtc_source_set("generic_frame_descriptor") { + sources = [ + "generic_frame_info.cc", + "generic_frame_info.h", + ] + + deps = [ + "../../api:array_view", + "../../rtc_base:checks", + "//third_party/abseil-cpp/absl/container:inlined_vector", + "//third_party/abseil-cpp/absl/strings:strings", + ] +} diff --git a/common_video/generic_frame_descriptor/OWNERS b/common_video/generic_frame_descriptor/OWNERS new file mode 100644 index 0000000000..2f874a3a75 --- /dev/null +++ b/common_video/generic_frame_descriptor/OWNERS @@ -0,0 +1,7 @@ +philipel@webrtc.org +danilchap@webrtc.org + +# These are for the common case of adding or renaming files. If you're doing +# structural changes, please get a review from a reviewer in this file. +per-file *.gn=* +per-file *.gni=* \ No newline at end of file diff --git a/common_video/generic_frame_descriptor/generic_frame_info.cc b/common_video/generic_frame_descriptor/generic_frame_info.cc new file mode 100644 index 0000000000..4294e9521a --- /dev/null +++ b/common_video/generic_frame_descriptor/generic_frame_info.cc @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "common_video/generic_frame_descriptor/generic_frame_info.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +GenericFrameInfo::GenericFrameInfo() = default; +GenericFrameInfo::GenericFrameInfo(const GenericFrameInfo&) = default; +GenericFrameInfo::~GenericFrameInfo() = default; + +GenericFrameInfo::Builder::Builder() = default; +GenericFrameInfo::Builder::~Builder() = default; + +GenericFrameInfo GenericFrameInfo::Builder::Build() const { + return info_; +} + +GenericFrameInfo::Builder& GenericFrameInfo::Builder::T(int temporal_id) { + info_.temporal_id = temporal_id; + return *this; +} + +GenericFrameInfo::Builder& GenericFrameInfo::Builder::S(int spatial_id) { + info_.spatial_id = spatial_id; + return *this; +} + +GenericFrameInfo::Builder& GenericFrameInfo::Builder::Dtis( + absl::string_view indication_symbols) { + for (const auto& symbol : indication_symbols) { + OperatingPointIndication indication; + switch (symbol) { + case '-': indication = OperatingPointIndication::kNotPresent; break; + case 'D': indication = OperatingPointIndication::kDiscardable; break; + case 'R': indication = OperatingPointIndication::kRequired; break; + case 'S': indication = OperatingPointIndication::kSwitch; break; + default: RTC_NOTREACHED(); + } + + info_.operating_points.push_back(indication); + } + + return *this; +} + +GenericFrameInfo::Builder& GenericFrameInfo::Builder::Fdiffs( + std::initializer_list frame_diffs) { + info_.frame_diffs.insert(info_.frame_diffs.end(), frame_diffs.begin(), + frame_diffs.end()); + return *this; +} + +TemplateStructure::TemplateStructure() = default; +TemplateStructure::TemplateStructure(const TemplateStructure&) = default; +TemplateStructure::TemplateStructure(TemplateStructure&&) = default; +TemplateStructure& TemplateStructure::operator=(const TemplateStructure&) = + default; +TemplateStructure::~TemplateStructure() = default; +} // namespace webrtc diff --git a/common_video/generic_frame_descriptor/generic_frame_info.h b/common_video/generic_frame_descriptor/generic_frame_info.h new file mode 100644 index 0000000000..b800adc9de --- /dev/null +++ b/common_video/generic_frame_descriptor/generic_frame_info.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_ +#define COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_ + +#include +#include + +#include "absl/container/inlined_vector.h" +#include "absl/strings/string_view.h" +#include "api/array_view.h" + +namespace webrtc { + +struct GenericFrameInfo { + enum class OperatingPointIndication { + kNotPresent, // GenericFrameInfo::Builder symbol '-' + kDiscardable, // GenericFrameInfo::Builder symbol 'D' + kSwitch, // GenericFrameInfo::Builder symbol 'S' + kRequired // GenericFrameInfo::Builder symbol 'R' + }; + + class Builder; + + GenericFrameInfo(); + GenericFrameInfo(const GenericFrameInfo&); + ~GenericFrameInfo(); + + int temporal_id = 0; + int spatial_id = 0; + absl::InlinedVector frame_diffs; + absl::InlinedVector operating_points; +}; + +class GenericFrameInfo::Builder { + public: + Builder(); + ~Builder(); + + GenericFrameInfo Build() const; + Builder& T(int temporal_id); + Builder& S(int spatial_id); + Builder& Dtis(absl::string_view indication_symbols); + Builder& Fdiffs(std::initializer_list frame_diffs); + + private: + GenericFrameInfo info_; +}; + +struct TemplateStructure { + TemplateStructure(); + TemplateStructure(const TemplateStructure&); + TemplateStructure(TemplateStructure&&); + TemplateStructure& operator=(const TemplateStructure&); + ~TemplateStructure(); + + int num_operating_points = 0; + std::vector templates; +}; +} // namespace webrtc + +#endif // COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_ diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index ffd58a24eb..fb67178dc8 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -185,6 +185,7 @@ rtc_static_library("video_coding") { rtc_source_set("video_codec_interface") { visibility = [ "*" ] sources = [ + "include/video_codec_interface.cc", "include/video_codec_interface.h", "include/video_coding_defines.h", "include/video_error_codes.h", @@ -196,6 +197,7 @@ rtc_source_set("video_codec_interface") { "../../api/video:video_frame", "../../api/video_codecs:video_codecs_api", "../../common_video:common_video", + "../../common_video/generic_frame_descriptor:generic_frame_descriptor", "//third_party/abseil-cpp/absl/types:optional", ] } diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 47b1bef33d..61dca0909f 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -444,7 +444,7 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, - CodecSpecificInfoVP8* vp8_info) { + CodecSpecificInfo* info) { RTC_DCHECK_GT(num_layers_, 0); auto pending_frame = pending_frames_.find(rtp_timestamp); @@ -463,15 +463,16 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp, } #endif + CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8; if (num_layers_ == 1) { - vp8_info->temporalIdx = kNoTemporalIdx; - vp8_info->layerSync = false; + vp8_info.temporalIdx = kNoTemporalIdx; + vp8_info.layerSync = false; } else { if (is_keyframe) { // Restart the temporal pattern on keyframes. pattern_idx_ = 0; - vp8_info->temporalIdx = 0; - vp8_info->layerSync = true; // Keyframes are always sync frames. + vp8_info.temporalIdx = 0; + vp8_info.layerSync = true; // Keyframes are always sync frames. for (Vp8BufferReference buffer : kAllBuffers) { if (kf_buffers_.find(buffer) != kf_buffers_.end()) { @@ -486,29 +487,35 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp, } } else { // Delta frame, update codec specifics with temporal id and sync flag. - vp8_info->temporalIdx = frame.frame_config.packetizer_temporal_idx; - vp8_info->layerSync = frame.frame_config.layer_sync; + vp8_info.temporalIdx = frame.frame_config.packetizer_temporal_idx; + vp8_info.layerSync = frame.frame_config.layer_sync; } } - vp8_info->useExplicitDependencies = true; - RTC_DCHECK_EQ(vp8_info->referencedBuffersCount, 0u); - RTC_DCHECK_EQ(vp8_info->updatedBuffersCount, 0u); + vp8_info.useExplicitDependencies = true; + RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u); + RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u); for (int i = 0; i < static_cast(Buffer::kCount); ++i) { if (!is_keyframe && frame.frame_config.References(static_cast(i))) { - RTC_DCHECK_LT(vp8_info->referencedBuffersCount, + RTC_DCHECK_LT(vp8_info.referencedBuffersCount, arraysize(CodecSpecificInfoVP8::referencedBuffers)); - vp8_info->referencedBuffers[vp8_info->referencedBuffersCount++] = i; + vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; } if (is_keyframe || frame.frame_config.Updates(static_cast(i))) { - RTC_DCHECK_LT(vp8_info->updatedBuffersCount, + RTC_DCHECK_LT(vp8_info.updatedBuffersCount, arraysize(CodecSpecificInfoVP8::updatedBuffers)); - vp8_info->updatedBuffers[vp8_info->updatedBuffersCount++] = i; + vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; } } + // The templates are always present on keyframes, and then refered to by + // subsequent frames. + if (is_keyframe) { + info->template_structure = GetTemplateStructure(num_layers_); + } + if (!frame.expired) { for (Vp8BufferReference buffer : kAllBuffers) { if (frame.updated_buffer_mask & static_cast(buffer)) { @@ -518,6 +525,65 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp, } } +TemplateStructure DefaultTemporalLayers::GetTemplateStructure( + int num_layers) const { + RTC_CHECK_LT(num_layers, 5); + RTC_CHECK_GT(num_layers, 0); + + TemplateStructure template_structure; + template_structure.num_operating_points = num_layers; + + using Builder = GenericFrameInfo::Builder; + switch (num_layers) { + case 1: { + template_structure.templates = { + Builder().T(0).Dtis("S").Build(), + Builder().T(0).Dtis("S").Fdiffs({1}).Build(), + }; + return template_structure; + } + case 2: { + template_structure.templates = { + Builder().T(0).Dtis("SS").Build(), + Builder().T(0).Dtis("SS").Fdiffs({2}).Build(), + Builder().T(0).Dtis("SR").Fdiffs({2}).Build(), + Builder().T(1).Dtis("-S").Fdiffs({1}).Build(), + Builder().T(1).Dtis("-D").Fdiffs({1, 2}).Build(), + }; + return template_structure; + } + case 3: { + template_structure.templates = { + Builder().T(0).Dtis("SSS").Build(), + Builder().T(0).Dtis("SSS").Fdiffs({4}).Build(), + Builder().T(0).Dtis("SRR").Fdiffs({4}).Build(), + Builder().T(1).Dtis("-SR").Fdiffs({2}).Build(), + Builder().T(1).Dtis("-DR").Fdiffs({2, 4}).Build(), + Builder().T(2).Dtis("--D").Fdiffs({1}).Build(), + Builder().T(2).Dtis("--D").Fdiffs({1, 3}).Build(), + }; + return template_structure; + } + case 4: { + template_structure.templates = { + Builder().T(0).Dtis("SSSS").Build(), + Builder().T(0).Dtis("SSSS").Fdiffs({8}).Build(), + Builder().T(1).Dtis("-SRR").Fdiffs({4}).Build(), + Builder().T(1).Dtis("-SRR").Fdiffs({4, 8}).Build(), + Builder().T(2).Dtis("--SR").Fdiffs({2}).Build(), + Builder().T(2).Dtis("--SR").Fdiffs({2, 4}).Build(), + Builder().T(3).Dtis("---D").Fdiffs({1}).Build(), + Builder().T(3).Dtis("---D").Fdiffs({1, 3}).Build(), + }; + return template_structure; + } + default: + RTC_NOTREACHED(); + // To make the compiler happy! + return template_structure; + } +} + // Returns list of temporal dependencies for each frame in the temporal pattern. // Values are lists of indecies in the pattern. std::vector> GetTemporalDependencies( diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index d04e4f4862..8deeb40b77 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -49,7 +49,7 @@ class DefaultTemporalLayers : public Vp8TemporalLayers { size_t size_bytes, bool is_keyframe, int qp, - CodecSpecificInfoVP8* vp8_info) override; + CodecSpecificInfo* info) override; private: static constexpr size_t kKeyframeBuffer = std::numeric_limits::max(); @@ -64,6 +64,7 @@ class DefaultTemporalLayers : public Vp8TemporalLayers { const std::vector temporal_pattern_; // Set of buffers that are never updated except by keyframes. const std::set kf_buffers_; + TemplateStructure GetTemplateStructure(int num_layers) const; uint8_t pattern_idx_; // Updated cumulative bitrates, per temporal layer. diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index ae72ec9a40..37e662068e 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -107,9 +107,9 @@ class TemporalLayersTest : public ::testing::Test { public: ~TemporalLayersTest() override = default; - CodecSpecificInfoVP8* IgnoredCodecSpecificInfoVp8() { + CodecSpecificInfo* IgnoredCodecSpecificInfo() { codec_specific_info_ = absl::make_unique(); - return &codec_specific_info_->codecSpecific.VP8; + return codec_specific_info_.get(); } private: @@ -142,17 +142,17 @@ TEST_F(TemporalLayersTest, 2Layers) { for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) { const size_t ind = i % kPatternSize; CodecSpecificInfo info; - CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[ind], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, - &vp8_info); + &info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); - EXPECT_EQ(expected_temporal_idx[ind], vp8_info.temporalIdx); + EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx); EXPECT_EQ(expected_temporal_idx[ind], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[ind], tl_config.encoder_layer_id); - EXPECT_EQ(i == 0 || expected_layer_sync[ind], vp8_info.layerSync); + EXPECT_EQ(i == 0 || expected_layer_sync[ind], + info.codecSpecific.VP8.layerSync); EXPECT_EQ(expected_layer_sync[ind], tl_config.layer_sync); timestamp += 3000; } @@ -196,16 +196,16 @@ TEST_F(TemporalLayersTest, 3Layers) { unsigned int timestamp = 0; for (int i = 0; i < 16; ++i) { CodecSpecificInfo info; - CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, - &vp8_info); + &info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); - EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); + EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); - EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync); + EXPECT_EQ(i == 0 || expected_layer_sync[i], + info.codecSpecific.VP8.layerSync); EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); timestamp += 3000; } @@ -238,16 +238,16 @@ TEST_F(TemporalLayersTest, Alternative3Layers) { unsigned int timestamp = 0; for (int i = 0; i < 8; ++i) { CodecSpecificInfo info; - CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, - &vp8_info); + &info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); - EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); + EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); - EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync); + EXPECT_EQ(i == 0 || expected_layer_sync[i], + info.codecSpecific.VP8.layerSync); EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); timestamp += 3000; } @@ -271,19 +271,19 @@ TEST_F(TemporalLayersTest, SearchOrder) { uint32_t timestamp = 0; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame. First one only references TL0. Updates altref. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); // TL1 frame. Can only reference TL0. Updated golden. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); @@ -291,7 +291,7 @@ TEST_F(TemporalLayersTest, SearchOrder) { // updated, the next to last was altref. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref); } @@ -314,12 +314,12 @@ TEST_F(TemporalLayersTest, SearchOrderWithDrop) { uint32_t timestamp = 0; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame. First one only references TL0. Updates altref. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); @@ -331,7 +331,7 @@ TEST_F(TemporalLayersTest, SearchOrderWithDrop) { // been populated this cycle. Altref was last to be updated, before that last. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast); } @@ -373,16 +373,16 @@ TEST_F(TemporalLayersTest, 4Layers) { uint32_t timestamp = 0; for (int i = 0; i < 16; ++i) { CodecSpecificInfo info; - CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, - &vp8_info); + &info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); - EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx); + EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx); EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); - EXPECT_EQ(i == 0 || expected_layer_sync[i], vp8_info.layerSync); + EXPECT_EQ(i == 0 || expected_layer_sync[i], + info.codecSpecific.VP8.layerSync); EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); timestamp += 3000; } @@ -405,7 +405,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { uint32_t timestamp = 0; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Dropped TL2 frame. tl_config = tl.UpdateLayerConfig(++timestamp); @@ -419,7 +419,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // both contain the last keyframe. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference); @@ -429,23 +429,23 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // TL0 base layer frame, updating and referencing last. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame, updating altref. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL1 frame, updating golden. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame. Can still reference all buffer since they have been update this // cycle. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference); @@ -455,7 +455,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // TL0 base layer frame, updating and referencing last. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Dropped TL2 frame. tl_config = tl.UpdateLayerConfig(++timestamp); @@ -469,7 +469,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // and cannot be referenced. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference); @@ -491,24 +491,24 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) { uint32_t timestamp = 0; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Do a full cycle of the pattern. for (int i = 0; i < 7; ++i) { tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); } // TL0 base layer frame, starting the cycle over. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Encoder has a hiccup and builds a queue, so frame encoding is delayed. // TL1 frame, updating golden. @@ -528,13 +528,13 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) { // buffers are now OK to reference. // Enqueued TL1 frame ready. tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Enqueued TL2 frame. tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Enqueued TL0 frame. tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame, all buffers are now in a known good state, OK to reference. tl_config = tl.UpdateLayerConfig(++timestamp + 1); @@ -560,24 +560,24 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) { uint32_t timestamp = 0; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Do a full cycle of the pattern. for (int i = 0; i < 3; ++i) { tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); } // TL0 base layer frame, starting the cycle over. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame. tl_config = tl.UpdateLayerConfig(++timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Encoder has a hiccup and builds a queue, so frame encoding is delayed. // Encoded, but delayed frames in TL 1, 2. @@ -592,10 +592,10 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) { // TL1 frame from last cycle is ready. tl.OnEncodeDone(timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame from last cycle is ready. tl.OnEncodeDone(timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // TL2 frame, that should be referencing all buffers, but altref and golden // haven not been updated this cycle. (Don't be fooled by the late frames from @@ -641,7 +641,7 @@ TEST_F(TemporalLayersTest, KeyFrame) { EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config)) << j; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config)); EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx); EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id); @@ -650,12 +650,11 @@ TEST_F(TemporalLayersTest, KeyFrame) { } CodecSpecificInfo info; - CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, - &vp8_info); - EXPECT_TRUE(vp8_info.layerSync) << "Key frame should be marked layer sync."; - EXPECT_EQ(0, vp8_info.temporalIdx) + tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, &info); + EXPECT_TRUE(info.codecSpecific.VP8.layerSync) + << "Key frame should be marked layer sync."; + EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx) << "Key frame should always be packetized as layer 0"; EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config)); } @@ -741,7 +740,7 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) { for (int i = 0; i < kMaxPatternLength; ++i) { Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_); tl.OnEncodeDone(timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); ++timestamp_; EXPECT_FALSE(tl_config.drop_frame); tl_configs.push_back(tl_config); diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 1dc0837e3d..e8a6d25c3b 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -850,15 +850,16 @@ void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, uint32_t timestamp) { assert(codec_specific != NULL); codec_specific->codecType = kVideoCodecVP8; - CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8); - vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this - vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0; + codec_specific->codecSpecific.VP8.keyIdx = + kNoKeyIdx; // TODO(hlundin) populate this + codec_specific->codecSpecific.VP8.nonReference = + (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0; int qp = 0; vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); temporal_layers_[stream_idx]->OnEncodeDone( timestamp, encoded_images_[encoder_idx].size(), - (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, vp8Info); + (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, codec_specific); } int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image) { diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index d3f4d426ca..17b1aed4d9 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -265,7 +265,7 @@ void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, - CodecSpecificInfoVP8* vp8_info) { + CodecSpecificInfo* info) { if (size_bytes == 0) { layers_[active_layer_].state = TemporalLayer::State::kDropped; ++stats_.num_overshoots_; @@ -283,44 +283,47 @@ void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp, } } + CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8; if (number_of_temporal_layers_ == 1) { - vp8_info->temporalIdx = kNoTemporalIdx; - vp8_info->layerSync = false; + vp8_info.temporalIdx = kNoTemporalIdx; + vp8_info.layerSync = false; } else { int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp); if (frame_config) { - vp8_info->temporalIdx = frame_config->packetizer_temporal_idx; - vp8_info->layerSync = frame_config->layer_sync; + vp8_info.temporalIdx = frame_config->packetizer_temporal_idx; + vp8_info.layerSync = frame_config->layer_sync; } else { RTC_DCHECK(is_keyframe); } if (is_keyframe) { - vp8_info->temporalIdx = 0; + vp8_info.temporalIdx = 0; last_sync_timestamp_ = unwrapped_timestamp; - vp8_info->layerSync = true; + vp8_info.layerSync = true; layers_[0].state = TemporalLayer::State::kKeyFrame; layers_[1].state = TemporalLayer::State::kKeyFrame; active_layer_ = 1; + info->template_structure = + GetTemplateStructure(number_of_temporal_layers_); } - vp8_info->useExplicitDependencies = true; - RTC_DCHECK_EQ(vp8_info->referencedBuffersCount, 0u); - RTC_DCHECK_EQ(vp8_info->updatedBuffersCount, 0u); + vp8_info.useExplicitDependencies = true; + RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u); + RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u); // Note that |frame_config| is not derefernced if |is_keyframe|, // meaning it's never dereferenced if the optional may be unset. for (int i = 0; i < static_cast(Buffer::kCount); ++i) { if (!is_keyframe && frame_config->References(static_cast(i))) { - RTC_DCHECK_LT(vp8_info->referencedBuffersCount, + RTC_DCHECK_LT(vp8_info.referencedBuffersCount, arraysize(CodecSpecificInfoVP8::referencedBuffers)); - vp8_info->referencedBuffers[vp8_info->referencedBuffersCount++] = i; + vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; } if (is_keyframe || frame_config->Updates(static_cast(i))) { - RTC_DCHECK_LT(vp8_info->updatedBuffersCount, + RTC_DCHECK_LT(vp8_info.updatedBuffersCount, arraysize(CodecSpecificInfoVP8::updatedBuffers)); - vp8_info->updatedBuffers[vp8_info->updatedBuffersCount++] = i; + vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; } } } @@ -352,6 +355,38 @@ void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp, } } +TemplateStructure ScreenshareLayers::GetTemplateStructure( + int num_layers) const { + RTC_CHECK_LT(num_layers, 3); + RTC_CHECK_GT(num_layers, 0); + + TemplateStructure template_structure; + template_structure.num_operating_points = num_layers; + + using Builder = GenericFrameInfo::Builder; + switch (num_layers) { + case 1: { + template_structure.templates = { + Builder().T(0).Dtis("S").Build(), + Builder().T(0).Dtis("S").Fdiffs({1}).Build(), + }; + return template_structure; + } + case 2: { + template_structure.templates = { + Builder().T(0).Dtis("SS").Build(), + Builder().T(0).Dtis("SS").Fdiffs({1}).Build(), + Builder().T(1).Dtis("-S").Fdiffs({1}).Build(), + }; + return template_structure; + } + default: + RTC_NOTREACHED(); + // To make the compiler happy! + return template_structure; + } +} + bool ScreenshareLayers::TimeToSync(int64_t timestamp) const { RTC_DCHECK_EQ(1, active_layer_); RTC_DCHECK_NE(-1, layers_[0].last_qp); diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index b1dbde892e..1c96796cc7 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -16,6 +16,7 @@ #include "api/video_codecs/vp8_frame_config.h" #include "api/video_codecs/vp8_temporal_layers.h" #include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h" +#include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/utility/frame_dropper.h" #include "rtc_base/rate_statistics.h" #include "rtc_base/time_utils.h" @@ -52,7 +53,7 @@ class ScreenshareLayers : public Vp8TemporalLayers { size_t size_bytes, bool is_keyframe, int qp, - CodecSpecificInfoVP8* vp8_info) override; + CodecSpecificInfo* info) override; private: enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync }; @@ -77,6 +78,7 @@ class ScreenshareLayers : public Vp8TemporalLayers { absl::optional target_framerate_; // Incoming framerate from capturer. absl::optional capture_framerate_; + // Tracks what framerate we actually encode, and drops frames on overshoot. RateStatistics encode_framerate_; bool bitrate_updated_; @@ -107,6 +109,8 @@ class ScreenshareLayers : public Vp8TemporalLayers { } layers_[kMaxNumTemporalLayers]; void UpdateHistograms(); + TemplateStructure GetTemplateStructure(int num_layers) const; + // Data for histogram statistics. struct Stats { int64_t first_frame_time_ms_ = -1; diff --git a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index 5f4f9af350..16e4d468c7 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -80,8 +80,7 @@ class ScreenshareLayerTest : public ::testing::Test { int flags = ConfigureFrame(base_sync); if (flags != -1) layers_->OnEncodeDone(timestamp_, frame_size_, base_sync, kDefaultQp, - &info->codecSpecific.VP8); - + info); return flags; } @@ -133,10 +132,9 @@ class ScreenshareLayerTest : public ::testing::Test { bool got_tl1 = false; for (int i = 0; i < 10; ++i) { CodecSpecificInfo info; - const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; EXPECT_NE(-1, EncodeFrame(false, &info)); timestamp_ += kTimestampDelta5Fps; - if (vp8_info.temporalIdx == 0) { + if (info.codecSpecific.VP8.temporalIdx == 0) { got_tl0 = true; } else { got_tl1 = true; @@ -164,9 +162,8 @@ class ScreenshareLayerTest : public ::testing::Test { if (tl_config_.packetizer_temporal_idx != layer || (sync && *sync != tl_config_.layer_sync)) { CodecSpecificInfo info; - CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8; layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, - vp8_info); + &info); timestamp_ += kTimestampDelta5Fps; } else { // Found frame from sought after layer. @@ -188,9 +185,9 @@ class ScreenshareLayerTest : public ::testing::Test { Vp8EncoderConfig cfg_; bool config_updated_; - CodecSpecificInfoVP8* IgnoredCodecSpecificInfoVp8() { + CodecSpecificInfo* IgnoredCodecSpecificInfo() { ignored_codec_specific_info_ = absl::make_unique(); - return &ignored_codec_specific_info_->codecSpecific.VP8; + return ignored_codec_specific_info_.get(); } private: @@ -223,10 +220,10 @@ TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) { const int kNumFrames = kSyncPeriodSeconds * kFrameRate * 2 - 1; for (int i = 0; i < kNumFrames; ++i) { CodecSpecificInfo info; - const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; EncodeFrame(false, &info); timestamp_ += kTimestampDelta5Fps; - if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) { + if (info.codecSpecific.VP8.temporalIdx == 1 && + info.codecSpecific.VP8.layerSync) { sync_times.push_back(timestamp_); } } @@ -240,21 +237,20 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1; for (int i = 0; i < kNumFrames; ++i) { CodecSpecificInfo info; - CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8; tl_config_ = UpdateLayerConfig(timestamp_); config_updated_ = layers_->UpdateConfiguration(&cfg_); // Simulate TL1 being at least 8 qp steps better. if (tl_config_.packetizer_temporal_idx == 0) { - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, - vp8_info); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &info); } else { layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, - vp8_info); + &info); } - if (vp8_info->temporalIdx == 1 && vp8_info->layerSync) + if (info.codecSpecific.VP8.temporalIdx == 1 && + info.codecSpecific.VP8.layerSync) sync_times.push_back(timestamp_); timestamp_ += kTimestampDelta5Fps; @@ -272,20 +268,19 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { kFrameRate; for (int i = 0; i < kNumFrames; ++i) { CodecSpecificInfo info; - CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8; ConfigureFrame(false); // Simulate TL1 being at least 8 qp steps better. if (tl_config_.packetizer_temporal_idx == 0) { - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, - vp8_info); + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &info); } else { layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, - vp8_info); + &info); } - if (vp8_info->temporalIdx == 1 && vp8_info->layerSync) + if (info.codecSpecific.VP8.temporalIdx == 1 && + info.codecSpecific.VP8.layerSync) sync_times.push_back(timestamp_); timestamp_ += kTimestampDelta5Fps; @@ -296,17 +291,16 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { bool bumped_tl0_quality = false; for (int i = 0; i < 3; ++i) { CodecSpecificInfo info; - CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8; int flags = ConfigureFrame(false); layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, - vp8_info); - if (vp8_info->temporalIdx == 0) { + &info); + if (info.codecSpecific.VP8.temporalIdx == 0) { // Bump TL0 to same quality as TL1. bumped_tl0_quality = true; } else { if (bumped_tl0_quality) { - EXPECT_TRUE(vp8_info->layerSync); + EXPECT_TRUE(info.codecSpecific.VP8.layerSync); EXPECT_EQ(kTl1SyncFlags, flags); return; } @@ -324,10 +318,9 @@ TEST_F(ScreenshareLayerTest, 2LayersToggling) { int tl1_frames = 0; for (int i = 0; i < 50; ++i) { CodecSpecificInfo info; - const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; EncodeFrame(false, &info); timestamp_ += kTimestampDelta5Fps; - switch (vp8_info.temporalIdx) { + switch (info.codecSpecific.VP8.temporalIdx) { case 0: ++tl0_frames; break; @@ -348,11 +341,10 @@ TEST_F(ScreenshareLayerTest, AllFitsLayer0) { // Insert 50 frames, small enough that all fits in TL0. for (int i = 0; i < 50; ++i) { CodecSpecificInfo info; - const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; int flags = EncodeFrame(false, &info); timestamp_ += kTimestampDelta5Fps; EXPECT_EQ(kTl0Flags, flags); - EXPECT_EQ(0, vp8_info.temporalIdx); + EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx); } } @@ -365,13 +357,12 @@ TEST_F(ScreenshareLayerTest, TooHighBitrate) { int dropped_frames = 0; for (int i = 0; i < 100; ++i) { CodecSpecificInfo info; - const CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; int flags = EncodeFrame(false, &info); timestamp_ += kTimestampDelta5Fps; if (flags == -1) { ++dropped_frames; } else { - switch (vp8_info.temporalIdx) { + switch (info.codecSpecific.VP8.temporalIdx) { case 0: ++tl0_frames; break; @@ -428,7 +419,7 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { SkipUntilTl(0); // Size 0 indicates dropped frame. - layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfoVp8()); + layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfo()); // Re-encode frame (so don't advance timestamp). int flags = EncodeFrame(false); @@ -441,19 +432,19 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { EXPECT_TRUE(config_updated_); EXPECT_LT(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); timestamp_ += kTimestampDelta5Fps; // ...then back to standard setup. SkipUntilTl(0); layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); timestamp_ += kTimestampDelta5Fps; EXPECT_EQ(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); // Next drop in TL1. SkipUntilTl(1); - layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfoVp8()); + layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfo()); // Re-encode frame (so don't advance timestamp). flags = EncodeFrame(false); @@ -466,14 +457,14 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { EXPECT_TRUE(config_updated_); EXPECT_LT(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); timestamp_ += kTimestampDelta5Fps; // ...and back to normal. SkipUntilTl(1); EXPECT_EQ(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); timestamp_ += kTimestampDelta5Fps; } @@ -489,7 +480,7 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { EXPECT_EQ(kTl0Flags, LibvpxVp8Encoder::EncodeFlags(UpdateLayerConfig(kStartTimestamp))); layers_->OnEncodeDone(kStartTimestamp, kLargeFrameSizeBytes, false, - kDefaultQp, IgnoredCodecSpecificInfoVp8()); + kDefaultQp, IgnoredCodecSpecificInfo()); const uint32_t kTwoSecondsLater = kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90); @@ -539,15 +530,15 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) { if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) { // Simulate a too large frame, to cause frame drop. layers_->OnEncodeDone(timestamp, frame_size_ * 10, false, kTl0Qp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); trigger_drop = true; } else { layers_->OnEncodeDone(timestamp, frame_size_, false, kTl0Qp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); } } else if (flags == kTl1Flags || flags == kTl1SyncFlags) { layers_->OnEncodeDone(timestamp, frame_size_, false, kTl1Qp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); } else if (flags == -1) { dropped_frame = true; } else { @@ -614,7 +605,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); } timestamp += kFrameIntervalsMs * 90; clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalsMs)); @@ -632,7 +623,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); } timestamp += kFrameIntervalsMs * 90 / 2; clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalsMs)); @@ -657,10 +648,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) { config_updated_ = layers_->UpdateConfiguration(&cfg_); EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_)); - CodecSpecificInfo info; - CodecSpecificInfoVP8* vp8_info = &info.codecSpecific.VP8; - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, vp8_info); - EXPECT_TRUE(vp8_info->layerSync); + CodecSpecificInfo new_info; + layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &new_info); + EXPECT_TRUE(new_info.codecSpecific.VP8.layerSync); } TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) { @@ -671,7 +661,7 @@ TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) { timestamp_ += kTimestampDelta5Fps * 3; EXPECT_FALSE(UpdateLayerConfig(timestamp_).drop_frame); layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Frame interval below 90% if desired time is not allowed, try inserting // frame just before this limit. @@ -735,7 +725,7 @@ TEST_F(ScreenshareLayerTest, DISABLED_MaxQpRestoredAfterDoubleDrop) { // Simulate re-encoded frame. layers_->OnEncodeDone(timestamp_, 1, false, max_qp_, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // Next frame, expect boosted quality. // Slightly alter bitrate between each frame. @@ -752,7 +742,7 @@ TEST_F(ScreenshareLayerTest, DISABLED_MaxQpRestoredAfterDoubleDrop) { // Simulate re-encoded frame. layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // A third frame, expect boosted quality. layers_->OnRatesUpdated(kDefault2TlBitratesBps, kFrameRate); @@ -763,7 +753,7 @@ TEST_F(ScreenshareLayerTest, DISABLED_MaxQpRestoredAfterDoubleDrop) { // Frame encoded. layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_, - IgnoredCodecSpecificInfoVp8()); + IgnoredCodecSpecificInfo()); // A fourth frame, max qp should be restored. layers_->OnRatesUpdated(kDefault2TlBitratesBpsAlt, kFrameRate); diff --git a/modules/video_coding/include/video_codec_interface.cc b/modules/video_coding/include/video_codec_interface.cc new file mode 100644 index 0000000000..bd033b6c57 --- /dev/null +++ b/modules/video_coding/include/video_codec_interface.cc @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { +CodecSpecificInfo::CodecSpecificInfo() : codecType(kVideoCodecGeneric) { + memset(&codecSpecific, 0, sizeof(codecSpecific)); +} + +CodecSpecificInfo::CodecSpecificInfo(const CodecSpecificInfo&) = default; +CodecSpecificInfo::~CodecSpecificInfo() = default; + +} // namespace webrtc diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h index cde551212f..fb440be664 100644 --- a/modules/video_coding/include/video_codec_interface.h +++ b/modules/video_coding/include/video_codec_interface.h @@ -13,10 +13,12 @@ #include +#include "absl/types/optional.h" #include "api/video/video_frame.h" #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_encoder.h" #include "common_types.h" // NOLINT(build/include) +#include "common_video/generic_frame_descriptor/generic_frame_info.h" #include "modules/include/module_common_types.h" #include "modules/video_coding/include/video_error_codes.h" @@ -96,16 +98,18 @@ union CodecSpecificInfoUnion { }; static_assert(std::is_pod::value, ""); -// Note: If any pointers are added to this struct or its sub-structs, it +// Note: if any pointers are added to this struct or its sub-structs, it // must be fitted with a copy-constructor. This is because it is copied // in the copy-constructor of VCMEncodedFrame. struct CodecSpecificInfo { - CodecSpecificInfo() : codecType(kVideoCodecGeneric) { - memset(&codecSpecific, 0, sizeof(codecSpecific)); - } + CodecSpecificInfo(); + CodecSpecificInfo(const CodecSpecificInfo&); + ~CodecSpecificInfo(); VideoCodecType codecType; CodecSpecificInfoUnion codecSpecific; + absl::optional generic_frame_info; + absl::optional template_structure; }; } // namespace webrtc diff --git a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index 690c3faaab..4744e38d25 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -41,7 +41,7 @@ class MockTemporalLayers : public Vp8TemporalLayers { MOCK_METHOD2(OnRatesUpdated, void(const std::vector&, int)); MOCK_METHOD1(UpdateConfiguration, bool(Vp8EncoderConfig*)); MOCK_METHOD5(OnEncodeDone, - void(uint32_t, size_t, bool, int, CodecSpecificInfoVP8*)); + void(uint32_t, size_t, bool, int, CodecSpecificInfo*)); MOCK_METHOD3(FrameEncoded, void(uint32_t, size_t, int)); MOCK_CONST_METHOD0(Tl0PicIdx, uint8_t()); MOCK_CONST_METHOD1(GetTemporalLayerId, int(const Vp8FrameConfig&)); diff --git a/test/fake_vp8_encoder.cc b/test/fake_vp8_encoder.cc index c368eb50f0..cb60091d1d 100644 --- a/test/fake_vp8_encoder.cc +++ b/test/fake_vp8_encoder.cc @@ -106,11 +106,10 @@ void FakeVP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, uint32_t timestamp) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); codec_specific->codecType = kVideoCodecVP8; - CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8); - vp8Info->keyIdx = kNoKeyIdx; - vp8Info->nonReference = false; + codec_specific->codecSpecific.VP8.keyIdx = kNoKeyIdx; + codec_specific->codecSpecific.VP8.nonReference = false; temporal_layers_[stream_idx]->OnEncodeDone( - timestamp, size_bytes, frame_type == kVideoFrameKey, -1, vp8Info); + timestamp, size_bytes, frame_type == kVideoFrameKey, -1, codec_specific); } EncodedImageCallback::Result FakeVP8Encoder::OnEncodedImage(