diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index 71cd615591..b45470ced3 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -28,12 +28,15 @@ rtc_source_set("video_codecs_api") { "video_encoder_config.cc", "video_encoder_config.h", "video_encoder_factory.h", + "vp8_frame_config.cc", + "vp8_frame_config.h", "vp8_temporal_layers.h", ] deps = [ "..:scoped_refptr", "../..:webrtc_common", + "../../modules/video_coding:codec_globals_headers", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", "../../rtc_base/system:rtc_export", diff --git a/api/video_codecs/vp8_frame_config.cc b/api/video_codecs/vp8_frame_config.cc new file mode 100644 index 0000000000..40e2c57c2f --- /dev/null +++ b/api/video_codecs/vp8_frame_config.cc @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/vp8_frame_config.h" + +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +Vp8FrameConfig::Vp8FrameConfig() : Vp8FrameConfig(kNone, kNone, kNone, false) {} + +Vp8FrameConfig::Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf) + : Vp8FrameConfig(last, golden, arf, false) {} + +Vp8FrameConfig::Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + FreezeEntropy) + : Vp8FrameConfig(last, golden, arf, true) {} + +Vp8FrameConfig::Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + bool freeze_entropy) + : drop_frame(last == BufferFlags::kNone && golden == BufferFlags::kNone && + arf == BufferFlags::kNone), + last_buffer_flags(last), + golden_buffer_flags(golden), + arf_buffer_flags(arf), + encoder_layer_id(0), + packetizer_temporal_idx(kNoTemporalIdx), + layer_sync(false), + freeze_entropy(freeze_entropy), + first_reference(Vp8BufferReference::kNone), + second_reference(Vp8BufferReference::kNone) {} + +bool Vp8FrameConfig::References(Buffer buffer) const { + switch (buffer) { + case Buffer::kLast: + return (last_buffer_flags & kReference) != 0; + case Buffer::kGolden: + return (golden_buffer_flags & kReference) != 0; + case Buffer::kArf: + return (arf_buffer_flags & kReference) != 0; + case Buffer::kCount: + break; + } + RTC_NOTREACHED(); + return false; +} + +bool Vp8FrameConfig::Updates(Buffer buffer) const { + switch (buffer) { + case Buffer::kLast: + return (last_buffer_flags & kUpdate) != 0; + case Buffer::kGolden: + return (golden_buffer_flags & kUpdate) != 0; + case Buffer::kArf: + return (arf_buffer_flags & kUpdate) != 0; + case Buffer::kCount: + break; + } + RTC_NOTREACHED(); + return false; +} + +} // namespace webrtc diff --git a/api/video_codecs/vp8_frame_config.h b/api/video_codecs/vp8_frame_config.h new file mode 100644 index 0000000000..e610a87672 --- /dev/null +++ b/api/video_codecs/vp8_frame_config.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VP8_FRAME_CONFIG_H_ +#define API_VIDEO_CODECS_VP8_FRAME_CONFIG_H_ + +#include + +namespace webrtc { + +struct Vp8FrameConfig { + enum BufferFlags : int { + kNone = 0, + kReference = 1, + kUpdate = 2, + kReferenceAndUpdate = kReference | kUpdate, + }; + + enum FreezeEntropy { kFreezeEntropy }; + + // Defined bit-maskable reference to the three buffers available in VP8. + enum class Vp8BufferReference : uint8_t { + kNone = 0, + kLast = 1, + kGolden = 2, + kAltref = 4 + }; + + Vp8FrameConfig(); + + Vp8FrameConfig(BufferFlags last, BufferFlags golden, BufferFlags arf); + Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + FreezeEntropy); + + enum class Buffer : int { kLast = 0, kGolden = 1, kArf = 2, kCount }; + + bool References(Buffer buffer) const; + + bool Updates(Buffer buffer) const; + + bool drop_frame; + BufferFlags last_buffer_flags; + BufferFlags golden_buffer_flags; + BufferFlags arf_buffer_flags; + + // The encoder layer ID is used to utilize the correct bitrate allocator + // inside the encoder. It does not control references nor determine which + // "actual" temporal layer this is. The packetizer temporal index determines + // which layer the encoded frame should be packetized into. + // Normally these are the same, but current temporal-layer strategies for + // screenshare use one bitrate allocator for all layers, but attempt to + // packetize / utilize references to split a stream into multiple layers, + // with different quantizer settings, to hit target bitrate. + // TODO(sprang): Screenshare layers are being reconsidered at the time of + // writing, we might be able to remove this distinction, and have a temporal + // layer imply both (the normal case). + int encoder_layer_id; + // TODO(eladalon/sprang): Move out of this class. + int packetizer_temporal_idx; + + // TODO(eladalon/sprang): Move out of this class. + bool layer_sync; + + bool freeze_entropy; + + // Indicates in which order the encoder should search the reference buffers + // when doing motion prediction. Set to kNone to use unspecified order. Any + // buffer indicated here must not have the corresponding no_ref bit set. + // If all three buffers can be reference, the one not listed here should be + // searched last. + Vp8BufferReference first_reference; + Vp8BufferReference second_reference; + + private: + Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + bool freeze_entropy); +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VP8_FRAME_CONFIG_H_ diff --git a/api/video_codecs/vp8_temporal_layers.h b/api/video_codecs/vp8_temporal_layers.h index ecc7e3df5e..67e9ebda89 100644 --- a/api/video_codecs/vp8_temporal_layers.h +++ b/api/video_codecs/vp8_temporal_layers.h @@ -14,7 +14,7 @@ #include #include -#include "rtc_base/checks.h" +#include "api/video_codecs/vp8_frame_config.h" namespace webrtc { @@ -72,111 +72,16 @@ struct Vp8EncoderConfig { uint32_t rc_max_quantizer; }; -// Defined bit-maskable reference to the three buffers available in VP8. -enum class Vp8BufferReference : uint8_t { - kNone = 0, - kLast = 1, - kGolden = 2, - kAltref = 4 -}; - // This interface defines a way of getting the encoder settings needed to // realize a temporal layer structure. class Vp8TemporalLayers { public: - enum BufferFlags : int { - kNone = 0, - kReference = 1, - kUpdate = 2, - kReferenceAndUpdate = kReference | kUpdate, - }; - enum FreezeEntropy { kFreezeEntropy }; - - struct FrameConfig { - FrameConfig(); - - FrameConfig(BufferFlags last, BufferFlags golden, BufferFlags arf); - FrameConfig(BufferFlags last, - BufferFlags golden, - BufferFlags arf, - FreezeEntropy); - - enum class Buffer : int { kLast = 0, kGolden = 1, kArf = 2, kCount }; - - bool References(Buffer buffer) const { - switch (buffer) { - case Buffer::kLast: - return (last_buffer_flags & kReference) != 0; - case Buffer::kGolden: - return (golden_buffer_flags & kReference) != 0; - case Buffer::kArf: - return (arf_buffer_flags & kReference) != 0; - case Buffer::kCount: - break; - } - RTC_NOTREACHED(); - return false; - } - - bool Updates(Buffer buffer) const { - switch (buffer) { - case Buffer::kLast: - return (last_buffer_flags & kUpdate) != 0; - case Buffer::kGolden: - return (golden_buffer_flags & kUpdate) != 0; - case Buffer::kArf: - return (arf_buffer_flags & kUpdate) != 0; - case Buffer::kCount: - break; - } - RTC_NOTREACHED(); - return false; - } - - bool drop_frame; - BufferFlags last_buffer_flags; - BufferFlags golden_buffer_flags; - BufferFlags arf_buffer_flags; - - // The encoder layer ID is used to utilize the correct bitrate allocator - // inside the encoder. It does not control references nor determine which - // "actual" temporal layer this is. The packetizer temporal index determines - // which layer the encoded frame should be packetized into. - // Normally these are the same, but current temporal-layer strategies for - // screenshare use one bitrate allocator for all layers, but attempt to - // packetize / utilize references to split a stream into multiple layers, - // with different quantizer settings, to hit target bitrate. - // TODO(pbos): Screenshare layers are being reconsidered at the time of - // writing, we might be able to remove this distinction, and have a temporal - // layer imply both (the normal case). - int encoder_layer_id; - int packetizer_temporal_idx; - - bool layer_sync; - - bool freeze_entropy; - - // Indicates in which order the encoder should search the reference buffers - // when doing motion prediction. Set to kNone to use unspecified order. Any - // buffer indicated here must not have the corresponding no_ref bit set. - // If all three buffers can be reference, the one not listed here should be - // searched last. - Vp8BufferReference first_reference; - Vp8BufferReference second_reference; - - private: - FrameConfig(BufferFlags last, - BufferFlags golden, - BufferFlags arf, - bool freeze_entropy); - }; - virtual ~Vp8TemporalLayers() = default; // If this method returns true, the encoder is free to drop frames for // instance in an effort to uphold encoding bitrate. // If this return false, the encoder must not drop any frames unless: - // 1. Requested to do so via FrameConfig.drop_frame + // 1. Requested to do so via Vp8FrameConfig.drop_frame // 2. The frame to be encoded is requested to be a keyframe // 3. The encoded detected a large overshoot and decided to drop and then // re-encode the image at a low bitrate. In this case the encoder should @@ -202,7 +107,7 @@ class Vp8TemporalLayers { // The timestamp uses a 90kHz RTP clock. // After calling this method, first call the actual encoder with the provided // frame configuration, and then OnEncodeDone() below. - virtual FrameConfig UpdateLayerConfig(uint32_t rtp_timestamp) = 0; + virtual Vp8FrameConfig UpdateLayerConfig(uint32_t rtp_timestamp) = 0; // Called after the encode step is done. |rtp_timestamp| must match the // parameter use in the UpdateLayerConfig() call. diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 1114234793..ee06bfa941 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -26,52 +26,27 @@ #include "system_wrappers/include/field_trial.h" namespace webrtc { -using Buffer = Vp8TemporalLayers::FrameConfig::Buffer; - -Vp8TemporalLayers::FrameConfig::FrameConfig() - : FrameConfig(kNone, kNone, kNone, false) {} - -Vp8TemporalLayers::FrameConfig::FrameConfig( - Vp8TemporalLayers::BufferFlags last, - Vp8TemporalLayers::BufferFlags golden, - Vp8TemporalLayers::BufferFlags arf) - : FrameConfig(last, golden, arf, false) {} - -Vp8TemporalLayers::FrameConfig::FrameConfig( - Vp8TemporalLayers::BufferFlags last, - Vp8TemporalLayers::BufferFlags golden, - Vp8TemporalLayers::BufferFlags arf, - FreezeEntropy) - : FrameConfig(last, golden, arf, true) {} - -Vp8TemporalLayers::FrameConfig::FrameConfig( - Vp8TemporalLayers::BufferFlags last, - Vp8TemporalLayers::BufferFlags golden, - Vp8TemporalLayers::BufferFlags arf, - bool freeze_entropy) - : drop_frame(last == Vp8TemporalLayers::kNone && - golden == Vp8TemporalLayers::kNone && - arf == Vp8TemporalLayers::kNone), - last_buffer_flags(last), - golden_buffer_flags(golden), - arf_buffer_flags(arf), - encoder_layer_id(0), - packetizer_temporal_idx(kNoTemporalIdx), - layer_sync(false), - freeze_entropy(freeze_entropy), - first_reference(Vp8BufferReference::kNone), - second_reference(Vp8BufferReference::kNone) {} - DefaultTemporalLayers::PendingFrame::PendingFrame() = default; DefaultTemporalLayers::PendingFrame::PendingFrame( bool expired, uint8_t updated_buffers_mask, - const FrameConfig& frame_config) + const Vp8FrameConfig& frame_config) : expired(expired), updated_buffer_mask(updated_buffers_mask), frame_config(frame_config) {} namespace { +using Buffer = Vp8FrameConfig::Buffer; +using BufferFlags = Vp8FrameConfig::BufferFlags; +using FreezeEntropy = Vp8FrameConfig::FreezeEntropy; +using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference; + +constexpr BufferFlags kNone = BufferFlags::kNone; +constexpr BufferFlags kReference = BufferFlags::kReference; +constexpr BufferFlags kUpdate = BufferFlags::kUpdate; +constexpr BufferFlags kReferenceAndUpdate = BufferFlags::kReferenceAndUpdate; +constexpr FreezeEntropy kFreezeEntropy = FreezeEntropy::kFreezeEntropy; + static constexpr uint8_t kUninitializedPatternIndex = std::numeric_limits::max(); static constexpr std::array kAllBuffers = { @@ -110,15 +85,15 @@ std::vector GetTemporalIds(size_t num_layers) { return {0}; } -uint8_t GetUpdatedBuffers(const Vp8TemporalLayers::FrameConfig& config) { +uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) { uint8_t flags = 0; - if (config.last_buffer_flags & Vp8TemporalLayers::BufferFlags::kUpdate) { + if (config.last_buffer_flags & BufferFlags::kUpdate) { flags |= static_cast(Vp8BufferReference::kLast); } - if (config.golden_buffer_flags & Vp8TemporalLayers::BufferFlags::kUpdate) { + if (config.golden_buffer_flags & BufferFlags::kUpdate) { flags |= static_cast(Vp8BufferReference::kGolden); } - if (config.arf_buffer_flags & Vp8TemporalLayers::BufferFlags::kUpdate) { + if (config.arf_buffer_flags & BufferFlags::kUpdate) { flags |= static_cast(Vp8BufferReference::kAltref); } return flags; @@ -126,10 +101,10 @@ uint8_t GetUpdatedBuffers(const Vp8TemporalLayers::FrameConfig& config) { // Find the set of buffers that are never updated by the given pattern. std::set FindKfBuffers( - const std::vector& frame_configs) { + const std::vector& frame_configs) { std::set kf_buffers(kAllBuffers.begin(), kAllBuffers.end()); - for (Vp8TemporalLayers::FrameConfig config : frame_configs) { + for (Vp8FrameConfig config : frame_configs) { // Get bit-masked set of update buffers for this frame config. uint8_t updated_buffers = GetUpdatedBuffers(config); for (Vp8BufferReference buffer : kAllBuffers) { @@ -142,8 +117,8 @@ std::set FindKfBuffers( } } // namespace -std::vector -DefaultTemporalLayers::GetTemporalPattern(size_t num_layers) { +std::vector DefaultTemporalLayers::GetTemporalPattern( + size_t num_layers) { // For indexing in the patterns described below (which temporal layers they // belong to), see the diagram above. // Layer sync is done similarly for all patterns (except single stream) and @@ -160,7 +135,7 @@ DefaultTemporalLayers::GetTemporalPattern(size_t num_layers) { switch (num_layers) { case 1: // All frames reference all buffers and the 'last' buffer is updated. - return {FrameConfig(kReferenceAndUpdate, kReference, kReference)}; + return {Vp8FrameConfig(kReferenceAndUpdate, kReference, kReference)}; case 2: // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. @@ -172,24 +147,24 @@ DefaultTemporalLayers::GetTemporalPattern(size_t num_layers) { // / / / / // 0---0---0---0 ... return { - FrameConfig(kReferenceAndUpdate, kNone, kReference), - FrameConfig(kReference, kUpdate, kReference), - FrameConfig(kReferenceAndUpdate, kNone, kReference), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; + Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), + Vp8FrameConfig(kReference, kUpdate, kReference), + Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; } else { // "Default" 8-frame pattern: // 1---1---1---1 1---1---1---1 ... // / / / / / / / / // 0---0---0---0---0---0---0---0 ... return { - FrameConfig(kReferenceAndUpdate, kNone, kReference), - FrameConfig(kReference, kUpdate, kReference), - FrameConfig(kReferenceAndUpdate, kNone, kReference), - FrameConfig(kReference, kReferenceAndUpdate, kReference), - FrameConfig(kReferenceAndUpdate, kNone, kReference), - FrameConfig(kReference, kReferenceAndUpdate, kReference), - FrameConfig(kReferenceAndUpdate, kNone, kReference), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; + Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), + Vp8FrameConfig(kReference, kUpdate, kReference), + Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), + Vp8FrameConfig(kReference, kReferenceAndUpdate, kReference), + Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), + Vp8FrameConfig(kReference, kReferenceAndUpdate, kReference), + Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; } case 3: if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { @@ -210,10 +185,10 @@ DefaultTemporalLayers::GetTemporalPattern(size_t num_layers) { // TL2 references both 'last' & 'golden' and references and updates // 'arf'. return { - FrameConfig(kReferenceAndUpdate, kNone, kNone), - FrameConfig(kReference, kNone, kUpdate), - FrameConfig(kReference, kUpdate, kNone), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; + Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), + Vp8FrameConfig(kReference, kNone, kUpdate), + Vp8FrameConfig(kReference, kUpdate, kNone), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; } else { // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. @@ -221,42 +196,43 @@ DefaultTemporalLayers::GetTemporalPattern(size_t num_layers) { // TL1 also references 'last' and references and updates 'golden'. // TL2 references both 'last' and 'golden' but updates no buffer. return { - FrameConfig(kReferenceAndUpdate, kNone, kReference), - FrameConfig(kReference, kNone, kReference, kFreezeEntropy), - FrameConfig(kReference, kUpdate, kReference), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - FrameConfig(kReferenceAndUpdate, kNone, kReference), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - FrameConfig(kReference, kReferenceAndUpdate, kReference), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; + Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), + Vp8FrameConfig(kReference, kNone, kReference, kFreezeEntropy), + Vp8FrameConfig(kReference, kUpdate, kReference), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), + Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), + Vp8FrameConfig(kReference, kReferenceAndUpdate, kReference), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; } case 4: // TL0 references and updates only the 'last' buffer. // TL1 references 'last' and updates and references 'golden'. // TL2 references 'last' and 'golden', and references and updates 'arf'. // TL3 references all buffers but update none of them. - return {FrameConfig(kReferenceAndUpdate, kNone, kNone), - FrameConfig(kReference, kNone, kNone, kFreezeEntropy), - FrameConfig(kReference, kNone, kUpdate), - FrameConfig(kReference, kNone, kReference, kFreezeEntropy), - FrameConfig(kReference, kUpdate, kNone), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - FrameConfig(kReference, kReference, kReferenceAndUpdate), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - FrameConfig(kReferenceAndUpdate, kNone, kNone), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - FrameConfig(kReference, kReference, kReferenceAndUpdate), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - FrameConfig(kReference, kReferenceAndUpdate, kNone), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - FrameConfig(kReference, kReference, kReferenceAndUpdate), - FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; + return { + Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), + Vp8FrameConfig(kReference, kNone, kNone, kFreezeEntropy), + Vp8FrameConfig(kReference, kNone, kUpdate), + Vp8FrameConfig(kReference, kNone, kReference, kFreezeEntropy), + Vp8FrameConfig(kReference, kUpdate, kNone), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), + Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), + Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), + Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), + Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), + Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), + Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; default: RTC_NOTREACHED(); break; } RTC_NOTREACHED(); - return {FrameConfig(kNone, kNone, kNone)}; + return {Vp8FrameConfig(kNone, kNone, kNone)}; } DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) @@ -326,7 +302,7 @@ bool DefaultTemporalLayers::UpdateConfiguration(Vp8EncoderConfig* cfg) { return true; } -bool DefaultTemporalLayers::IsSyncFrame(const FrameConfig& config) const { +bool DefaultTemporalLayers::IsSyncFrame(const Vp8FrameConfig& config) const { // Since we always assign TL0 to 'last' in these patterns, we can infer layer // sync by checking if temporal id > 0 and we only reference TL0 or buffers // containing the last key-frame. @@ -354,13 +330,12 @@ bool DefaultTemporalLayers::IsSyncFrame(const FrameConfig& config) const { return true; } -Vp8TemporalLayers::FrameConfig DefaultTemporalLayers::UpdateLayerConfig( - uint32_t timestamp) { +Vp8FrameConfig DefaultTemporalLayers::UpdateLayerConfig(uint32_t timestamp) { RTC_DCHECK_GT(num_layers_, 0); RTC_DCHECK_LT(0, temporal_pattern_.size()); pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size(); - Vp8TemporalLayers::FrameConfig tl_config = temporal_pattern_[pattern_idx_]; + Vp8FrameConfig tl_config = temporal_pattern_[pattern_idx_]; tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx = temporal_ids_[pattern_idx_ % temporal_ids_.size()]; @@ -428,7 +403,7 @@ void DefaultTemporalLayers::ValidateReferences(BufferFlags* flags, } } -void DefaultTemporalLayers::UpdateSearchOrder(FrameConfig* config) { +void DefaultTemporalLayers::UpdateSearchOrder(Vp8FrameConfig* config) { // Figure out which of the buffers we can reference, and order them so that // the most recently refreshed is first. Otherwise prioritize last first, // golden second, and altref third. @@ -592,7 +567,7 @@ DefaultTemporalLayersChecker::~DefaultTemporalLayersChecker() = default; bool DefaultTemporalLayersChecker::CheckTemporalConfig( bool frame_is_keyframe, - const Vp8TemporalLayers::FrameConfig& frame_config) { + const Vp8FrameConfig& frame_config) { if (!TemporalLayersChecker::CheckTemporalConfig(frame_is_keyframe, frame_config)) { return false; @@ -642,8 +617,7 @@ bool DefaultTemporalLayersChecker::CheckTemporalConfig( temporal_ids_[pattern_idx_] != kNoTemporalIdx; std::vector dependencies; - if (frame_config.last_buffer_flags & - Vp8TemporalLayers::BufferFlags::kReference) { + if (frame_config.last_buffer_flags & BufferFlags::kReference) { uint8_t referenced_layer = temporal_ids_[last_.pattern_idx]; if (referenced_layer > 0) { need_sync = false; @@ -658,8 +632,7 @@ bool DefaultTemporalLayersChecker::CheckTemporalConfig( return false; } - if (frame_config.arf_buffer_flags & - Vp8TemporalLayers::BufferFlags::kReference) { + if (frame_config.arf_buffer_flags & BufferFlags::kReference) { uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx]; if (referenced_layer > 0) { need_sync = false; @@ -674,8 +647,7 @@ bool DefaultTemporalLayersChecker::CheckTemporalConfig( return false; } - if (frame_config.golden_buffer_flags & - Vp8TemporalLayers::BufferFlags::kReference) { + if (frame_config.golden_buffer_flags & BufferFlags::kReference) { uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx]; if (referenced_layer > 0) { need_sync = false; @@ -711,19 +683,17 @@ bool DefaultTemporalLayersChecker::CheckTemporalConfig( } } - if (frame_config.last_buffer_flags & - Vp8TemporalLayers::BufferFlags::kUpdate) { + if (frame_config.last_buffer_flags & BufferFlags::kUpdate) { last_.is_updated_this_cycle = true; last_.pattern_idx = pattern_idx_; last_.is_keyframe = false; } - if (frame_config.arf_buffer_flags & Vp8TemporalLayers::BufferFlags::kUpdate) { + if (frame_config.arf_buffer_flags & BufferFlags::kUpdate) { arf_.is_updated_this_cycle = true; arf_.pattern_idx = pattern_idx_; arf_.is_keyframe = false; } - if (frame_config.golden_buffer_flags & - Vp8TemporalLayers::BufferFlags::kUpdate) { + if (frame_config.golden_buffer_flags & BufferFlags::kUpdate) { golden_.is_updated_this_cycle = true; golden_.pattern_idx = pattern_idx_; golden_.is_keyframe = false; diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index 16d84f22c0..d04e4f4862 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -21,6 +21,7 @@ #include #include "absl/types/optional.h" +#include "api/video_codecs/vp8_frame_config.h" #include "api/video_codecs/vp8_temporal_layers.h" #include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -36,7 +37,7 @@ class DefaultTemporalLayers : public Vp8TemporalLayers { // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - Vp8TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) override; + Vp8FrameConfig UpdateLayerConfig(uint32_t timestamp) override; // New target bitrate, per temporal layer. void OnRatesUpdated(const std::vector& bitrates_bps, @@ -52,17 +53,17 @@ class DefaultTemporalLayers : public Vp8TemporalLayers { private: static constexpr size_t kKeyframeBuffer = std::numeric_limits::max(); - static std::vector GetTemporalPattern( - size_t num_layers); - bool IsSyncFrame(const FrameConfig& config) const; - void ValidateReferences(BufferFlags* flags, Vp8BufferReference ref) const; - void UpdateSearchOrder(FrameConfig* config); + static std::vector GetTemporalPattern(size_t num_layers); + bool IsSyncFrame(const Vp8FrameConfig& config) const; + void ValidateReferences(Vp8FrameConfig::BufferFlags* flags, + Vp8FrameConfig::Vp8BufferReference ref) const; + void UpdateSearchOrder(Vp8FrameConfig* config); const size_t num_layers_; const std::vector temporal_ids_; - const std::vector temporal_pattern_; + const std::vector temporal_pattern_; // Set of buffers that are never updated except by keyframes. - const std::set kf_buffers_; + const std::set kf_buffers_; uint8_t pattern_idx_; // Updated cumulative bitrates, per temporal layer. @@ -72,7 +73,7 @@ class DefaultTemporalLayers : public Vp8TemporalLayers { PendingFrame(); PendingFrame(bool expired, uint8_t updated_buffers_mask, - const FrameConfig& frame_config); + const Vp8FrameConfig& frame_config); // Flag indicating if this frame has expired, ie it belongs to a previous // iteration of the temporal pattern. bool expired = false; @@ -80,7 +81,7 @@ class DefaultTemporalLayers : public Vp8TemporalLayers { // updates. uint8_t updated_buffer_mask = 0; // The frame config return by UpdateLayerConfig() for this frame. - FrameConfig frame_config; + Vp8FrameConfig frame_config; }; // Map from rtp timestamp to pending frame status. Reset on pattern loop. std::map pending_frames_; @@ -88,7 +89,8 @@ class DefaultTemporalLayers : public Vp8TemporalLayers { // One counter per Vp8BufferReference, indicating number of frames since last // refresh. For non-base-layer frames (ie golden, altref buffers), this is // reset when the pattern loops. - std::map frames_since_buffer_refresh_; + std::map + frames_since_buffer_refresh_; // Optional utility used to verify reference validity. std::unique_ptr checker_; @@ -99,9 +101,8 @@ class DefaultTemporalLayersChecker : public TemporalLayersChecker { explicit DefaultTemporalLayersChecker(int number_of_temporal_layers); ~DefaultTemporalLayersChecker() override; - bool CheckTemporalConfig( - bool frame_is_keyframe, - const Vp8TemporalLayers::FrameConfig& frame_config) override; + bool CheckTemporalConfig(bool frame_is_keyframe, + const Vp8FrameConfig& frame_config) override; private: struct BufferState { diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index c3291abebf..d2ddac3e40 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -15,6 +15,7 @@ #include "absl/memory/memory.h" #include "api/video/video_bitrate_allocation.h" #include "api/video_codecs/video_codec.h" +#include "api/video_codecs/vp8_frame_config.h" #include "common_types.h" // NOLINT(build/include) #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -82,7 +83,8 @@ constexpr int kDefaultBytesPerFrame = constexpr int kDefaultQp = 2; } // namespace -using BufferFlags = Vp8TemporalLayers::BufferFlags; +using BufferFlags = Vp8FrameConfig::BufferFlags; +using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference; class TemporalLayersTest : public ::testing::Test { public: @@ -136,7 +138,7 @@ TEST_F(TemporalLayersTest, 2Layers) { for (int i = 0; i < 16; ++i) { CodecSpecificInfo info; CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, &vp8_info); @@ -189,7 +191,7 @@ TEST_F(TemporalLayersTest, 3Layers) { for (int i = 0; i < 16; ++i) { CodecSpecificInfo info; CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, &vp8_info); @@ -231,7 +233,7 @@ TEST_F(TemporalLayersTest, Alternative3Layers) { for (int i = 0; i < 8; ++i) { CodecSpecificInfo info; CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, &vp8_info); @@ -261,7 +263,7 @@ TEST_F(TemporalLayersTest, SearchOrder) { // Start with a key-frame. tl_config flags can be ignored. uint32_t timestamp = 0; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfoVp8()); @@ -304,7 +306,7 @@ TEST_F(TemporalLayersTest, SearchOrderWithDrop) { // Start with a key-frame. tl_config flags can be ignored. uint32_t timestamp = 0; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfoVp8()); @@ -366,7 +368,7 @@ TEST_F(TemporalLayersTest, 4Layers) { for (int i = 0; i < 16; ++i) { CodecSpecificInfo info; CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, &vp8_info); @@ -395,7 +397,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // Start with a keyframe. uint32_t timestamp = 0; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfoVp8()); @@ -481,7 +483,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) { // Start with a keyframe. uint32_t timestamp = 0; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfoVp8()); @@ -550,7 +552,7 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) { // Start with a keyframe. uint32_t timestamp = 0; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfoVp8()); @@ -629,8 +631,7 @@ TEST_F(TemporalLayersTest, KeyFrame) { for (int j = 1; j <= i; ++j) { // Since last frame was always a keyframe and thus index 0 in the pattern, // this loop starts at index 1. - Vp8TemporalLayers::FrameConfig tl_config = - tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config)) << j; tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, @@ -644,7 +645,7 @@ TEST_F(TemporalLayersTest, KeyFrame) { CodecSpecificInfo info; CodecSpecificInfoVP8& vp8_info = info.codecSpecific.VP8; - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, &vp8_info); EXPECT_TRUE(vp8_info.layerSync) << "Key frame should be marked layer sync."; @@ -675,9 +676,8 @@ class TemporalLayersReferenceTest : public TemporalLayersTest, bool sync; }; - bool UpdateSyncRefState(const Vp8TemporalLayers::BufferFlags& flags, - BufferState* buffer_state) { - if (flags & Vp8TemporalLayers::kReference) { + bool UpdateSyncRefState(const BufferFlags& flags, BufferState* buffer_state) { + if (flags & BufferFlags::kReference) { if (buffer_state->temporal_idx == -1) return true; // References key-frame. if (buffer_state->temporal_idx == 0) { @@ -691,10 +691,10 @@ class TemporalLayersReferenceTest : public TemporalLayersTest, return true; // No reference, does not affect sync frame status. } - void ValidateReference(const Vp8TemporalLayers::BufferFlags& flags, + void ValidateReference(const BufferFlags& flags, const BufferState& buffer_state, int temporal_layer) { - if (flags & Vp8TemporalLayers::kReference) { + if (flags & BufferFlags::kReference) { if (temporal_layer > 0 && buffer_state.timestamp > 0) { // Check that high layer reference does not go past last sync frame. EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_); @@ -731,9 +731,9 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) { // (any). If a given buffer is never updated, it is legal to reference it // even for sync frames. In order to be general, don't assume TL0 always // updates |last|. - std::vector tl_configs(kMaxPatternLength); + std::vector tl_configs(kMaxPatternLength); for (int i = 0; i < kMaxPatternLength; ++i) { - Vp8TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_); tl.OnEncodeDone(timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp, IgnoredCodecSpecificInfoVp8()); ++timestamp_; @@ -774,11 +774,11 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) { // Update the current layer state. BufferState state = {temporal_idx, timestamp_, is_sync_frame}; - if (tl_config.last_buffer_flags & Vp8TemporalLayers::kUpdate) + if (tl_config.last_buffer_flags & BufferFlags::kUpdate) last_state = state; - if (tl_config.golden_buffer_flags & Vp8TemporalLayers::kUpdate) + if (tl_config.golden_buffer_flags & BufferFlags::kUpdate) golden_state = state; - if (tl_config.arf_buffer_flags & Vp8TemporalLayers::kUpdate) + if (tl_config.arf_buffer_flags & BufferFlags::kUpdate) altref_state = state; } } diff --git a/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h b/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h index ae14f68236..3d1671a676 100644 --- a/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h +++ b/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h @@ -15,6 +15,7 @@ #include +#include "api/video_codecs/vp8_frame_config.h" #include "api/video_codecs/vp8_temporal_layers.h" namespace webrtc { @@ -27,9 +28,8 @@ class TemporalLayersChecker { explicit TemporalLayersChecker(int num_temporal_layers); virtual ~TemporalLayersChecker() {} - virtual bool CheckTemporalConfig( - bool frame_is_keyframe, - const Vp8TemporalLayers::FrameConfig& frame_config); + virtual bool CheckTemporalConfig(bool frame_is_keyframe, + const Vp8FrameConfig& frame_config); static std::unique_ptr CreateTemporalLayersChecker( Vp8TemporalLayersType type, @@ -46,7 +46,7 @@ class TemporalLayersChecker { bool* need_sync, bool frame_is_keyframe, uint8_t temporal_layer, - webrtc::Vp8TemporalLayers::BufferFlags flags, + Vp8FrameConfig::BufferFlags flags, uint32_t sequence_number, uint32_t* lowest_sequence_referenced); BufferState last_; diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 7af2b520c7..07c0167124 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -132,22 +132,27 @@ std::unique_ptr VP8Encoder::Create() { } vpx_enc_frame_flags_t LibvpxVp8Encoder::EncodeFlags( - const Vp8TemporalLayers::FrameConfig& references) { + const Vp8FrameConfig& references) { RTC_DCHECK(!references.drop_frame); vpx_enc_frame_flags_t flags = 0; - if ((references.last_buffer_flags & Vp8TemporalLayers::kReference) == 0) + if ((references.last_buffer_flags & + Vp8FrameConfig::BufferFlags::kReference) == 0) flags |= VP8_EFLAG_NO_REF_LAST; - if ((references.last_buffer_flags & Vp8TemporalLayers::kUpdate) == 0) + if ((references.last_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == + 0) flags |= VP8_EFLAG_NO_UPD_LAST; - if ((references.golden_buffer_flags & Vp8TemporalLayers::kReference) == 0) + if ((references.golden_buffer_flags & + Vp8FrameConfig::BufferFlags::kReference) == 0) flags |= VP8_EFLAG_NO_REF_GF; - if ((references.golden_buffer_flags & Vp8TemporalLayers::kUpdate) == 0) + if ((references.golden_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == + 0) flags |= VP8_EFLAG_NO_UPD_GF; - if ((references.arf_buffer_flags & Vp8TemporalLayers::kReference) == 0) + if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kReference) == + 0) flags |= VP8_EFLAG_NO_REF_ARF; - if ((references.arf_buffer_flags & Vp8TemporalLayers::kUpdate) == 0) + if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == 0) flags |= VP8_EFLAG_NO_UPD_ARF; if (references.freeze_entropy) flags |= VP8_EFLAG_NO_UPD_ENTROPY; @@ -757,7 +762,7 @@ int LibvpxVp8Encoder::Encode(const VideoFrame& frame, } } vpx_enc_frame_flags_t flags[kMaxSimulcastStreams]; - Vp8TemporalLayers::FrameConfig tl_configs[kMaxSimulcastStreams]; + Vp8FrameConfig tl_configs[kMaxSimulcastStreams]; for (size_t i = 0; i < encoders_.size(); ++i) { tl_configs[i] = temporal_layers_[i]->UpdateLayerConfig(frame.timestamp()); if (tl_configs[i].drop_frame) { diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h index df2dbcee59..4be58093f7 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h @@ -17,6 +17,7 @@ #include "api/video/encoded_image.h" #include "api/video/video_frame.h" #include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/vp8_frame_config.h" #include "api/video_codecs/vp8_temporal_layers.h" #include "common_types.h" // NOLINT(build/include) #include "modules/video_coding/codecs/vp8/include/vp8.h" @@ -52,8 +53,7 @@ class LibvpxVp8Encoder : public VideoEncoder { EncoderInfo GetEncoderInfo() const override; - static vpx_enc_frame_flags_t EncodeFlags( - const Vp8TemporalLayers::FrameConfig& references); + static vpx_enc_frame_flags_t EncodeFlags(const Vp8FrameConfig& references); private: void SetupTemporalLayers(const VideoCodec& codec); diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index ede21e554d..900a48edcc 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -22,11 +22,20 @@ namespace webrtc { namespace { -static const int kOneSecond90Khz = 90000; -static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 2; -static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4; -static const int kQpDeltaThresholdForSync = 8; -static const int kMinBitrateKbpsForQpBoost = 500; +using Buffer = Vp8FrameConfig::Buffer; +using BufferFlags = Vp8FrameConfig::BufferFlags; + +constexpr BufferFlags kNone = Vp8FrameConfig::BufferFlags::kNone; +constexpr BufferFlags kReference = Vp8FrameConfig::BufferFlags::kReference; +constexpr BufferFlags kUpdate = Vp8FrameConfig::BufferFlags::kUpdate; +constexpr BufferFlags kReferenceAndUpdate = + Vp8FrameConfig::BufferFlags::kReferenceAndUpdate; + +constexpr int kOneSecond90Khz = 90000; +constexpr int kMinTimeBetweenSyncs = kOneSecond90Khz * 2; +constexpr int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4; +constexpr int kQpDeltaThresholdForSync = 8; +constexpr int kMinBitrateKbpsForQpBoost = 500; } // namespace const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5; @@ -68,8 +77,7 @@ bool ScreenshareLayers::SupportsEncoderFrameDropping() const { return false; } -Vp8TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( - uint32_t timestamp) { +Vp8FrameConfig ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) { auto it = pending_frame_configs_.find(timestamp); if (it != pending_frame_configs_.end()) { // Drop and re-encode, reuse the previous config. @@ -79,8 +87,8 @@ Vp8TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( if (number_of_temporal_layers_ <= 1) { // No flags needed for 1 layer screenshare. // TODO(pbos): Consider updating only last, and not all buffers. - Vp8TemporalLayers::FrameConfig tl_config( - kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate); + Vp8FrameConfig tl_config(kReferenceAndUpdate, kReferenceAndUpdate, + kReferenceAndUpdate); pending_frame_configs_[timestamp] = tl_config; return tl_config; } @@ -100,7 +108,7 @@ Vp8TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( // averaging window, or if frame interval is below 90% of desired value, // drop frame. if (encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) - return Vp8TemporalLayers::FrameConfig(kNone, kNone, kNone); + return Vp8FrameConfig(kNone, kNone, kNone); // Primarily check if frame interval is too short using frame timestamps, // as if they are correct they won't be affected by queuing in webrtc. @@ -108,7 +116,7 @@ Vp8TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( kOneSecond90Khz / *target_framerate_; if (last_timestamp_ != -1 && ts_diff > 0) { if (ts_diff < 85 * expected_frame_interval_90khz / 100) { - return Vp8TemporalLayers::FrameConfig(kNone, kNone, kNone); + return Vp8FrameConfig(kNone, kNone, kNone); } } else { // Timestamps looks off, use realtime clock here instead. @@ -116,7 +124,7 @@ Vp8TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( if (last_frame_time_ms_ != -1 && now_ms - last_frame_time_ms_ < (85 * expected_frame_interval_ms) / 100) { - return Vp8TemporalLayers::FrameConfig(kNone, kNone, kNone); + return Vp8FrameConfig(kNone, kNone, kNone); } } } @@ -182,30 +190,28 @@ Vp8TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( RTC_NOTREACHED(); } - Vp8TemporalLayers::FrameConfig tl_config; + Vp8FrameConfig tl_config; // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all // layers. switch (layer_state) { case TemporalLayerState::kDrop: - tl_config = Vp8TemporalLayers::FrameConfig(kNone, kNone, kNone); + tl_config = Vp8FrameConfig(kNone, kNone, kNone); break; case TemporalLayerState::kTl0: // TL0 only references and updates 'last'. - tl_config = - Vp8TemporalLayers::FrameConfig(kReferenceAndUpdate, kNone, kNone); + tl_config = Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone); tl_config.packetizer_temporal_idx = 0; break; case TemporalLayerState::kTl1: // TL1 references both 'last' and 'golden' but only updates 'golden'. - tl_config = Vp8TemporalLayers::FrameConfig(kReference, - kReferenceAndUpdate, kNone); + tl_config = Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone); tl_config.packetizer_temporal_idx = 1; break; case TemporalLayerState::kTl1Sync: // Predict from only TL0 to allow participants to switch to the high // bitrate stream. Updates 'golden' so that TL1 can continue to refer to // and update 'golden' from this point on. - tl_config = Vp8TemporalLayers::FrameConfig(kReference, kUpdate, kNone); + tl_config = Vp8FrameConfig(kReference, kUpdate, kNone); tl_config.packetizer_temporal_idx = 1; break; } @@ -266,7 +272,7 @@ void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp, return; } - absl::optional frame_config; + absl::optional frame_config; auto it = pending_frame_configs_.find(rtp_timestamp); if (it != pending_frame_configs_.end()) { frame_config = it->second; diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index acaa6d82d7..b586f2b2fb 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -13,6 +13,7 @@ #include #include +#include "api/video_codecs/vp8_frame_config.h" #include "api/video_codecs/vp8_temporal_layers.h" #include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h" #include "modules/video_coding/utility/frame_dropper.h" @@ -38,8 +39,7 @@ class ScreenshareLayers : public Vp8TemporalLayers { // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - Vp8TemporalLayers::FrameConfig UpdateLayerConfig( - uint32_t rtp_timestamp) override; + Vp8FrameConfig UpdateLayerConfig(uint32_t rtp_timestamp) override; // New target bitrate, per temporal layer. void OnRatesUpdated(const std::vector& bitrates_bps, @@ -74,7 +74,7 @@ class ScreenshareLayers : public Vp8TemporalLayers { int max_qp_; uint32_t max_debt_bytes_; - std::map pending_frame_configs_; + std::map pending_frame_configs_; // Configured max framerate. absl::optional target_framerate_; diff --git a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index e0bb464919..6daca703f5 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -14,6 +14,7 @@ #include #include +#include "api/video_codecs/vp8_frame_config.h" #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h" #include "modules/video_coding/codecs/vp8/screenshare_layers.h" @@ -93,7 +94,7 @@ class ScreenshareLayerTest : public ::testing::Test { return flags; } - Vp8TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) { + Vp8FrameConfig UpdateLayerConfig(uint32_t timestamp) { int64_t timestamp_ms = timestamp / 90; clock_.AdvanceTimeMilliseconds(timestamp_ms - clock_.TimeInMilliseconds()); return layers_->UpdateLayerConfig(timestamp); @@ -173,7 +174,7 @@ class ScreenshareLayerTest : public ::testing::Test { std::unique_ptr layers_; uint32_t timestamp_; - Vp8TemporalLayers::FrameConfig tl_config_; + Vp8FrameConfig tl_config_; Vp8EncoderConfig cfg_; bool config_updated_; CodecSpecificInfoVP8 vp8_info_; diff --git a/modules/video_coding/codecs/vp8/temporal_layers_checker.cc b/modules/video_coding/codecs/vp8/temporal_layers_checker.cc index 4dba7a366a..15ecce514d 100644 --- a/modules/video_coding/codecs/vp8/temporal_layers_checker.cc +++ b/modules/video_coding/codecs/vp8/temporal_layers_checker.cc @@ -40,10 +40,10 @@ bool TemporalLayersChecker::CheckAndUpdateBufferState( bool* need_sync, bool frame_is_keyframe, uint8_t temporal_layer, - webrtc::Vp8TemporalLayers::BufferFlags flags, + Vp8FrameConfig::BufferFlags flags, uint32_t sequence_number, uint32_t* lowest_sequence_referenced) { - if (flags & Vp8TemporalLayers::BufferFlags::kReference) { + if (flags & Vp8FrameConfig::BufferFlags::kReference) { if (state->temporal_layer > 0 && !state->is_keyframe) { *need_sync = false; } @@ -57,7 +57,7 @@ bool TemporalLayersChecker::CheckAndUpdateBufferState( return false; } } - if ((flags & Vp8TemporalLayers::BufferFlags::kUpdate)) { + if ((flags & Vp8FrameConfig::BufferFlags::kUpdate)) { state->temporal_layer = temporal_layer; state->sequence_number = sequence_number; state->is_keyframe = frame_is_keyframe; @@ -69,7 +69,7 @@ bool TemporalLayersChecker::CheckAndUpdateBufferState( bool TemporalLayersChecker::CheckTemporalConfig( bool frame_is_keyframe, - const Vp8TemporalLayers::FrameConfig& frame_config) { + const Vp8FrameConfig& frame_config) { if (frame_config.drop_frame || frame_config.packetizer_temporal_idx == kNoTemporalIdx) { return true; diff --git a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index bcfc4a8fbc..46098f4ad3 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -15,6 +15,7 @@ #include #include +#include "api/video_codecs/vp8_frame_config.h" #include "api/video_codecs/vp8_temporal_layers.h" #include "rtc_base/checks.h" #include "test/field_trial.h" @@ -36,15 +37,14 @@ constexpr uint32_t kSimulcastScreenshareMaxBitrateKbps = 1250; class MockTemporalLayers : public Vp8TemporalLayers { public: - MOCK_METHOD1(UpdateLayerConfig, Vp8TemporalLayers::FrameConfig(uint32_t)); + MOCK_METHOD1(UpdateLayerConfig, Vp8FrameConfig(uint32_t)); MOCK_METHOD2(OnRatesUpdated, void(const std::vector&, int)); MOCK_METHOD1(UpdateConfiguration, bool(Vp8EncoderConfig*)); MOCK_METHOD5(OnEncodeDone, void(uint32_t, size_t, bool, int, CodecSpecificInfoVP8*)); MOCK_METHOD3(FrameEncoded, void(uint32_t, size_t, int)); MOCK_CONST_METHOD0(Tl0PicIdx, uint8_t()); - MOCK_CONST_METHOD1(GetTemporalLayerId, - int(const Vp8TemporalLayers::FrameConfig&)); + MOCK_CONST_METHOD1(GetTemporalLayerId, int(const Vp8FrameConfig&)); }; } // namespace