diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index 50e3d02ca6..24b1298bd2 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -31,6 +31,7 @@ rtc_source_set("video_codecs_api") { "vp8_frame_buffer_controller.h", "vp8_frame_config.cc", "vp8_frame_config.h", + "vp8_temporal_layers.cc", "vp8_temporal_layers.h", ] @@ -95,20 +96,19 @@ rtc_static_library("builtin_video_encoder_factory") { ] } -rtc_static_library("create_vp8_temporal_layers") { +rtc_static_library("vp8_temporal_layers_factory") { visibility = [ "*" ] allow_poison = [ "software_video_codecs" ] sources = [ - "create_vp8_temporal_layers.cc", - "create_vp8_temporal_layers.h", + "vp8_temporal_layers_factory.cc", + "vp8_temporal_layers_factory.h", ] deps = [ ":video_codecs_api", - "../..:webrtc_common", - "../../modules/video_coding:video_codec_interface", + "../../modules/video_coding:video_coding_utility", "../../modules/video_coding:webrtc_vp8_temporal_layers", - "../../system_wrappers:system_wrappers", + "../../rtc_base:checks", "//third_party/abseil-cpp/absl/memory", ] } diff --git a/api/video_codecs/create_vp8_temporal_layers.cc b/api/video_codecs/create_vp8_temporal_layers.cc deleted file mode 100644 index 26f734a503..0000000000 --- a/api/video_codecs/create_vp8_temporal_layers.cc +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "api/video_codecs/create_vp8_temporal_layers.h" -#include "absl/memory/memory.h" -#include "api/video_codecs/vp8_temporal_layers.h" -#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" -#include "modules/video_coding/codecs/vp8/screenshare_layers.h" - -namespace webrtc { - -std::unique_ptr CreateVp8TemporalLayers( - Vp8TemporalLayersType type, - int num_temporal_layers) { - switch (type) { - case Vp8TemporalLayersType::kFixedPattern: - return absl::make_unique(num_temporal_layers); - case Vp8TemporalLayersType::kBitrateDynamic: - // Conference mode temporal layering for screen content in base stream. - return absl::make_unique(num_temporal_layers); - } -} - -} // namespace webrtc diff --git a/api/video_codecs/video_encoder.cc b/api/video_codecs/video_encoder.cc index a8b6f427c6..233ca24e49 100644 --- a/api/video_codecs/video_encoder.cc +++ b/api/video_codecs/video_encoder.cc @@ -125,6 +125,10 @@ int32_t VideoEncoder::SetRateAllocation( return SetRates(allocation.get_sum_kbps(), framerate); } +void VideoEncoder::OnPacketLossRateUpdate(float packet_loss_rate) {} + +void VideoEncoder::OnRttUpdate(int64_t rtt_ms) {} + // TODO(webrtc:9722): Remove and make pure virtual. VideoEncoder::EncoderInfo VideoEncoder::GetEncoderInfo() const { return EncoderInfo(); diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h index dc68928492..9ff8faacda 100644 --- a/api/video_codecs/video_encoder.h +++ b/api/video_codecs/video_encoder.h @@ -263,6 +263,16 @@ class RTC_EXPORT VideoEncoder { virtual int32_t SetRateAllocation(const VideoBitrateAllocation& allocation, uint32_t framerate); + // Inform the encoder when the packet loss rate changes. + // + // Input: - packet_loss_rate : The packet loss rate (0.0 to 1.0). + virtual void OnPacketLossRateUpdate(float packet_loss_rate); + + // Inform the encoder when the round trip time changes. + // + // Input: - rtt_ms : The new RTT, in milliseconds. + virtual void OnRttUpdate(int64_t rtt_ms); + // Returns meta-data about the encoder, such as implementation name. // The output of this method may change during runtime. For instance if a // hardware encoder fails, it may fall back to doing software encoding using diff --git a/api/video_codecs/vp8_frame_buffer_controller.h b/api/video_codecs/vp8_frame_buffer_controller.h index 93ed6dab45..7f4c282ded 100644 --- a/api/video_codecs/vp8_frame_buffer_controller.h +++ b/api/video_codecs/vp8_frame_buffer_controller.h @@ -11,8 +11,10 @@ #ifndef API_VIDEO_CODECS_VP8_FRAME_BUFFER_CONTROLLER_H_ #define API_VIDEO_CODECS_VP8_FRAME_BUFFER_CONTROLLER_H_ +#include #include +#include "api/video_codecs/video_codec.h" #include "api/video_codecs/vp8_frame_config.h" namespace webrtc { @@ -66,10 +68,15 @@ struct Vp8EncoderConfig { }; // This interface defines a way of delegating the logic of buffer management. +// Multiple streams may be controlled by a single controller, demuxing between +// them using stream_index. class Vp8FrameBufferController { public: virtual ~Vp8FrameBufferController() = default; + // Number of streamed controlled by |this|. + virtual size_t StreamCount() const = 0; + // If this method returns true, the encoder is free to drop frames for // instance in an effort to uphold encoding bitrate. // If this return false, the encoder must not drop any frames unless: @@ -79,10 +86,11 @@ class Vp8FrameBufferController { // re-encode the image at a low bitrate. In this case the encoder should // call OnEncodeDone() once with size = 0 to indicate drop, and then call // OnEncodeDone() again when the frame has actually been encoded. - virtual bool SupportsEncoderFrameDropping() const = 0; + virtual bool SupportsEncoderFrameDropping(size_t stream_index) const = 0; // New target bitrate, per temporal layer. - virtual void OnRatesUpdated(const std::vector& bitrates_bps, + virtual void OnRatesUpdated(size_t stream_index, + const std::vector& bitrates_bps, int framerate_fps) = 0; // Called by the encoder before encoding a frame. |cfg| contains the current @@ -90,7 +98,8 @@ class Vp8FrameBufferController { // to be changed before the encode step, |cfg| should be changed and then // return true. If false is returned, the encoder will proceed without // updating the configuration. - virtual bool UpdateConfiguration(Vp8EncoderConfig* cfg) = 0; + virtual bool UpdateConfiguration(size_t stream_index, + Vp8EncoderConfig* cfg) = 0; // Returns the recommended VP8 encode flags needed, and moves the temporal // pattern to the next frame. @@ -99,7 +108,8 @@ class Vp8FrameBufferController { // The timestamp uses a 90kHz RTP clock. // After calling this method, first call the actual encoder with the provided // frame configuration, and then OnEncodeDone() below. - virtual Vp8FrameConfig UpdateLayerConfig(uint32_t rtp_timestamp) = 0; + virtual Vp8FrameConfig UpdateLayerConfig(size_t stream_index, + uint32_t rtp_timestamp) = 0; // Called after the encode step is done. |rtp_timestamp| must match the // parameter use in the UpdateLayerConfig() call. @@ -114,11 +124,28 @@ class Vp8FrameBufferController { // If |size_bytes| > 0, |qp| should indicate the frame-level QP this frame was // encoded at. If the encoder does not support extracting this, |qp| should be // set to 0. - virtual void OnEncodeDone(uint32_t rtp_timestamp, + virtual void OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, CodecSpecificInfo* info) = 0; + + // Called by the encoder when the packet loss rate changes. + // |packet_loss_rate| runs between 0.0 (no loss) and 1.0 (everything lost). + virtual void OnPacketLossRateUpdate(float packet_loss_rate) = 0; + + // Called by the encoder when the round trip time changes. + virtual void OnRttUpdate(int64_t rtt_ms) = 0; +}; + +// Interface for a factory of Vp8FrameBufferController instances. +class Vp8FrameBufferControllerFactory { + public: + virtual ~Vp8FrameBufferControllerFactory() = default; + + virtual std::unique_ptr Create( + const VideoCodec& codec) = 0; }; } // namespace webrtc diff --git a/api/video_codecs/vp8_temporal_layers.cc b/api/video_codecs/vp8_temporal_layers.cc new file mode 100644 index 0000000000..a3c0db1a04 --- /dev/null +++ b/api/video_codecs/vp8_temporal_layers.cc @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/vp8_temporal_layers.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +Vp8TemporalLayers::Vp8TemporalLayers( + std::vector>&& controllers) + : controllers_(std::move(controllers)) { + RTC_DCHECK(!controllers_.empty()); + RTC_DCHECK(std::none_of( + controllers_.begin(), controllers_.end(), + [](const std::unique_ptr& controller) { + return controller.get() == nullptr; + })); +} + +size_t Vp8TemporalLayers::StreamCount() const { + return controllers_.size(); +} + +bool Vp8TemporalLayers::SupportsEncoderFrameDropping( + size_t stream_index) const { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->SupportsEncoderFrameDropping(0); +} + +void Vp8TemporalLayers::OnRatesUpdated( + size_t stream_index, + const std::vector& bitrates_bps, + int framerate_fps) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->OnRatesUpdated(0, bitrates_bps, + framerate_fps); +} + +bool Vp8TemporalLayers::UpdateConfiguration(size_t stream_index, + Vp8EncoderConfig* cfg) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->UpdateConfiguration(0, cfg); +} + +Vp8FrameConfig Vp8TemporalLayers::UpdateLayerConfig(size_t stream_index, + uint32_t rtp_timestamp) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->UpdateLayerConfig(0, rtp_timestamp); +} + +void Vp8TemporalLayers::OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->OnEncodeDone(0, rtp_timestamp, size_bytes, + is_keyframe, qp, info); +} + +void Vp8TemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) { + for (auto& controller : controllers_) { + controller->OnPacketLossRateUpdate(packet_loss_rate); + } +} + +void Vp8TemporalLayers::OnRttUpdate(int64_t rtt_ms) { + for (auto& controller : controllers_) { + controller->OnRttUpdate(rtt_ms); + } +} + +} // namespace webrtc diff --git a/api/video_codecs/vp8_temporal_layers.h b/api/video_codecs/vp8_temporal_layers.h index 61a9defea5..d5d29cdb6c 100644 --- a/api/video_codecs/vp8_temporal_layers.h +++ b/api/video_codecs/vp8_temporal_layers.h @@ -11,8 +11,10 @@ #ifndef API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_H_ #define API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_H_ +#include #include +#include "api/video_codecs/video_codec.h" #include "api/video_codecs/vp8_frame_buffer_controller.h" #include "api/video_codecs/vp8_frame_config.h" @@ -22,28 +24,43 @@ namespace webrtc { // kFixedPattern uses a fixed repeating pattern of 1-4 layers. // kBitrateDynamic can allocate frames dynamically to 1 or 2 layers, based on // the bitrate produced. +// TODO(eladalon): Remove this enum. enum class Vp8TemporalLayersType { kFixedPattern, kBitrateDynamic }; // This interface defines a way of getting the encoder settings needed to // realize a temporal layer structure. -class Vp8TemporalLayers : public Vp8FrameBufferController { +class Vp8TemporalLayers final : public Vp8FrameBufferController { public: + explicit Vp8TemporalLayers( + std::vector>&& controllers); ~Vp8TemporalLayers() override = default; - bool SupportsEncoderFrameDropping() const override = 0; + size_t StreamCount() const override; - void OnRatesUpdated(const std::vector& bitrates_bps, - int framerate_fps) override = 0; + bool SupportsEncoderFrameDropping(size_t stream_index) const override; - bool UpdateConfiguration(Vp8EncoderConfig* cfg) override = 0; + void OnRatesUpdated(size_t stream_index, + const std::vector& bitrates_bps, + int framerate_fps) override; - Vp8FrameConfig UpdateLayerConfig(uint32_t rtp_timestamp) override = 0; + bool UpdateConfiguration(size_t stream_index, Vp8EncoderConfig* cfg) override; - void OnEncodeDone(uint32_t rtp_timestamp, + Vp8FrameConfig UpdateLayerConfig(size_t stream_index, + uint32_t rtp_timestamp) override; + + void OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, - CodecSpecificInfo* info) override = 0; + CodecSpecificInfo* info) override; + + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + + private: + std::vector> controllers_; }; } // namespace webrtc diff --git a/api/video_codecs/vp8_temporal_layers_factory.cc b/api/video_codecs/vp8_temporal_layers_factory.cc new file mode 100644 index 0000000000..2eb7bb9416 --- /dev/null +++ b/api/video_codecs/vp8_temporal_layers_factory.cc @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/vp8_temporal_layers_factory.h" + +#include +#include +#include + +#include "absl/memory/memory.h" +#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" +#include "modules/video_coding/codecs/vp8/screenshare_layers.h" +#include "modules/video_coding/utility/simulcast_utility.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +std::unique_ptr Vp8TemporalLayersFactory::Create( + const VideoCodec& codec) { + std::vector> controllers; + const int num_streams = SimulcastUtility::NumberOfSimulcastStreams(codec); + RTC_DCHECK_GE(num_streams, 1); + controllers.reserve(num_streams); + + for (int i = 0; i < num_streams; ++i) { + int num_temporal_layers = + SimulcastUtility::NumberOfTemporalLayers(codec, i); + if (SimulcastUtility::IsConferenceModeScreenshare(codec) && i == 0) { + // Legacy screenshare layers supports max 2 layers. + num_temporal_layers = std::max(2, num_temporal_layers); + controllers.push_back( + absl::make_unique(num_temporal_layers)); + } else { + controllers.push_back( + absl::make_unique(num_temporal_layers)); + } + } + + return absl::make_unique(std::move(controllers)); +} + +} // namespace webrtc diff --git a/api/video_codecs/create_vp8_temporal_layers.h b/api/video_codecs/vp8_temporal_layers_factory.h similarity index 50% rename from api/video_codecs/create_vp8_temporal_layers.h rename to api/video_codecs/vp8_temporal_layers_factory.h index 52f14c8eda..9c65d4c2fc 100644 --- a/api/video_codecs/create_vp8_temporal_layers.h +++ b/api/video_codecs/vp8_temporal_layers_factory.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef API_VIDEO_CODECS_CREATE_VP8_TEMPORAL_LAYERS_H_ -#define API_VIDEO_CODECS_CREATE_VP8_TEMPORAL_LAYERS_H_ +#ifndef API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_FACTORY_H_ +#define API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_FACTORY_H_ #include @@ -17,10 +17,14 @@ namespace webrtc { -std::unique_ptr CreateVp8TemporalLayers( - Vp8TemporalLayersType type, - int num_temporal_layers); +class Vp8TemporalLayersFactory : public Vp8FrameBufferControllerFactory { + public: + ~Vp8TemporalLayersFactory() override = default; + + std::unique_ptr Create( + const VideoCodec& codec) override; +}; } // namespace webrtc -#endif // API_VIDEO_CODECS_CREATE_VP8_TEMPORAL_LAYERS_H_ +#endif // API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_FACTORY_H_ diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index b6314bd04a..afd68372d7 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -373,8 +373,8 @@ rtc_static_library("webrtc_vp8") { "../../api/video:encoded_image", "../../api/video:video_frame", "../../api/video:video_frame_i420", - "../../api/video_codecs:create_vp8_temporal_layers", "../../api/video_codecs:video_codecs_api", + "../../api/video_codecs:vp8_temporal_layers_factory", "../../common_video", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", @@ -838,8 +838,8 @@ if (rtc_include_tests) { "../../api/video:video_bitrate_allocator_factory", "../../api/video:video_frame", "../../api/video:video_frame_i420", - "../../api/video_codecs:create_vp8_temporal_layers", "../../api/video_codecs:video_codecs_api", + "../../api/video_codecs:vp8_temporal_layers_factory", "../../common_video:common_video", "../../media:rtc_media_base", "../../rtc_base:checks", diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 61dca0909f..70e78a7062 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -7,6 +7,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" + #include #include @@ -18,7 +20,6 @@ #include #include "modules/include/module_common_types.h" -#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" @@ -260,14 +261,22 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) DefaultTemporalLayers::~DefaultTemporalLayers() = default; -bool DefaultTemporalLayers::SupportsEncoderFrameDropping() const { +size_t DefaultTemporalLayers::StreamCount() const { + return 1; +} + +bool DefaultTemporalLayers::SupportsEncoderFrameDropping( + size_t stream_index) const { + RTC_DCHECK_LT(stream_index, StreamCount()); // This class allows the encoder drop frames as it sees fit. return true; } void DefaultTemporalLayers::OnRatesUpdated( + size_t stream_index, const std::vector& bitrates_bps, int framerate_fps) { + RTC_DCHECK_LT(stream_index, StreamCount()); RTC_DCHECK_GT(bitrates_bps.size(), 0); RTC_DCHECK_LE(bitrates_bps.size(), num_layers_); // |bitrates_bps| uses individual rate per layer, but Vp8EncoderConfig wants @@ -279,7 +288,10 @@ void DefaultTemporalLayers::OnRatesUpdated( } } -bool DefaultTemporalLayers::UpdateConfiguration(Vp8EncoderConfig* cfg) { +bool DefaultTemporalLayers::UpdateConfiguration(size_t stream_index, + Vp8EncoderConfig* cfg) { + RTC_DCHECK_LT(stream_index, StreamCount()); + if (!new_bitrates_bps_) { return false; } @@ -328,9 +340,11 @@ bool DefaultTemporalLayers::IsSyncFrame(const Vp8FrameConfig& config) const { return true; } -Vp8FrameConfig DefaultTemporalLayers::UpdateLayerConfig(uint32_t timestamp) { +Vp8FrameConfig DefaultTemporalLayers::UpdateLayerConfig(size_t stream_index, + uint32_t timestamp) { + RTC_DCHECK_LT(stream_index, StreamCount()); RTC_DCHECK_GT(num_layers_, 0); - RTC_DCHECK_LT(0, temporal_pattern_.size()); + RTC_DCHECK_GT(temporal_pattern_.size(), 0); pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size(); Vp8FrameConfig tl_config = temporal_pattern_[pattern_idx_]; @@ -440,11 +454,13 @@ void DefaultTemporalLayers::UpdateSearchOrder(Vp8FrameConfig* config) { } } -void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp, +void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, CodecSpecificInfo* info) { + RTC_DCHECK_LT(stream_index, StreamCount()); RTC_DCHECK_GT(num_layers_, 0); auto pending_frame = pending_frames_.find(rtp_timestamp); @@ -525,6 +541,10 @@ void DefaultTemporalLayers::OnEncodeDone(uint32_t rtp_timestamp, } } +void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} + +void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {} + TemplateStructure DefaultTemporalLayers::GetTemplateStructure( int num_layers) const { RTC_CHECK_LT(num_layers, 5); diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index f94d9086ba..ee2333ec2d 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -28,29 +28,38 @@ namespace webrtc { -class DefaultTemporalLayers : public Vp8TemporalLayers { +class DefaultTemporalLayers final : public Vp8FrameBufferController { public: explicit DefaultTemporalLayers(int number_of_temporal_layers); ~DefaultTemporalLayers() override; - bool SupportsEncoderFrameDropping() const override; + size_t StreamCount() const override; + + bool SupportsEncoderFrameDropping(size_t stream_index) const override; // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - Vp8FrameConfig UpdateLayerConfig(uint32_t timestamp) override; + Vp8FrameConfig UpdateLayerConfig(size_t stream_index, + uint32_t timestamp) override; // New target bitrate, per temporal layer. - void OnRatesUpdated(const std::vector& bitrates_bps, + void OnRatesUpdated(size_t stream_index, + const std::vector& bitrates_bps, int framerate_fps) override; - bool UpdateConfiguration(Vp8EncoderConfig* cfg) override; + bool UpdateConfiguration(size_t stream_index, Vp8EncoderConfig* cfg) override; - void OnEncodeDone(uint32_t rtp_timestamp, + void OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, CodecSpecificInfo* info) override; + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + private: static std::vector GetTemporalPattern(size_t num_layers); bool IsSyncFrame(const Vp8FrameConfig& config) const; diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index 37e662068e..b8625ad0e8 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -121,10 +121,11 @@ TEST_F(TemporalLayersTest, 2Layers) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); constexpr size_t kPatternSize = 4; constexpr size_t kRepetitions = 4; @@ -142,10 +143,10 @@ TEST_F(TemporalLayersTest, 2Layers) { for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) { const size_t ind = i % kPatternSize; CodecSpecificInfo info; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); EXPECT_EQ(expected_flags[ind], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, &info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx); @@ -163,10 +164,11 @@ TEST_F(TemporalLayersTest, 3Layers) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); int expected_flags[16] = { kTemporalUpdateLastRefAltRef, @@ -196,9 +198,9 @@ TEST_F(TemporalLayersTest, 3Layers) { unsigned int timestamp = 0; for (int i = 0; i < 16; ++i) { CodecSpecificInfo info; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, &info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx); @@ -217,10 +219,11 @@ TEST_F(TemporalLayersTest, Alternative3Layers) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); int expected_flags[8] = {kTemporalUpdateLast, kTemporalUpdateAltrefWithoutDependency, @@ -238,9 +241,9 @@ TEST_F(TemporalLayersTest, Alternative3Layers) { unsigned int timestamp = 0; for (int i = 0; i < 8; ++i) { CodecSpecificInfo info; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, &info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx); @@ -259,38 +262,39 @@ TEST_F(TemporalLayersTest, SearchOrder) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); // Use a repeating pattern of tl 0, 2, 1, 2. // Tl 0, 1, 2 update last, golden, altref respectively. // Start with a key-frame. tl_config flags can be ignored. uint32_t timestamp = 0; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame. First one only references TL0. Updates altref. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); // TL1 frame. Can only reference TL0. Updated golden. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); // TL2 frame. Can reference all three buffers. Golden was the last to be // updated, the next to last was altref. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref); @@ -302,35 +306,36 @@ TEST_F(TemporalLayersTest, SearchOrderWithDrop) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); // Use a repeating pattern of tl 0, 2, 1, 2. // Tl 0, 1, 2 update last, golden, altref respectively. // Start with a key-frame. tl_config flags can be ignored. uint32_t timestamp = 0; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame. First one only references TL0. Updates altref. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); // Dropped TL1 frame. Can only reference TL0. Should have updated golden. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); // TL2 frame. Can normally reference all three buffers, but golden has not // been populated this cycle. Altref was last to be updated, before that last. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref); EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast); @@ -341,10 +346,11 @@ TEST_F(TemporalLayersTest, 4Layers) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); int expected_flags[16] = { kTemporalUpdateLast, kTemporalUpdateNoneNoRefGoldenAltRef, @@ -373,9 +379,9 @@ TEST_F(TemporalLayersTest, 4Layers) { uint32_t timestamp = 0; for (int i = 0; i < 16; ++i) { CodecSpecificInfo info; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); EXPECT_EQ(expected_flags[i], LibvpxVp8Encoder::EncodeFlags(tl_config)) << i; - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, i == 0, kDefaultQp, &info); EXPECT_TRUE(checker.CheckTemporalConfig(i == 0, tl_config)); EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx); @@ -396,29 +402,30 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); // Start with a keyframe. uint32_t timestamp = 0; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfo()); // Dropped TL2 frame. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); // Dropped TL1 frame. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); // TL2 frame. Can reference all three buffers, valid since golden and altref // both contain the last keyframe. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); @@ -427,24 +434,24 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // Restart of cycle! // TL0 base layer frame, updating and referencing last. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame, updating altref. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // TL1 frame, updating golden. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame. Can still reference all buffer since they have been update this // cycle. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); @@ -453,22 +460,22 @@ TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { // Restart of cycle! // TL0 base layer frame, updating and referencing last. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // Dropped TL2 frame. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); // Dropped TL1 frame. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, 0, false, 0, nullptr); + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); // TL2 frame. This time golden and altref contain data from the previous cycle // and cannot be referenced. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference); @@ -482,62 +489,63 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); // Start with a keyframe. uint32_t timestamp = 0; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfo()); // Do a full cycle of the pattern. for (int i = 0; i < 7; ++i) { - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); } // TL0 base layer frame, starting the cycle over. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // Encoder has a hiccup and builds a queue, so frame encoding is delayed. // TL1 frame, updating golden. - tl_config = tl.UpdateLayerConfig(++timestamp); + tl_config = tl.UpdateLayerConfig(0, ++timestamp); // TL2 frame, that should be referencing golden, but we can't be certain it's // not going to be dropped, so that is not allowed. - tl_config = tl.UpdateLayerConfig(timestamp + 1); + tl_config = tl.UpdateLayerConfig(0, timestamp + 1); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference); // TL0 base layer frame. - tl_config = tl.UpdateLayerConfig(timestamp + 2); + tl_config = tl.UpdateLayerConfig(0, timestamp + 2); // The previous four enqueued frames finally get encoded, and the updated // buffers are now OK to reference. // Enqueued TL1 frame ready. - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // Enqueued TL2 frame. - tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // Enqueued TL0 frame. - tl.OnEncodeDone(++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame, all buffers are now in a known good state, OK to reference. - tl_config = tl.UpdateLayerConfig(++timestamp + 1); + tl_config = tl.UpdateLayerConfig(0, ++timestamp + 1); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference); @@ -551,56 +559,57 @@ TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); // Start with a keyframe. uint32_t timestamp = 0; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, IgnoredCodecSpecificInfo()); // Do a full cycle of the pattern. for (int i = 0; i < 3; ++i) { - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); } // TL0 base layer frame, starting the cycle over. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame. - tl_config = tl.UpdateLayerConfig(++timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl_config = tl.UpdateLayerConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // Encoder has a hiccup and builds a queue, so frame encoding is delayed. // Encoded, but delayed frames in TL 1, 2. - tl_config = tl.UpdateLayerConfig(timestamp + 1); - tl_config = tl.UpdateLayerConfig(timestamp + 2); + tl_config = tl.UpdateLayerConfig(0, timestamp + 1); + tl_config = tl.UpdateLayerConfig(0, timestamp + 2); // Restart of the pattern! // Encoded, but delayed frames in TL 2, 1. - tl_config = tl.UpdateLayerConfig(timestamp + 3); - tl_config = tl.UpdateLayerConfig(timestamp + 4); + tl_config = tl.UpdateLayerConfig(0, timestamp + 3); + tl_config = tl.UpdateLayerConfig(0, timestamp + 4); // TL1 frame from last cycle is ready. - tl.OnEncodeDone(timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp, + tl.OnEncodeDone(0, timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame from last cycle is ready. - tl.OnEncodeDone(timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp, + tl.OnEncodeDone(0, timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); // TL2 frame, that should be referencing all buffers, but altref and golden // haven not been updated this cycle. (Don't be fooled by the late frames from // the last cycle!) - tl_config = tl.UpdateLayerConfig(timestamp + 5); + tl_config = tl.UpdateLayerConfig(0, timestamp + 5); EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference); EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference); @@ -611,10 +620,11 @@ TEST_F(TemporalLayersTest, KeyFrame) { DefaultTemporalLayers tl(kNumLayers); DefaultTemporalLayersChecker checker(kNumLayers); Vp8EncoderConfig cfg; - tl.OnRatesUpdated(GetTemporalLayerRates(kDefaultBytesPerFrame, + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, kNumLayers), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); int expected_flags[8] = { kTemporalUpdateLastRefAltRef, @@ -637,10 +647,10 @@ TEST_F(TemporalLayersTest, KeyFrame) { for (int j = 1; j <= i; ++j) { // Since last frame was always a keyframe and thus index 0 in the pattern, // this loop starts at index 1. - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config)) << j; - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, IgnoredCodecSpecificInfo()); EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config)); EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx); @@ -650,8 +660,9 @@ TEST_F(TemporalLayersTest, KeyFrame) { } CodecSpecificInfo info; - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp); - tl.OnEncodeDone(timestamp, kDefaultBytesPerFrame, true, kDefaultQp, &info); + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + &info); EXPECT_TRUE(info.codecSpecific.VP8.layerSync) << "Key frame should be marked layer sync."; EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx) @@ -727,9 +738,9 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) { DefaultTemporalLayers tl(num_layers); Vp8EncoderConfig cfg; tl.OnRatesUpdated( - GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, 1), + 0, GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, 1), kDefaultFramerate); - tl.UpdateConfiguration(&cfg); + tl.UpdateConfiguration(0, &cfg); // Run through the pattern and store the frame dependencies, plus keep track // of the buffer state; which buffers references which temporal layers (if @@ -738,8 +749,8 @@ TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) { // updates |last|. std::vector tl_configs(kMaxPatternLength); for (int i = 0; i < kMaxPatternLength; ++i) { - Vp8FrameConfig tl_config = tl.UpdateLayerConfig(timestamp_); - tl.OnEncodeDone(timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp, + Vp8FrameConfig tl_config = tl.UpdateLayerConfig(0, timestamp_); + tl.OnEncodeDone(0, timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp, IgnoredCodecSpecificInfo()); ++timestamp_; EXPECT_FALSE(tl_config.drop_frame); diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 98d5e3c44d..580f706f8b 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -25,8 +25,8 @@ #include "api/video/video_content_type.h" #include "api/video/video_frame_buffer.h" #include "api/video/video_timing.h" -#include "api/video_codecs/create_vp8_temporal_layers.h" #include "api/video_codecs/vp8_temporal_layers.h" +#include "api/video_codecs/vp8_temporal_layers_factory.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" @@ -122,10 +122,12 @@ static void FillInEncoderConfig(vpx_codec_enc_cfg* vpx_config, vpx_config->rc_max_quantizer = config.rc_max_quantizer; } -bool UpdateVpxConfiguration(Vp8FrameBufferController* frame_buffer_controller, +bool UpdateVpxConfiguration(size_t stream_index, + Vp8FrameBufferController* frame_buffer_controller, vpx_codec_enc_cfg_t* cfg) { Vp8EncoderConfig config = GetEncoderConfig(cfg); - const bool res = frame_buffer_controller->UpdateConfiguration(&config); + const bool res = + frame_buffer_controller->UpdateConfiguration(stream_index, &config); if (res) FillInEncoderConfig(cfg, config); return res; @@ -185,7 +187,6 @@ LibvpxVp8Encoder::LibvpxVp8Encoder(std::unique_ptr interface) "WebRTC-VP8VariableFramerateScreenshare")), framerate_controller_(variable_framerate_experiment_.framerate_limit), num_steady_state_frames_(0) { - frame_buffer_controllers_.reserve(kMaxSimulcastStreams); raw_images_.reserve(kMaxSimulcastStreams); encoded_images_.reserve(kMaxSimulcastStreams); send_stream_.reserve(kMaxSimulcastStreams); @@ -220,7 +221,7 @@ int LibvpxVp8Encoder::Release() { libvpx_->img_free(&raw_images_.back()); raw_images_.pop_back(); } - frame_buffer_controllers_.clear(); + frame_buffer_controller_.reset(); inited_ = false; return ret_val; } @@ -279,11 +280,12 @@ int LibvpxVp8Encoder::SetRateAllocation(const VideoBitrateAllocation& bitrate, configurations_[i].rc_target_bitrate = target_bitrate_kbps; if (send_stream) { - frame_buffer_controllers_[stream_idx]->OnRatesUpdated( - bitrate.GetTemporalLayerAllocation(stream_idx), new_framerate); + frame_buffer_controller_->OnRatesUpdated( + stream_idx, bitrate.GetTemporalLayerAllocation(stream_idx), + new_framerate); } - UpdateVpxConfiguration(frame_buffer_controllers_[stream_idx].get(), + UpdateVpxConfiguration(stream_idx, frame_buffer_controller_.get(), &configurations_[i]); if (libvpx_->codec_enc_config_set(&encoders_[i], &configurations_[i])) { @@ -293,6 +295,20 @@ int LibvpxVp8Encoder::SetRateAllocation(const VideoBitrateAllocation& bitrate, return WEBRTC_VIDEO_CODEC_OK; } +void LibvpxVp8Encoder::OnPacketLossRateUpdate(float packet_loss_rate) { + // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK. + if (frame_buffer_controller_) { + frame_buffer_controller_->OnPacketLossRateUpdate(packet_loss_rate); + } +} + +void LibvpxVp8Encoder::OnRttUpdate(int64_t rtt_ms) { + // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK. + if (frame_buffer_controller_) { + frame_buffer_controller_->OnRttUpdate(rtt_ms); + } +} + void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) { if (send_stream && !send_stream_[stream_idx]) { // Need a key frame if we have not sent this stream before. @@ -301,26 +317,6 @@ void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) { send_stream_[stream_idx] = send_stream; } -void LibvpxVp8Encoder::SetupTemporalLayers(const VideoCodec& codec) { - RTC_DCHECK(frame_buffer_controllers_.empty()); - const int num_streams = SimulcastUtility::NumberOfSimulcastStreams(codec); - for (int i = 0; i < num_streams; ++i) { - Vp8TemporalLayersType type; - int num_frame_buffer_controllers = - SimulcastUtility::NumberOfTemporalLayers(codec, i); - if (SimulcastUtility::IsConferenceModeScreenshare(codec) && i == 0) { - type = Vp8TemporalLayersType::kBitrateDynamic; - // Legacy screenshare layers supports max 2 layers. - num_frame_buffer_controllers = - std::max(2, num_frame_buffer_controllers); - } else { - type = Vp8TemporalLayersType::kFixedPattern; - } - frame_buffer_controllers_.emplace_back( - CreateVp8TemporalLayers(type, num_frame_buffer_controllers)); - } -} - int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, int number_of_cores, size_t /*maxPayloadSize */) { @@ -354,7 +350,10 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } - SetupTemporalLayers(*inst); + RTC_DCHECK(!frame_buffer_controller_); + // TODO(bugs.webrtc.org/10382): Inject the factory. + Vp8TemporalLayersFactory factory; + frame_buffer_controller_ = factory.Create(*inst); number_of_cores_ = number_of_cores; timestamp_ = 0; @@ -492,10 +491,11 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx]; if (stream_bitrates[stream_idx] > 0) { - frame_buffer_controllers_[stream_idx]->OnRatesUpdated( - allocation.GetTemporalLayerAllocation(stream_idx), inst->maxFramerate); + frame_buffer_controller_->OnRatesUpdated( + stream_idx, allocation.GetTemporalLayerAllocation(stream_idx), + inst->maxFramerate); } - UpdateVpxConfiguration(frame_buffer_controllers_[stream_idx].get(), + UpdateVpxConfiguration(stream_idx, frame_buffer_controller_.get(), &configurations_[0]); configurations_[0].rc_dropframe_thresh = FrameDropThreshold(stream_idx); @@ -522,11 +522,11 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx]; if (stream_bitrates[stream_idx] > 0) { - frame_buffer_controllers_[stream_idx]->OnRatesUpdated( - allocation.GetTemporalLayerAllocation(stream_idx), + frame_buffer_controller_->OnRatesUpdated( + stream_idx, allocation.GetTemporalLayerAllocation(stream_idx), inst->maxFramerate); } - UpdateVpxConfiguration(frame_buffer_controllers_[stream_idx].get(), + UpdateVpxConfiguration(stream_idx, frame_buffer_controller_.get(), &configurations_[i]); } @@ -695,9 +695,10 @@ uint32_t LibvpxVp8Encoder::FrameDropThreshold(size_t spatial_idx) const { // setting, as eg. ScreenshareLayers does not work as intended with frame // dropping on and DefaultTemporalLayers will have performance issues with // frame dropping off. - RTC_CHECK_LT(spatial_idx, frame_buffer_controllers_.size()); + RTC_DCHECK(frame_buffer_controller_); + RTC_DCHECK_LT(spatial_idx, frame_buffer_controller_->StreamCount()); enable_frame_dropping = - frame_buffer_controllers_[spatial_idx]->SupportsEncoderFrameDropping(); + frame_buffer_controller_->SupportsEncoderFrameDropping(spatial_idx); return enable_frame_dropping ? 30 : 0; } @@ -811,7 +812,7 @@ int LibvpxVp8Encoder::Encode(const VideoFrame& frame, Vp8FrameConfig tl_configs[kMaxSimulcastStreams]; for (size_t i = 0; i < encoders_.size(); ++i) { tl_configs[i] = - frame_buffer_controllers_[i]->UpdateLayerConfig(frame.timestamp()); + frame_buffer_controller_->UpdateLayerConfig(i, frame.timestamp()); if (tl_configs[i].drop_frame) { if (send_key_frame) { continue; @@ -840,10 +841,9 @@ int LibvpxVp8Encoder::Encode(const VideoFrame& frame, } // Set the encoder frame flags and temporal layer_id for each spatial stream. - // Note that |frame_buffer_controllers_| are defined starting from lowest - // resolution at position 0 to highest resolution at position - // |encoders_.size() - 1|, whereas |encoder_| is from highest to lowest - // resolution. + // Note that streams are defined starting from lowest resolution at + // position 0 to highest resolution at position |encoders_.size() - 1|, + // whereas |encoder_| is from highest to lowest resolution. size_t stream_idx = encoders_.size() - 1; for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) { // Allow the layers adapter to temporarily modify the configuration. This @@ -851,7 +851,7 @@ int LibvpxVp8Encoder::Encode(const VideoFrame& frame, // the next update. vpx_codec_enc_cfg_t temp_config; memcpy(&temp_config, &configurations_[i], sizeof(vpx_codec_enc_cfg_t)); - if (UpdateVpxConfiguration(frame_buffer_controllers_[stream_idx].get(), + if (UpdateVpxConfiguration(stream_idx, frame_buffer_controller_.get(), &temp_config)) { if (libvpx_->codec_enc_config_set(&encoders_[i], &temp_config)) return WEBRTC_VIDEO_CODEC_ERROR; @@ -913,8 +913,8 @@ void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, int qp = 0; vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); - frame_buffer_controllers_[stream_idx]->OnEncodeDone( - timestamp, encoded_images_[encoder_idx].size(), + frame_buffer_controller_->OnEncodeDone( + stream_idx, timestamp, encoded_images_[encoder_idx].size(), (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, qp, codec_specific); } @@ -988,13 +988,13 @@ int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image) { } else { ++num_steady_state_frames_; } - } else if (!frame_buffer_controllers_[stream_idx] - ->SupportsEncoderFrameDropping()) { + } else if (!frame_buffer_controller_->SupportsEncoderFrameDropping( + stream_idx)) { result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT; if (encoded_images_[encoder_idx].size() == 0) { // Dropped frame that will be re-encoded. - frame_buffer_controllers_[stream_idx]->OnEncodeDone( - input_image.timestamp(), 0, false, 0, nullptr); + frame_buffer_controller_->OnEncodeDone( + stream_idx, input_image.timestamp(), 0, false, 0, nullptr); } } } diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h index b0e1afb358..11a2e6a6ea 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h @@ -53,13 +53,15 @@ class LibvpxVp8Encoder : public VideoEncoder { int SetRateAllocation(const VideoBitrateAllocation& bitrate, uint32_t new_framerate) override; + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + EncoderInfo GetEncoderInfo() const override; static vpx_enc_frame_flags_t EncodeFlags(const Vp8FrameConfig& references); private: - void SetupTemporalLayers(const VideoCodec& codec); - // Get the cpu_speed setting for encoder based on resolution and/or platform. int GetCpuSpeed(int width, int height); @@ -100,8 +102,7 @@ class LibvpxVp8Encoder : public VideoEncoder { int cpu_speed_default_; int number_of_cores_; uint32_t rc_max_intra_target_; - std::vector> - frame_buffer_controllers_; + std::unique_ptr frame_buffer_controller_; std::vector key_frame_request_; std::vector send_stream_; std::vector cpu_speed_; diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index 17b1aed4d9..594afad605 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -72,12 +72,21 @@ ScreenshareLayers::~ScreenshareLayers() { UpdateHistograms(); } -bool ScreenshareLayers::SupportsEncoderFrameDropping() const { +size_t ScreenshareLayers::StreamCount() const { + return 1; +} + +bool ScreenshareLayers::SupportsEncoderFrameDropping( + size_t stream_index) const { + RTC_DCHECK_LT(stream_index, StreamCount()); // Frame dropping is handled internally by this class. return false; } -Vp8FrameConfig ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) { +Vp8FrameConfig ScreenshareLayers::UpdateLayerConfig(size_t stream_index, + uint32_t timestamp) { + RTC_DCHECK_LT(stream_index, StreamCount()); + auto it = pending_frame_configs_.find(timestamp); if (it != pending_frame_configs_.end()) { // Drop and re-encode, reuse the previous config. @@ -222,8 +231,10 @@ Vp8FrameConfig ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) { } void ScreenshareLayers::OnRatesUpdated( + size_t stream_index, const std::vector& bitrates_bps, int framerate_fps) { + RTC_DCHECK_LT(stream_index, StreamCount()); RTC_DCHECK_GT(framerate_fps, 0); RTC_DCHECK_GE(bitrates_bps.size(), 1); RTC_DCHECK_LE(bitrates_bps.size(), 2); @@ -261,11 +272,14 @@ void ScreenshareLayers::OnRatesUpdated( layers_[1].target_rate_kbps_ = tl1_kbps; } -void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp, +void ScreenshareLayers::OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, CodecSpecificInfo* info) { + RTC_DCHECK_LT(stream_index, StreamCount()); + if (size_bytes == 0) { layers_[active_layer_].state = TemporalLayer::State::kDropped; ++stats_.num_overshoots_; @@ -355,6 +369,10 @@ void ScreenshareLayers::OnEncodeDone(uint32_t rtp_timestamp, } } +void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} + +void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {} + TemplateStructure ScreenshareLayers::GetTemplateStructure( int num_layers) const { RTC_CHECK_LT(num_layers, 3); @@ -428,7 +446,10 @@ uint32_t ScreenshareLayers::GetCodecTargetBitrateKbps() const { return std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps); } -bool ScreenshareLayers::UpdateConfiguration(Vp8EncoderConfig* cfg) { +bool ScreenshareLayers::UpdateConfiguration(size_t stream_index, + Vp8EncoderConfig* cfg) { + RTC_DCHECK_LT(stream_index, StreamCount()); + if (min_qp_ == -1 || max_qp_ == -1) { // Store the valid qp range. This must not change during the lifetime of // this class. diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index 1c96796cc7..6b1bc7ec25 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -26,7 +26,7 @@ namespace webrtc { struct CodecSpecificInfoVP8; class Clock; -class ScreenshareLayers : public Vp8TemporalLayers { +class ScreenshareLayers final : public Vp8FrameBufferController { public: static const double kMaxTL0FpsReduction; static const double kAcceptableTargetOvershoot; @@ -35,26 +35,35 @@ class ScreenshareLayers : public Vp8TemporalLayers { explicit ScreenshareLayers(int num_temporal_layers); ~ScreenshareLayers() override; - bool SupportsEncoderFrameDropping() const override; + size_t StreamCount() const override; + + bool SupportsEncoderFrameDropping(size_t stream_index) const override; // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - Vp8FrameConfig UpdateLayerConfig(uint32_t rtp_timestamp) override; + Vp8FrameConfig UpdateLayerConfig(size_t stream_index, + uint32_t rtp_timestamp) override; // New target bitrate, per temporal layer. - void OnRatesUpdated(const std::vector& bitrates_bps, + void OnRatesUpdated(size_t stream_index, + const std::vector& bitrates_bps, int framerate_fps) override; // Update the encoder configuration with target bitrates or other parameters. // Returns true iff the configuration was actually modified. - bool UpdateConfiguration(Vp8EncoderConfig* cfg) override; + bool UpdateConfiguration(size_t stream_index, Vp8EncoderConfig* cfg) override; - void OnEncodeDone(uint32_t rtp_timestamp, + void OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, CodecSpecificInfo* info) override; + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + private: enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync }; diff --git a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index 16e4d468c7..b8aa43539f 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -79,13 +79,13 @@ class ScreenshareLayerTest : public ::testing::Test { int flags = ConfigureFrame(base_sync); if (flags != -1) - layers_->OnEncodeDone(timestamp_, frame_size_, base_sync, kDefaultQp, + layers_->OnEncodeDone(0, timestamp_, frame_size_, base_sync, kDefaultQp, info); return flags; } int ConfigureFrame(bool key_frame) { - tl_config_ = UpdateLayerConfig(timestamp_); + tl_config_ = UpdateLayerConfig(0, timestamp_); EXPECT_EQ(0, tl_config_.encoder_layer_id) << "ScreenshareLayers always encodes using the bitrate allocator for " "layer 0, but may reference different buffers and packetize " @@ -93,16 +93,16 @@ class ScreenshareLayerTest : public ::testing::Test { if (tl_config_.drop_frame) { return -1; } - config_updated_ = layers_->UpdateConfiguration(&cfg_); + config_updated_ = layers_->UpdateConfiguration(0, &cfg_); int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_); EXPECT_NE(-1, frame_size_); return flags; } - Vp8FrameConfig UpdateLayerConfig(uint32_t timestamp) { + Vp8FrameConfig UpdateLayerConfig(size_t stream_index, uint32_t timestamp) { int64_t timestamp_ms = timestamp / 90; clock_.AdvanceTime(TimeDelta::ms(timestamp_ms - rtc::TimeMillis())); - return layers_->UpdateLayerConfig(timestamp); + return layers_->UpdateLayerConfig(stream_index, timestamp); } int FrameSizeForBitrate(int bitrate_kbps) { @@ -114,8 +114,8 @@ class ScreenshareLayerTest : public ::testing::Test { memset(&vp8_cfg, 0, sizeof(Vp8EncoderConfig)); vp8_cfg.rc_min_quantizer = min_qp_; vp8_cfg.rc_max_quantizer = max_qp_; - layers_->OnRatesUpdated(kDefault2TlBitratesBps, kFrameRate); - EXPECT_TRUE(layers_->UpdateConfiguration(&vp8_cfg)); + layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate); + EXPECT_TRUE(layers_->UpdateConfiguration(0, &vp8_cfg)); frame_size_ = FrameSizeForBitrate(vp8_cfg.rc_target_bitrate); return vp8_cfg; } @@ -162,7 +162,7 @@ class ScreenshareLayerTest : public ::testing::Test { if (tl_config_.packetizer_temporal_idx != layer || (sync && *sync != tl_config_.layer_sync)) { CodecSpecificInfo info; - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, &info); timestamp_ += kTimestampDelta5Fps; } else { @@ -238,14 +238,15 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { for (int i = 0; i < kNumFrames; ++i) { CodecSpecificInfo info; - tl_config_ = UpdateLayerConfig(timestamp_); - config_updated_ = layers_->UpdateConfiguration(&cfg_); + tl_config_ = UpdateLayerConfig(0, timestamp_); + config_updated_ = layers_->UpdateConfiguration(0, &cfg_); // Simulate TL1 being at least 8 qp steps better. if (tl_config_.packetizer_temporal_idx == 0) { - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &info); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + &info); } else { - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8, &info); } @@ -273,9 +274,10 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { // Simulate TL1 being at least 8 qp steps better. if (tl_config_.packetizer_temporal_idx == 0) { - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &info); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + &info); } else { - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8, &info); } @@ -293,7 +295,7 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { CodecSpecificInfo info; int flags = ConfigureFrame(false); - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp - 8, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8, &info); if (info.codecSpecific.VP8.temporalIdx == 0) { // Bump TL0 to same quality as TL1. @@ -384,8 +386,8 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { const int kTl1_kbps = 1000; const std::vector layer_rates = {kTl0_kbps * 1000, (kTl1_kbps - kTl0_kbps) * 1000}; - layers_->OnRatesUpdated(layer_rates, kFrameRate); - EXPECT_TRUE(layers_->UpdateConfiguration(&cfg_)); + layers_->OnRatesUpdated(0, layer_rates, kFrameRate); + EXPECT_TRUE(layers_->UpdateConfiguration(0, &cfg_)); EXPECT_EQ(static_cast( ScreenshareLayers::kMaxTL0FpsReduction * kTl0_kbps + 0.5), @@ -397,8 +399,8 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { const int kTl1_kbps = 450; const std::vector layer_rates = {kTl0_kbps * 1000, (kTl1_kbps - kTl0_kbps) * 1000}; - layers_->OnRatesUpdated(layer_rates, kFrameRate); - EXPECT_TRUE(layers_->UpdateConfiguration(&cfg_)); + layers_->OnRatesUpdated(0, layer_rates, kFrameRate); + EXPECT_TRUE(layers_->UpdateConfiguration(0, &cfg_)); EXPECT_EQ(static_cast( kTl1_kbps / ScreenshareLayers::kAcceptableTargetOvershoot), @@ -408,8 +410,8 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { const int kTl0_kbps = 100; const std::vector layer_rates = {kTl0_kbps * 1000}; - layers_->OnRatesUpdated(layer_rates, kFrameRate); - EXPECT_TRUE(layers_->UpdateConfiguration(&cfg_)); + layers_->OnRatesUpdated(0, layer_rates, kFrameRate); + EXPECT_TRUE(layers_->UpdateConfiguration(0, &cfg_)); EXPECT_EQ(static_cast(kTl0_kbps), cfg_.rc_target_bitrate); } @@ -419,7 +421,7 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { SkipUntilTl(0); // Size 0 indicates dropped frame. - layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfo()); + layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo()); // Re-encode frame (so don't advance timestamp). int flags = EncodeFrame(false); @@ -431,20 +433,20 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { SkipUntilTl(0); EXPECT_TRUE(config_updated_); EXPECT_LT(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, IgnoredCodecSpecificInfo()); timestamp_ += kTimestampDelta5Fps; // ...then back to standard setup. SkipUntilTl(0); - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, IgnoredCodecSpecificInfo()); timestamp_ += kTimestampDelta5Fps; EXPECT_EQ(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); // Next drop in TL1. SkipUntilTl(1); - layers_->OnEncodeDone(timestamp_, 0, false, 0, IgnoredCodecSpecificInfo()); + layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo()); // Re-encode frame (so don't advance timestamp). flags = EncodeFrame(false); @@ -456,14 +458,14 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { SkipUntilTl(1); EXPECT_TRUE(config_updated_); EXPECT_LT(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, IgnoredCodecSpecificInfo()); timestamp_ += kTimestampDelta5Fps; // ...and back to normal. SkipUntilTl(1); EXPECT_EQ(cfg_.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, IgnoredCodecSpecificInfo()); timestamp_ += kTimestampDelta5Fps; } @@ -474,12 +476,12 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { const uint32_t kStartTimestamp = 1234; const std::vector layer_rates = {kLowBitrateKbps * 1000}; - layers_->OnRatesUpdated(layer_rates, kFrameRate); - layers_->UpdateConfiguration(&cfg_); + layers_->OnRatesUpdated(0, layer_rates, kFrameRate); + layers_->UpdateConfiguration(0, &cfg_); - EXPECT_EQ(kTl0Flags, - LibvpxVp8Encoder::EncodeFlags(UpdateLayerConfig(kStartTimestamp))); - layers_->OnEncodeDone(kStartTimestamp, kLargeFrameSizeBytes, false, + EXPECT_EQ(kTl0Flags, LibvpxVp8Encoder::EncodeFlags( + UpdateLayerConfig(0, kStartTimestamp))); + layers_->OnEncodeDone(0, kStartTimestamp, kLargeFrameSizeBytes, false, kDefaultQp, IgnoredCodecSpecificInfo()); const uint32_t kTwoSecondsLater = @@ -493,12 +495,12 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { // any later, the frame will be dropped anyway by the frame rate throttling // logic. EXPECT_TRUE( - UpdateLayerConfig(kTwoSecondsLater - kTimestampDelta5Fps).drop_frame); + UpdateLayerConfig(0, kTwoSecondsLater - kTimestampDelta5Fps).drop_frame); // More than two seconds has passed since last frame, one should be emitted // even if bitrate target is then exceeded. EXPECT_EQ(kTl0Flags, LibvpxVp8Encoder::EncodeFlags( - UpdateLayerConfig(kTwoSecondsLater + 90))); + UpdateLayerConfig(0, kTwoSecondsLater + 90))); } TEST_F(ScreenshareLayerTest, UpdatesHistograms) { @@ -511,33 +513,33 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) { for (int64_t timestamp = 0; timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds; timestamp += kTimestampDelta5Fps) { - tl_config_ = UpdateLayerConfig(timestamp); + tl_config_ = UpdateLayerConfig(0, timestamp); if (tl_config_.drop_frame) { dropped_frame = true; continue; } int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_); if (flags != -1) - layers_->UpdateConfiguration(&cfg_); + layers_->UpdateConfiguration(0, &cfg_); if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) { // Simulate one overshoot. - layers_->OnEncodeDone(timestamp, 0, false, 0, nullptr); + layers_->OnEncodeDone(0, timestamp, 0, false, 0, nullptr); overshoot = true; } if (flags == kTl0Flags) { if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) { // Simulate a too large frame, to cause frame drop. - layers_->OnEncodeDone(timestamp, frame_size_ * 10, false, kTl0Qp, + layers_->OnEncodeDone(0, timestamp, frame_size_ * 10, false, kTl0Qp, IgnoredCodecSpecificInfo()); trigger_drop = true; } else { - layers_->OnEncodeDone(timestamp, frame_size_, false, kTl0Qp, + layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl0Qp, IgnoredCodecSpecificInfo()); } } else if (flags == kTl1Flags || flags == kTl1SyncFlags) { - layers_->OnEncodeDone(timestamp, frame_size_, false, kTl1Qp, + layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl1Qp, IgnoredCodecSpecificInfo()); } else if (flags == -1) { dropped_frame = true; @@ -587,7 +589,7 @@ TEST_F(ScreenshareLayerTest, AllowsUpdateConfigBeforeSetRates) { layers_.reset(new ScreenshareLayers(2)); // New layer instance, OnRatesUpdated() never called. // UpdateConfiguration() call should not cause crash. - layers_->UpdateConfiguration(&cfg_); + layers_->UpdateConfiguration(0, &cfg_); } TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { @@ -600,11 +602,11 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { // Send at regular rate - no drops expected. for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs) { - if (UpdateLayerConfig(timestamp).drop_frame) { + if (UpdateLayerConfig(0, timestamp).drop_frame) { ++num_discarded_frames; } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; - layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp, + layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp, IgnoredCodecSpecificInfo()); } timestamp += kFrameIntervalsMs * 90; @@ -618,11 +620,11 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { num_input_frames = 0; num_discarded_frames = 0; for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs / 2) { - if (UpdateLayerConfig(timestamp).drop_frame) { + if (UpdateLayerConfig(0, timestamp).drop_frame) { ++num_discarded_frames; } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; - layers_->OnEncodeDone(timestamp, frame_size_bytes, false, kDefaultQp, + layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp, IgnoredCodecSpecificInfo()); } timestamp += kFrameIntervalsMs * 90 / 2; @@ -643,13 +645,14 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) { ASSERT_TRUE(tl_config_.layer_sync); // Simulate overshoot of this frame. - layers_->OnEncodeDone(timestamp_, 0, false, 0, nullptr); + layers_->OnEncodeDone(0, timestamp_, 0, false, 0, nullptr); - config_updated_ = layers_->UpdateConfiguration(&cfg_); + config_updated_ = layers_->UpdateConfiguration(0, &cfg_); EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_)); CodecSpecificInfo new_info; - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, &new_info); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + &new_info); EXPECT_TRUE(new_info.codecSpecific.VP8.layerSync); } @@ -659,26 +662,26 @@ TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) { // Add a large gap, so there's plenty of room in the rate tracker. timestamp_ += kTimestampDelta5Fps * 3; - EXPECT_FALSE(UpdateLayerConfig(timestamp_).drop_frame); - layers_->OnEncodeDone(timestamp_, frame_size_, false, kDefaultQp, + EXPECT_FALSE(UpdateLayerConfig(0, timestamp_).drop_frame); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, IgnoredCodecSpecificInfo()); // Frame interval below 90% if desired time is not allowed, try inserting // frame just before this limit. const int64_t kMinFrameInterval = (kTimestampDelta5Fps * 85) / 100; timestamp_ += kMinFrameInterval - 90; - EXPECT_TRUE(UpdateLayerConfig(timestamp_).drop_frame); + EXPECT_TRUE(UpdateLayerConfig(0, timestamp_).drop_frame); // Try again at the limit, now it should pass. timestamp_ += 90; - EXPECT_FALSE(UpdateLayerConfig(timestamp_).drop_frame); + EXPECT_FALSE(UpdateLayerConfig(0, timestamp_).drop_frame); } TEST_F(ScreenshareLayerTest, AdjustsBitrateWhenDroppingFrames) { const uint32_t kTimestampDelta10Fps = kTimestampDelta5Fps / 2; const int kNumFrames = 30; uint32_t default_bitrate = cfg_.rc_target_bitrate; - layers_->OnRatesUpdated(kDefault2TlBitratesBps, 10); + layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, 10); int num_dropped_frames = 0; for (int i = 0; i < kNumFrames; ++i) { @@ -686,7 +689,7 @@ TEST_F(ScreenshareLayerTest, AdjustsBitrateWhenDroppingFrames) { ++num_dropped_frames; timestamp_ += kTimestampDelta10Fps; } - layers_->UpdateConfiguration(&cfg_); + layers_->UpdateConfiguration(0, &cfg_); EXPECT_EQ(num_dropped_frames, kNumFrames / 2); EXPECT_EQ(cfg_.rc_target_bitrate, default_bitrate * 2); @@ -694,21 +697,21 @@ TEST_F(ScreenshareLayerTest, AdjustsBitrateWhenDroppingFrames) { TEST_F(ScreenshareLayerTest, UpdatesConfigurationAfterRateChange) { // Set inital rate again, no need to update configuration. - layers_->OnRatesUpdated(kDefault2TlBitratesBps, kFrameRate); - EXPECT_FALSE(layers_->UpdateConfiguration(&cfg_)); + layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate); + EXPECT_FALSE(layers_->UpdateConfiguration(0, &cfg_)); // Rate changed, now update config. std::vector bitrates = kDefault2TlBitratesBps; bitrates[1] -= 100000; - layers_->OnRatesUpdated(bitrates, 5); - EXPECT_TRUE(layers_->UpdateConfiguration(&cfg_)); + layers_->OnRatesUpdated(0, bitrates, 5); + EXPECT_TRUE(layers_->UpdateConfiguration(0, &cfg_)); // Changed rate, but then set changed rate again before trying to update // configuration, update should still apply. bitrates[1] -= 100000; - layers_->OnRatesUpdated(bitrates, 5); - layers_->OnRatesUpdated(bitrates, 5); - EXPECT_TRUE(layers_->UpdateConfiguration(&cfg_)); + layers_->OnRatesUpdated(0, bitrates, 5); + layers_->OnRatesUpdated(0, bitrates, 5); + EXPECT_TRUE(layers_->UpdateConfiguration(0, &cfg_)); } // TODO(bugs.webrtc.org/10260): Fix. @@ -721,42 +724,42 @@ TEST_F(ScreenshareLayerTest, DISABLED_MaxQpRestoredAfterDoubleDrop) { ASSERT_TRUE(tl_config_.layer_sync); // Simulate overshoot of this frame. - layers_->OnEncodeDone(timestamp_, 0, false, -1, nullptr); + layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr); // Simulate re-encoded frame. - layers_->OnEncodeDone(timestamp_, 1, false, max_qp_, + layers_->OnEncodeDone(0, timestamp_, 1, false, max_qp_, IgnoredCodecSpecificInfo()); // Next frame, expect boosted quality. // Slightly alter bitrate between each frame. std::vector kDefault2TlBitratesBpsAlt = kDefault2TlBitratesBps; kDefault2TlBitratesBpsAlt[1] += 4000; - layers_->OnRatesUpdated(kDefault2TlBitratesBpsAlt, kFrameRate); + layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate); EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false)); EXPECT_TRUE(config_updated_); EXPECT_LT(cfg_.rc_max_quantizer, max_qp_); uint32_t adjusted_qp = cfg_.rc_max_quantizer; // Simulate overshoot of this frame. - layers_->OnEncodeDone(timestamp_, 0, false, -1, nullptr); + layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr); // Simulate re-encoded frame. - layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_, IgnoredCodecSpecificInfo()); // A third frame, expect boosted quality. - layers_->OnRatesUpdated(kDefault2TlBitratesBps, kFrameRate); + layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate); EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false)); EXPECT_TRUE(config_updated_); EXPECT_LT(cfg_.rc_max_quantizer, max_qp_); EXPECT_EQ(adjusted_qp, cfg_.rc_max_quantizer); // Frame encoded. - layers_->OnEncodeDone(timestamp_, frame_size_, false, max_qp_, + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_, IgnoredCodecSpecificInfo()); // A fourth frame, max qp should be restored. - layers_->OnRatesUpdated(kDefault2TlBitratesBpsAlt, kFrameRate); + layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate); EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false)); EXPECT_EQ(cfg_.rc_max_quantizer, max_qp_); } diff --git a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index 4744e38d25..6a6604bd55 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -35,14 +35,14 @@ constexpr uint32_t kLegacyScreenshareMaxBitrateKbps = 1000; constexpr uint32_t kSimulcastScreenshareMinBitrateKbps = 600; constexpr uint32_t kSimulcastScreenshareMaxBitrateKbps = 1250; -class MockTemporalLayers : public Vp8TemporalLayers { +class MockTemporalLayers : public Vp8FrameBufferController { public: - MOCK_METHOD1(UpdateLayerConfig, Vp8FrameConfig(uint32_t)); - MOCK_METHOD2(OnRatesUpdated, void(const std::vector&, int)); - MOCK_METHOD1(UpdateConfiguration, bool(Vp8EncoderConfig*)); - MOCK_METHOD5(OnEncodeDone, - void(uint32_t, size_t, bool, int, CodecSpecificInfo*)); - MOCK_METHOD3(FrameEncoded, void(uint32_t, size_t, int)); + MOCK_METHOD2(UpdateLayerConfig, Vp8FrameConfig(size_t, uint32_t)); + MOCK_METHOD3(OnRatesUpdated, void(size_t, const std::vector&, int)); + MOCK_METHOD2(UpdateConfiguration, bool(size_t, Vp8EncoderConfig*)); + MOCK_METHOD6(OnEncodeDone, + void(size_t, uint32_t, size_t, bool, int, CodecSpecificInfo*)); + MOCK_METHOD4(FrameEncoded, void(size_t, uint32_t, size_t, int)); MOCK_CONST_METHOD0(Tl0PicIdx, uint8_t()); MOCK_CONST_METHOD1(GetTemporalLayerId, int(const Vp8FrameConfig&)); }; diff --git a/modules/video_coding/video_codec_initializer_unittest.cc b/modules/video_coding/video_codec_initializer_unittest.cc index 0f2d9f19b9..0eca06de79 100644 --- a/modules/video_coding/video_codec_initializer_unittest.cc +++ b/modules/video_coding/video_codec_initializer_unittest.cc @@ -20,9 +20,9 @@ #include "api/video/video_bitrate_allocation.h" #include "api/video/video_bitrate_allocator.h" #include "api/video/video_bitrate_allocator_factory.h" -#include "api/video_codecs/create_vp8_temporal_layers.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/vp8_temporal_layers.h" +#include "api/video_codecs/vp8_temporal_layers_factory.h" #include "common_types.h" // NOLINT(build/include) #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "rtc_base/checks.h" @@ -86,7 +86,7 @@ class VideoCodecInitializerTest : public ::testing::Test { bool InitializeCodec() { codec_out_ = VideoCodec(); - temporal_layers_.clear(); + frame_buffer_controller_.reset(); if (!VideoCodecInitializer::SetupCodec(config_, streams_, &codec_out_)) { return false; } @@ -98,11 +98,8 @@ class VideoCodecInitializerTest : public ::testing::Test { // Make sure temporal layers instances have been created. if (codec_out_.codecType == VideoCodecType::kVideoCodecVP8) { - for (int i = 0; i < codec_out_.numberOfSimulcastStreams; ++i) { - temporal_layers_.emplace_back( - CreateVp8TemporalLayers(Vp8TemporalLayersType::kFixedPattern, - codec_out_.VP8()->numberOfTemporalLayers)); - } + Vp8TemporalLayersFactory factory; + frame_buffer_controller_ = factory.Create(codec_out_); } return true; } @@ -139,7 +136,7 @@ class VideoCodecInitializerTest : public ::testing::Test { // Output. VideoCodec codec_out_; std::unique_ptr bitrate_allocator_; - std::vector> temporal_layers_; + std::unique_ptr frame_buffer_controller_; }; TEST_F(VideoCodecInitializerTest, SingleStreamVp8Screenshare) { diff --git a/test/BUILD.gn b/test/BUILD.gn index b07a1f8a4d..bce875fce9 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -648,8 +648,8 @@ rtc_source_set("fake_video_codecs") { "../api/video:video_bitrate_allocation", "../api/video:video_frame", "../api/video:video_frame_i420", - "../api/video_codecs:create_vp8_temporal_layers", "../api/video_codecs:video_codecs_api", + "../api/video_codecs:vp8_temporal_layers_factory", "../common_video:common_video", "../modules:module_api", "../modules/video_coding:codec_globals_headers", diff --git a/test/fake_vp8_encoder.cc b/test/fake_vp8_encoder.cc index 6ad40947a4..c9adee4242 100644 --- a/test/fake_vp8_encoder.cc +++ b/test/fake_vp8_encoder.cc @@ -13,8 +13,8 @@ #include #include "absl/types/optional.h" -#include "api/video_codecs/create_vp8_temporal_layers.h" #include "api/video_codecs/vp8_temporal_layers.h" +#include "api/video_codecs/vp8_temporal_layers_factory.h" #include "common_types.h" // NOLINT(build/include) #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -59,7 +59,8 @@ int32_t FakeVP8Encoder::InitEncode(const VideoCodec* config, return result; } - SetupTemporalLayers(*config); + Vp8TemporalLayersFactory factory; + frame_buffer_controller_ = factory.Create(*config); return WEBRTC_VIDEO_CODEC_OK; } @@ -70,26 +71,6 @@ int32_t FakeVP8Encoder::Release() { return result; } -void FakeVP8Encoder::SetupTemporalLayers(const VideoCodec& codec) { - RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); - - int num_streams = SimulcastUtility::NumberOfSimulcastStreams(codec); - for (int i = 0; i < num_streams; ++i) { - Vp8TemporalLayersType type; - int num_temporal_layers = - SimulcastUtility::NumberOfTemporalLayers(codec, i); - if (SimulcastUtility::IsConferenceModeScreenshare(codec) && i == 0) { - type = Vp8TemporalLayersType::kBitrateDynamic; - // Legacy screenshare layers supports max 2 layers. - num_temporal_layers = std::max(2, num_temporal_layers); - } else { - type = Vp8TemporalLayersType::kFixedPattern; - } - temporal_layers_.emplace_back( - CreateVp8TemporalLayers(type, num_temporal_layers)); - } -} - void FakeVP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, size_t size_bytes, VideoFrameType frame_type, @@ -99,8 +80,9 @@ void FakeVP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, codec_specific->codecType = kVideoCodecVP8; codec_specific->codecSpecific.VP8.keyIdx = kNoKeyIdx; codec_specific->codecSpecific.VP8.nonReference = false; - temporal_layers_[stream_idx]->OnEncodeDone( - timestamp, size_bytes, frame_type == kVideoFrameKey, -1, codec_specific); + frame_buffer_controller_->OnEncodeDone(stream_idx, timestamp, size_bytes, + frame_type == kVideoFrameKey, -1, + codec_specific); } std::unique_ptr FakeVP8Encoder::EncodeHook( @@ -108,7 +90,8 @@ std::unique_ptr FakeVP8Encoder::EncodeHook( CodecSpecificInfo* codec_specific) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); uint8_t stream_idx = encoded_image->SpatialIndex().value_or(0); - temporal_layers_[stream_idx]->UpdateLayerConfig(encoded_image->Timestamp()); + frame_buffer_controller_->UpdateLayerConfig(stream_idx, + encoded_image->Timestamp()); PopulateCodecSpecific(codec_specific, encoded_image->size(), encoded_image->_frameType, stream_idx, encoded_image->Timestamp()); diff --git a/test/fake_vp8_encoder.h b/test/fake_vp8_encoder.h index 1906e7418d..23c6551a31 100644 --- a/test/fake_vp8_encoder.h +++ b/test/fake_vp8_encoder.h @@ -13,12 +13,13 @@ #include #include + #include -#include #include "api/video/encoded_image.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/vp8_frame_buffer_controller.h" #include "api/video_codecs/vp8_temporal_layers.h" #include "common_types.h" // NOLINT(build/include) #include "modules/include/module_common_types.h" @@ -45,7 +46,6 @@ class FakeVP8Encoder : public FakeEncoder { EncoderInfo GetEncoderInfo() const override; private: - void SetupTemporalLayers(const VideoCodec& codec); void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, size_t size_bytes, VideoFrameType frame_type, @@ -58,7 +58,7 @@ class FakeVP8Encoder : public FakeEncoder { rtc::SequencedTaskChecker sequence_checker_; - std::vector> temporal_layers_ + std::unique_ptr frame_buffer_controller_ RTC_GUARDED_BY(sequence_checker_); }; diff --git a/video/BUILD.gn b/video/BUILD.gn index a9a3c04954..7a6537c85f 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -518,8 +518,8 @@ if (rtc_include_tests) { "../api/video:video_bitrate_allocation", "../api/video:video_frame", "../api/video:video_frame_i420", - "../api/video_codecs:create_vp8_temporal_layers", "../api/video_codecs:video_codecs_api", + "../api/video_codecs:vp8_temporal_layers_factory", "../call:call_interfaces", "../call:fake_network", "../call:mock_bitrate_allocator", diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 449286fd7a..d68af8db2c 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1455,6 +1455,11 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, has_seen_first_significant_bwe_change_ = true; } + if (encoder_) { + encoder_->OnPacketLossRateUpdate(static_cast(fraction_lost) / 256.f); + encoder_->OnRttUpdate(round_trip_time_ms); + } + uint32_t framerate_fps = GetInputFramerateFps(); frame_dropper_.SetRates((target_bitrate.bps() + 500) / 1000, framerate_fps); SetEncoderRates(GetBitrateAllocationAndNotifyObserver(target_bitrate.bps(), diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index e2453a42a6..a9f1c4d9be 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -18,8 +18,8 @@ #include "api/video/builtin_video_bitrate_allocator_factory.h" #include "api/video/i420_buffer.h" #include "api/video/video_bitrate_allocation.h" -#include "api/video_codecs/create_vp8_temporal_layers.h" #include "api/video_codecs/vp8_temporal_layers.h" +#include "api/video_codecs/vp8_temporal_layers_factory.h" #include "media/base/video_adapter.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/utility/default_video_bitrate_allocator.h" @@ -678,12 +678,8 @@ class VideoStreamEncoderTest : public ::testing::Test { if (config->codecType == kVideoCodecVP8) { // Simulate setting up temporal layers, in order to validate the life // cycle of these objects. - int num_streams = std::max(1, config->numberOfSimulcastStreams); - for (int i = 0; i < num_streams; ++i) { - allocated_temporal_layers_.emplace_back( - CreateVp8TemporalLayers(Vp8TemporalLayersType::kFixedPattern, - config->VP8().numberOfTemporalLayers)); - } + Vp8TemporalLayersFactory factory; + frame_buffer_controller_ = factory.Create(*config); } if (force_init_encode_failed_) { initialized_ = EncoderState::kInitializationFailed; @@ -736,7 +732,7 @@ class VideoStreamEncoderTest : public ::testing::Test { int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0; bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true; bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false; - std::vector> allocated_temporal_layers_ + std::unique_ptr frame_buffer_controller_ RTC_GUARDED_BY(local_crit_sect_); bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false; double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;