From fb2a66a58ac3e61162804d5fcdc377e465852d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Spr=C3=A5ng?= Date: Mon, 10 Sep 2018 10:48:01 +0200 Subject: [PATCH] libvpx vp8 encoder: get frame drop setting from temporal layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Today, the internal frame dropper in libvpx vp8 encoder is enabled or disabled based on video or screen content. This is then expected to match up with screenshare vs default temporal layers implementation. This cl makes libvpx query the temporal layers implementation as well, breaking this implicit dependency and allows frames to be dropped if default temporal layers is used with screen content. Bug: webrtc:9734 Change-Id: If2523a211f4929f16e65a02fa7a6b4edf7328571 Reviewed-on: https://webrtc-review.googlesource.com/99062 Commit-Queue: Erik Språng Reviewed-by: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#24645} --- .../codecs/vp8/default_temporal_layers.cc | 5 +++++ .../codecs/vp8/default_temporal_layers.h | 2 ++ .../codecs/vp8/libvpx_vp8_encoder.cc | 18 +++++++++++++++++- .../codecs/vp8/libvpx_vp8_encoder.h | 2 ++ .../codecs/vp8/screenshare_layers.cc | 5 +++++ .../codecs/vp8/screenshare_layers.h | 4 +++- .../video_coding/codecs/vp8/temporal_layers.h | 2 ++ 7 files changed, 36 insertions(+), 2 deletions(-) diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 8389eff16e..b6884da8fb 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -266,6 +266,11 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) } } +bool DefaultTemporalLayers::SupportsEncoderFrameDropping() const { + // This class allows the encoder drop frames as it sees fit. + return true; +} + void DefaultTemporalLayers::OnRatesUpdated( const std::vector& bitrates_bps, int framerate_fps) { diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index 9978a9ae24..691447964c 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -28,6 +28,8 @@ class DefaultTemporalLayers : public TemporalLayers { explicit DefaultTemporalLayers(int number_of_temporal_layers); virtual ~DefaultTemporalLayers() {} + bool SupportsEncoderFrameDropping() const override; + // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) override; diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index 3d2a5c9542..72e5f872d8 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -399,7 +399,7 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, (num_temporal_layers > 1) ? VPX_ERROR_RESILIENT_DEFAULT : 0; // rate control settings - configurations_[0].rc_dropframe_thresh = inst->VP8().frameDroppingOn ? 30 : 0; + configurations_[0].rc_dropframe_thresh = FrameDropThreshold(0); configurations_[0].rc_end_usage = VPX_CBR; configurations_[0].g_pass = VPX_RC_ONE_PASS; // Handle resizing outside of libvpx. @@ -481,6 +481,7 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, } UpdateVpxConfiguration(temporal_layers_[stream_idx].get(), &configurations_[0]); + configurations_[0].rc_dropframe_thresh = FrameDropThreshold(stream_idx); --stream_idx; for (size_t i = 1; i < encoders_.size(); ++i, --stream_idx) { @@ -493,6 +494,8 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, // Use 1 thread for lower resolutions. configurations_[i].g_threads = 1; + configurations_[i].rc_dropframe_thresh = FrameDropThreshold(stream_idx); + // Setting alignment to 32 - as that ensures at least 16 for all // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for // the y plane, but only half of it to the u and v planes. @@ -654,6 +657,19 @@ uint32_t LibvpxVp8Encoder::MaxIntraTarget(uint32_t optimalBuffersize) { return (targetPct < minIntraTh) ? minIntraTh : targetPct; } +uint32_t LibvpxVp8Encoder::FrameDropThreshold(size_t spatial_idx) const { + bool enable_frame_dropping = codec_.VP8().frameDroppingOn; + // If temporal layers are used, they get to override the frame dropping + // setting, as eg. ScreenshareLayers does not work as intended with frame + // dropping on and DefaultTemporalLayers will have performance issues with + // frame dropping off. + if (temporal_layers_.size() <= spatial_idx) { + enable_frame_dropping = + temporal_layers_[spatial_idx]->SupportsEncoderFrameDropping(); + } + return enable_frame_dropping ? 30 : 0; +} + int LibvpxVp8Encoder::Encode(const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) { diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h index 4e9d374c4c..b1889fcd75 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h @@ -84,6 +84,8 @@ class LibvpxVp8Encoder : public VP8Encoder { uint32_t MaxIntraTarget(uint32_t optimal_buffer_size); + uint32_t FrameDropThreshold(size_t spatial_idx) const; + const bool use_gf_boost_; EncodedImageCallback* encoded_complete_callback_; diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index ccdaa10f1c..0117d12ccb 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -60,6 +60,11 @@ ScreenshareLayers::~ScreenshareLayers() { UpdateHistograms(); } +bool ScreenshareLayers::SupportsEncoderFrameDropping() const { + // Frame dropping is handled internally by this class. + return false; +} + TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig( uint32_t timestamp) { if (number_of_temporal_layers_ <= 1) { diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index 873d525938..fcc3aca650 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -29,7 +29,9 @@ class ScreenshareLayers : public TemporalLayers { ScreenshareLayers(int num_temporal_layers, Clock* clock); - virtual ~ScreenshareLayers(); + ~ScreenshareLayers() override; + + bool SupportsEncoderFrameDropping() const override; // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. diff --git a/modules/video_coding/codecs/vp8/temporal_layers.h b/modules/video_coding/codecs/vp8/temporal_layers.h index 20da1a44fa..91b5531128 100644 --- a/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/modules/video_coding/codecs/vp8/temporal_layers.h @@ -147,6 +147,8 @@ class TemporalLayers { virtual ~TemporalLayers() = default; + virtual bool SupportsEncoderFrameDropping() const = 0; + // New target bitrate, per temporal layer. virtual void OnRatesUpdated(const std::vector& bitrates_bps, int framerate_fps) = 0;