From be71a1ee08c9ee56ba0f92955ea946556ae6593e Mon Sep 17 00:00:00 2001 From: Sergey Silkin Date: Thu, 17 May 2018 16:46:43 +0200 Subject: [PATCH] Replace VP9 screen sharing. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove referencing control from encoder wrapper. Use fixed temporal prediction structure. - Remove flexible mode from encoder wrapper. It only worked with referencing control which this CL removes. - Remove external framerate/bitrate controller. Keep codec's internal frame dropping enabled at screen sharing. - Use GetSvcConfig() to configure layering. Bug: webrtc:9261 Change-Id: I355baa6aab7b98ac5028b3851d1f8ccc82a308e0 Reviewed-on: https://webrtc-review.googlesource.com/76801 Reviewed-by: Stefan Holmer Reviewed-by: Erik Språng Commit-Queue: Sergey Silkin Cr-Commit-Position: refs/heads/master@{#23311} --- media/engine/webrtcvideoengine.cc | 45 ++- media/engine/webrtcvideoengine_unittest.cc | 6 +- modules/video_coding/BUILD.gn | 5 - .../codecs/vp9/screenshare_layers.cc | 93 ----- .../codecs/vp9/screenshare_layers.h | 66 ---- modules/video_coding/codecs/vp9/vp9_impl.cc | 195 ++--------- modules/video_coding/codecs/vp9/vp9_impl.h | 32 +- .../vp9/vp9_screenshare_layers_unittest.cc | 324 ------------------ .../video_coding/video_codec_initializer.cc | 78 ++--- video/video_send_stream_tests.cc | 5 +- 10 files changed, 92 insertions(+), 757 deletions(-) delete mode 100644 modules/video_coding/codecs/vp9/screenshare_layers.cc delete mode 100644 modules/video_coding/codecs/vp9/screenshare_layers.h delete mode 100644 modules/video_coding/codecs/vp9/vp9_screenshare_layers_unittest.cc diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc index 29766a771d..2b20be059b 100644 --- a/media/engine/webrtcvideoengine.cc +++ b/media/engine/webrtcvideoengine.cc @@ -455,37 +455,32 @@ WebRtcVideoChannel::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( if (CodecNamesEq(codec.name, kVp9CodecName)) { webrtc::VideoCodecVP9 vp9_settings = webrtc::VideoEncoder::GetDefaultVp9Settings(); - if (is_screencast) { - // TODO(asapersson): Set to 2 for now since there is a DCHECK in - // VideoSendStream::ReconfigureVideoEncoder. - vp9_settings.numberOfSpatialLayers = 2; - vp9_settings.numberOfTemporalLayers = 1; - } else { - const size_t default_num_spatial_layers = - parameters_.config.rtp.ssrcs.size(); - const size_t num_spatial_layers = - GetVp9SpatialLayersFromFieldTrial().value_or( - default_num_spatial_layers); + const size_t default_num_spatial_layers = + parameters_.config.rtp.ssrcs.size(); + const size_t num_spatial_layers = + GetVp9SpatialLayersFromFieldTrial().value_or( + default_num_spatial_layers); - const size_t default_num_temporal_layers = - num_spatial_layers > 1 ? kConferenceDefaultNumTemporalLayers : 1; - const size_t num_temporal_layers = - GetVp9TemporalLayersFromFieldTrial().value_or( - default_num_temporal_layers); + const size_t default_num_temporal_layers = + num_spatial_layers > 1 ? kConferenceDefaultNumTemporalLayers : 1; + const size_t num_temporal_layers = + GetVp9TemporalLayersFromFieldTrial().value_or( + default_num_temporal_layers); - vp9_settings.numberOfSpatialLayers = std::min( - num_spatial_layers, kConferenceMaxNumSpatialLayers); - vp9_settings.numberOfTemporalLayers = std::min( - num_temporal_layers, kConferenceMaxNumTemporalLayers); - - // Limit inter-layer prediction to key pictures. - vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOnKeyPic; - } + vp9_settings.numberOfSpatialLayers = std::min( + num_spatial_layers, kConferenceMaxNumSpatialLayers); + vp9_settings.numberOfTemporalLayers = std::min( + num_temporal_layers, kConferenceMaxNumTemporalLayers); // VP9 denoising is disabled by default. vp9_settings.denoisingOn = codec_default_denoising ? true : denoising; - vp9_settings.frameDroppingOn = frame_dropping; vp9_settings.automaticResizeOn = automatic_resize; + // Ensure frame dropping is always enabled. + RTC_DCHECK(vp9_settings.frameDroppingOn); + if (!is_screencast) { + // Limit inter-layer prediction to key pictures. + vp9_settings.interLayerPred = webrtc::InterLayerPredMode::kOnKeyPic; + } return new rtc::RefCountedObject< webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings); } diff --git a/media/engine/webrtcvideoengine_unittest.cc b/media/engine/webrtcvideoengine_unittest.cc index 14723cde91..b368abe9ce 100644 --- a/media/engine/webrtcvideoengine_unittest.cc +++ b/media/engine/webrtcvideoengine_unittest.cc @@ -2962,14 +2962,14 @@ TEST_F(Vp9SettingsTest, VerifyVp9SpecificSettings) { ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; EXPECT_FALSE(vp9_settings.denoisingOn); - // Frame dropping always off for screen sharing. - EXPECT_FALSE(vp9_settings.frameDroppingOn); + // Frame dropping always on for screen sharing. + EXPECT_TRUE(vp9_settings.frameDroppingOn); stream = SetDenoisingOption(last_ssrc_, &capturer, false); ASSERT_TRUE(stream->GetVp9Settings(&vp9_settings)) << "No VP9 config set."; EXPECT_FALSE(vp9_settings.denoisingOn); - EXPECT_FALSE(vp9_settings.frameDroppingOn); + EXPECT_TRUE(vp9_settings.frameDroppingOn); EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); } diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index d5a76a0ed8..269b2ca7e8 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -477,8 +477,6 @@ rtc_static_library("webrtc_vp9") { if (rtc_libvpx_build_vp9) { sources = [ "codecs/vp9/include/vp9.h", - "codecs/vp9/screenshare_layers.cc", - "codecs/vp9/screenshare_layers.h", "codecs/vp9/vp9_frame_buffer_pool.cc", "codecs/vp9/vp9_frame_buffer_pool.h", "codecs/vp9/vp9_impl.cc", @@ -816,9 +814,6 @@ if (rtc_include_tests) { "video_receiver_unittest.cc", "video_sender_unittest.cc", ] - if (rtc_libvpx_build_vp9) { - sources += [ "codecs/vp9/vp9_screenshare_layers_unittest.cc" ] - } if (rtc_use_h264) { sources += [ "codecs/h264/h264_encoder_impl_unittest.cc" ] } diff --git a/modules/video_coding/codecs/vp9/screenshare_layers.cc b/modules/video_coding/codecs/vp9/screenshare_layers.cc deleted file mode 100644 index 158b0e52dc..0000000000 --- a/modules/video_coding/codecs/vp9/screenshare_layers.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -*/ - -#include "modules/video_coding/codecs/vp9/screenshare_layers.h" -#include -#include "rtc_base/checks.h" - -namespace webrtc { - -ScreenshareLayersVP9::ScreenshareLayersVP9(uint8_t num_layers) - : num_layers_(num_layers), - start_layer_(0), - last_timestamp_(0), - timestamp_initialized_(false) { - RTC_DCHECK_GT(num_layers, 0); - RTC_DCHECK_LE(num_layers, kMaxVp9NumberOfSpatialLayers); - memset(bits_used_, 0, sizeof(bits_used_)); - memset(threshold_kbps_, 0, sizeof(threshold_kbps_)); -} - -uint8_t ScreenshareLayersVP9::GetStartLayer() const { - return start_layer_; -} - -void ScreenshareLayersVP9::ConfigureBitrate(int threshold_kbps, - uint8_t layer_id) { - // The upper layer is always the layer we spill frames - // to when the bitrate becomes to high, therefore setting - // a max limit is not allowed. The top layer bitrate is - // never used either so configuring it makes no difference. - RTC_DCHECK_LT(layer_id, num_layers_ - 1); - threshold_kbps_[layer_id] = threshold_kbps; -} - -void ScreenshareLayersVP9::LayerFrameEncoded(unsigned int size_bytes, - uint8_t layer_id) { - RTC_DCHECK_LT(layer_id, num_layers_); - bits_used_[layer_id] += size_bytes * 8; -} - -VP9EncoderImpl::SuperFrameRefSettings -ScreenshareLayersVP9::GetSuperFrameSettings(uint32_t timestamp, - bool is_keyframe) { - VP9EncoderImpl::SuperFrameRefSettings settings; - if (!timestamp_initialized_) { - last_timestamp_ = timestamp; - timestamp_initialized_ = true; - } - float time_diff = (timestamp - last_timestamp_) / 90.f; - float total_bits_used = 0; - float total_threshold_kbps = 0; - start_layer_ = 0; - - // Up to (num_layers - 1) because we only have - // (num_layers - 1) thresholds to check. - for (int layer_id = 0; layer_id < num_layers_ - 1; ++layer_id) { - bits_used_[layer_id] = std::max( - 0.f, bits_used_[layer_id] - time_diff * threshold_kbps_[layer_id]); - total_bits_used += bits_used_[layer_id]; - total_threshold_kbps += threshold_kbps_[layer_id]; - - // If this is a keyframe then there should be no - // references to any previous frames. - if (!is_keyframe) { - settings.layer[layer_id].ref_buf1 = layer_id; - if (total_bits_used > total_threshold_kbps * 1000) - start_layer_ = layer_id + 1; - } - - settings.layer[layer_id].upd_buf = layer_id; - } - // Since the above loop does not iterate over the last layer - // the reference of the last layer has to be set after the loop, - // and if this is a keyframe there should be no references to - // any previous frames. - if (!is_keyframe) - settings.layer[num_layers_ - 1].ref_buf1 = num_layers_ - 1; - - settings.layer[num_layers_ - 1].upd_buf = num_layers_ - 1; - settings.is_keyframe = is_keyframe; - settings.start_layer = start_layer_; - settings.stop_layer = num_layers_ - 1; - last_timestamp_ = timestamp; - return settings; -} - -} // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/screenshare_layers.h b/modules/video_coding/codecs/vp9/screenshare_layers.h deleted file mode 100644 index 6725ab76f9..0000000000 --- a/modules/video_coding/codecs/vp9/screenshare_layers.h +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -*/ - -#ifndef MODULES_VIDEO_CODING_CODECS_VP9_SCREENSHARE_LAYERS_H_ -#define MODULES_VIDEO_CODING_CODECS_VP9_SCREENSHARE_LAYERS_H_ - -#include "modules/video_coding/codecs/vp9/vp9_impl.h" - -namespace webrtc { - -class ScreenshareLayersVP9 { - public: - explicit ScreenshareLayersVP9(uint8_t num_layers); - - // The target bitrate for layer with id layer_id. - void ConfigureBitrate(int threshold_kbps, uint8_t layer_id); - - // The current start layer. - uint8_t GetStartLayer() const; - - // Update the layer with the size of the layer frame. - void LayerFrameEncoded(unsigned int size_bytes, uint8_t layer_id); - - // Get the layer settings for the next superframe. - // - // In short, each time the GetSuperFrameSettings is called the - // bitrate of every layer is calculated and if the cummulative - // bitrate exceeds the configured cummulative bitrates - // (ConfigureBitrate to configure) up to and including that - // layer then the resulting encoding settings for the - // superframe will only encode layers above that layer. - VP9EncoderImpl::SuperFrameRefSettings GetSuperFrameSettings( - uint32_t timestamp, - bool is_keyframe); - - private: - // How many layers that are used. - uint8_t num_layers_; - - // The index of the first layer to encode. - uint8_t start_layer_; - - // Cummulative target kbps for the different layers. - float threshold_kbps_[kMaxVp9NumberOfSpatialLayers - 1]; - - // How many bits that has been used for a certain layer. Increased in - // FrameEncoded() by the size of the encoded frame and decreased in - // GetSuperFrameSettings() depending on the time between frames. - float bits_used_[kMaxVp9NumberOfSpatialLayers]; - - // Timestamp of last frame. - uint32_t last_timestamp_; - - // If the last_timestamp_ has been set. - bool timestamp_initialized_; -}; - -} // namespace webrtc - -#endif // MODULES_VIDEO_CODING_CODECS_VP9_SCREENSHARE_LAYERS_H_ diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index 408ec15a9b..6ff7e328a1 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -23,7 +23,6 @@ #include "common_video/include/video_frame_buffer.h" #include "common_video/libyuv/include/webrtc_libyuv.h" -#include "modules/video_coding/codecs/vp9/screenshare_layers.h" #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" #include "rtc_base/checks.h" #include "rtc_base/keep_ref_until_done.h" @@ -79,10 +78,7 @@ VP9EncoderImpl::VP9EncoderImpl() num_temporal_layers_(0), num_spatial_layers_(0), inter_layer_pred_(InterLayerPredMode::kOn), - is_flexible_mode_(false), - frames_encoded_(0), - // Use two spatial when screensharing with flexible mode. - spatial_layer_(new ScreenshareLayersVP9(2)) { + is_flexible_mode_(false) { memset(&codec_, 0, sizeof(codec_)); memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t)); } @@ -130,7 +126,6 @@ bool VP9EncoderImpl::SetSvcRates( uint8_t i = 0; config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps(); - spatial_layer_->ConfigureBitrate(bitrate_allocation.get_sum_kbps(), 0); if (ExplicitlyConfiguredSpatialLayers()) { for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) { @@ -342,12 +337,11 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, // TODO(asapersson): Check configuration of temporal switch up and increase // pattern length. is_flexible_mode_ = inst->VP9().flexibleMode; - if (is_flexible_mode_) { - config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_BYPASS; - config_->ts_number_layers = num_temporal_layers_; - if (codec_.mode == kScreensharing) - spatial_layer_->ConfigureBitrate(inst->startBitrate, 0); - } else if (num_temporal_layers_ == 1) { + + // TODO(ssilkin): Only non-flexible mode is supported for now. + RTC_DCHECK(!is_flexible_mode_); + + if (num_temporal_layers_ == 1) { gof_.SetGofInfoVP9(kTemporalStructureMode1); config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING; config_->ts_number_layers = 1; @@ -490,6 +484,22 @@ int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) { default: RTC_NOTREACHED(); } + + if (!is_flexible_mode_) { + // In RTP non-flexible mode, frame dropping of individual layers in a + // superframe leads to incorrect reference picture ID values in the + // RTP header. Dropping the entire superframe if the base is dropped + // or not dropping upper layers if base is not dropped mitigates + // the problem. + vpx_svc_frame_drop_t svc_drop_frame; + svc_drop_frame.framedrop_mode = CONSTRAINED_LAYER_DROP; + for (size_t i = 0; i < num_spatial_layers_; ++i) { + svc_drop_frame.framedrop_thresh[i] = + (i == 0) ? config_->rc_dropframe_thresh : 0; + } + vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, + &svc_drop_frame); + } } // Register callback for getting each spatial layer. @@ -582,41 +592,6 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, flags = VPX_EFLAG_FORCE_KF; } - if (is_flexible_mode_) { - SuperFrameRefSettings settings; - - // These structs are copied when calling vpx_codec_control, - // therefore it is ok for them to go out of scope. - vpx_svc_ref_frame_config enc_layer_conf; - vpx_svc_layer_id layer_id; - - if (codec_.mode == kRealtimeVideo) { - // Real time video not yet implemented in flexible mode. - RTC_NOTREACHED(); - } else { - settings = spatial_layer_->GetSuperFrameSettings(input_image.timestamp(), - force_key_frame_); - } - enc_layer_conf = GenerateRefsAndFlags(settings); - layer_id.temporal_layer_id = 0; - layer_id.spatial_layer_id = settings.start_layer; - vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id); - vpx_codec_control(encoder_, VP9E_SET_SVC_REF_FRAME_CONFIG, &enc_layer_conf); - } else if (codec_.mode == kRealtimeVideo && num_spatial_layers_ > 1) { - // In RTP non-flexible mode, frame dropping of individual layers in a - // superframe leads to incorrect reference picture ID values in the - // RTP header. Dropping the entire superframe if the base is dropped - // or not dropping upper layers if base is not dropped mitigates - // the problem. - vpx_svc_frame_drop_t svc_drop_frame; - svc_drop_frame.framedrop_mode = CONSTRAINED_LAYER_DROP; - for (size_t i = 0; i < num_spatial_layers_; ++i) { - svc_drop_frame.framedrop_thresh[i] = - (i == 0) ? config_->rc_dropframe_thresh : 0; - } - vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, &svc_drop_frame); - } - RTC_CHECK_GT(codec_.maxFramerate, 0); uint32_t duration = 90000 / codec_.maxFramerate; if (vpx_codec_encode(encoder_, raw_, timestamp_, duration, flags, @@ -703,18 +678,12 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, // bit. vp9_info->num_spatial_layers = num_spatial_layers_; - vp9_info->num_ref_pics = 0; - if (vp9_info->flexible_mode) { - vp9_info->gof_idx = kNoGofIdx; - vp9_info->num_ref_pics = num_ref_pics_[layer_id.spatial_layer_id]; - for (int i = 0; i < num_ref_pics_[layer_id.spatial_layer_id]; ++i) { - vp9_info->p_diff[i] = p_diff_[layer_id.spatial_layer_id][i]; - } - } else { - vp9_info->gof_idx = - static_cast(pics_since_key_ % gof_.num_frames_in_gof); - vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx]; - } + RTC_DCHECK(!vp9_info->flexible_mode); + vp9_info->gof_idx = + static_cast(pics_since_key_ % gof_.num_frames_in_gof); + vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx]; + vp9_info->num_ref_pics = + is_key_pic ? 0 : gof_.num_ref_pics[vp9_info->gof_idx]; if (vp9_info->ss_data_available) { vp9_info->spatial_layer_resolution_present = true; @@ -758,12 +727,6 @@ int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) { memcpy(encoded_image_._buffer, pkt->data.frame.buf, pkt->data.frame.sz); encoded_image_._length = pkt->data.frame.sz; - if (is_flexible_mode_ && codec_.mode == kScreensharing) { - spatial_layer_->LayerFrameEncoded( - static_cast(encoded_image_._length), - layer_id.spatial_layer_id); - } - const bool is_key_frame = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) ? true : false; // Ensure encoder issued key frame on request. @@ -819,108 +782,6 @@ void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) { } } -vpx_svc_ref_frame_config VP9EncoderImpl::GenerateRefsAndFlags( - const SuperFrameRefSettings& settings) { - static const vpx_enc_frame_flags_t kAllFlags = - VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_LAST | - VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_GF; - vpx_svc_ref_frame_config sf_conf = {}; - if (settings.is_keyframe) { - // Used later on to make sure we don't make any invalid references. - memset(buffer_updated_at_frame_, -1, sizeof(buffer_updated_at_frame_)); - for (int layer = settings.start_layer; layer <= settings.stop_layer; - ++layer) { - num_ref_pics_[layer] = 0; - buffer_updated_at_frame_[settings.layer[layer].upd_buf] = frames_encoded_; - // When encoding a keyframe only the alt_fb_idx is used - // to specify which layer ends up in which buffer. - sf_conf.alt_fb_idx[layer] = settings.layer[layer].upd_buf; - } - } else { - for (int layer_idx = settings.start_layer; layer_idx <= settings.stop_layer; - ++layer_idx) { - vpx_enc_frame_flags_t layer_flags = kAllFlags; - num_ref_pics_[layer_idx] = 0; - int8_t refs[3] = {settings.layer[layer_idx].ref_buf1, - settings.layer[layer_idx].ref_buf2, - settings.layer[layer_idx].ref_buf3}; - - for (unsigned int ref_idx = 0; ref_idx < kMaxVp9RefPics; ++ref_idx) { - if (refs[ref_idx] == -1) - continue; - - RTC_DCHECK_GE(refs[ref_idx], 0); - RTC_DCHECK_LE(refs[ref_idx], 7); - // Easier to remove flags from all flags rather than having to - // build the flags from 0. - switch (num_ref_pics_[layer_idx]) { - case 0: { - sf_conf.lst_fb_idx[layer_idx] = refs[ref_idx]; - layer_flags &= ~VP8_EFLAG_NO_REF_LAST; - break; - } - case 1: { - sf_conf.gld_fb_idx[layer_idx] = refs[ref_idx]; - layer_flags &= ~VP8_EFLAG_NO_REF_GF; - break; - } - case 2: { - sf_conf.alt_fb_idx[layer_idx] = refs[ref_idx]; - layer_flags &= ~VP8_EFLAG_NO_REF_ARF; - break; - } - } - // Make sure we don't reference a buffer that hasn't been - // used at all or hasn't been used since a keyframe. - RTC_DCHECK_NE(buffer_updated_at_frame_[refs[ref_idx]], -1); - - p_diff_[layer_idx][num_ref_pics_[layer_idx]] = - frames_encoded_ - buffer_updated_at_frame_[refs[ref_idx]]; - num_ref_pics_[layer_idx]++; - } - - bool upd_buf_same_as_a_ref = false; - if (settings.layer[layer_idx].upd_buf != -1) { - for (unsigned int ref_idx = 0; ref_idx < kMaxVp9RefPics; ++ref_idx) { - if (settings.layer[layer_idx].upd_buf == refs[ref_idx]) { - switch (ref_idx) { - case 0: { - layer_flags &= ~VP8_EFLAG_NO_UPD_LAST; - break; - } - case 1: { - layer_flags &= ~VP8_EFLAG_NO_UPD_GF; - break; - } - case 2: { - layer_flags &= ~VP8_EFLAG_NO_UPD_ARF; - break; - } - } - upd_buf_same_as_a_ref = true; - break; - } - } - if (!upd_buf_same_as_a_ref) { - // If we have three references and a buffer is specified to be - // updated, then that buffer must be the same as one of the - // three references. - RTC_CHECK_LT(num_ref_pics_[layer_idx], kMaxVp9RefPics); - - sf_conf.alt_fb_idx[layer_idx] = settings.layer[layer_idx].upd_buf; - layer_flags ^= VP8_EFLAG_NO_UPD_ARF; - } - - int updated_buffer = settings.layer[layer_idx].upd_buf; - buffer_updated_at_frame_[updated_buffer] = frames_encoded_; - sf_conf.frame_flags[layer_idx] = layer_flags; - } - } - } - ++frames_encoded_; - return sf_conf; -} - int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) { return WEBRTC_VIDEO_CODEC_OK; } diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index 6da3dce5b6..509f04eedf 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -24,8 +24,6 @@ namespace webrtc { -class ScreenshareLayersVP9; - class VP9EncoderImpl : public VP9Encoder { public: VP9EncoderImpl(); @@ -51,20 +49,6 @@ class VP9EncoderImpl : public VP9Encoder { const char* ImplementationName() const override; - struct LayerFrameRefSettings { - int8_t upd_buf = -1; // -1 - no update, 0..7 - update buffer 0..7 - int8_t ref_buf1 = -1; // -1 - no reference, 0..7 - reference buffer 0..7 - int8_t ref_buf2 = -1; // -1 - no reference, 0..7 - reference buffer 0..7 - int8_t ref_buf3 = -1; // -1 - no reference, 0..7 - reference buffer 0..7 - }; - - struct SuperFrameRefSettings { - LayerFrameRefSettings layer[kMaxVp9NumberOfSpatialLayers]; - uint8_t start_layer = 0; // The first spatial layer to be encoded. - uint8_t stop_layer = 0; // The last spatial layer to be encoded. - bool is_keyframe = false; - }; - private: // Determine number of encoder threads to use. int NumberOfThreads(int width, int height, int number_of_cores); @@ -80,15 +64,6 @@ class VP9EncoderImpl : public VP9Encoder { bool ExplicitlyConfiguredSpatialLayers() const; bool SetSvcRates(const VideoBitrateAllocation& bitrate_allocation); - // Used for flexible mode to set the flags and buffer references used - // by the encoder. Also calculates the references used by the RTP - // packetizer. - // - // Has to be called for every frame (keyframes included) to update the - // state used to calculate references. - vpx_svc_ref_frame_config GenerateRefsAndFlags( - const SuperFrameRefSettings& settings); - virtual int GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt); // Callback function for outputting packets per spatial layer. @@ -97,6 +72,8 @@ class VP9EncoderImpl : public VP9Encoder { void DeliverBufferedFrame(bool end_of_picture); + bool DropFrame(uint32_t rtp_timestamp); + // Determine maximum target for Intra frames // // Input: @@ -128,11 +105,6 @@ class VP9EncoderImpl : public VP9Encoder { // Used for flexible mode. bool is_flexible_mode_; - int64_t buffer_updated_at_frame_[kNumVp9Buffers]; - int64_t frames_encoded_; - uint8_t num_ref_pics_[kMaxVp9NumberOfSpatialLayers]; - uint8_t p_diff_[kMaxVp9NumberOfSpatialLayers][kMaxVp9RefPics]; - std::unique_ptr spatial_layer_; }; class VP9DecoderImpl : public VP9Decoder { diff --git a/modules/video_coding/codecs/vp9/vp9_screenshare_layers_unittest.cc b/modules/video_coding/codecs/vp9/vp9_screenshare_layers_unittest.cc deleted file mode 100644 index 5ef7470c3b..0000000000 --- a/modules/video_coding/codecs/vp9/vp9_screenshare_layers_unittest.cc +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include -#include - -#include "vpx/vp8cx.h" -#include "modules/video_coding/codecs/vp9/screenshare_layers.h" -#include "modules/video_coding/codecs/vp9/vp9_impl.h" -#include "rtc_base/logging.h" -#include "system_wrappers/include/clock.h" -#include "test/gtest.h" - -namespace webrtc { - -typedef VP9EncoderImpl::SuperFrameRefSettings Settings; - -const uint32_t kTickFrequency = 90000; - -class ScreenshareLayerTestVP9 : public ::testing::Test { - protected: - ScreenshareLayerTestVP9() : clock_(0) {} - virtual ~ScreenshareLayerTestVP9() {} - - void InitScreenshareLayers(int layers) { - layers_.reset(new ScreenshareLayersVP9(layers)); - } - - void ConfigureBitrateForLayer(int kbps, uint8_t layer_id) { - layers_->ConfigureBitrate(kbps, layer_id); - } - - void AdvanceTime(int64_t milliseconds) { - clock_.AdvanceTimeMilliseconds(milliseconds); - } - - void AddKilobitsToLayer(int kilobits, uint8_t layer_id) { - layers_->LayerFrameEncoded(kilobits * 1000 / 8, layer_id); - } - - void EqualRefsForLayer(const Settings& actual, uint8_t layer_id) { - EXPECT_EQ(expected_.layer[layer_id].upd_buf, - actual.layer[layer_id].upd_buf); - EXPECT_EQ(expected_.layer[layer_id].ref_buf1, - actual.layer[layer_id].ref_buf1); - EXPECT_EQ(expected_.layer[layer_id].ref_buf2, - actual.layer[layer_id].ref_buf2); - EXPECT_EQ(expected_.layer[layer_id].ref_buf3, - actual.layer[layer_id].ref_buf3); - } - - void EqualRefs(const Settings& actual) { - for (unsigned int layer_id = 0; layer_id < kMaxVp9NumberOfSpatialLayers; - ++layer_id) { - EqualRefsForLayer(actual, layer_id); - } - } - - void EqualStartStopKeyframe(const Settings& actual) { - EXPECT_EQ(expected_.start_layer, actual.start_layer); - EXPECT_EQ(expected_.stop_layer, actual.stop_layer); - EXPECT_EQ(expected_.is_keyframe, actual.is_keyframe); - } - - // Check that the settings returned by GetSuperFrameSettings() is - // equal to the expected_ settings. - void EqualToExpected() { - uint32_t frame_timestamp_ = - clock_.TimeInMilliseconds() * (kTickFrequency / 1000); - Settings actual = - layers_->GetSuperFrameSettings(frame_timestamp_, expected_.is_keyframe); - EqualRefs(actual); - EqualStartStopKeyframe(actual); - } - - Settings expected_; - SimulatedClock clock_; - std::unique_ptr layers_; -}; - -TEST_F(ScreenshareLayerTestVP9, NoRefsOnKeyFrame) { - const int kNumLayers = kMaxVp9NumberOfSpatialLayers; - InitScreenshareLayers(kNumLayers); - expected_.start_layer = 0; - expected_.stop_layer = kNumLayers - 1; - - for (int l = 0; l < kNumLayers; ++l) { - expected_.layer[l].upd_buf = l; - } - expected_.is_keyframe = true; - EqualToExpected(); - - for (int l = 0; l < kNumLayers; ++l) { - expected_.layer[l].ref_buf1 = l; - } - expected_.is_keyframe = false; - EqualToExpected(); -} - -// Test if it is possible to send at a high bitrate (over the threshold) -// after a longer period of low bitrate. This should not be possible. -TEST_F(ScreenshareLayerTestVP9, DontAccumelateAvailableBitsOverTime) { - InitScreenshareLayers(2); - ConfigureBitrateForLayer(100, 0); - - expected_.layer[0].upd_buf = 0; - expected_.layer[0].ref_buf1 = 0; - expected_.layer[1].upd_buf = 1; - expected_.layer[1].ref_buf1 = 1; - expected_.start_layer = 0; - expected_.stop_layer = 1; - - // Send 10 frames at a low bitrate (50 kbps) - for (int i = 0; i < 10; ++i) { - AdvanceTime(200); - EqualToExpected(); - AddKilobitsToLayer(10, 0); - } - - AdvanceTime(200); - EqualToExpected(); - AddKilobitsToLayer(301, 0); - - // Send 10 frames at a high bitrate (200 kbps) - expected_.start_layer = 1; - for (int i = 0; i < 10; ++i) { - AdvanceTime(200); - EqualToExpected(); - AddKilobitsToLayer(40, 1); - } -} - -// Test if used bits are accumelated over layers, as they should; -TEST_F(ScreenshareLayerTestVP9, AccumelateUsedBitsOverLayers) { - const int kNumLayers = kMaxVp9NumberOfSpatialLayers; - InitScreenshareLayers(kNumLayers); - for (int l = 0; l < kNumLayers - 1; ++l) - ConfigureBitrateForLayer(100, l); - for (int l = 0; l < kNumLayers; ++l) { - expected_.layer[l].upd_buf = l; - expected_.layer[l].ref_buf1 = l; - } - - expected_.start_layer = 0; - expected_.stop_layer = kNumLayers - 1; - EqualToExpected(); - - for (int layer = 0; layer < kNumLayers - 1; ++layer) { - expected_.start_layer = layer; - EqualToExpected(); - AddKilobitsToLayer(101, layer); - } -} - -// General testing of the bitrate controller. -TEST_F(ScreenshareLayerTestVP9, 2LayerBitrate) { - InitScreenshareLayers(2); - ConfigureBitrateForLayer(100, 0); - - expected_.layer[0].upd_buf = 0; - expected_.layer[1].upd_buf = 1; - expected_.layer[0].ref_buf1 = -1; - expected_.layer[1].ref_buf1 = -1; - expected_.start_layer = 0; - expected_.stop_layer = 1; - - expected_.is_keyframe = true; - EqualToExpected(); - AddKilobitsToLayer(100, 0); - - expected_.layer[0].ref_buf1 = 0; - expected_.layer[1].ref_buf1 = 1; - expected_.is_keyframe = false; - AdvanceTime(199); - EqualToExpected(); - AddKilobitsToLayer(100, 0); - - expected_.start_layer = 1; - for (int frame = 0; frame < 3; ++frame) { - AdvanceTime(200); - EqualToExpected(); - AddKilobitsToLayer(100, 1); - } - - // Just before enough bits become available for L0 @0.999 seconds. - AdvanceTime(199); - EqualToExpected(); - AddKilobitsToLayer(100, 1); - - // Just after enough bits become available for L0 @1.0001 seconds. - expected_.start_layer = 0; - AdvanceTime(2); - EqualToExpected(); - AddKilobitsToLayer(100, 0); - - // Keyframes always encode all layers, even if it is over budget. - expected_.layer[0].ref_buf1 = -1; - expected_.layer[1].ref_buf1 = -1; - expected_.is_keyframe = true; - AdvanceTime(499); - EqualToExpected(); - expected_.layer[0].ref_buf1 = 0; - expected_.layer[1].ref_buf1 = 1; - expected_.start_layer = 1; - expected_.is_keyframe = false; - EqualToExpected(); - AddKilobitsToLayer(100, 0); - - // 400 kb in L0 --> @3 second mark to fall below the threshold.. - // just before @2.999 seconds. - expected_.is_keyframe = false; - AdvanceTime(1499); - EqualToExpected(); - AddKilobitsToLayer(100, 1); - - // just after @3.001 seconds. - expected_.start_layer = 0; - AdvanceTime(2); - EqualToExpected(); - AddKilobitsToLayer(100, 0); -} - -// General testing of the bitrate controller. -TEST_F(ScreenshareLayerTestVP9, 3LayerBitrate) { - InitScreenshareLayers(3); - ConfigureBitrateForLayer(100, 0); - ConfigureBitrateForLayer(100, 1); - - for (int l = 0; l < 3; ++l) { - expected_.layer[l].upd_buf = l; - expected_.layer[l].ref_buf1 = l; - } - expected_.start_layer = 0; - expected_.stop_layer = 2; - - EqualToExpected(); - AddKilobitsToLayer(105, 0); - AddKilobitsToLayer(30, 1); - - AdvanceTime(199); - EqualToExpected(); - AddKilobitsToLayer(105, 0); - AddKilobitsToLayer(30, 1); - - expected_.start_layer = 1; - AdvanceTime(200); - EqualToExpected(); - AddKilobitsToLayer(130, 1); - - expected_.start_layer = 2; - AdvanceTime(200); - EqualToExpected(); - - // 400 kb in L1 --> @1.0 second mark to fall below threshold. - // 210 kb in L0 --> @1.1 second mark to fall below threshold. - // Just before L1 @0.999 seconds. - AdvanceTime(399); - EqualToExpected(); - - // Just after L1 @1.001 seconds. - expected_.start_layer = 1; - AdvanceTime(2); - EqualToExpected(); - - // Just before L0 @1.099 seconds. - AdvanceTime(99); - EqualToExpected(); - - // Just after L0 @1.101 seconds. - expected_.start_layer = 0; - AdvanceTime(2); - EqualToExpected(); - - // @1.1 seconds - AdvanceTime(99); - EqualToExpected(); - AddKilobitsToLayer(200, 1); - - expected_.is_keyframe = true; - for (int l = 0; l < 3; ++l) - expected_.layer[l].ref_buf1 = -1; - AdvanceTime(200); - EqualToExpected(); - - expected_.is_keyframe = false; - expected_.start_layer = 2; - for (int l = 0; l < 3; ++l) - expected_.layer[l].ref_buf1 = l; - AdvanceTime(200); - EqualToExpected(); -} - -// Test that the bitrate calculations are -// correct when the timestamp wrap. -TEST_F(ScreenshareLayerTestVP9, TimestampWrap) { - InitScreenshareLayers(2); - ConfigureBitrateForLayer(100, 0); - - expected_.layer[0].upd_buf = 0; - expected_.layer[0].ref_buf1 = 0; - expected_.layer[1].upd_buf = 1; - expected_.layer[1].ref_buf1 = 1; - expected_.start_layer = 0; - expected_.stop_layer = 1; - - // Advance time to just before the timestamp wraps. - AdvanceTime(std::numeric_limits::max() / (kTickFrequency / 1000)); - EqualToExpected(); - AddKilobitsToLayer(200, 0); - - // Wrap - expected_.start_layer = 1; - AdvanceTime(1); - EqualToExpected(); -} - -} // namespace webrtc diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc index adc7f32ba9..f122070282 100644 --- a/modules/video_coding/video_codec_initializer.cc +++ b/modules/video_coding/video_codec_initializer.cc @@ -195,54 +195,46 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers, kMaxTemporalStreams); - if (video_codec.mode == kScreensharing && - config.encoder_specific_settings) { - video_codec.VP9()->flexibleMode = true; - // For now VP9 screensharing use 1 temporal and 2 spatial layers. - RTC_DCHECK_EQ(1, video_codec.VP9()->numberOfTemporalLayers); - RTC_DCHECK_EQ(2, video_codec.VP9()->numberOfSpatialLayers); + RTC_DCHECK(config.spatial_layers.empty() || + config.spatial_layers.size() == + video_codec.VP9()->numberOfSpatialLayers); + + std::vector spatial_layers; + if (!config.spatial_layers.empty()) { + // Layering is set explicitly. + spatial_layers = config.spatial_layers; } else { - RTC_DCHECK(config.spatial_layers.empty() || - config.spatial_layers.size() == - video_codec.VP9()->numberOfSpatialLayers); + spatial_layers = GetSvcConfig(video_codec.width, video_codec.height, + video_codec.VP9()->numberOfSpatialLayers, + video_codec.VP9()->numberOfTemporalLayers, + video_codec.mode == kScreensharing); - std::vector spatial_layers; - if (!config.spatial_layers.empty()) { - // Layering is set explicitly. - spatial_layers = config.spatial_layers; - } else { - spatial_layers = - GetSvcConfig(video_codec.width, video_codec.height, - video_codec.VP9()->numberOfSpatialLayers, - video_codec.VP9()->numberOfTemporalLayers, false); - - const bool no_spatial_layering = (spatial_layers.size() == 1); - if (no_spatial_layering) { - // Use codec's bitrate limits. - spatial_layers.back().minBitrate = video_codec.minBitrate; - spatial_layers.back().maxBitrate = video_codec.maxBitrate; - } + const bool no_spatial_layering = (spatial_layers.size() == 1); + if (no_spatial_layering) { + // Use codec's bitrate limits. + spatial_layers.back().minBitrate = video_codec.minBitrate; + spatial_layers.back().maxBitrate = video_codec.maxBitrate; } - - RTC_DCHECK(!spatial_layers.empty()); - for (size_t i = 0; i < spatial_layers.size(); ++i) { - video_codec.spatialLayers[i] = spatial_layers[i]; - } - - // Update layering settings. - video_codec.VP9()->numberOfSpatialLayers = - static_cast(spatial_layers.size()); - RTC_DCHECK_GE(video_codec.VP9()->numberOfSpatialLayers, 1); - RTC_DCHECK_LE(video_codec.VP9()->numberOfSpatialLayers, - kMaxSpatialLayers); - - video_codec.VP9()->numberOfTemporalLayers = static_cast( - spatial_layers.back().numberOfTemporalLayers); - RTC_DCHECK_GE(video_codec.VP9()->numberOfTemporalLayers, 1); - RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers, - kMaxTemporalStreams); } + RTC_DCHECK(!spatial_layers.empty()); + for (size_t i = 0; i < spatial_layers.size(); ++i) { + video_codec.spatialLayers[i] = spatial_layers[i]; + } + + // Update layering settings. + video_codec.VP9()->numberOfSpatialLayers = + static_cast(spatial_layers.size()); + RTC_DCHECK_GE(video_codec.VP9()->numberOfSpatialLayers, 1); + RTC_DCHECK_LE(video_codec.VP9()->numberOfSpatialLayers, + kMaxSpatialLayers); + + video_codec.VP9()->numberOfTemporalLayers = static_cast( + spatial_layers.back().numberOfTemporalLayers); + RTC_DCHECK_GE(video_codec.VP9()->numberOfTemporalLayers, 1); + RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers, + kMaxTemporalStreams); + break; } case kVideoCodecH264: { diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index 4a72e2520c..67758136a2 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -3571,7 +3571,10 @@ TEST_F(VideoSendStreamTest, Vp9NonFlexModeSmallResolution) { // Crashes on Android; bugs.webrtc.org/7401 #define MAYBE_Vp9FlexModeRefCount DISABLED_Vp9FlexModeRefCount #else -#define MAYBE_Vp9FlexModeRefCount Vp9FlexModeRefCount +// TODO(webrtc:9270): Support of flexible mode is temporarily disabled. Enable +// the test after webrtc:9270 is implemented. +#define MAYBE_Vp9FlexModeRefCount DISABLED_Vp9FlexModeRefCount +// #define MAYBE_Vp9FlexModeRefCount Vp9FlexModeRefCount #endif TEST_F(VideoSendStreamTest, MAYBE_Vp9FlexModeRefCount) { class FlexibleMode : public Vp9HeaderObserver {