From 24dbb213833b0346c4f2a0bfbc355c2949027c70 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Mon, 2 Mar 2020 20:23:50 +0100 Subject: [PATCH] Enable quality scaler for simulcast and SVC if only one stream is active Also, make sure active flags are not lost in simulcast encoder adapter which is needed in case of simulcast encoder adapter is used. VP9 libvpx encoder currently ignores scaling setting for SVC, but libvpx fix is incoming. TESTED=On a manually patched chrome with singlecast-simulcast vp8 stream. Bug: webrtc:11396 Change-Id: Ic81f014bec1bdaaf6d5d173743933e5d77d71ea2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169547 Reviewed-by: Evan Shrubsole Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#30681} --- media/engine/simulcast_encoder_adapter.cc | 12 ++++++---- media/engine/webrtc_video_engine.cc | 13 ++++++++++- .../codecs/vp8/libvpx_vp8_encoder.cc | 23 +++++++++++++++---- .../codecs/vp8/libvpx_vp8_encoder.h | 1 + 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 9ca6afe5f2..a5454eb582 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -97,7 +97,7 @@ int VerifyCodec(const webrtc::VideoCodec* inst) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (inst->codecType == webrtc::kVideoCodecVP8 && - inst->VP8().automaticResizeOn && inst->numberOfSimulcastStreams > 1) { + inst->VP8().automaticResizeOn && NumActiveStreams(*inst) > 1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } return WEBRTC_VIDEO_CODEC_OK; @@ -585,6 +585,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec( stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate; stream_codec->maxFramerate = inst.simulcastStream[stream_index].maxFramerate; stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax; + stream_codec->active = inst.simulcastStream[stream_index].active; // Settings that are based on stream/resolution. if (stream_resolution == StreamResolution::LOWEST) { // Settings for lowest spatial resolutions. @@ -644,14 +645,14 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { return encoder_info; } + encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff; + int num_active_streams = NumActiveStreams(codec_); + for (size_t i = 0; i < streaminfos_.size(); ++i) { VideoEncoder::EncoderInfo encoder_impl_info = streaminfos_[i].encoder->GetEncoderInfo(); if (i == 0) { - // Quality scaling not enabled for simulcast. - encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff; - // Encoder name indicates names of all sub-encoders. encoder_info.implementation_name += " ("; encoder_info.implementation_name += encoder_impl_info.implementation_name; @@ -689,6 +690,9 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple( encoder_info.requested_resolution_alignment, encoder_impl_info.requested_resolution_alignment); + if (num_active_streams == 1 && codec_.simulcastStream[i].active) { + encoder_info.scaling_settings = encoder_impl_info.scaling_settings; + } } encoder_info.implementation_name += ")"; diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 0c23ff8b51..a93a509e0a 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -313,6 +313,16 @@ size_t FindRequiredActiveLayers( return 0; } +int NumActiveStreams(const webrtc::RtpParameters& rtp_parameters) { + int res = 0; + for (size_t i = 0; i < rtp_parameters.encodings.size(); ++i) { + if (rtp_parameters.encodings[i].active) { + ++res; + } + } + return res; +} + } // namespace // This constant is really an on/off, lower-level configurable NACK history @@ -331,7 +341,8 @@ WebRtcVideoChannel::WebRtcVideoSendStream::ConfigureVideoEncoderSettings( bool is_screencast = parameters_.options.is_screencast.value_or(false); // No automatic resizing when using simulcast or screencast. bool automatic_resize = - !is_screencast && parameters_.config.rtp.ssrcs.size() == 1; + !is_screencast && (parameters_.config.rtp.ssrcs.size() == 1 || + NumActiveStreams(rtp_parameters_) == 1); bool frame_dropping = !is_screencast; bool denoising; bool codec_default_denoising = false; diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index e3776aac92..c0a9e545c8 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -288,6 +288,7 @@ LibvpxVp8Encoder::LibvpxVp8Encoder( cpu_speed_default_(-6), number_of_cores_(0), rc_max_intra_target_(0), + num_active_streams_(0), frame_buffer_controller_factory_( std::move(frame_buffer_controller_factory)), key_frame_request_(kMaxSimulcastStreams, false), @@ -475,9 +476,21 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, if (settings.number_of_cores < 1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - if (inst->VP8().automaticResizeOn && inst->numberOfSimulcastStreams > 1) { + + num_active_streams_ = 0; + for (int i = 0; i < inst->numberOfSimulcastStreams; ++i) { + if (inst->simulcastStream[i].active) { + ++num_active_streams_; + } + } + if (inst->numberOfSimulcastStreams == 0 && inst->active) { + num_active_streams_ = 1; + } + + if (inst->VP8().automaticResizeOn && num_active_streams_ > 1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } + int retVal = Release(); if (retVal < 0) { return retVal; @@ -1232,9 +1245,11 @@ VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const { info.has_internal_source = false; info.supports_simulcast = true; - const bool enable_scaling = encoders_.size() == 1 && - vpx_configs_[0].rc_dropframe_thresh > 0 && - codec_.VP8().automaticResizeOn; + const bool enable_scaling = + num_active_streams_ == 1 && + (vpx_configs_.empty() || vpx_configs_[0].rc_dropframe_thresh > 0) && + codec_.VP8().automaticResizeOn; + info.scaling_settings = enable_scaling ? VideoEncoder::ScalingSettings( kLowVp8QpThreshold, kHighVp8QpThreshold) diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h index 675d386456..a283a9472e 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h @@ -113,6 +113,7 @@ class LibvpxVp8Encoder : public VideoEncoder { int cpu_speed_default_; int number_of_cores_; uint32_t rc_max_intra_target_; + int num_active_streams_; const std::unique_ptr frame_buffer_controller_factory_; std::unique_ptr frame_buffer_controller_;