diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index 30f61dc7ce..309dac12da 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -1288,6 +1288,47 @@ TEST_F(TestVp9ImplFrameDropping, DifferentFrameratePerSpatialLayer) { } } +TEST_F(TestVp9ImplFrameDropping, LayerMaxFramerateIsCappedByCodecMaxFramerate) { + const float input_framerate_fps = 30.0; + const float layer_max_framerate_fps = 30.0; + const uint32_t codec_max_framerate_fps = layer_max_framerate_fps / 3; + const size_t video_duration_secs = 3; + const size_t num_input_frames = video_duration_secs * input_framerate_fps; + + VideoBitrateAllocation bitrate_allocation; + codec_settings_.spatialLayers[0].width = codec_settings_.width; + codec_settings_.spatialLayers[0].height = codec_settings_.height; + codec_settings_.spatialLayers[0].maxFramerate = layer_max_framerate_fps; + codec_settings_.spatialLayers[0].minBitrate = codec_settings_.startBitrate; + codec_settings_.spatialLayers[0].maxBitrate = codec_settings_.startBitrate; + codec_settings_.spatialLayers[0].targetBitrate = codec_settings_.startBitrate; + codec_settings_.spatialLayers[0].active = true; + + bitrate_allocation.SetBitrate( + 0, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, 1 /* number of cores */, + 0 /* max payload size (unused) */)); + + encoder_->SetRates(VideoEncoder::RateControlParameters( + bitrate_allocation, codec_max_framerate_fps)); + + VideoFrame* input_frame = NextInputFrame(); + for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(*input_frame, nullptr)); + const size_t timestamp = input_frame->timestamp() + + kVideoPayloadTypeFrequency / input_framerate_fps; + input_frame->set_timestamp(static_cast(timestamp)); + } + + const size_t num_encoded_frames = GetNumEncodedFrames(); + const float encoded_framerate_fps = num_encoded_frames / video_duration_secs; + const float max_framerate_error_fps = codec_max_framerate_fps * 0.1f; + EXPECT_NEAR(encoded_framerate_fps, codec_max_framerate_fps, + max_framerate_error_fps); +} + class TestVp9ImplProfile2 : public TestVp9Impl { protected: void SetUp() override { diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index e0fe754631..f974932289 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -316,7 +316,8 @@ bool VP9EncoderImpl::SetSvcRates( } framerate_controller_[sl_idx].SetTargetRate( - codec_.spatialLayers[sl_idx].maxFramerate); + std::min(static_cast(codec_.maxFramerate), + codec_.spatialLayers[sl_idx].maxFramerate)); } } else { float rate_ratio[VPX_MAX_LAYERS] = {0}; @@ -534,17 +535,24 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, inter_layer_pred_ = inst->VP9().interLayerPred; - if (num_spatial_layers_ > 1 && - codec_.mode == VideoCodecMode::kScreensharing && !is_flexible_mode_) { - RTC_LOG(LS_ERROR) << "Flexible mode is required for screenshare with " - "several spatial layers"; + different_framerates_used_ = false; + for (size_t sl_idx = 1; sl_idx < num_spatial_layers_; ++sl_idx) { + if (std::abs(codec_.spatialLayers[sl_idx].maxFramerate - + codec_.spatialLayers[0].maxFramerate) > 1e-9) { + different_framerates_used_ = true; + } + } + + if (different_framerates_used_ && !is_flexible_mode_) { + RTC_LOG(LS_ERROR) << "Flexible mode required for different framerates on " + "different spatial layers"; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } // External reference control is required for different frame rate on spatial // layers because libvpx generates rtp incompatible references in this case. external_ref_control_ = field_trial::IsEnabled("WebRTC-Vp9ExternalRefCtrl") || - is_flexible_mode_ || + different_framerates_used_ || inter_layer_pred_ == InterLayerPredMode::kOn; if (num_temporal_layers_ == 1) { @@ -581,8 +589,7 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, if (external_ref_control_) { config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_BYPASS; - if (num_temporal_layers_ > 1 && num_spatial_layers_ > 1 && - codec_.mode == VideoCodecMode::kScreensharing) { + if (num_temporal_layers_ > 1 && different_framerates_used_) { // External reference control for several temporal layers with different // frame rates on spatial layers is not implemented yet. return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -959,8 +966,7 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, if (VideoCodecMode::kScreensharing == codec_.mode) { for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) { ref_config.duration[sl_idx] = static_cast( - 90000 / (std::min(static_cast(codec_.maxFramerate), - framerate_controller_[sl_idx].GetTargetRate()))); + 90000 / framerate_controller_[sl_idx].GetTargetRate()); } } @@ -979,9 +985,8 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, RTC_DCHECK_GE(framerate_controller_.size(), num_active_spatial_layers_); float target_framerate_fps = (codec_.mode == VideoCodecMode::kScreensharing) - ? std::min(static_cast(codec_.maxFramerate), - framerate_controller_[num_active_spatial_layers_ - 1] - .GetTargetRate()) + ? framerate_controller_[num_active_spatial_layers_ - 1] + .GetTargetRate() : codec_.maxFramerate; uint32_t duration = static_cast(90000 / target_framerate_fps); const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_, @@ -1196,8 +1201,6 @@ void VP9EncoderImpl::FillReferenceIndices(const vpx_codec_cx_pkt& pkt, // It is safe to ignore this requirement if inter-layer prediction is // enabled for all frames when all base frames are relayed to receiver. RTC_DCHECK_EQ(ref_buf.spatial_layer_id, layer_id.spatial_layer_id); - } else { - RTC_DCHECK_LE(ref_buf.spatial_layer_id, layer_id.spatial_layer_id); } RTC_DCHECK_LE(ref_buf.temporal_layer_id, layer_id.temporal_layer_id); @@ -1317,7 +1320,7 @@ vpx_svc_ref_frame_config_t VP9EncoderImpl::SetReferences( const bool same_spatial_layer = ref_buf_[buf_idx].spatial_layer_id == sl_idx; bool correct_pid = false; - if (is_flexible_mode_) { + if (different_framerates_used_) { correct_pid = pid_diff < kMaxAllowedPidDIff; } else { // Below code assumes single temporal referecence. @@ -1516,8 +1519,7 @@ size_t VP9EncoderImpl::SteadyStateSize(int sid, int tid) { const size_t bitrate_bps = current_bitrate_allocation_.GetBitrate( sid, tid == kNoTemporalIdx ? 0 : tid); const float fps = (codec_.mode == VideoCodecMode::kScreensharing) - ? std::min(static_cast(codec_.maxFramerate), - framerate_controller_[sid].GetTargetRate()) + ? framerate_controller_[sid].GetTargetRate() : codec_.maxFramerate; return static_cast( bitrate_bps / (8 * fps) * diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index 73bca263ef..acc03bff5c 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -113,6 +113,7 @@ class VP9EncoderImpl : public VP9Encoder { GofInfoVP9 gof_; // Contains each frame's temporal information for // non-flexible mode. bool force_key_frame_; + bool different_framerates_used_; size_t pics_since_key_; uint8_t num_temporal_layers_; uint8_t num_spatial_layers_; // Number of configured SLs diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index cc30c79b26..5415f96401 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -843,7 +843,7 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, params_.ss[video_idx].num_spatial_layers); vp9_settings.interLayerPred = params_.ss[video_idx].inter_layer_pred; // High FPS vp9 screenshare requires flexible mode. - if (params_.ss[video_idx].num_spatial_layers > 1) { + if (params_.video[video_idx].fps > 5) { vp9_settings.flexibleMode = true; } video_encoder_configs_[video_idx].encoder_specific_settings =