From 4a5a3da00a372a88838f64be12d3ba7fc11d09a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20M=C3=B6ller?= Date: Fri, 17 Jun 2022 11:26:02 +0200 Subject: [PATCH] Change VideoStreamConfig to use vector of scalability modes. Each entry represents one simulcast stream. Bug: webrtc:11607 Change-Id: If78ff334fdb99322deded57f0cbe7ebad7de5abc Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/265960 Commit-Queue: Niels Moller Reviewed-by: Artem Titov Reviewed-by: Florent Castelli Cr-Commit-Position: refs/heads/main@{#37272} --- test/scenario/BUILD.gn | 2 + test/scenario/probing_test.cc | 4 +- test/scenario/scenario_config.h | 12 ++--- test/scenario/video_stream.cc | 63 ++++++++++++-------------- test/scenario/video_stream_unittest.cc | 9 ++-- 5 files changed, 44 insertions(+), 46 deletions(-) diff --git a/test/scenario/BUILD.gn b/test/scenario/BUILD.gn index ddcce69c84..dc38247736 100644 --- a/test/scenario/BUILD.gn +++ b/test/scenario/BUILD.gn @@ -97,6 +97,7 @@ if (rtc_include_tests && !build_with_chromium) { "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:video_frame", "../../api/video:video_rtp_headers", + "../../api/video_codecs:scalability_mode", "../../api/video_codecs:video_codecs_api", "../../audio", "../../call", @@ -123,6 +124,7 @@ if (rtc_include_tests && !build_with_chromium) { "../../modules/video_coding:webrtc_multiplex", "../../modules/video_coding:webrtc_vp8", "../../modules/video_coding:webrtc_vp9", + "../../modules/video_coding/svc:scalability_mode_util", "../../rtc_base", "../../rtc_base:checks", "../../rtc_base:copy_on_write_buffer", diff --git a/test/scenario/probing_test.cc b/test/scenario/probing_test.cc index f08a003d5c..74b68fc044 100644 --- a/test/scenario/probing_test.cc +++ b/test/scenario/probing_test.cc @@ -86,7 +86,9 @@ TEST(ProbingTest, ProbesRampsUpWhenVideoEncoderConfigChanges) { VideoStreamConfig video_config; video_config.encoder.codec = VideoStreamConfig::Encoder::Codec::kVideoCodecVP8; - video_config.encoder.layers.spatial = 3; + video_config.encoder.simulcast_streams = {webrtc::ScalabilityMode::kL1T3, + webrtc::ScalabilityMode::kL1T3, + webrtc::ScalabilityMode::kL1T3}; video_config.source.generator.width = 1280; video_config.source.generator.height = 720; diff --git a/test/scenario/scenario_config.h b/test/scenario/scenario_config.h index c0dcd65e0c..5ff622f522 100644 --- a/test/scenario/scenario_config.h +++ b/test/scenario/scenario_config.h @@ -23,6 +23,7 @@ #include "api/units/data_size.h" #include "api/units/time_delta.h" #include "api/video/video_codec_type.h" +#include "api/video_codecs/scalability_mode.h" #include "test/scenario/performance_stats.h" namespace webrtc { @@ -138,15 +139,8 @@ struct VideoStreamConfig { bool denoising = true; bool automatic_scaling = true; } single; - struct Layers { - int temporal = 1; - int spatial = 1; - enum class Prediction { - kTemporalOnly, - kSpatialOnKey, - kFull, - } prediction = Prediction::kFull; - } layers; + std::vector simulcast_streams = { + webrtc::ScalabilityMode::kL1T1}; DegradationPreference degradation_preference = DegradationPreference::MAINTAIN_FRAMERATE; diff --git a/test/scenario/video_stream.cc b/test/scenario/video_stream.cc index 3fcef570c7..3d1253ac92 100644 --- a/test/scenario/video_stream.cc +++ b/test/scenario/video_stream.cc @@ -22,6 +22,7 @@ #include "media/engine/internal_decoder_factory.h" #include "media/engine/internal_encoder_factory.h" #include "media/engine/webrtc_video_engine.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "test/call_test.h" #include "test/fake_encoder.h" #include "test/scenario/hardware_codecs.h" @@ -77,18 +78,7 @@ VideoEncoderConfig::ContentType ConvertContentType( return VideoEncoderConfig::ContentType::kScreen; } } -InterLayerPredMode ToInterLayerPredMode( - VideoStreamConfig::Encoder::Layers::Prediction value) { - using Pred = VideoStreamConfig::Encoder::Layers::Prediction; - switch (value) { - case Pred::kTemporalOnly: - return InterLayerPredMode::kOff; - case Pred::kSpatialOnKey: - return InterLayerPredMode::kOnKeyPic; - case Pred::kFull: - return InterLayerPredMode::kOn; - } -} + std::vector GetVideoRtpExtensions( const VideoStreamConfig config) { std::vector res = { @@ -155,18 +145,23 @@ CreateVp9SpecificSettings(VideoStreamConfig video_config) { constexpr auto kScreen = VideoStreamConfig::Encoder::ContentType::kScreen; VideoStreamConfig::Encoder conf = video_config.encoder; VideoCodecVP9 vp9 = VideoEncoder::GetDefaultVp9Settings(); + // TODO(bugs.webrtc.org/11607): Support separate scalability mode per + // simulcast stream. + ScalabilityMode scalability_mode = conf.simulcast_streams[0]; vp9.keyFrameInterval = conf.key_frame_interval.value_or(0); - vp9.numberOfTemporalLayers = static_cast(conf.layers.temporal); - vp9.numberOfSpatialLayers = static_cast(conf.layers.spatial); - vp9.interLayerPred = ToInterLayerPredMode(conf.layers.prediction); + vp9.numberOfTemporalLayers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + vp9.numberOfSpatialLayers = + ScalabilityModeToNumSpatialLayers(scalability_mode); + vp9.interLayerPred = ScalabilityModeToInterLayerPredMode(scalability_mode); if (conf.content_type == kScreen && - (video_config.source.framerate > 5 || conf.layers.spatial >= 3)) { + (video_config.source.framerate > 5 || vp9.numberOfSpatialLayers >= 3)) { vp9.flexibleMode = true; } - if (conf.content_type == kScreen || - conf.layers.temporal * conf.layers.spatial) { + if (conf.content_type == kScreen || vp9.numberOfTemporalLayers > 1 || + vp9.numberOfSpatialLayers > 1) { vp9.automaticResizeOn = false; vp9.denoisingOn = false; } else { @@ -181,8 +176,13 @@ rtc::scoped_refptr CreateVp8SpecificSettings(VideoStreamConfig config) { VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); vp8_settings.keyFrameInterval = config.encoder.key_frame_interval.value_or(0); - vp8_settings.numberOfTemporalLayers = config.encoder.layers.temporal; - if (config.encoder.layers.spatial * config.encoder.layers.temporal > 1) { + // TODO(bugs.webrtc.org/11607): Support separate scalability mode per + // simulcast stream. + ScalabilityMode scalability_mode = config.encoder.simulcast_streams[0]; + vp8_settings.numberOfTemporalLayers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + if (vp8_settings.numberOfTemporalLayers > 1 || + config.encoder.simulcast_streams.size() > 1) { vp8_settings.automaticResizeOn = false; vp8_settings.denoisingOn = false; } else { @@ -195,9 +195,8 @@ CreateVp8SpecificSettings(VideoStreamConfig config) { rtc::scoped_refptr CreateH264SpecificSettings(VideoStreamConfig config) { - RTC_DCHECK_EQ(config.encoder.layers.temporal, 1); - RTC_DCHECK_EQ(config.encoder.layers.spatial, 1); - + RTC_DCHECK_EQ(config.encoder.simulcast_streams.size(), 1); + RTC_DCHECK(config.encoder.simulcast_streams[0] == ScalabilityMode::kL1T1); // TODO(bugs.webrtc.org/6883): Set a key frame interval as a setting that // isn't codec specific. RTC_CHECK_EQ(0, config.encoder.key_frame_interval.value_or(0)); @@ -230,12 +229,9 @@ VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) { encoder_config.video_format = SdpVideoFormat(CodecTypeToPayloadString(config.encoder.codec), {}); - encoder_config.number_of_streams = 1; - if (config.encoder.codec == VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) - encoder_config.number_of_streams = - static_cast(config.encoder.layers.spatial); + encoder_config.number_of_streams = config.encoder.simulcast_streams.size(); encoder_config.simulcast_layers = - std::vector(config.encoder.layers.spatial); + std::vector(encoder_config.number_of_streams); encoder_config.min_transmit_bitrate_bps = config.stream.pad_to_rate.bps(); std::string cricket_codec = CodecTypeToCodecName(config.encoder.codec); @@ -258,11 +254,14 @@ VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) { encoder_config.frame_drop_enabled = config.encoder.frame_dropping; encoder_config.encoder_specific_settings = CreateEncoderSpecificSettings(config); - if (config.encoder.max_framerate) { - for (auto& layer : encoder_config.simulcast_layers) { + + for (size_t i = 0; i < encoder_config.number_of_streams; ++i) { + auto& layer = encoder_config.simulcast_layers[i]; + if (config.encoder.max_framerate) { layer.max_framerate = *config.encoder.max_framerate; layer.min_bitrate_bps = config.encoder.min_data_rate->bps_or(-1); } + layer.scalability_mode = config.encoder.simulcast_streams[i]; } return encoder_config; @@ -549,9 +548,7 @@ ReceiveVideoStream::ReceiveVideoStream(CallClient* receiver, VideoReceiveStreamInterface::Decoder decoder = CreateMatchingDecoder(CodecTypeToPayloadType(config.encoder.codec), CodecTypeToPayloadString(config.encoder.codec)); - size_t num_streams = 1; - if (config.encoder.codec == VideoStreamConfig::Encoder::Codec::kVideoCodecVP8) - num_streams = config.encoder.layers.spatial; + size_t num_streams = config.encoder.simulcast_streams.size(); for (size_t i = 0; i < num_streams; ++i) { rtc::VideoSinkInterface* renderer = &fake_renderer_; if (matcher->Active()) { diff --git a/test/scenario/video_stream_unittest.cc b/test/scenario/video_stream_unittest.cc index b37530dc50..e53af4ef2b 100644 --- a/test/scenario/video_stream_unittest.cc +++ b/test/scenario/video_stream_unittest.cc @@ -102,8 +102,11 @@ TEST(VideoStreamTest, ReceivesVp8SimulcastFrames) { c->source.generator.height = 768; c->encoder.implementation = CodecImpl::kSoftware; c->encoder.codec = Codec::kVideoCodecVP8; - // By enabling multiple spatial layers, simulcast will be enabled for VP8. - c->encoder.layers.spatial = 3; + // Enable simulcast. + c->encoder.simulcast_streams = {webrtc::ScalabilityMode::kL1T1, + webrtc::ScalabilityMode::kL1T1, + webrtc::ScalabilityMode::kL1T1}; + }); s.RunFor(kRunTime); } @@ -213,7 +216,7 @@ TEST(VideoStreamTest, ResolutionAdaptsToAvailableBandwidth) { c->encoder.implementation = CodecImpl::kSoftware; c->encoder.codec = Codec::kVideoCodecVP9; // Enable SVC. - c->encoder.layers.spatial = 2; + c->encoder.simulcast_streams = {webrtc::ScalabilityMode::kL2T1}; }); // Run for a few seconds, until streams have stabilized,