diff --git a/webrtc/media/engine/simulcast.cc b/webrtc/media/engine/simulcast.cc index a89ac5265b..62587f682d 100644 --- a/webrtc/media/engine/simulcast.cc +++ b/webrtc/media/engine/simulcast.cc @@ -13,6 +13,7 @@ #include "webrtc/base/arraysize.h" #include "webrtc/base/logging.h" #include "webrtc/media/base/streamparams.h" +#include "webrtc/media/engine/constants.h" #include "webrtc/media/engine/simulcast.h" #include "webrtc/system_wrappers/include/field_trial.h" @@ -48,6 +49,8 @@ const SimulcastFormat kSimulcastFormats[] = { {0, 0, 1, 200, 150, 30} }; +const int kDefaultScreenshareSimulcastStreams = 2; + // Multiway: Number of temporal layers for each simulcast stream, for maximum // possible number of simulcast streams |kMaxSimulcastStreams|. The array // goes from lowest resolution at position 0 to highest resolution. @@ -78,8 +81,8 @@ int FindSimulcastFormatIndex(int width, int height) { MaybeExchangeWidthHeight(&width, &height); for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) { - if (width >= kSimulcastFormats[i].width && - height >= kSimulcastFormats[i].height) { + if (width * height >= + kSimulcastFormats[i].width * kSimulcastFormats[i].height) { return i; } } @@ -90,8 +93,8 @@ int FindSimulcastFormatIndex(int width, int height, size_t max_layers) { MaybeExchangeWidthHeight(&width, &height); for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) { - if (width >= kSimulcastFormats[i].width && - height >= kSimulcastFormats[i].height && + if (width * height >= + kSimulcastFormats[i].width * kSimulcastFormats[i].height && max_layers == kSimulcastFormats[i].max_layers) { return i; } @@ -117,7 +120,7 @@ size_t FindSimulcastMaxLayers(int width, int height) { // TODO(marpan): Investigate if we should return 0 instead of -1 in // FindSimulcast[Max/Target/Min]Bitrate functions below, since the // codec struct max/min/targeBitrates are unsigned. -int FindSimulcastMaxBitrateBps(int width, int height, size_t max_layers) { +int FindSimulcastMaxBitrateBps(int width, int height) { const int format_index = FindSimulcastFormatIndex(width, height); if (format_index == -1) { return -1; @@ -125,9 +128,7 @@ int FindSimulcastMaxBitrateBps(int width, int height, size_t max_layers) { return kSimulcastFormats[format_index].max_bitrate_kbps * 1000; } -int FindSimulcastTargetBitrateBps(int width, - int height, - size_t max_layers) { +int FindSimulcastTargetBitrateBps(int width, int height) { const int format_index = FindSimulcastFormatIndex(width, height); if (format_index == -1) { return -1; @@ -135,7 +136,7 @@ int FindSimulcastTargetBitrateBps(int width, return kSimulcastFormats[format_index].target_bitrate_kbps * 1000; } -int FindSimulcastMinBitrateBps(int width, int height, size_t max_layers) { +int FindSimulcastMinBitrateBps(int width, int height) { const int format_index = FindSimulcastFormatIndex(width, height); if (format_index == -1) { return -1; @@ -166,52 +167,73 @@ int GetTotalMaxBitrateBps(const std::vector& streams) { return total_max_bitrate_bps; } -std::vector GetSimulcastConfig( - size_t max_streams, - int width, - int height, - int max_bitrate_bps, - int max_qp, - int max_framerate) { - size_t simulcast_layers = FindSimulcastMaxLayers(width, height); - if (simulcast_layers > max_streams) { +std::vector GetSimulcastConfig(size_t max_streams, + int width, + int height, + int max_bitrate_bps, + int max_qp, + int max_framerate, + bool is_screencast) { + size_t num_simulcast_layers; + if (is_screencast) { + num_simulcast_layers = + UseSimulcastScreenshare() ? kDefaultScreenshareSimulcastStreams : 1; + } else { + num_simulcast_layers = FindSimulcastMaxLayers(width, height); + } + + if (num_simulcast_layers > max_streams) { // If the number of SSRCs in the group differs from our target // number of simulcast streams for current resolution, switch down // to a resolution that matches our number of SSRCs. if (!SlotSimulcastMaxResolution(max_streams, &width, &height)) { return std::vector(); } - simulcast_layers = max_streams; + num_simulcast_layers = max_streams; } std::vector streams; - streams.resize(simulcast_layers); + streams.resize(num_simulcast_layers); - // Format width and height has to be divisible by |2 ^ number_streams - 1|. - width = NormalizeSimulcastSize(width, simulcast_layers); - height = NormalizeSimulcastSize(height, simulcast_layers); + if (!is_screencast) { + // Format width and height has to be divisible by |2 ^ number_streams - 1|. + width = NormalizeSimulcastSize(width, num_simulcast_layers); + height = NormalizeSimulcastSize(height, num_simulcast_layers); + } // Add simulcast sub-streams from lower resolution to higher resolutions. // Add simulcast streams, from highest resolution (|s| = number_streams -1) // to lowest resolution at |s| = 0. - for (size_t s = simulcast_layers - 1;; --s) { + for (size_t s = num_simulcast_layers - 1;; --s) { streams[s].width = width; streams[s].height = height; // TODO(pbos): Fill actual temporal-layer bitrate thresholds. - streams[s].temporal_layer_thresholds_bps.resize( - kDefaultConferenceNumberOfTemporalLayers[s] - 1); - streams[s].max_bitrate_bps = - FindSimulcastMaxBitrateBps(width, height, simulcast_layers); - streams[s].target_bitrate_bps = - FindSimulcastTargetBitrateBps(width, height, simulcast_layers); - streams[s].min_bitrate_bps = - FindSimulcastMinBitrateBps(width, height, simulcast_layers); streams[s].max_qp = max_qp; - streams[s].max_framerate = max_framerate; + if (is_screencast && s == 0) { + ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); + // For legacy screenshare in conference mode, tl0 and tl1 bitrates are + // piggybacked on the VideoCodec struct as target and max bitrates, + // respectively. See eg. webrtc::VP8EncoderImpl::SetRates(). + streams[s].min_bitrate_bps = kMinVideoBitrateKbps * 1000; + streams[s].target_bitrate_bps = config.tl0_bitrate_kbps * 1000; + streams[s].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; + streams[s].temporal_layer_thresholds_bps.clear(); + streams[s].temporal_layer_thresholds_bps.push_back( + config.tl0_bitrate_kbps * 1000); + streams[s].max_framerate = 5; + } else { + streams[s].temporal_layer_thresholds_bps.resize( + kDefaultConferenceNumberOfTemporalLayers[s] - 1); + streams[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height); + streams[s].target_bitrate_bps = + FindSimulcastTargetBitrateBps(width, height); + streams[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height); + streams[s].max_framerate = max_framerate; + } + width /= 2; height /= 2; - if (s == 0) { + if (s == 0) break; - } } // Spend additional bits to boost the max stream. @@ -230,6 +252,8 @@ static const int kScreenshareDefaultTl1BitrateKbps = 1000; static const char* kScreencastLayerFieldTrialName = "WebRTC-ScreenshareLayerRates"; +static const char* kSimulcastScreenshareFieldTrialName = + "WebRTC-SimulcastScreenshare"; ScreenshareLayerConfig::ScreenshareLayerConfig(int tl0_bitrate, int tl1_bitrate) : tl0_bitrate_kbps(tl0_bitrate), tl1_bitrate_kbps(tl1_bitrate) { @@ -272,4 +296,9 @@ bool ScreenshareLayerConfig::FromFieldTrialGroup( return true; } +bool UseSimulcastScreenshare() { + return webrtc::field_trial::FindFullName( + kSimulcastScreenshareFieldTrialName) == "Enabled"; +} + } // namespace cricket diff --git a/webrtc/media/engine/simulcast.h b/webrtc/media/engine/simulcast.h index 20be4c487e..3aae70f59d 100644 --- a/webrtc/media/engine/simulcast.h +++ b/webrtc/media/engine/simulcast.h @@ -19,6 +19,7 @@ namespace cricket { struct StreamParams; +// TODO(sprang): Remove this, as we're moving away from temporal layer mode. // Config for use with screen cast when temporal layers are enabled. struct ScreenshareLayerConfig { public: @@ -45,12 +46,16 @@ int GetTotalMaxBitrateBps(const std::vector& streams); void GetSimulcastSsrcs(const StreamParams& sp, std::vector* ssrcs); // Get simulcast settings. +// TODO(sprang): Remove default parameter when it's not longer referenced. std::vector GetSimulcastConfig(size_t max_streams, int width, int height, int max_bitrate_bps, int max_qp, - int max_framerate); + int max_framerate, + bool is_screencast = false); + +bool UseSimulcastScreenshare(); } // namespace cricket diff --git a/webrtc/media/engine/webrtcvideoengine2.cc b/webrtc/media/engine/webrtcvideoengine2.cc index 9e7f5c4bd4..0cd31937cc 100644 --- a/webrtc/media/engine/webrtcvideoengine2.cc +++ b/webrtc/media/engine/webrtcvideoengine2.cc @@ -301,11 +301,16 @@ class EncoderStreamFactory int width, int height, const webrtc::VideoEncoderConfig& encoder_config) override { - RTC_DCHECK(encoder_config.number_of_streams > 1 ? !is_screencast_ : true); - if (encoder_config.number_of_streams > 1) { + if (is_screencast_ && + (!conference_mode_ || !cricket::UseSimulcastScreenshare())) { + RTC_DCHECK_EQ(1, encoder_config.number_of_streams); + } + if (encoder_config.number_of_streams > 1 || + (CodecNamesEq(codec_name_, kVp8CodecName) && is_screencast_ && + conference_mode_)) { return GetSimulcastConfig(encoder_config.number_of_streams, width, height, encoder_config.max_bitrate_bps, max_qp_, - max_framerate_); + max_framerate_, is_screencast_); } // For unset max bitrates set default bitrate for non-simulcast. @@ -322,20 +327,6 @@ class EncoderStreamFactory stream.target_bitrate_bps = stream.max_bitrate_bps = max_bitrate_bps; stream.max_qp = max_qp_; - // Conference mode screencast uses 2 temporal layers split at 100kbit. - if (conference_mode_ && is_screencast_) { - ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); - // For screenshare in conference mode, tl0 and tl1 bitrates are - // piggybacked - // on the VideoCodec struct as target and max bitrates, respectively. - // See eg. webrtc::VP8EncoderImpl::SetRates(). - stream.target_bitrate_bps = config.tl0_bitrate_kbps * 1000; - stream.max_bitrate_bps = config.tl1_bitrate_kbps * 1000; - stream.temporal_layer_thresholds_bps.clear(); - stream.temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps * - 1000); - } - if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screencast_) { stream.temporal_layer_thresholds_bps.resize( GetDefaultVp9TemporalLayers() - 1); @@ -1551,6 +1542,7 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( enable_cpu_overuse_detection_(enable_cpu_overuse_detection), source_(nullptr), external_encoder_factory_(external_encoder_factory), + internal_encoder_factory_(new InternalEncoderFactory()), stream_(nullptr), encoder_sink_(nullptr), parameters_(std::move(config), options, max_bitrate_bps, codec_settings), @@ -1678,10 +1670,20 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder( } // Try creating internal encoder. - InternalEncoderFactory internal_encoder_factory; - if (FindMatchingCodec(internal_encoder_factory.supported_codecs(), codec)) { - return AllocatedEncoder(internal_encoder_factory.CreateVideoEncoder(codec), - codec, false /* is_external */); + if (FindMatchingCodec(internal_encoder_factory_->supported_codecs(), codec)) { + if (parameters_.encoder_config.content_type == + webrtc::VideoEncoderConfig::ContentType::kScreen && + parameters_.conference_mode && UseSimulcastScreenshare()) { + // TODO(sprang): Remove this adapter once libvpx supports simulcast with + // same-resolution substreams. + WebRtcSimulcastEncoderFactory adapter_factory( + internal_encoder_factory_.get()); + return AllocatedEncoder(adapter_factory.CreateVideoEncoder(codec), codec, + false /* is_external */); + } + return AllocatedEncoder( + internal_encoder_factory_->CreateVideoEncoder(codec), codec, + false /* is_external */); } // This shouldn't happen, we should not be trying to create something we don't @@ -1858,9 +1860,11 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoderConfig( // By default, the stream count for the codec configuration should match the // number of negotiated ssrcs. But if the codec is blacklisted for simulcast - // or a screencast, only configure a single stream. + // or a screencast (and not in simulcast screenshare experiment), only + // configure a single stream. encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size(); - if (IsCodecBlacklistedForSimulcast(codec.name) || is_screencast) { + if (IsCodecBlacklistedForSimulcast(codec.name) || + (is_screencast && !UseSimulcastScreenshare())) { encoder_config.number_of_streams = 1; } diff --git a/webrtc/media/engine/webrtcvideoengine2.h b/webrtc/media/engine/webrtcvideoengine2.h index 2ee9bc79b8..7043e031a4 100644 --- a/webrtc/media/engine/webrtcvideoengine2.h +++ b/webrtc/media/engine/webrtcvideoengine2.h @@ -330,6 +330,8 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, public webrtc::Transport { ACCESS_ON(&thread_checker_); WebRtcVideoEncoderFactory* const external_encoder_factory_ ACCESS_ON(&thread_checker_); + const std::unique_ptr internal_encoder_factory_ + ACCESS_ON(&thread_checker_); webrtc::VideoSendStream* stream_ ACCESS_ON(&thread_checker_); rtc::VideoSinkInterface* encoder_sink_ diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc index f72d7ae86c..a93d2d98c2 100644 --- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc @@ -21,6 +21,7 @@ #include "webrtc/media/base/mediaconstants.h" #include "webrtc/media/base/testutils.h" #include "webrtc/media/base/videoengine_unittest.h" +#include "webrtc/media/engine/constants.h" #include "webrtc/media/engine/fakewebrtccall.h" #include "webrtc/media/engine/fakewebrtcvideoengine.h" #include "webrtc/media/engine/simulcast.h" @@ -3960,7 +3961,7 @@ TEST_F(WebRtcVideoChannel2Test, ConfiguresLocalSsrcOnExistingReceivers) { class WebRtcVideoChannel2SimulcastTest : public testing::Test { public: WebRtcVideoChannel2SimulcastTest() - : fake_call_(webrtc::Call::Config(&event_log_)) {} + : fake_call_(webrtc::Call::Config(&event_log_)), last_ssrc_(0) {} void SetUp() override { engine_.Init(); @@ -3975,9 +3976,16 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test { int capture_width, int capture_height, size_t num_configured_streams, - size_t expected_num_streams) { + size_t expected_num_streams, + bool screenshare, + bool conference_mode) { cricket::VideoSendParameters parameters; + VideoOptions options; parameters.codecs.push_back(codec); + parameters.conference_mode = conference_mode; + if (screenshare) { + options.is_screencast = rtc::Optional(screenshare); + } ASSERT_TRUE(channel_->SetSendParameters(parameters)); std::vector ssrcs = MAKE_VECTOR(kSsrcs3); @@ -3990,7 +3998,7 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test { // expected simulcast layers. cricket::FakeVideoCapturer capturer; EXPECT_TRUE( - channel_->SetVideoSend(ssrcs.front(), true, nullptr, &capturer)); + channel_->SetVideoSend(ssrcs.front(), true, &options, &capturer)); EXPECT_EQ(cricket::CS_RUNNING, capturer.Start(cricket::VideoFormat( capture_width, capture_height, cricket::VideoFormat::FpsToInterval(30), @@ -4001,9 +4009,32 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test { std::vector video_streams = stream->GetVideoStreams(); ASSERT_EQ(expected_num_streams, video_streams.size()); - std::vector expected_streams = GetSimulcastConfig( - num_configured_streams, capture_width, capture_height, 0, kDefaultQpMax, - kDefaultVideoMaxFramerate); + std::vector expected_streams; + if (conference_mode) { + expected_streams = GetSimulcastConfig( + num_configured_streams, capture_width, capture_height, 0, + kDefaultQpMax, kDefaultVideoMaxFramerate, screenshare); + } else { + webrtc::VideoStream stream; + stream.width = capture_width; + stream.height = capture_height; + stream.max_framerate = kDefaultVideoMaxFramerate; + stream.min_bitrate_bps = cricket::kMinVideoBitrateKbps * 1000; + int max_bitrate_kbps; + if (capture_width * capture_height <= 320 * 240) { + max_bitrate_kbps = 600; + } else if (capture_width * capture_height <= 640 * 480) { + max_bitrate_kbps = 1700; + } else if (capture_width * capture_height <= 960 * 540) { + max_bitrate_kbps = 2000; + } else { + max_bitrate_kbps = 2500; + } + stream.target_bitrate_bps = stream.max_bitrate_bps = + max_bitrate_kbps * 1000; + stream.max_qp = kDefaultQpMax; + expected_streams.push_back(stream); + } ASSERT_EQ(expected_streams.size(), video_streams.size()); @@ -4032,7 +4063,8 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test { EXPECT_GT(video_streams[i].max_qp, 0); EXPECT_EQ(expected_streams[i].max_qp, video_streams[i].max_qp); - EXPECT_FALSE(expected_streams[i].temporal_layer_thresholds_bps.empty()); + EXPECT_EQ(!conference_mode, + expected_streams[i].temporal_layer_thresholds_bps.empty()); EXPECT_EQ(expected_streams[i].temporal_layer_thresholds_bps, video_streams[i].temporal_layer_thresholds_bps); @@ -4086,15 +4118,37 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test { }; TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWith2SimulcastStreams) { - VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2); + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2, false, + true); } TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWith3SimulcastStreams) { - VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3); + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3, false, + true); } // Test that we normalize send codec format size in simulcast. TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWithOddSizeInSimulcast) { - VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2); + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2, false, + true); } + +TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsForScreenshare) { + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 1, true, + false); +} + +TEST_F(WebRtcVideoChannel2SimulcastTest, + SetSendCodecsForConferenceModeScreenshare) { + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 1, true, + true); +} + +TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsForSimulcastScreenshare) { + webrtc::test::ScopedFieldTrials override_field_trials_( + "WebRTC-SimulcastScreenshare/Enabled/"); + VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 2, true, + true); +} + } // namespace cricket diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index fc4d3aff50..e09fd9f2d6 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -371,6 +371,7 @@ if (rtc_include_tests) { "utility/moving_average_unittest.cc", "utility/quality_scaler_unittest.cc", "utility/simulcast_rate_allocator_unittest.cc", + "video_codec_initializer_unittest.cc", "video_coding_robustness_unittest.cc", "video_packet_buffer_unittest.cc", "video_receiver_unittest.cc", diff --git a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc index 7807d45d7a..f842fb4213 100644 --- a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc @@ -94,11 +94,11 @@ class RealTimeTemporalLayers : public TemporalLayers { timestamp_(0), last_base_layer_sync_(0), layer_ids_length_(0), - layer_ids_(NULL), + layer_ids_(nullptr), encode_flags_length_(0), - encode_flags_(NULL) { + encode_flags_(nullptr) { RTC_CHECK_GE(max_temporal_layers_, 1); - RTC_CHECK_GE(max_temporal_layers_, 3); + RTC_CHECK_LE(max_temporal_layers_, 3); } virtual ~RealTimeTemporalLayers() {} diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index feb8da9f28..aca4514279 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -30,6 +30,8 @@ static const int kQpDeltaThresholdForSync = 8; const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5; const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0; +constexpr int ScreenshareLayers::kMaxNumTemporalLayers; + // Since this is TL0 we only allow updating and predicting from the LAST // reference frame. const int ScreenshareLayers::kTl0Flags = @@ -55,8 +57,14 @@ webrtc::TemporalLayers* ScreenshareTemporalLayersFactory::Create( int simulcast_id, int num_temporal_layers, uint8_t initial_tl0_pic_idx) const { - webrtc::TemporalLayers* tl = new webrtc::ScreenshareLayers( - num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock()); + webrtc::TemporalLayers* tl; + if (simulcast_id == 0) { + tl = new webrtc::ScreenshareLayers(num_temporal_layers, rand(), + webrtc::Clock::GetRealTimeClock()); + } else { + RealTimeTemporalLayersFactory rt_tl_factory; + tl = rt_tl_factory.Create(simulcast_id, num_temporal_layers, rand()); + } if (listener_) listener_->OnTemporalLayersCreated(simulcast_id, tl); return tl; @@ -66,7 +74,8 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, uint8_t initial_tl0_pic_idx, Clock* clock) : clock_(clock), - number_of_temporal_layers_(num_temporal_layers), + number_of_temporal_layers_( + std::min(kMaxNumTemporalLayers, num_temporal_layers)), last_base_layer_sync_(false), tl0_pic_idx_(initial_tl0_pic_idx), active_layer_(-1), @@ -78,8 +87,8 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, max_debt_bytes_(0), encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale. bitrate_updated_(false) { - RTC_CHECK_GT(num_temporal_layers, 0); - RTC_CHECK_LE(num_temporal_layers, 2); + RTC_CHECK_GT(number_of_temporal_layers_, 0); + RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers); } ScreenshareLayers::~ScreenshareLayers() { diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index de7665ca95..53818a2a97 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -84,7 +84,7 @@ class ScreenshareLayers : public TemporalLayers { RateStatistics encode_framerate_; bool bitrate_updated_; - static const int kMaxNumTemporalLayers = 2; + static constexpr int kMaxNumTemporalLayers = 2; struct TemporalLayer { TemporalLayer() : state(State::kNormal), diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc index 311568b6a2..0a81550781 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -106,23 +106,24 @@ BitrateAllocation SimulcastRateAllocator::GetAllocation( const int num_temporal_streams = std::max( 1, codec_.numberOfSimulcastStreams == 0 ? codec_.VP8().numberOfTemporalLayers - : codec_.simulcastStream[0].numberOfTemporalLayers); + : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers); uint32_t max_bitrate_kbps; - if (num_spatial_streams == 1) { - max_bitrate_kbps = codec_.maxBitrate; - - // TODO(holmer): This is a temporary hack for screensharing, where we + // Legacy temporal-layered only screenshare, or simulcast screenshare + // with legacy mode for simulcast stream 0. + if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 && + ((num_spatial_streams == 1 && num_temporal_streams == 2) || // Legacy. + (num_spatial_streams > 1 && simulcast_id == 0))) { // Simulcast. + // TODO(holmer): This is a "temporary" hack for screensharing, where we // interpret the startBitrate as the encoder target bitrate. This is // to allow for a different max bitrate, so if the codec can't meet // the target we still allow it to overshoot up to the max before dropping // frames. This hack should be improved. - if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 && - num_temporal_streams == 2) { - int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps); - max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps); - target_bitrate_kbps = tl0_bitrate; - } + int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps); + max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps); + target_bitrate_kbps = tl0_bitrate; + } else if (num_spatial_streams == 1) { + max_bitrate_kbps = codec_.maxBitrate; } else { max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate; } diff --git a/webrtc/modules/video_coding/video_codec_initializer.cc b/webrtc/modules/video_coding/video_codec_initializer.cc index dbdede07ad..96a00334ab 100644 --- a/webrtc/modules/video_coding/video_codec_initializer.cc +++ b/webrtc/modules/video_coding/video_codec_initializer.cc @@ -38,8 +38,9 @@ bool VideoCodecInitializer::SetupCodec( case kVideoCodecVP8: { if (!codec->VP8()->tl_factory) { if (codec->mode == kScreensharing && - codec->numberOfSimulcastStreams == 1 && - codec->VP8()->numberOfTemporalLayers == 2) { + (codec->numberOfSimulcastStreams > 1 || + (codec->numberOfSimulcastStreams == 1 && + codec->VP8()->numberOfTemporalLayers == 2))) { // Conference mode temporal layering for screen content. tl_factory.reset(new ScreenshareTemporalLayersFactory()); } else { @@ -102,7 +103,7 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( break; case VideoEncoderConfig::ContentType::kScreen: video_codec.mode = kScreensharing; - if (streams.size() == 1 && + if (streams.size() >= 1 && streams[0].temporal_layer_thresholds_bps.size() == 1) { video_codec.targetBitrate = streams[0].temporal_layer_thresholds_bps[0] / 1000; @@ -180,8 +181,12 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( RTC_DCHECK_GT(streams[i].width, 0); RTC_DCHECK_GT(streams[i].height, 0); RTC_DCHECK_GT(streams[i].max_framerate, 0); - // Different framerates not supported per stream at the moment. - RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); + // Different framerates not supported per stream at the moment, unless it's + // screenshare where there is an exception and a simulcast encoder adapter, + // which supports different framerates, is used instead. + if (config.content_type != VideoEncoderConfig::ContentType::kScreen) { + RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); + } RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); diff --git a/webrtc/modules/video_coding/video_codec_initializer_unittest.cc b/webrtc/modules/video_coding/video_codec_initializer_unittest.cc new file mode 100644 index 0000000000..d9d1d03e37 --- /dev/null +++ b/webrtc/modules/video_coding/video_codec_initializer_unittest.cc @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/common_video/include/video_bitrate_allocator.h" +#include "webrtc/common_types.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/modules/video_coding/include/video_codec_initializer.h" +#include "webrtc/test/gtest.h" +#include "webrtc/video_encoder.h" + +namespace webrtc { + +namespace { +static const char* kVp8PayloadName = "VP8"; +static const int kVp8PayloadType = 100; +static const int kDefaultWidth = 1280; +static const int kDefaultHeight = 720; +static const int kDefaultFrameRate = 30; +static const uint32_t kDefaultMinBitrateBps = 60000; +static const uint32_t kDefaultTargetBitrateBps = 2000000; +static const uint32_t kDefaultMaxBitrateBps = 2000000; +static const uint32_t kDefaultMinTransmitBitrateBps = 400000; +static const int kDefaultMaxQp = 48; +static const uint32_t kScreenshareTl0BitrateBps = 100000; +static const uint32_t kScreenshareCodecTargetBitrateBps = 200000; +static const uint32_t kScreenshareDefaultFramerate = 5; +// Bitrates for the temporal layers of the higher screenshare simulcast stream. +static const uint32_t kHighScreenshareTl0Bps = 800000; +static const uint32_t kHighScreenshareTl1Bps = 1200000; +} // namespace + +/* + * static bool SetupCodec( + const VideoEncoderConfig& config, + const VideoSendStream::Config::EncoderSettings settings, + const std::vector& streams, + bool nack_enabled, + VideoCodec* codec, + std::unique_ptr* bitrate_allocator); + + // Create a bitrate allocator for the specified codec. |tl_factory| is + // optional, if it is populated, ownership of that instance will be + // transferred to the VideoBitrateAllocator instance. + static std::unique_ptr CreateBitrateAllocator( + const VideoCodec& codec, + std::unique_ptr tl_factory); + */ + +// TODO(sprang): Extend coverage to handle the rest of the codec initializer. +class VideoCodecInitializerTest : public ::testing::Test { + public: + VideoCodecInitializerTest() : nack_enabled_(false) {} + virtual ~VideoCodecInitializerTest() {} + + protected: + void SetUpFor(VideoCodecType type, + int num_spatial_streams, + int num_temporal_streams, + bool screenshare) { + config_ = VideoEncoderConfig(); + if (screenshare) { + config_.min_transmit_bitrate_bps = kDefaultMinTransmitBitrateBps; + config_.content_type = VideoEncoderConfig::ContentType::kScreen; + } + + if (type == VideoCodecType::kVideoCodecVP8) { + config_.number_of_streams = num_spatial_streams; + VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); + vp8_settings.numberOfTemporalLayers = num_temporal_streams; + config_.encoder_specific_settings = new rtc::RefCountedObject< + webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings); + settings_.payload_name = kVp8PayloadName; + settings_.payload_type = kVp8PayloadType; + } else { + ADD_FAILURE() << "Unexpected codec type: " << type; + } + } + + bool InitializeCodec() { + codec_out_ = VideoCodec(); + bitrate_allocator_out_.reset(); + temporal_layers_.clear(); + if (!VideoCodecInitializer::SetupCodec(config_, settings_, streams_, + nack_enabled_, &codec_out_, + &bitrate_allocator_out_)) { + return false; + } + + // Make sure temporal layers instances have been created. + if (codec_out_.codecType == VideoCodecType::kVideoCodecVP8) { + if (!codec_out_.VP8()->tl_factory) + return false; + + for (int i = 0; i < codec_out_.numberOfSimulcastStreams; ++i) { + temporal_layers_.emplace_back(codec_out_.VP8()->tl_factory->Create( + i, streams_[i].temporal_layer_thresholds_bps.size() + 1, 0)); + } + } + return true; + } + + VideoStream DefaultStream() { + VideoStream stream; + stream.width = kDefaultWidth; + stream.height = kDefaultHeight; + stream.max_framerate = kDefaultFrameRate; + stream.min_bitrate_bps = kDefaultMinBitrateBps; + stream.target_bitrate_bps = kDefaultTargetBitrateBps; + stream.max_bitrate_bps = kDefaultMaxBitrateBps; + stream.max_qp = kDefaultMaxQp; + return stream; + } + + VideoStream DefaultScreenshareStream() { + VideoStream stream = DefaultStream(); + stream.min_bitrate_bps = 30000; + stream.target_bitrate_bps = kScreenshareTl0BitrateBps; + stream.max_bitrate_bps = 1000000; + stream.max_framerate = kScreenshareDefaultFramerate; + stream.temporal_layer_thresholds_bps.push_back(kScreenshareTl0BitrateBps); + return stream; + } + + // Input settings. + VideoEncoderConfig config_; + VideoSendStream::Config::EncoderSettings settings_; + std::vector streams_; + bool nack_enabled_; + + // Output. + VideoCodec codec_out_; + std::unique_ptr bitrate_allocator_out_; + std::vector> temporal_layers_; +}; + +TEST_F(VideoCodecInitializerTest, SingleStreamVp8Screenshare) { + SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 1, true); + streams_.push_back(DefaultStream()); + EXPECT_TRUE(InitializeCodec()); + + BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation( + kDefaultTargetBitrateBps, kDefaultFrameRate); + EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams); + EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers); + EXPECT_EQ(kDefaultTargetBitrateBps, bitrate_allocation.get_sum_bps()); +} + +TEST_F(VideoCodecInitializerTest, TemporalLayeredVp8Screenshare) { + SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 2, true); + streams_.push_back(DefaultScreenshareStream()); + EXPECT_TRUE(InitializeCodec()); + + EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams); + EXPECT_EQ(2u, codec_out_.VP8()->numberOfTemporalLayers); + BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation( + kScreenshareCodecTargetBitrateBps, kScreenshareDefaultFramerate); + EXPECT_EQ(kScreenshareCodecTargetBitrateBps, + bitrate_allocation.get_sum_bps()); + EXPECT_EQ(kScreenshareTl0BitrateBps, bitrate_allocation.GetBitrate(0, 0)); +} + +TEST_F(VideoCodecInitializerTest, SimlucastVp8Screenshare) { + SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 1, true); + streams_.push_back(DefaultScreenshareStream()); + VideoStream video_stream = DefaultStream(); + video_stream.max_framerate = kScreenshareDefaultFramerate; + streams_.push_back(video_stream); + EXPECT_TRUE(InitializeCodec()); + + EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams); + EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers); + const uint32_t max_bitrate_bps = + streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps; + BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation( + max_bitrate_bps, kScreenshareDefaultFramerate); + EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps()); + EXPECT_EQ(static_cast(streams_[0].target_bitrate_bps), + bitrate_allocation.GetSpatialLayerSum(0)); + EXPECT_EQ(static_cast(streams_[1].max_bitrate_bps), + bitrate_allocation.GetSpatialLayerSum(1)); +} + +TEST_F(VideoCodecInitializerTest, HighFpsSimlucastVp8Screenshare) { + // Two simulcast streams, the lower one using legacy settings (two temporal + // streams, 5fps), the higher one using 3 temporal streams and 30fps. + SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 3, true); + streams_.push_back(DefaultScreenshareStream()); + VideoStream video_stream = DefaultStream(); + video_stream.temporal_layer_thresholds_bps.push_back(kHighScreenshareTl0Bps); + video_stream.temporal_layer_thresholds_bps.push_back(kHighScreenshareTl1Bps); + streams_.push_back(video_stream); + EXPECT_TRUE(InitializeCodec()); + + EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams); + EXPECT_EQ(3u, codec_out_.VP8()->numberOfTemporalLayers); + const uint32_t max_bitrate_bps = + streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps; + BitrateAllocation bitrate_allocation = + bitrate_allocator_out_->GetAllocation(max_bitrate_bps, kDefaultFrameRate); + EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps()); + EXPECT_EQ(static_cast(streams_[0].target_bitrate_bps), + bitrate_allocation.GetSpatialLayerSum(0)); + EXPECT_EQ(static_cast(streams_[1].max_bitrate_bps), + bitrate_allocation.GetSpatialLayerSum(1)); + EXPECT_EQ(kHighScreenshareTl0Bps, bitrate_allocation.GetBitrate(1, 0)); + EXPECT_EQ(kHighScreenshareTl1Bps - kHighScreenshareTl0Bps, + bitrate_allocation.GetBitrate(1, 1)); +} + +} // namespace webrtc