From 1370e309e720b83b18f8dc7e3430e439481d8c51 Mon Sep 17 00:00:00 2001 From: Seth Hampson Date: Wed, 7 Feb 2018 08:50:36 -0800 Subject: [PATCH] Refactor of GetSimulcastConfig & EncoderStreamFactory. The main pieces of this refactor are splitting up the creation of simulcast layers for screenshare or the normal case, more consistent naming, renaming streams to layers and trying to be more explicit with some of the logic. Also added TODOs for future work to put more application control into creating simulcast streams. Bug: webrtc:8785 Change-Id: Ibf49fa0cc6d890ff96f8ee11c89d93a2c94119d6 Reviewed-on: https://webrtc-review.googlesource.com/47580 Commit-Queue: Seth Hampson Reviewed-by: Peter Thatcher Cr-Commit-Position: refs/heads/master@{#21989} --- media/engine/simulcast.cc | 243 +++++++++++++++++------------- media/engine/simulcast.h | 40 +++-- media/engine/webrtcvideoengine.cc | 38 +++-- media/engine/webrtcvideoengine.h | 10 +- 4 files changed, 201 insertions(+), 130 deletions(-) diff --git a/media/engine/simulcast.cc b/media/engine/simulcast.cc index f2909d3f4c..956e31d4ec 100644 --- a/media/engine/simulcast.cc +++ b/media/engine/simulcast.cc @@ -51,7 +51,7 @@ const SimulcastFormat kSimulcastFormats[] = { {0, 0, 1, 200, 150, 30} }; -const int kMaxScreenshareSimulcastStreams = 2; +const int kMaxScreenshareSimulcastLayers = 2; // Multiway: Number of temporal layers for each simulcast stream, for maximum // possible number of simulcast streams |kMaxSimulcastStreams|. The array @@ -124,125 +124,164 @@ void SlotSimulcastMaxResolution(size_t max_layers, int* width, int* height) { << " height:" << *height; } -int GetTotalMaxBitrateBps(const std::vector& streams) { - int total_max_bitrate_bps = 0; - for (size_t s = 0; s < streams.size() - 1; ++s) { - total_max_bitrate_bps += streams[s].target_bitrate_bps; +void BoostMaxSimulcastLayer(int max_bitrate_bps, + std::vector* layers) { + // Spend additional bits to boost the max layer. + int bitrate_left_bps = max_bitrate_bps - GetTotalMaxBitrateBps(*layers); + if (bitrate_left_bps > 0) { + layers->back().max_bitrate_bps += bitrate_left_bps; } - total_max_bitrate_bps += streams.back().max_bitrate_bps; +} + +int GetTotalMaxBitrateBps(const std::vector& layers) { + int total_max_bitrate_bps = 0; + for (size_t s = 0; s < layers.size() - 1; ++s) { + total_max_bitrate_bps += layers[s].target_bitrate_bps; + } + total_max_bitrate_bps += layers.back().max_bitrate_bps; return total_max_bitrate_bps; } -std::vector GetSimulcastConfig(size_t max_streams, +std::vector GetSimulcastConfig(size_t max_layers, int width, int height, int max_bitrate_bps, double bitrate_priority, int max_qp, int max_framerate, - bool is_screencast) { - size_t num_simulcast_layers; - if (is_screencast) { - if (UseSimulcastScreenshare()) { - num_simulcast_layers = - std::min(max_streams, kMaxScreenshareSimulcastStreams); - } else { - num_simulcast_layers = 1; - } + bool is_screenshare) { + if (is_screenshare) { + return GetScreenshareLayers(max_layers, width, height, max_bitrate_bps, + bitrate_priority, max_qp, max_framerate, + ScreenshareSimulcastFieldTrialEnabled()); } else { - num_simulcast_layers = FindSimulcastMaxLayers(width, height); + return GetNormalSimulcastLayers(max_layers, width, height, max_bitrate_bps, + bitrate_priority, max_qp, max_framerate); } +} - if (num_simulcast_layers > max_streams) { - // If the number of SSRCs in the group differs from our target - // number of simulcast streams for current resolution, switch down - // to a resolution that matches our number of SSRCs. - SlotSimulcastMaxResolution(max_streams, &width, &height); - num_simulcast_layers = max_streams; +std::vector GetNormalSimulcastLayers( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate) { + // TODO(bugs.webrtc.org/8785): Currently if the resolution isn't large enough + // (defined in kSimulcastFormats) we scale down the number of simulcast + // layers. Consider changing this so that the application can have more + // control over exactly how many simulcast layers are used. + size_t num_simulcast_layers = FindSimulcastMaxLayers(width, height); + if (num_simulcast_layers > max_layers) { + // TODO(bugs.webrtc.org/8486): This scales down the resolution if the + // number of simulcast layers created by the application isn't sufficient + // (defined in kSimulcastFormats). For example if the input frame's + // resolution is HD, but there are only 2 simulcast layers, the + // resolution gets scaled down to VGA. Consider taking this logic out to + // allow the application more control over the resolutions. + SlotSimulcastMaxResolution(max_layers, &width, &height); + num_simulcast_layers = max_layers; } - std::vector streams; - streams.resize(num_simulcast_layers); + std::vector layers(num_simulcast_layers); - if (is_screencast) { - ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); - // For legacy screenshare in conference mode, tl0 and tl1 bitrates are - // piggybacked on the VideoCodec struct as target and max bitrates, - // respectively. See eg. webrtc::VP8EncoderImpl::SetRates(). - streams[0].width = width; - streams[0].height = height; - streams[0].max_qp = max_qp; - streams[0].max_framerate = 5; - streams[0].min_bitrate_bps = kMinVideoBitrateBps; - streams[0].target_bitrate_bps = config.tl0_bitrate_kbps * 1000; - streams[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; - streams[0].temporal_layer_thresholds_bps.clear(); - streams[0].temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps * - 1000); + // Format width and height has to be divisible by |2 ^ num_simulcast_layers - + // 1|. + width = NormalizeSimulcastSize(width, num_simulcast_layers); + height = NormalizeSimulcastSize(height, num_simulcast_layers); + // Add simulcast streams, from highest resolution (|s| = num_simulcast_layers + // -1) to lowest resolution at |s| = 0. + for (size_t s = num_simulcast_layers - 1;; --s) { + layers[s].width = width; + layers[s].height = height; + // TODO(pbos): Fill actual temporal-layer bitrate thresholds. + layers[s].max_qp = max_qp; + layers[s].temporal_layer_thresholds_bps.resize( + kDefaultConferenceNumberOfTemporalLayers[s] - 1); + layers[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height); + layers[s].target_bitrate_bps = FindSimulcastTargetBitrateBps(width, height); + layers[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height); + layers[s].max_framerate = max_framerate; - // With simulcast enabled, add another spatial layer. This one will have a - // more normal layout, with the regular 3 temporal layer pattern and no fps - // restrictions. The base simulcast stream will still use legacy setup. - if (num_simulcast_layers == kMaxScreenshareSimulcastStreams) { - // Add optional upper simulcast layer. - // Lowest temporal layers of a 3 layer setup will have 40% of the total - // bitrate allocation for that stream. Make sure the gap between the - // target of the lower stream and first temporal layer of the higher one - // is at most 2x the bitrate, so that upswitching is not hampered by - // stalled bitrate estimates. - int max_bitrate_bps = 2 * ((streams[0].target_bitrate_bps * 10) / 4); - // Cap max bitrate so it isn't overly high for the given resolution. - max_bitrate_bps = std::min( - max_bitrate_bps, FindSimulcastMaxBitrateBps(width, height)); + width /= 2; + height /= 2; - streams[1].width = width; - streams[1].height = height; - streams[1].max_qp = max_qp; - streams[1].max_framerate = max_framerate; - // Three temporal layers means two thresholds. - streams[1].temporal_layer_thresholds_bps.resize(2); - streams[1].min_bitrate_bps = streams[0].target_bitrate_bps * 2; - streams[1].target_bitrate_bps = max_bitrate_bps; - streams[1].max_bitrate_bps = max_bitrate_bps; - } - } else { - // Format width and height has to be divisible by |2 ^ number_streams - 1|. - width = NormalizeSimulcastSize(width, num_simulcast_layers); - height = NormalizeSimulcastSize(height, num_simulcast_layers); - - // Add simulcast sub-streams from lower resolution to higher resolutions. - // Add simulcast streams, from highest resolution (|s| = number_streams -1) - // to lowest resolution at |s| = 0. - for (size_t s = num_simulcast_layers - 1;; --s) { - streams[s].width = width; - streams[s].height = height; - // TODO(pbos): Fill actual temporal-layer bitrate thresholds. - streams[s].max_qp = max_qp; - streams[s].temporal_layer_thresholds_bps.resize( - kDefaultConferenceNumberOfTemporalLayers[s] - 1); - streams[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height); - streams[s].target_bitrate_bps = - FindSimulcastTargetBitrateBps(width, height); - streams[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height); - streams[s].max_framerate = max_framerate; - - width /= 2; - height /= 2; - - if (s == 0) - break; - } - - // Spend additional bits to boost the max stream. - int bitrate_left_bps = max_bitrate_bps - GetTotalMaxBitrateBps(streams); - if (bitrate_left_bps > 0) { - streams.back().max_bitrate_bps += bitrate_left_bps; + if (s == 0) { + break; } } + // If there is bitrate leftover, give it to the largest layer. + BoostMaxSimulcastLayer(max_bitrate_bps, &layers); + // Currently the relative bitrate priority of the sender is controlled by + // the value of the lowest VideoStream. + // TODO(bugs.webrtc.org/8630): The web specification describes being able to + // control relative bitrate for each individual simulcast layer, but this + // is currently just implemented per rtp sender. + layers[0].bitrate_priority = bitrate_priority; + return layers; +} +std::vector GetScreenshareLayers( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate, + bool screenshare_simulcast_enabled) { + auto max_screenshare_layers = + screenshare_simulcast_enabled ? kMaxScreenshareSimulcastLayers : 1; + size_t num_simulcast_layers = + std::min(max_layers, max_screenshare_layers); + + std::vector layers(num_simulcast_layers); + ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); + // For legacy screenshare in conference mode, tl0 and tl1 bitrates are + // piggybacked on the VideoCodec struct as target and max bitrates, + // respectively. See eg. webrtc::VP8EncoderImpl::SetRates(). + layers[0].width = width; + layers[0].height = height; + layers[0].max_qp = max_qp; + layers[0].max_framerate = 5; + layers[0].min_bitrate_bps = kMinVideoBitrateBps; + layers[0].target_bitrate_bps = config.tl0_bitrate_kbps * 1000; + layers[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; + layers[0].temporal_layer_thresholds_bps.clear(); + layers[0].temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps * + 1000); + + // With simulcast enabled, add another spatial layer. This one will have a + // more normal layout, with the regular 3 temporal layer pattern and no fps + // restrictions. The base simulcast layer will still use legacy setup. + if (num_simulcast_layers == kMaxScreenshareSimulcastLayers) { + // Add optional upper simulcast layer. + // Lowest temporal layers of a 3 layer setup will have 40% of the total + // bitrate allocation for that simulcast layer. Make sure the gap between + // the target of the lower simulcast layer and first temporal layer of the + // higher one is at most 2x the bitrate, so that upswitching is not hampered + // by stalled bitrate estimates. + int max_bitrate_bps = 2 * ((layers[0].target_bitrate_bps * 10) / 4); + // Cap max bitrate so it isn't overly high for the given resolution. + max_bitrate_bps = std::min(max_bitrate_bps, + FindSimulcastMaxBitrateBps(width, height)); + + layers[1].width = width; + layers[1].height = height; + layers[1].max_qp = max_qp; + layers[1].max_framerate = max_framerate; + // Three temporal layers means two thresholds. + layers[1].temporal_layer_thresholds_bps.resize(2); + layers[1].min_bitrate_bps = layers[0].target_bitrate_bps * 2; + layers[1].target_bitrate_bps = max_bitrate_bps; + layers[1].max_bitrate_bps = max_bitrate_bps; + } + + BoostMaxSimulcastLayer(max_bitrate_bps, &layers); // The bitrate priority currently implemented on a per-sender level, so we - // just set it for the first video stream. - streams[0].bitrate_priority = bitrate_priority; - return streams; + // just set it for the first simulcast layer. + layers[0].bitrate_priority = bitrate_priority; + return layers; } static const int kScreenshareMinBitrateKbps = 50; @@ -250,7 +289,7 @@ static const int kScreenshareMaxBitrateKbps = 6000; static const int kScreenshareDefaultTl0BitrateKbps = 200; static const int kScreenshareDefaultTl1BitrateKbps = 1000; -static const char* kScreencastLayerFieldTrialName = +static const char* kScreenshareLayerFieldTrialName = "WebRTC-ScreenshareLayerRates"; static const char* kSimulcastScreenshareFieldTrialName = "WebRTC-SimulcastScreenshare"; @@ -261,7 +300,7 @@ ScreenshareLayerConfig::ScreenshareLayerConfig(int tl0_bitrate, int tl1_bitrate) ScreenshareLayerConfig ScreenshareLayerConfig::GetDefault() { std::string group = - webrtc::field_trial::FindFullName(kScreencastLayerFieldTrialName); + webrtc::field_trial::FindFullName(kScreenshareLayerFieldTrialName); ScreenshareLayerConfig config(kScreenshareDefaultTl0BitrateKbps, kScreenshareDefaultTl1BitrateKbps); @@ -297,7 +336,7 @@ bool ScreenshareLayerConfig::FromFieldTrialGroup( return true; } -bool UseSimulcastScreenshare() { +bool ScreenshareSimulcastFieldTrialEnabled() { return webrtc::field_trial::IsEnabled(kSimulcastScreenshareFieldTrialName); } diff --git a/media/engine/simulcast.h b/media/engine/simulcast.h index a2d27ed7b2..005c80e944 100644 --- a/media/engine/simulcast.h +++ b/media/engine/simulcast.h @@ -45,16 +45,38 @@ int GetTotalMaxBitrateBps(const std::vector& streams); // Get simulcast settings. // TODO(sprang): Remove default parameter when it's not longer referenced. -std::vector GetSimulcastConfig(size_t max_streams, - int width, - int height, - int max_bitrate_bps, - double bitrate_priority, - int max_qp, - int max_framerate, - bool is_screencast = false); +std::vector GetSimulcastConfig( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate, + bool is_screenshare = false); -bool UseSimulcastScreenshare(); +// Gets the simulcast config layers for a non-screensharing case. +std::vector GetNormalSimulcastLayers( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate); + +// Get simulcast config layers for screenshare settings. +std::vector GetScreenshareLayers( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate, + bool screenshare_simulcast_enabled); + +bool ScreenshareSimulcastFieldTrialEnabled(); } // namespace cricket diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc index dbf862a86b..8aa5aa222a 100644 --- a/media/engine/webrtcvideoengine.cc +++ b/media/engine/webrtcvideoengine.cc @@ -1896,8 +1896,8 @@ WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig( // configure a single stream. encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size(); if (IsCodecBlacklistedForSimulcast(codec.name) || - (is_screencast && - (!UseSimulcastScreenshare() || !parameters_.conference_mode))) { + (is_screencast && (!ScreenshareSimulcastFieldTrialEnabled() || + !parameters_.conference_mode))) { encoder_config.number_of_streams = 1; } @@ -2614,23 +2614,31 @@ WebRtcVideoChannel::MapCodecs(const std::vector& codecs) { return video_codecs; } -EncoderStreamFactory::EncoderStreamFactory(std::string codec_name, - int max_qp, - int max_framerate, - bool is_screencast, - bool conference_mode) +// TODO(bugs.webrtc.org/8785): Consider removing max_qp and max_framerate +// as members of EncoderStreamFactory and instead set these values individually +// for each stream in the VideoEncoderConfig.simulcast_layers. +EncoderStreamFactory::EncoderStreamFactory( + std::string codec_name, + int max_qp, + int max_framerate, + bool is_screenshare, + bool screenshare_config_explicitly_enabled) + : codec_name_(codec_name), max_qp_(max_qp), max_framerate_(max_framerate), - is_screencast_(is_screencast), - conference_mode_(conference_mode) {} + is_screenshare_(is_screenshare), + screenshare_config_explicitly_enabled_( + screenshare_config_explicitly_enabled) {} std::vector EncoderStreamFactory::CreateEncoderStreams( int width, int height, const webrtc::VideoEncoderConfig& encoder_config) { - if (is_screencast_ && - (!conference_mode_ || !cricket::UseSimulcastScreenshare())) { + bool screenshare_simulcast_enabled = + screenshare_config_explicitly_enabled_ && + cricket::ScreenshareSimulcastFieldTrialEnabled(); + if (is_screenshare_ && !screenshare_simulcast_enabled) { RTC_DCHECK_EQ(1, encoder_config.number_of_streams); } RTC_DCHECK_EQ(encoder_config.simulcast_layers.size(), @@ -2638,12 +2646,12 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( std::vector layers; if (encoder_config.number_of_streams > 1 || - (CodecNamesEq(codec_name_, kVp8CodecName) && is_screencast_ && - conference_mode_)) { + (CodecNamesEq(codec_name_, kVp8CodecName) && is_screenshare_ && + screenshare_config_explicitly_enabled_)) { layers = GetSimulcastConfig(encoder_config.number_of_streams, width, height, encoder_config.max_bitrate_bps, encoder_config.bitrate_priority, max_qp_, - max_framerate_, is_screencast_); + max_framerate_, is_screenshare_); // Update the active simulcast layers. for (size_t i = 0; i < layers.size(); ++i) { layers[i].active = encoder_config.simulcast_layers[i].active; @@ -2666,7 +2674,7 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( layer.max_qp = max_qp_; layer.bitrate_priority = encoder_config.bitrate_priority; - if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screencast_) { + if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screenshare_) { layer.temporal_layer_thresholds_bps.resize(GetDefaultVp9TemporalLayers() - 1); } diff --git a/media/engine/webrtcvideoengine.h b/media/engine/webrtcvideoengine.h index 03fa3e8eab..f1264932a1 100644 --- a/media/engine/webrtcvideoengine.h +++ b/media/engine/webrtcvideoengine.h @@ -516,8 +516,8 @@ class EncoderStreamFactory EncoderStreamFactory(std::string codec_name, int max_qp, int max_framerate, - bool is_screencast, - bool conference_mode); + bool is_screenshare, + bool screenshare_config_explicitly_enabled); private: std::vector CreateEncoderStreams( @@ -528,8 +528,10 @@ class EncoderStreamFactory const std::string codec_name_; const int max_qp_; const int max_framerate_; - const bool is_screencast_; - const bool conference_mode_; + const bool is_screenshare_; + // Allows a screenshare specific configuration, which enables temporal + // layering and allows simulcast. + const bool screenshare_config_explicitly_enabled_; }; } // namespace cricket