diff --git a/media/engine/simulcast.cc b/media/engine/simulcast.cc index f2909d3f4c..956e31d4ec 100644 --- a/media/engine/simulcast.cc +++ b/media/engine/simulcast.cc @@ -51,7 +51,7 @@ const SimulcastFormat kSimulcastFormats[] = { {0, 0, 1, 200, 150, 30} }; -const int kMaxScreenshareSimulcastStreams = 2; +const int kMaxScreenshareSimulcastLayers = 2; // Multiway: Number of temporal layers for each simulcast stream, for maximum // possible number of simulcast streams |kMaxSimulcastStreams|. The array @@ -124,125 +124,164 @@ void SlotSimulcastMaxResolution(size_t max_layers, int* width, int* height) { << " height:" << *height; } -int GetTotalMaxBitrateBps(const std::vector& streams) { - int total_max_bitrate_bps = 0; - for (size_t s = 0; s < streams.size() - 1; ++s) { - total_max_bitrate_bps += streams[s].target_bitrate_bps; +void BoostMaxSimulcastLayer(int max_bitrate_bps, + std::vector* layers) { + // Spend additional bits to boost the max layer. + int bitrate_left_bps = max_bitrate_bps - GetTotalMaxBitrateBps(*layers); + if (bitrate_left_bps > 0) { + layers->back().max_bitrate_bps += bitrate_left_bps; } - total_max_bitrate_bps += streams.back().max_bitrate_bps; +} + +int GetTotalMaxBitrateBps(const std::vector& layers) { + int total_max_bitrate_bps = 0; + for (size_t s = 0; s < layers.size() - 1; ++s) { + total_max_bitrate_bps += layers[s].target_bitrate_bps; + } + total_max_bitrate_bps += layers.back().max_bitrate_bps; return total_max_bitrate_bps; } -std::vector GetSimulcastConfig(size_t max_streams, +std::vector GetSimulcastConfig(size_t max_layers, int width, int height, int max_bitrate_bps, double bitrate_priority, int max_qp, int max_framerate, - bool is_screencast) { - size_t num_simulcast_layers; - if (is_screencast) { - if (UseSimulcastScreenshare()) { - num_simulcast_layers = - std::min(max_streams, kMaxScreenshareSimulcastStreams); - } else { - num_simulcast_layers = 1; - } + bool is_screenshare) { + if (is_screenshare) { + return GetScreenshareLayers(max_layers, width, height, max_bitrate_bps, + bitrate_priority, max_qp, max_framerate, + ScreenshareSimulcastFieldTrialEnabled()); } else { - num_simulcast_layers = FindSimulcastMaxLayers(width, height); + return GetNormalSimulcastLayers(max_layers, width, height, max_bitrate_bps, + bitrate_priority, max_qp, max_framerate); } +} - if (num_simulcast_layers > max_streams) { - // If the number of SSRCs in the group differs from our target - // number of simulcast streams for current resolution, switch down - // to a resolution that matches our number of SSRCs. - SlotSimulcastMaxResolution(max_streams, &width, &height); - num_simulcast_layers = max_streams; +std::vector GetNormalSimulcastLayers( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate) { + // TODO(bugs.webrtc.org/8785): Currently if the resolution isn't large enough + // (defined in kSimulcastFormats) we scale down the number of simulcast + // layers. Consider changing this so that the application can have more + // control over exactly how many simulcast layers are used. + size_t num_simulcast_layers = FindSimulcastMaxLayers(width, height); + if (num_simulcast_layers > max_layers) { + // TODO(bugs.webrtc.org/8486): This scales down the resolution if the + // number of simulcast layers created by the application isn't sufficient + // (defined in kSimulcastFormats). For example if the input frame's + // resolution is HD, but there are only 2 simulcast layers, the + // resolution gets scaled down to VGA. Consider taking this logic out to + // allow the application more control over the resolutions. + SlotSimulcastMaxResolution(max_layers, &width, &height); + num_simulcast_layers = max_layers; } - std::vector streams; - streams.resize(num_simulcast_layers); + std::vector layers(num_simulcast_layers); - if (is_screencast) { - ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); - // For legacy screenshare in conference mode, tl0 and tl1 bitrates are - // piggybacked on the VideoCodec struct as target and max bitrates, - // respectively. See eg. webrtc::VP8EncoderImpl::SetRates(). - streams[0].width = width; - streams[0].height = height; - streams[0].max_qp = max_qp; - streams[0].max_framerate = 5; - streams[0].min_bitrate_bps = kMinVideoBitrateBps; - streams[0].target_bitrate_bps = config.tl0_bitrate_kbps * 1000; - streams[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; - streams[0].temporal_layer_thresholds_bps.clear(); - streams[0].temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps * - 1000); + // Format width and height has to be divisible by |2 ^ num_simulcast_layers - + // 1|. + width = NormalizeSimulcastSize(width, num_simulcast_layers); + height = NormalizeSimulcastSize(height, num_simulcast_layers); + // Add simulcast streams, from highest resolution (|s| = num_simulcast_layers + // -1) to lowest resolution at |s| = 0. + for (size_t s = num_simulcast_layers - 1;; --s) { + layers[s].width = width; + layers[s].height = height; + // TODO(pbos): Fill actual temporal-layer bitrate thresholds. + layers[s].max_qp = max_qp; + layers[s].temporal_layer_thresholds_bps.resize( + kDefaultConferenceNumberOfTemporalLayers[s] - 1); + layers[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height); + layers[s].target_bitrate_bps = FindSimulcastTargetBitrateBps(width, height); + layers[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height); + layers[s].max_framerate = max_framerate; - // With simulcast enabled, add another spatial layer. This one will have a - // more normal layout, with the regular 3 temporal layer pattern and no fps - // restrictions. The base simulcast stream will still use legacy setup. - if (num_simulcast_layers == kMaxScreenshareSimulcastStreams) { - // Add optional upper simulcast layer. - // Lowest temporal layers of a 3 layer setup will have 40% of the total - // bitrate allocation for that stream. Make sure the gap between the - // target of the lower stream and first temporal layer of the higher one - // is at most 2x the bitrate, so that upswitching is not hampered by - // stalled bitrate estimates. - int max_bitrate_bps = 2 * ((streams[0].target_bitrate_bps * 10) / 4); - // Cap max bitrate so it isn't overly high for the given resolution. - max_bitrate_bps = std::min( - max_bitrate_bps, FindSimulcastMaxBitrateBps(width, height)); + width /= 2; + height /= 2; - streams[1].width = width; - streams[1].height = height; - streams[1].max_qp = max_qp; - streams[1].max_framerate = max_framerate; - // Three temporal layers means two thresholds. - streams[1].temporal_layer_thresholds_bps.resize(2); - streams[1].min_bitrate_bps = streams[0].target_bitrate_bps * 2; - streams[1].target_bitrate_bps = max_bitrate_bps; - streams[1].max_bitrate_bps = max_bitrate_bps; - } - } else { - // Format width and height has to be divisible by |2 ^ number_streams - 1|. - width = NormalizeSimulcastSize(width, num_simulcast_layers); - height = NormalizeSimulcastSize(height, num_simulcast_layers); - - // Add simulcast sub-streams from lower resolution to higher resolutions. - // Add simulcast streams, from highest resolution (|s| = number_streams -1) - // to lowest resolution at |s| = 0. - for (size_t s = num_simulcast_layers - 1;; --s) { - streams[s].width = width; - streams[s].height = height; - // TODO(pbos): Fill actual temporal-layer bitrate thresholds. - streams[s].max_qp = max_qp; - streams[s].temporal_layer_thresholds_bps.resize( - kDefaultConferenceNumberOfTemporalLayers[s] - 1); - streams[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height); - streams[s].target_bitrate_bps = - FindSimulcastTargetBitrateBps(width, height); - streams[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height); - streams[s].max_framerate = max_framerate; - - width /= 2; - height /= 2; - - if (s == 0) - break; - } - - // Spend additional bits to boost the max stream. - int bitrate_left_bps = max_bitrate_bps - GetTotalMaxBitrateBps(streams); - if (bitrate_left_bps > 0) { - streams.back().max_bitrate_bps += bitrate_left_bps; + if (s == 0) { + break; } } + // If there is bitrate leftover, give it to the largest layer. + BoostMaxSimulcastLayer(max_bitrate_bps, &layers); + // Currently the relative bitrate priority of the sender is controlled by + // the value of the lowest VideoStream. + // TODO(bugs.webrtc.org/8630): The web specification describes being able to + // control relative bitrate for each individual simulcast layer, but this + // is currently just implemented per rtp sender. + layers[0].bitrate_priority = bitrate_priority; + return layers; +} +std::vector GetScreenshareLayers( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate, + bool screenshare_simulcast_enabled) { + auto max_screenshare_layers = + screenshare_simulcast_enabled ? kMaxScreenshareSimulcastLayers : 1; + size_t num_simulcast_layers = + std::min(max_layers, max_screenshare_layers); + + std::vector layers(num_simulcast_layers); + ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault(); + // For legacy screenshare in conference mode, tl0 and tl1 bitrates are + // piggybacked on the VideoCodec struct as target and max bitrates, + // respectively. See eg. webrtc::VP8EncoderImpl::SetRates(). + layers[0].width = width; + layers[0].height = height; + layers[0].max_qp = max_qp; + layers[0].max_framerate = 5; + layers[0].min_bitrate_bps = kMinVideoBitrateBps; + layers[0].target_bitrate_bps = config.tl0_bitrate_kbps * 1000; + layers[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000; + layers[0].temporal_layer_thresholds_bps.clear(); + layers[0].temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps * + 1000); + + // With simulcast enabled, add another spatial layer. This one will have a + // more normal layout, with the regular 3 temporal layer pattern and no fps + // restrictions. The base simulcast layer will still use legacy setup. + if (num_simulcast_layers == kMaxScreenshareSimulcastLayers) { + // Add optional upper simulcast layer. + // Lowest temporal layers of a 3 layer setup will have 40% of the total + // bitrate allocation for that simulcast layer. Make sure the gap between + // the target of the lower simulcast layer and first temporal layer of the + // higher one is at most 2x the bitrate, so that upswitching is not hampered + // by stalled bitrate estimates. + int max_bitrate_bps = 2 * ((layers[0].target_bitrate_bps * 10) / 4); + // Cap max bitrate so it isn't overly high for the given resolution. + max_bitrate_bps = std::min(max_bitrate_bps, + FindSimulcastMaxBitrateBps(width, height)); + + layers[1].width = width; + layers[1].height = height; + layers[1].max_qp = max_qp; + layers[1].max_framerate = max_framerate; + // Three temporal layers means two thresholds. + layers[1].temporal_layer_thresholds_bps.resize(2); + layers[1].min_bitrate_bps = layers[0].target_bitrate_bps * 2; + layers[1].target_bitrate_bps = max_bitrate_bps; + layers[1].max_bitrate_bps = max_bitrate_bps; + } + + BoostMaxSimulcastLayer(max_bitrate_bps, &layers); // The bitrate priority currently implemented on a per-sender level, so we - // just set it for the first video stream. - streams[0].bitrate_priority = bitrate_priority; - return streams; + // just set it for the first simulcast layer. + layers[0].bitrate_priority = bitrate_priority; + return layers; } static const int kScreenshareMinBitrateKbps = 50; @@ -250,7 +289,7 @@ static const int kScreenshareMaxBitrateKbps = 6000; static const int kScreenshareDefaultTl0BitrateKbps = 200; static const int kScreenshareDefaultTl1BitrateKbps = 1000; -static const char* kScreencastLayerFieldTrialName = +static const char* kScreenshareLayerFieldTrialName = "WebRTC-ScreenshareLayerRates"; static const char* kSimulcastScreenshareFieldTrialName = "WebRTC-SimulcastScreenshare"; @@ -261,7 +300,7 @@ ScreenshareLayerConfig::ScreenshareLayerConfig(int tl0_bitrate, int tl1_bitrate) ScreenshareLayerConfig ScreenshareLayerConfig::GetDefault() { std::string group = - webrtc::field_trial::FindFullName(kScreencastLayerFieldTrialName); + webrtc::field_trial::FindFullName(kScreenshareLayerFieldTrialName); ScreenshareLayerConfig config(kScreenshareDefaultTl0BitrateKbps, kScreenshareDefaultTl1BitrateKbps); @@ -297,7 +336,7 @@ bool ScreenshareLayerConfig::FromFieldTrialGroup( return true; } -bool UseSimulcastScreenshare() { +bool ScreenshareSimulcastFieldTrialEnabled() { return webrtc::field_trial::IsEnabled(kSimulcastScreenshareFieldTrialName); } diff --git a/media/engine/simulcast.h b/media/engine/simulcast.h index a2d27ed7b2..005c80e944 100644 --- a/media/engine/simulcast.h +++ b/media/engine/simulcast.h @@ -45,16 +45,38 @@ int GetTotalMaxBitrateBps(const std::vector& streams); // Get simulcast settings. // TODO(sprang): Remove default parameter when it's not longer referenced. -std::vector GetSimulcastConfig(size_t max_streams, - int width, - int height, - int max_bitrate_bps, - double bitrate_priority, - int max_qp, - int max_framerate, - bool is_screencast = false); +std::vector GetSimulcastConfig( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate, + bool is_screenshare = false); -bool UseSimulcastScreenshare(); +// Gets the simulcast config layers for a non-screensharing case. +std::vector GetNormalSimulcastLayers( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate); + +// Get simulcast config layers for screenshare settings. +std::vector GetScreenshareLayers( + size_t max_layers, + int width, + int height, + int max_bitrate_bps, + double bitrate_priority, + int max_qp, + int max_framerate, + bool screenshare_simulcast_enabled); + +bool ScreenshareSimulcastFieldTrialEnabled(); } // namespace cricket diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc index dbf862a86b..8aa5aa222a 100644 --- a/media/engine/webrtcvideoengine.cc +++ b/media/engine/webrtcvideoengine.cc @@ -1896,8 +1896,8 @@ WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig( // configure a single stream. encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size(); if (IsCodecBlacklistedForSimulcast(codec.name) || - (is_screencast && - (!UseSimulcastScreenshare() || !parameters_.conference_mode))) { + (is_screencast && (!ScreenshareSimulcastFieldTrialEnabled() || + !parameters_.conference_mode))) { encoder_config.number_of_streams = 1; } @@ -2614,23 +2614,31 @@ WebRtcVideoChannel::MapCodecs(const std::vector& codecs) { return video_codecs; } -EncoderStreamFactory::EncoderStreamFactory(std::string codec_name, - int max_qp, - int max_framerate, - bool is_screencast, - bool conference_mode) +// TODO(bugs.webrtc.org/8785): Consider removing max_qp and max_framerate +// as members of EncoderStreamFactory and instead set these values individually +// for each stream in the VideoEncoderConfig.simulcast_layers. +EncoderStreamFactory::EncoderStreamFactory( + std::string codec_name, + int max_qp, + int max_framerate, + bool is_screenshare, + bool screenshare_config_explicitly_enabled) + : codec_name_(codec_name), max_qp_(max_qp), max_framerate_(max_framerate), - is_screencast_(is_screencast), - conference_mode_(conference_mode) {} + is_screenshare_(is_screenshare), + screenshare_config_explicitly_enabled_( + screenshare_config_explicitly_enabled) {} std::vector EncoderStreamFactory::CreateEncoderStreams( int width, int height, const webrtc::VideoEncoderConfig& encoder_config) { - if (is_screencast_ && - (!conference_mode_ || !cricket::UseSimulcastScreenshare())) { + bool screenshare_simulcast_enabled = + screenshare_config_explicitly_enabled_ && + cricket::ScreenshareSimulcastFieldTrialEnabled(); + if (is_screenshare_ && !screenshare_simulcast_enabled) { RTC_DCHECK_EQ(1, encoder_config.number_of_streams); } RTC_DCHECK_EQ(encoder_config.simulcast_layers.size(), @@ -2638,12 +2646,12 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( std::vector layers; if (encoder_config.number_of_streams > 1 || - (CodecNamesEq(codec_name_, kVp8CodecName) && is_screencast_ && - conference_mode_)) { + (CodecNamesEq(codec_name_, kVp8CodecName) && is_screenshare_ && + screenshare_config_explicitly_enabled_)) { layers = GetSimulcastConfig(encoder_config.number_of_streams, width, height, encoder_config.max_bitrate_bps, encoder_config.bitrate_priority, max_qp_, - max_framerate_, is_screencast_); + max_framerate_, is_screenshare_); // Update the active simulcast layers. for (size_t i = 0; i < layers.size(); ++i) { layers[i].active = encoder_config.simulcast_layers[i].active; @@ -2666,7 +2674,7 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( layer.max_qp = max_qp_; layer.bitrate_priority = encoder_config.bitrate_priority; - if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screencast_) { + if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screenshare_) { layer.temporal_layer_thresholds_bps.resize(GetDefaultVp9TemporalLayers() - 1); } diff --git a/media/engine/webrtcvideoengine.h b/media/engine/webrtcvideoengine.h index 03fa3e8eab..f1264932a1 100644 --- a/media/engine/webrtcvideoengine.h +++ b/media/engine/webrtcvideoengine.h @@ -516,8 +516,8 @@ class EncoderStreamFactory EncoderStreamFactory(std::string codec_name, int max_qp, int max_framerate, - bool is_screencast, - bool conference_mode); + bool is_screenshare, + bool screenshare_config_explicitly_enabled); private: std::vector CreateEncoderStreams( @@ -528,8 +528,10 @@ class EncoderStreamFactory const std::string codec_name_; const int max_qp_; const int max_framerate_; - const bool is_screencast_; - const bool conference_mode_; + const bool is_screenshare_; + // Allows a screenshare specific configuration, which enables temporal + // layering and allows simulcast. + const bool screenshare_config_explicitly_enabled_; }; } // namespace cricket