admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

EncoderBitrateAdjuster: Clarify spatial can also mean simulcast layer.

Browse Source 
This class is used for both simulcast and SVC use cases. Update variable
names and code comments to reflect this fact.

Also add TODOs that we'll need to address for VP9 simulcast.

Bug: webrtc:14884
Change-Id: I814c8fa0097306b16d552f55ca391ac8f716348a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/294383
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39367}
This commit is contained in:
Henrik Boström 2023-02-22 08:37:47 +01:00 committed by WebRTC LUCI CQ
parent fe1b39a648
commit 3d5c6dd38a
2 changed files with 37 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
video/encoder_bitrate_adjuster.cc
View File

@ -48,6 +48,11 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings)
.BitrateAdjusterCanUseNetworkHeadroom()),
frames_since_layout_change_(0),
min_bitrates_bps_{} {
// TODO(https://crbug.com/webrtc/14884): In order to support simulcast VP9,
// this code needs to be updated to care about `numberOfSimulcastStreams` even
// in the case of VP9.
// TODO(https://crbug.com/webrtc/14891): This also needs to be updated in
// order to support a mix of simulcast and SVC.
if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9) {
for (size_t si = 0; si < codec_settings.VP9().numberOfSpatialLayers; ++si) {
if (codec_settings.spatialLayers[si].active) {
@ -73,8 +78,8 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
const VideoEncoder::RateControlParameters& rates) {
current_rate_control_parameters_ = rates;
// First check that overshoot detectors exist, and store per spatial layer
// how many active temporal layers we have.
// First check that overshoot detectors exist, and store per simulcast/spatial
// layer how many active temporal layers we have.
size_t active_tls[kMaxSpatialLayers] = {};
for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
active_tls[si] = 0;
@ -110,21 +115,22 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
layer_info.target_rate =
DataRate::BitsPerSec(rates.bitrate.GetSpatialLayerSum(si));
// Adjustment is done per spatial layer only (not per temporal layer).
// Adjustment is done per simulcast/spatial layer only (not per temporal
// layer).
if (frames_since_layout_change_ < kMinFramesSinceLayoutChange) {
layer_info.link_utilization_factor = kDefaultUtilizationFactor;
layer_info.media_utilization_factor = kDefaultUtilizationFactor;
} else if (active_tls[si] == 0 ||
layer_info.target_rate == DataRate::Zero()) {
// No signaled temporal layers, or no bitrate set. Could either be unused
// spatial layer or bitrate dynamic mode; pass bitrate through without any
// change.
// simulcast/spatial layer or bitrate dynamic mode; pass bitrate through
// without any change.
layer_info.link_utilization_factor = 1.0;
layer_info.media_utilization_factor = 1.0;
} else if (active_tls[si] == 1) {
// A single active temporal layer, this might mean single layer or that
// encoder does not support temporal layers. Merge target bitrates for
// this spatial layer.
// this simulcast/spatial layer.
RTC_DCHECK(overshoot_detectors_[si][0]);
layer_info.link_utilization_factor =
overshoot_detectors_[si][0]
@ -135,9 +141,9 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
->GetMediaRateUtilizationFactor(now_ms)
.value_or(kDefaultUtilizationFactor);
} else if (layer_info.target_rate > DataRate::Zero()) {
// Multiple temporal layers enabled for this spatial layer. Update rate
// for each of them and make a weighted average of utilization factors,
// with bitrate fraction used as weight.
// Multiple temporal layers enabled for this simulcast/spatial layer.
// Update rate for each of them and make a weighted average of utilization
// factors, with bitrate fraction used as weight.
// If any layer is missing a utilization factor, fall back to default.
layer_info.link_utilization_factor = 0.0;
layer_info.media_utilization_factor = 0.0;
@ -221,15 +227,13 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
}
if (layer_info.target_rate > DataRate::Zero()) {
RTC_LOG(LS_VERBOSE) << "Utilization factors for spatial index " << si
<< ": link = " << layer_info.link_utilization_factor
<< ", media = " << layer_info.media_utilization_factor
<< ", wanted overshoot = "
<< layer_info.WantedOvershoot().bps()
<< " bps, available headroom = "
<< available_headroom.bps()
<< " bps, total utilization factor = "
<< utilization_factor;
RTC_LOG(LS_VERBOSE)
<< "Utilization factors for simulcast/spatial index " << si
<< ": link = " << layer_info.link_utilization_factor
<< ", media = " << layer_info.media_utilization_factor
<< ", wanted overshoot = " << layer_info.WantedOvershoot().bps()
<< " bps, available headroom = " << available_headroom.bps()
<< " bps, total utilization factor = " << utilization_factor;
}
// Populate the adjusted allocation with determined utilization factor.
@ -295,7 +299,7 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
}
}
// Since no spatial layers or streams are toggled by the adjustment
// Since no simulcast/spatial layers or streams are toggled by the adjustment
// bw-limited flag stays the same.
adjusted_allocation.set_bw_limited(rates.bitrate.is_bw_limited());
@ -314,11 +318,11 @@ void EncoderBitrateAdjuster::OnEncoderInfo(
}
void EncoderBitrateAdjuster::OnEncodedFrame(DataSize size,
int spatial_index,
int stream_index,
int temporal_index) {
++frames_since_layout_change_;
// Detectors may not exist, for instance if ScreenshareLayers is used.
auto& detector = overshoot_detectors_[spatial_index][temporal_index];
auto& detector = overshoot_detectors_[stream_index][temporal_index];
if (detector) {
detector->OnEncodedFrame(size.bytes(), rtc::TimeMillis());
}

19
video/encoder_bitrate_adjuster.h
View File

@ -26,8 +26,8 @@ class EncoderBitrateAdjuster {
static constexpr int64_t kWindowSizeMs = 3000;
// Minimum number of frames since last layout change required to trust the
// overshoot statistics. Otherwise falls back to default utilization.
// By layout change, we mean any spatial/temporal layer being either enabled
// or disabled.
// By layout change, we mean any simulcast/spatial/temporal layer being either
// enabled or disabled.
static constexpr size_t kMinFramesSinceLayoutChange = 30;
// Default utilization, before reliable metrics are available, is set to 20%
// overshoot. This is conservative so that badly misbehaving encoders don't
@ -47,7 +47,11 @@ class EncoderBitrateAdjuster {
void OnEncoderInfo(const VideoEncoder::EncoderInfo& encoder_info);
// Updates the overuse detectors according to the encoded image size.
void OnEncodedFrame(DataSize size, int spatial_index, int temporal_index);
// `stream_index` is the spatial or simulcast index.
// TODO(https://crbug.com/webrtc/14891): If we want to support a mix of
// simulcast and SVC we'll also need to consider the case where we have both
// simulcast and spatial indices.
void OnEncodedFrame(DataSize size, int stream_index, int temporal_index);
void Reset();
@ -55,13 +59,14 @@ class EncoderBitrateAdjuster {
const bool utilize_bandwidth_headroom_;
VideoEncoder::RateControlParameters current_rate_control_parameters_;
// FPS allocation of temporal layers, per spatial layer. Represented as a Q8
// fraction; 0 = 0%, 255 = 100%. See VideoEncoder::EncoderInfo.fps_allocation.
// FPS allocation of temporal layers, per simulcast/spatial layer. Represented
// as a Q8 fraction; 0 = 0%, 255 = 100%. See
// VideoEncoder::EncoderInfo.fps_allocation.
absl::InlinedVector<uint8_t, kMaxTemporalStreams>
current_fps_allocation_[kMaxSpatialLayers];
// Frames since layout was changed, mean that any spatial or temporal layer
// was either disabled or enabled.
// Frames since layout was changed, mean that any simulcast, spatial or
// temporal layer was either disabled or enabled.
size_t frames_since_layout_change_;
std::unique_ptr<EncoderOvershootDetector>
overshoot_detectors_[kMaxSpatialLayers][kMaxTemporalStreams];