admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
webrtc_m130/video/adaptation/video_stream_encoder_resource_manager.cc
Henrik Boström 41fffaa6f4 Fix requested_resolution bug where we get stuck with old restrictions. 
Normally (scaleResolutionDownBy) restrictions are applied at the source
which changes the input frame size which triggers reconfiguration with
appropriate scaling factors.

But when requested_resolution is used, encoder settings are by
definition not relative to the input frame size. In order for
restrictions to have an effect, they are applied inside
ReconfigureEncoder(): you get the minimum between the requested
resolution and the restricted resolution.

ReconfigureEncoder() happens when you SetParameters(), but the bug
here is that we don't do it again once the restrictions are updated.
So if restrictions are 540p when you ask for 720p, you get 540p and
after restrictions change to unlimited you're still stuck in 540p.

The fix is to also trigger ReconfigureEncoder() inside
OnVideoSourceRestrictionsUpdated() when the restricted resolution is
changing and a requested_resolution is configured.

To ensure reconfiguring the encoder "on the fly" like this does not
reset initial frame dropping logic, InitialFrameDropper caring about
input frame size changing is made conditional on not using
requested_resolution.

# Slow purple bots failing but they are not affected by this change.
NOTRY=True

Bug: webrtc:361477261
Change-Id: I1389aa16cf408b0d14e0b5b6f68c2442db955be9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/360200
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42882}
2024-08-29 12:26:17 +00:00

806 lines
30 KiB
C++
Raw Blame History

/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/adaptation/video_stream_encoder_resource_manager.h"
#include <stdio.h>
#include <algorithm>
#include <cmath>
#include <limits>
#include <memory>
#include <utility>
#include "absl/algorithm/container.h"
#include "absl/base/macros.h"
#include "api/adaptation/resource.h"
#include "api/field_trials_view.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
#include "api/video/video_adaptation_reason.h"
#include "api/video/video_source_interface.h"
#include "call/adaptation/video_source_restrictions.h"
#include "modules/video_coding/svc/scalability_mode_util.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/time_utils.h"
#include "rtc_base/trace_event.h"
#include "video/adaptation/quality_scaler_resource.h"
namespace webrtc {
const int kDefaultInputPixelsWidth = 176;
const int kDefaultInputPixelsHeight = 144;
namespace {
constexpr const char* kPixelLimitResourceFieldTrialName =
"WebRTC-PixelLimitResource";
bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
degradation_preference == DegradationPreference::BALANCED;
}
bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION ||
degradation_preference == DegradationPreference::BALANCED;
}
std::string ToString(VideoAdaptationReason reason) {
switch (reason) {
case VideoAdaptationReason::kQuality:
return "quality";
case VideoAdaptationReason::kCpu:
return "cpu";
}
RTC_CHECK_NOTREACHED();
}
std::vector<bool> GetActiveLayersFlags(const VideoCodec& codec) {
std::vector<bool> flags;
if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
flags.resize(codec.VP9().numberOfSpatialLayers);
for (size_t i = 0; i < flags.size(); ++i) {
flags[i] = codec.spatialLayers[i].active;
}
} else {
flags.resize(codec.numberOfSimulcastStreams);
for (size_t i = 0; i < flags.size(); ++i) {
flags[i] = codec.simulcastStream[i].active;
}
}
return flags;
}
bool EqualFlags(const std::vector<bool>& a, const std::vector<bool>& b) {
if (a.size() != b.size())
return false;
return std::equal(a.begin(), a.end(), b.begin());
}
} // namespace
class VideoStreamEncoderResourceManager::InitialFrameDropper {
public:
explicit InitialFrameDropper(
rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource,
const FieldTrialsView& field_trials)
: quality_scaler_resource_(quality_scaler_resource),
quality_scaler_settings_(field_trials),
has_seen_first_bwe_drop_(false),
set_start_bitrate_(DataRate::Zero()),
set_start_bitrate_time_ms_(0),
initial_framedrop_(0),
use_bandwidth_allocation_(false),
bandwidth_allocation_(DataRate::Zero()),
last_input_width_(0),
last_input_height_(0),
last_stream_configuration_changed_(false) {
RTC_DCHECK(quality_scaler_resource_);
}
// Output signal.
bool DropInitialFrames() const {
return initial_framedrop_ < kMaxInitialFramedrop;
}
absl::optional<uint32_t> single_active_stream_pixels() const {
return single_active_stream_pixels_;
}
absl::optional<uint32_t> UseBandwidthAllocationBps() const {
return (use_bandwidth_allocation_ &&
bandwidth_allocation_ > DataRate::Zero())
? absl::optional<uint32_t>(bandwidth_allocation_.bps())
: absl::nullopt;
}
bool last_stream_configuration_changed() const {
return last_stream_configuration_changed_;
}
// Input signals.
void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
set_start_bitrate_ = start_bitrate;
set_start_bitrate_time_ms_ = now_ms;
}
void SetBandwidthAllocation(DataRate bandwidth_allocation) {
bandwidth_allocation_ = bandwidth_allocation;
}
void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) {
if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ &&
quality_scaler_resource_->is_started() &&
quality_scaler_settings_.InitialBitrateIntervalMs() &&
quality_scaler_settings_.InitialBitrateFactor()) {
int64_t diff_ms = now_ms - set_start_bitrate_time_ms_;
if (diff_ms <
quality_scaler_settings_.InitialBitrateIntervalMs().value() &&
(target_bitrate <
(set_start_bitrate_ *
quality_scaler_settings_.InitialBitrateFactor().value()))) {
RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: "
<< set_start_bitrate_.bps()
<< ", target bitrate: " << target_bitrate.bps();
initial_framedrop_ = 0;
has_seen_first_bwe_drop_ = true;
}
}
}
void OnEncoderSettingsUpdated(
const VideoCodec& codec,
const VideoAdaptationCounters& adaptation_counters,
bool has_requested_resolution) {
last_stream_configuration_changed_ = false;
std::vector<bool> active_flags = GetActiveLayersFlags(codec);
// Check if the source resolution has changed for the external reasons,
// i.e. without any adaptation from WebRTC. This only matters when encoding
// resolution is relative to input frame, which it isn't if the
// `requested_resolution` API is used.
const bool source_resolution_changed =
(last_input_width_ != codec.width ||
last_input_height_ != codec.height) &&
adaptation_counters.resolution_adaptations ==
last_adaptation_counters_.resolution_adaptations;
if (!EqualFlags(active_flags, last_active_flags_) ||
(!has_requested_resolution && source_resolution_changed)) {
// Streams configuration has changed.
last_stream_configuration_changed_ = true;
// Initial frame drop must be enabled because BWE might be way too low
// for the selected resolution.
if (quality_scaler_resource_->is_started()) {
RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed "
"stream parameters";
initial_framedrop_ = 0;
if (single_active_stream_pixels_ &&
VideoStreamAdapter::GetSingleActiveLayerPixels(codec) >
*single_active_stream_pixels_) {
// Resolution increased.
use_bandwidth_allocation_ = true;
}
}
}
last_adaptation_counters_ = adaptation_counters;
last_active_flags_ = active_flags;
last_input_width_ = codec.width;
last_input_height_ = codec.height;
single_active_stream_pixels_ =
VideoStreamAdapter::GetSingleActiveLayerPixels(codec);
}
void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
void Disable() {
initial_framedrop_ = kMaxInitialFramedrop;
use_bandwidth_allocation_ = false;
}
void OnQualityScalerSettingsUpdated() {
if (quality_scaler_resource_->is_started()) {
// Restart frame drops due to size.
initial_framedrop_ = 0;
} else {
// Quality scaling disabled so we shouldn't drop initial frames.
Disable();
}
}
private:
// The maximum number of frames to drop at beginning of stream to try and
// achieve desired bitrate.
static const int kMaxInitialFramedrop = 4;
const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
const QualityScalerSettings quality_scaler_settings_;
bool has_seen_first_bwe_drop_;
DataRate set_start_bitrate_;
int64_t set_start_bitrate_time_ms_;
// Counts how many frames we've dropped in the initial framedrop phase.
int initial_framedrop_;
absl::optional<uint32_t> single_active_stream_pixels_;
bool use_bandwidth_allocation_;
DataRate bandwidth_allocation_;
std::vector<bool> last_active_flags_;
VideoAdaptationCounters last_adaptation_counters_;
int last_input_width_;
int last_input_height_;
bool last_stream_configuration_changed_;
};
VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
VideoStreamInputStateProvider* input_state_provider,
VideoStreamEncoderObserver* encoder_stats_observer,
Clock* clock,
bool experiment_cpu_load_estimator,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
DegradationPreferenceProvider* degradation_preference_provider,
const FieldTrialsView& field_trials)
: field_trials_(field_trials),
degradation_preference_provider_(degradation_preference_provider),
bitrate_constraint_(std::make_unique<BitrateConstraint>()),
balanced_constraint_(
std::make_unique<BalancedConstraint>(degradation_preference_provider_,
field_trials)),
encode_usage_resource_(
EncodeUsageResource::Create(std::move(overuse_detector))),
quality_scaler_resource_(QualityScalerResource::Create()),
pixel_limit_resource_(nullptr),
bandwidth_quality_scaler_resource_(
BandwidthQualityScalerResource::Create()),
encoder_queue_(nullptr),
input_state_provider_(input_state_provider),
adaptation_processor_(nullptr),
encoder_stats_observer_(encoder_stats_observer),
degradation_preference_(DegradationPreference::DISABLED),
video_source_restrictions_(),
balanced_settings_(field_trials),
clock_(clock),
experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
initial_frame_dropper_(
std::make_unique<InitialFrameDropper>(quality_scaler_resource_,
field_trials)),
quality_scaling_experiment_enabled_(
QualityScalingExperiment::Enabled(field_trials_)),
pixel_limit_resource_experiment_enabled_(
field_trials.IsEnabled(kPixelLimitResourceFieldTrialName)),
encoder_target_bitrate_bps_(absl::nullopt),
encoder_settings_(absl::nullopt) {
TRACE_EVENT0(
"webrtc",
"VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager");
RTC_CHECK(degradation_preference_provider_);
RTC_CHECK(encoder_stats_observer_);
}
VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() =
default;
void VideoStreamEncoderResourceManager::Initialize(
TaskQueueBase* encoder_queue) {
RTC_DCHECK(!encoder_queue_);
RTC_DCHECK(encoder_queue);
encoder_queue_ = encoder_queue;
encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_);
quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_);
bandwidth_quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_);
}
void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor,
VideoStreamAdapter* stream_adapter) {
RTC_DCHECK_RUN_ON(encoder_queue_);
adaptation_processor_ = adaptation_processor;
stream_adapter_ = stream_adapter;
}
void VideoStreamEncoderResourceManager::SetDegradationPreferences(
DegradationPreference degradation_preference) {
RTC_DCHECK_RUN_ON(encoder_queue_);
degradation_preference_ = degradation_preference;
UpdateStatsAdaptationSettings();
}
DegradationPreference
VideoStreamEncoderResourceManager::degradation_preference() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
return degradation_preference_;
}
void VideoStreamEncoderResourceManager::ConfigureEncodeUsageResource() {
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(encoder_settings_.has_value());
if (encode_usage_resource_->is_started()) {
encode_usage_resource_->StopCheckForOveruse();
} else {
// If the resource has not yet started then it needs to be added.
AddResource(encode_usage_resource_, VideoAdaptationReason::kCpu);
}
encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions());
}
void VideoStreamEncoderResourceManager::MaybeInitializePixelLimitResource() {
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(adaptation_processor_);
RTC_DCHECK(!pixel_limit_resource_);
if (!pixel_limit_resource_experiment_enabled_) {
// The field trial is not running.
return;
}
int max_pixels = 0;
std::string pixel_limit_field_trial =
field_trials_.Lookup(kPixelLimitResourceFieldTrialName);
if (sscanf(pixel_limit_field_trial.c_str(), "Enabled-%d", &max_pixels) != 1) {
RTC_LOG(LS_ERROR) << "Couldn't parse " << kPixelLimitResourceFieldTrialName
<< " trial config: " << pixel_limit_field_trial;
return;
}
RTC_LOG(LS_INFO) << "Running field trial "
<< kPixelLimitResourceFieldTrialName << " configured to "
<< max_pixels << " max pixels";
// Configure the specified max pixels from the field trial. The pixel limit
// resource is active for the lifetme of the stream (until
// StopManagedResources() is called).
pixel_limit_resource_ =
PixelLimitResource::Create(encoder_queue_, input_state_provider_);
pixel_limit_resource_->SetMaxPixels(max_pixels);
AddResource(pixel_limit_resource_, VideoAdaptationReason::kCpu);
}
void VideoStreamEncoderResourceManager::StopManagedResources() {
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(adaptation_processor_);
if (encode_usage_resource_->is_started()) {
encode_usage_resource_->StopCheckForOveruse();
RemoveResource(encode_usage_resource_);
}
if (quality_scaler_resource_->is_started()) {
quality_scaler_resource_->StopCheckForOveruse();
RemoveResource(quality_scaler_resource_);
}
if (pixel_limit_resource_) {
RemoveResource(pixel_limit_resource_);
pixel_limit_resource_ = nullptr;
}
if (bandwidth_quality_scaler_resource_->is_started()) {
bandwidth_quality_scaler_resource_->StopCheckForOveruse();
RemoveResource(bandwidth_quality_scaler_resource_);
}
}
void VideoStreamEncoderResourceManager::AddResource(
rtc::scoped_refptr<Resource> resource,
VideoAdaptationReason reason) {
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(resource);
bool inserted;
std::tie(std::ignore, inserted) = resources_.emplace(resource, reason);
RTC_DCHECK(inserted) << "Resource " << resource->Name()
<< " already was inserted";
adaptation_processor_->AddResource(resource);
}
void VideoStreamEncoderResourceManager::RemoveResource(
rtc::scoped_refptr<Resource> resource) {
{
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(resource);
const auto& it = resources_.find(resource);
RTC_DCHECK(it != resources_.end())
<< "Resource \"" << resource->Name() << "\" not found.";
resources_.erase(it);
}
adaptation_processor_->RemoveResource(resource);
}
std::vector<AdaptationConstraint*>
VideoStreamEncoderResourceManager::AdaptationConstraints() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
return {bitrate_constraint_.get(), balanced_constraint_.get()};
}
void VideoStreamEncoderResourceManager::SetEncoderSettings(
EncoderSettings encoder_settings) {
RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_settings_ = std::move(encoder_settings);
bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_);
initial_frame_dropper_->OnEncoderSettingsUpdated(
encoder_settings_->video_codec(), current_adaptation_counters_,
encoder_settings.encoder_config().HasRequestedResolution());
MaybeUpdateTargetFrameRate();
}
void VideoStreamEncoderResourceManager::SetStartBitrate(
DataRate start_bitrate) {
RTC_DCHECK_RUN_ON(encoder_queue_);
if (!start_bitrate.IsZero()) {
encoder_target_bitrate_bps_ = start_bitrate.bps();
bitrate_constraint_->OnEncoderTargetBitrateUpdated(
encoder_target_bitrate_bps_);
balanced_constraint_->OnEncoderTargetBitrateUpdated(
encoder_target_bitrate_bps_);
}
initial_frame_dropper_->SetStartBitrate(start_bitrate,
clock_->TimeInMicroseconds());
}
void VideoStreamEncoderResourceManager::SetTargetBitrate(
DataRate target_bitrate) {
RTC_DCHECK_RUN_ON(encoder_queue_);
if (!target_bitrate.IsZero()) {
encoder_target_bitrate_bps_ = target_bitrate.bps();
bitrate_constraint_->OnEncoderTargetBitrateUpdated(
encoder_target_bitrate_bps_);
balanced_constraint_->OnEncoderTargetBitrateUpdated(
encoder_target_bitrate_bps_);
}
initial_frame_dropper_->SetTargetBitrate(target_bitrate,
clock_->TimeInMilliseconds());
}
void VideoStreamEncoderResourceManager::SetEncoderRates(
const VideoEncoder::RateControlParameters& encoder_rates) {
RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_rates_ = encoder_rates;
initial_frame_dropper_->SetBandwidthAllocation(
encoder_rates.bandwidth_allocation);
}
void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() {
RTC_DCHECK_RUN_ON(encoder_queue_);
initial_frame_dropper_->OnFrameDroppedDueToSize();
Adaptation reduce_resolution = stream_adapter_->GetAdaptDownResolution();
if (reduce_resolution.status() == Adaptation::Status::kValid) {
stream_adapter_->ApplyAdaptation(reduce_resolution,
quality_scaler_resource_);
}
}
void VideoStreamEncoderResourceManager::OnEncodeStarted(
const VideoFrame& cropped_frame,
int64_t time_when_first_seen_us) {
RTC_DCHECK_RUN_ON(encoder_queue_);
encode_usage_resource_->OnEncodeStarted(cropped_frame,
time_when_first_seen_us);
}
void VideoStreamEncoderResourceManager::OnEncodeCompleted(
const EncodedImage& encoded_image,
int64_t time_sent_in_us,
absl::optional<int> encode_duration_us,
DataSize frame_size) {
RTC_DCHECK_RUN_ON(encoder_queue_);
// Inform `encode_usage_resource_` of the encode completed event.
uint32_t timestamp = encoded_image.RtpTimestamp();
int64_t capture_time_us =
encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec;
encode_usage_resource_->OnEncodeCompleted(
timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
bandwidth_quality_scaler_resource_->OnEncodeCompleted(
encoded_image, time_sent_in_us, frame_size.bytes());
}
void VideoStreamEncoderResourceManager::OnFrameDropped(
EncodedImageCallback::DropReason reason) {
RTC_DCHECK_RUN_ON(encoder_queue_);
quality_scaler_resource_->OnFrameDropped(reason);
}
bool VideoStreamEncoderResourceManager::DropInitialFrames() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
return initial_frame_dropper_->DropInitialFrames();
}
absl::optional<uint32_t>
VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
return initial_frame_dropper_->single_active_stream_pixels();
}
absl::optional<uint32_t>
VideoStreamEncoderResourceManager::UseBandwidthAllocationBps() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
return initial_frame_dropper_->UseBandwidthAllocationBps();
}
void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
RTC_DCHECK_RUN_ON(encoder_queue_);
initial_frame_dropper_->Disable();
}
void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
RTC_DCHECK_RUN_ON(encoder_queue_);
if (qp_thresholds.has_value()) {
if (quality_scaler_resource_->is_started()) {
quality_scaler_resource_->SetQpThresholds(qp_thresholds.value());
} else {
quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value(),
field_trials_);
AddResource(quality_scaler_resource_, VideoAdaptationReason::kQuality);
}
} else if (quality_scaler_resource_->is_started()) {
quality_scaler_resource_->StopCheckForOveruse();
RemoveResource(quality_scaler_resource_);
}
initial_frame_dropper_->OnQualityScalerSettingsUpdated();
}
void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings(
bool bandwidth_quality_scaling_allowed,
const std::vector<VideoEncoder::ResolutionBitrateLimits>&
resolution_bitrate_limits) {
RTC_DCHECK_RUN_ON(encoder_queue_);
if (!bandwidth_quality_scaling_allowed) {
if (bandwidth_quality_scaler_resource_->is_started()) {
bandwidth_quality_scaler_resource_->StopCheckForOveruse();
RemoveResource(bandwidth_quality_scaler_resource_);
}
} else {
if (!bandwidth_quality_scaler_resource_->is_started()) {
// Before executing "StartCheckForOveruse",we must execute "AddResource"
// firstly,because it can make the listener valid.
AddResource(bandwidth_quality_scaler_resource_,
webrtc::VideoAdaptationReason::kQuality);
bandwidth_quality_scaler_resource_->StartCheckForOveruse(
resolution_bitrate_limits);
}
}
}
void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info) {
RTC_DCHECK_RUN_ON(encoder_queue_);
const auto scaling_settings = encoder_info.scaling_settings;
const bool quality_scaling_allowed =
IsResolutionScalingEnabled(degradation_preference_) &&
(scaling_settings.thresholds.has_value() ||
(encoder_settings_.has_value() &&
encoder_settings_->encoder_config().is_quality_scaling_allowed)) &&
encoder_info.is_qp_trusted.value_or(true);
// TODO(https://crbug.com/webrtc/11222): Should this move to
// QualityScalerResource?
if (quality_scaling_allowed) {
if (!quality_scaler_resource_->is_started()) {
// Quality scaler has not already been configured.
// Use experimental thresholds if available.
absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
if (quality_scaling_experiment_enabled_) {
experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
GetVideoCodecTypeOrGeneric(encoder_settings_), field_trials_);
}
UpdateQualityScalerSettings(experimental_thresholds.has_value()
? experimental_thresholds
: scaling_settings.thresholds);
}
} else {
UpdateQualityScalerSettings(absl::nullopt);
}
// Set the qp-thresholds to the balanced settings if balanced mode.
if (degradation_preference_ == DegradationPreference::BALANCED &&
quality_scaler_resource_->is_started()) {
absl::optional<VideoEncoder::QpThresholds> thresholds =
balanced_settings_.GetQpThresholds(
GetVideoCodecTypeOrGeneric(encoder_settings_),
LastFrameSizeOrDefault());
if (thresholds) {
quality_scaler_resource_->SetQpThresholds(*thresholds);
}
}
UpdateStatsAdaptationSettings();
}
void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info) {
RTC_DCHECK_RUN_ON(encoder_queue_);
const bool bandwidth_quality_scaling_allowed =
IsResolutionScalingEnabled(degradation_preference_) &&
(encoder_settings_.has_value() &&
encoder_settings_->encoder_config().is_quality_scaling_allowed) &&
!encoder_info.is_qp_trusted.value_or(true);
UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed,
encoder_info.resolution_bitrate_limits);
UpdateStatsAdaptationSettings();
}
VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource(
rtc::scoped_refptr<Resource> resource) const {
RTC_DCHECK_RUN_ON(encoder_queue_);
const auto& registered_resource = resources_.find(resource);
RTC_DCHECK(registered_resource != resources_.end())
<< resource->Name() << " not found.";
return registered_resource->second;
}
// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
// pipelining encoders better (multiple input frames before something comes
// out). This should effectively turn off CPU adaptations for systems that
// remotely cope with the load right now.
CpuOveruseOptions VideoStreamEncoderResourceManager::GetCpuOveruseOptions()
const {
RTC_DCHECK_RUN_ON(encoder_queue_);
// This is already ensured by the only caller of this method:
// StartResourceAdaptation().
RTC_DCHECK(encoder_settings_.has_value());
CpuOveruseOptions options;
// Hardware accelerated encoders are assumed to be pipelined; give them
// additional overuse time.
if (encoder_settings_->encoder_info().is_hardware_accelerated) {
options.low_encode_usage_threshold_percent = 150;
options.high_encode_usage_threshold_percent = 200;
}
if (experiment_cpu_load_estimator_) {
options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec;
}
return options;
}
int VideoStreamEncoderResourceManager::LastFrameSizeOrDefault() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
return input_state_provider_->InputState()
.single_active_stream_pixels()
.value_or(
input_state_provider_->InputState().frame_size_pixels().value_or(
kDefaultInputPixelsWidth * kDefaultInputPixelsHeight));
}
void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions,
const VideoAdaptationCounters& adaptation_counters,
rtc::scoped_refptr<Resource> reason,
const VideoSourceRestrictions& unfiltered_restrictions) {
RTC_DCHECK_RUN_ON(encoder_queue_);
current_adaptation_counters_ = adaptation_counters;
// TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback.
if (!reason && adaptation_counters.Total() == 0) {
// Adaptation was manually reset - clear the per-reason counters too.
encoder_stats_observer_->ClearAdaptationStats();
}
video_source_restrictions_ = FilterRestrictionsByDegradationPreference(
restrictions, degradation_preference_);
MaybeUpdateTargetFrameRate();
}
void VideoStreamEncoderResourceManager::OnResourceLimitationChanged(
rtc::scoped_refptr<Resource> resource,
const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
resource_limitations) {
RTC_DCHECK_RUN_ON(encoder_queue_);
if (!resource) {
encoder_stats_observer_->ClearAdaptationStats();
return;
}
std::map<VideoAdaptationReason, VideoAdaptationCounters> limitations;
for (auto& resource_counter : resource_limitations) {
std::map<VideoAdaptationReason, VideoAdaptationCounters>::iterator it;
bool inserted;
std::tie(it, inserted) = limitations.emplace(
GetReasonFromResource(resource_counter.first), resource_counter.second);
if (!inserted && it->second.Total() < resource_counter.second.Total()) {
it->second = resource_counter.second;
}
}
VideoAdaptationReason adaptation_reason = GetReasonFromResource(resource);
encoder_stats_observer_->OnAdaptationChanged(
adaptation_reason, limitations[VideoAdaptationReason::kCpu],
limitations[VideoAdaptationReason::kQuality]);
RTC_LOG(LS_INFO) << ActiveCountsToString(limitations);
}
void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() {
RTC_DCHECK_RUN_ON(encoder_queue_);
absl::optional<double> codec_max_frame_rate =
encoder_settings_.has_value()
? absl::optional<double>(
encoder_settings_->video_codec().maxFramerate)
: absl::nullopt;
// The current target framerate is the maximum frame rate as specified by
// the current codec configuration or any limit imposed by the adaptation
// module. This is used to make sure overuse detection doesn't needlessly
// trigger in low and/or variable framerate scenarios.
absl::optional<double> target_frame_rate =
video_source_restrictions_.max_frame_rate();
if (!target_frame_rate.has_value() ||
(codec_max_frame_rate.has_value() &&
codec_max_frame_rate.value() < target_frame_rate.value())) {
target_frame_rate = codec_max_frame_rate;
}
encode_usage_resource_->SetTargetFrameRate(target_frame_rate);
}
void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const {
RTC_DCHECK_RUN_ON(encoder_queue_);
VideoStreamEncoderObserver::AdaptationSettings cpu_settings(
IsResolutionScalingEnabled(degradation_preference_),
IsFramerateScalingEnabled(degradation_preference_));
VideoStreamEncoderObserver::AdaptationSettings quality_settings =
(quality_scaler_resource_->is_started() ||
bandwidth_quality_scaler_resource_->is_started())
? cpu_settings
: VideoStreamEncoderObserver::AdaptationSettings();
encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings,
quality_settings);
}
// static
std::string VideoStreamEncoderResourceManager::ActiveCountsToString(
const std::map<VideoAdaptationReason, VideoAdaptationCounters>&
active_counts) {
rtc::StringBuilder ss;
ss << "Downgrade counts: fps: {";
for (auto& reason_count : active_counts) {
ss << ToString(reason_count.first) << ":";
ss << reason_count.second.fps_adaptations;
}
ss << "}, resolution {";
for (auto& reason_count : active_counts) {
ss << ToString(reason_count.first) << ":";
ss << reason_count.second.resolution_adaptations;
}
ss << "}";
return ss.Release();
}
bool VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
const VideoEncoderConfig& encoder_config,
const VideoCodec& video_codec) {
const std::vector<VideoStream>& simulcast_layers =
encoder_config.simulcast_layers;
if (simulcast_layers.empty()) {
return false;
}
absl::optional<int> num_spatial_layers;
if (simulcast_layers[0].scalability_mode.has_value() &&
video_codec.numberOfSimulcastStreams == 1) {
num_spatial_layers = ScalabilityModeToNumSpatialLayers(
*simulcast_layers[0].scalability_mode);
}
if (simulcast_layers.size() == 1) {
// Check if multiple spatial layers are used.
return num_spatial_layers && *num_spatial_layers > 1;
}
bool svc_with_one_spatial_layer =
num_spatial_layers && *num_spatial_layers == 1;
if (simulcast_layers[0].active && !svc_with_one_spatial_layer) {
// We can't distinguish between simulcast and singlecast when only the
// lowest spatial layer is active. Treat this case as simulcast.
return true;
}
int num_active_layers =
std::count_if(simulcast_layers.begin(), simulcast_layers.end(),
[](const VideoStream& layer) { return layer.active; });
return num_active_layers > 1;
}
} // namespace webrtc
Reference in New Issue View Git Blame Copy Permalink