admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Overuse] Make VideoStreamAdapter responsible for executing adaptation.

Browse Source 
This CL moves GetAdaptUpTarget(), GetAdaptDownTarget() and
ApplyAdaptationTarget() - and related code - to the VideoStreamAdapter.

This includes pieces related to calculating how to adapt, including:
- DegradationPreference
- BalancedDegradationPreference
- AdaptationRequest and last_adaptation_request_
- CanAdaptUpResolution()

The VideoStreamAdapter's interface has changed: VideoSourceRestrictor
methods are now hidden in favor of methods exposing AdaptationTarget.

This CL also does some misc moves:
- GetEncoderBitrateLimits is moved and renamed to
  VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution.
- EncoderSettings moved to a separate file.

// For api/video_codecs/video_encoder.[cc/h] changes, which is the
// moving of a function.
TBR=sprang@webrtc.org

Bug: webrtc:11393
Change-Id: Ie6bd8ef644ce927d7eca6ab90a0a7bcace682f3c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169842
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30708}
This commit is contained in:
Henrik Boström 2020-03-06 13:32:03 +01:00 committed by Commit Bot
parent 74dadc1e8e
commit b0f2e0ced4
14 changed files with 647 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
api/video_codecs/video_encoder.cc
View File

@ -11,6 +11,7 @@
#include "api/video_codecs/video_encoder.h"
#include <string.h>
#include <algorithm>
#include "rtc_base/checks.h"
#include "rtc_base/strings/string_builder.h"
@ -208,6 +209,42 @@ bool VideoEncoder::EncoderInfo::operator==(const EncoderInfo& rhs) const {
return true;
}
absl::optional<VideoEncoder::ResolutionBitrateLimits>
VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution(
int frame_size_pixels) const {
std::vector<ResolutionBitrateLimits> bitrate_limits =
resolution_bitrate_limits;
// Sort the list of bitrate limits by resolution.
sort(bitrate_limits.begin(), bitrate_limits.end(),
[](const ResolutionBitrateLimits& lhs,
const ResolutionBitrateLimits& rhs) {
return lhs.frame_size_pixels < rhs.frame_size_pixels;
});
for (size_t i = 0; i < bitrate_limits.size(); ++i) {
RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0);
RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0);
RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
bitrate_limits[i].min_bitrate_bps);
if (i > 0) {
// The bitrate limits aren't expected to decrease with resolution.
RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps,
bitrate_limits[i - 1].min_bitrate_bps);
RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps,
bitrate_limits[i - 1].min_start_bitrate_bps);
RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
bitrate_limits[i - 1].max_bitrate_bps);
}
if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) {
return absl::optional<ResolutionBitrateLimits>(bitrate_limits[i]);
}
}
return absl::nullopt;
}
VideoEncoder::RateControlParameters::RateControlParameters()
: bitrate(VideoBitrateAllocation()),
framerate_fps(0.0),

5
api/video_codecs/video_encoder.h
View File

@ -236,6 +236,11 @@ class RTC_EXPORT VideoEncoder {
// Recommended bitrate limits for different resolutions.
std::vector<ResolutionBitrateLimits> resolution_bitrate_limits;
// Obtains the limits from |resolution_bitrate_limits| that best matches the
// |frame_size_pixels|.
absl::optional<ResolutionBitrateLimits>
GetEncoderBitrateLimitsForResolution(int frame_size_pixels) const;
// If true, this encoder has internal support for generating simulcast
// streams. Otherwise, an adapter class will be needed.
// Even if true, the config provided to InitEncode() might not be supported,

2
call/adaptation/BUILD.gn
View File

@ -10,6 +10,8 @@ import("../../webrtc.gni")
rtc_library("resource_adaptation") {
sources = [
"encoder_settings.cc",
"encoder_settings.h",
"resource.cc",
"resource.h",
"resource_adaptation_module_interface.cc",

42
call/adaptation/encoder_settings.cc Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "call/adaptation/encoder_settings.h"
#include <utility>
namespace webrtc {
EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
VideoEncoderConfig encoder_config,
VideoCodec video_codec)
: encoder_info_(std::move(encoder_info)),
encoder_config_(std::move(encoder_config)),
video_codec_(std::move(video_codec)) {}
const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const {
return encoder_info_;
}
const VideoEncoderConfig& EncoderSettings::encoder_config() const {
return encoder_config_;
}
const VideoCodec& EncoderSettings::video_codec() const {
return video_codec_;
}
VideoCodecType GetVideoCodecTypeOrGeneric(
const absl::optional<EncoderSettings>& settings) {
return settings.has_value() ? settings->encoder_config().codec_type
: kVideoCodecGeneric;
}
} // namespace webrtc

46
call/adaptation/encoder_settings.h Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef CALL_ADAPTATION_ENCODER_SETTINGS_H_
#define CALL_ADAPTATION_ENCODER_SETTINGS_H_
#include "absl/types/optional.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_config.h"
namespace webrtc {
// Information about an encoder available when reconfiguring the encoder.
class EncoderSettings {
public:
EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
VideoEncoderConfig encoder_config,
VideoCodec video_codec);
// Encoder capabilities, implementation info, etc.
const VideoEncoder::EncoderInfo& encoder_info() const;
// Configuration parameters, ultimately coming from the API and negotiation.
const VideoEncoderConfig& encoder_config() const;
// Lower level config, heavily based on the VideoEncoderConfig.
const VideoCodec& video_codec() const;
private:
VideoEncoder::EncoderInfo encoder_info_;
VideoEncoderConfig encoder_config_;
VideoCodec video_codec_;
};
VideoCodecType GetVideoCodecTypeOrGeneric(
const absl::optional<EncoderSettings>& settings);
} // namespace webrtc
#endif // CALL_ADAPTATION_ENCODER_SETTINGS_H_

21
call/adaptation/resource_adaptation_module_interface.cc
View File

@ -10,29 +10,8 @@
#include "call/adaptation/resource_adaptation_module_interface.h"
#include <utility>
namespace webrtc {
EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
VideoEncoderConfig encoder_config,
VideoCodec video_codec)
: encoder_info_(std::move(encoder_info)),
encoder_config_(std::move(encoder_config)),
video_codec_(std::move(video_codec)) {}
const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const {
return encoder_info_;
}
const VideoEncoderConfig& EncoderSettings::encoder_config() const {
return encoder_config_;
}
const VideoCodec& EncoderSettings::video_codec() const {
return video_codec_;
}
ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {}
ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {}

23
call/adaptation/resource_adaptation_module_interface.h
View File

@ -14,33 +14,12 @@
#include "absl/types/optional.h"
#include "api/rtp_parameters.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_config.h"
#include "call/adaptation/encoder_settings.h"
#include "call/adaptation/resource.h"
#include "call/adaptation/video_source_restrictions.h"
namespace webrtc {
// Information about an encoder available when reconfiguring the encoder.
class EncoderSettings {
public:
EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
VideoEncoderConfig encoder_config,
VideoCodec video_codec);
// Encoder capabilities, implementation info, etc.
const VideoEncoder::EncoderInfo& encoder_info() const;
// Configuration parameters, ultimately coming from the API and negotiation.
const VideoEncoderConfig& encoder_config() const;
// Lower level config, heavily based on the VideoEncoderConfig.
const VideoCodec& video_codec() const;
private:
VideoEncoder::EncoderInfo encoder_info_;
VideoEncoderConfig encoder_config_;
VideoCodec video_codec_;
};
// The listener is responsible for carrying out the reconfiguration of the video
// source such that the VideoSourceRestrictions are fulfilled.
class ResourceAdaptationModuleListener {

5
video/adaptation/BUILD.gn
View File

@ -17,12 +17,17 @@ rtc_library("video_adaptation") {
]
deps = [
"../../api:rtp_parameters",
"../../api/video:video_stream_encoder",
"../../api/video_codecs:video_codecs_api",
"../../call/adaptation:resource_adaptation",
"../../modules/video_coding:video_coding_utility",
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_event",
"../../rtc_base:rtc_numerics",
"../../rtc_base/experiments:balanced_degradation_settings",
"//third_party/abseil-cpp/absl/types:optional",
]
}

408
video/adaptation/video_stream_adapter.cc
View File

@ -14,50 +14,87 @@
#include <limits>
#include "absl/types/optional.h"
#include "api/video_codecs/video_encoder.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
namespace webrtc {
namespace {
const int kMinFramerateFps = 2;
// Generate suggested higher and lower frame rates and resolutions, to be
// applied to the VideoSourceRestrictor. These are used in "maintain-resolution"
// and "maintain-framerate". The "balanced" degradation preference also makes
// use of BalancedDegradationPreference when generating suggestions. The
// VideoSourceRestrictor decidedes whether or not a proposed adaptation is
// valid.
// For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
int GetLowerFrameRateThan(int fps) {
RTC_DCHECK(fps != std::numeric_limits<int>::max());
return (fps * 2) / 3;
}
// TODO(hbos): Use absl::optional<> instead?
int GetHigherFrameRateThan(int fps) {
return fps != std::numeric_limits<int>::max()
? (fps * 3) / 2
: std::numeric_limits<int>::max();
}
// For resolution, the steps we take are 3/5 (down) and 5/3 (up).
// Notice the asymmetry of which restriction property is set depending on if
// we are adapting up or down:
// - VideoSourceRestrictor::DecreaseResolution() sets the max_pixels_per_frame()
// to the desired target and target_pixels_per_frame() to null.
// - VideoSourceRestrictor::IncreaseResolutionTo() sets the
// target_pixels_per_frame() to the desired target, and max_pixels_per_frame()
// is set according to VideoSourceRestrictor::GetIncreasedMaxPixelsWanted().
int GetLowerResolutionThan(int pixel_count) {
RTC_DCHECK(pixel_count != std::numeric_limits<int>::max());
return (pixel_count * 3) / 5;
}
// TODO(hbos): Use absl::optional<> instead?
int GetHigherResolutionThan(int pixel_count) {
return pixel_count != std::numeric_limits<int>::max()
? (pixel_count * 5) / 3
: std::numeric_limits<int>::max();
}
// One of the conditions used in VideoStreamAdapter::GetAdaptUpTarget().
// TODO(hbos): Whether or not we can adapt up due to encoder settings and
// bitrate should be expressed as a bandwidth-related Resource.
bool CanAdaptUpResolution(
const absl::optional<EncoderSettings>& encoder_settings,
absl::optional<uint32_t> encoder_target_bitrate_bps,
int input_pixels) {
uint32_t bitrate_bps = encoder_target_bitrate_bps.value_or(0);
absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
encoder_settings.has_value()
? encoder_settings->encoder_info()
.GetEncoderBitrateLimitsForResolution(
GetHigherResolutionThan(input_pixels))
: absl::nullopt;
if (!bitrate_limits.has_value() || bitrate_bps == 0) {
return true; // No limit configured or bitrate provided.
}
RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, input_pixels);
return bitrate_bps >=
static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
}
} // namespace
VideoStreamAdapter::AdaptationTarget::AdaptationTarget(AdaptationAction action,
int value)
: action(action), value(value) {}
// VideoSourceRestrictor is responsible for keeping track of current
// VideoSourceRestrictions. It suggests higher and lower frame rates and
// resolutions (used by "maintain-resolution" and "maintain-framerate"), but is
// ultimately not reponsible for determining when or how we should adapt up or
// down (e.g. "balanced" mode also uses BalancedDegradationPreference).
// VideoSourceRestrictions.
class VideoStreamAdapter::VideoSourceRestrictor {
public:
// For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
static int GetLowerFrameRateThan(int fps) {
RTC_DCHECK(fps != std::numeric_limits<int>::max());
return (fps * 2) / 3;
}
// TODO(hbos): Use absl::optional<> instead?
static int GetHigherFrameRateThan(int fps) {
return fps != std::numeric_limits<int>::max()
? (fps * 3) / 2
: std::numeric_limits<int>::max();
}
// For resolution, the steps we take are 3/5 (down) and 5/3 (up).
// Notice the asymmetry of which restriction property is set depending on if
// we are adapting up or down:
// - DecreaseResolution() sets the max_pixels_per_frame() to the desired
// target and target_pixels_per_frame() to null.
// - IncreaseResolutionTo() sets the target_pixels_per_frame() to the desired
// target, and max_pixels_per_frame() is set according to
// GetIncreasedMaxPixelsWanted().
static int GetLowerResolutionThan(int pixel_count) {
RTC_DCHECK(pixel_count != std::numeric_limits<int>::max());
return (pixel_count * 3) / 5;
}
// TODO(hbos): Use absl::optional<> instead?
static int GetHigherResolutionThan(int pixel_count) {
return pixel_count != std::numeric_limits<int>::max()
? (pixel_count * 5) / 3
: std::numeric_limits<int>::max();
}
VideoSourceRestrictor() {}
VideoSourceRestrictions source_restrictions() const {
@ -168,30 +205,27 @@ class VideoStreamAdapter::VideoSourceRestrictor {
RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
};
const int VideoStreamAdapter::kMinFramerateFps = 2;
// static
int VideoStreamAdapter::GetLowerFrameRateThan(int fps) {
return VideoSourceRestrictor::GetLowerFrameRateThan(fps);
}
// static
int VideoStreamAdapter::GetHigherFrameRateThan(int fps) {
return VideoSourceRestrictor::GetHigherFrameRateThan(fps);
}
// static
int VideoStreamAdapter::GetLowerResolutionThan(int pixel_count) {
return VideoSourceRestrictor::GetLowerResolutionThan(pixel_count);
}
// static
int VideoStreamAdapter::GetHigherResolutionThan(int pixel_count) {
return VideoSourceRestrictor::GetHigherResolutionThan(pixel_count);
VideoStreamAdapter::AdaptationRequest::Mode
VideoStreamAdapter::AdaptationRequest::GetModeFromAdaptationAction(
VideoStreamAdapter::AdaptationAction action) {
switch (action) {
case AdaptationAction::kIncreaseResolution:
return AdaptationRequest::Mode::kAdaptUp;
case AdaptationAction::kDecreaseResolution:
return AdaptationRequest::Mode::kAdaptDown;
case AdaptationAction::kIncreaseFrameRate:
return AdaptationRequest::Mode::kAdaptUp;
case AdaptationAction::kDecreaseFrameRate:
return AdaptationRequest::Mode::kAdaptDown;
}
}
VideoStreamAdapter::VideoStreamAdapter()
: source_restrictor_(std::make_unique<VideoSourceRestrictor>()) {}
: source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
balanced_settings_(),
degradation_preference_(DegradationPreference::DISABLED),
last_adaptation_request_(absl::nullopt) {}
VideoStreamAdapter::~VideoStreamAdapter() {}
@ -203,43 +237,257 @@ const AdaptationCounters& VideoStreamAdapter::adaptation_counters() const {
return source_restrictor_->adaptation_counters();
}
const BalancedDegradationSettings& VideoStreamAdapter::balanced_settings()
const {
return balanced_settings_;
}
void VideoStreamAdapter::ClearRestrictions() {
source_restrictor_->ClearRestrictions();
last_adaptation_request_.reset();
}
bool VideoStreamAdapter::CanDecreaseResolutionTo(int target_pixels,
int min_pixels_per_frame) {
return source_restrictor_->CanDecreaseResolutionTo(target_pixels,
min_pixels_per_frame);
VideoStreamAdapter::SetDegradationPreferenceResult
VideoStreamAdapter::SetDegradationPreference(
DegradationPreference degradation_preference) {
bool did_clear = false;
if (degradation_preference_ != degradation_preference) {
if (degradation_preference == DegradationPreference::BALANCED ||
degradation_preference_ == DegradationPreference::BALANCED) {
ClearRestrictions();
did_clear = true;
}
}
degradation_preference_ = degradation_preference;
return did_clear ? SetDegradationPreferenceResult::kRestrictionsCleared
: SetDegradationPreferenceResult::kRestrictionsNotCleared;
}
void VideoStreamAdapter::DecreaseResolutionTo(int target_pixels,
int min_pixels_per_frame) {
source_restrictor_->DecreaseResolutionTo(target_pixels, min_pixels_per_frame);
DegradationPreference VideoStreamAdapter::EffectiveDegradationPreference(
VideoInputMode input_mode) const {
// Balanced mode for screenshare works via automatic animation detection:
// Resolution is capped for fullscreen animated content.
// Adapatation is done only via framerate downgrade.
// Thus effective degradation preference is MAINTAIN_RESOLUTION.
// TODO(hbos): Don't do this. This is not what "balanced" means. If the
// application wants to maintain resolution it should set that degradation
// preference rather than depend on non-standard behaviors.
return (input_mode == VideoInputMode::kScreenshareVideo &&
degradation_preference_ == DegradationPreference::BALANCED)
? DegradationPreference::MAINTAIN_RESOLUTION
: degradation_preference_;
}
bool VideoStreamAdapter::CanIncreaseResolutionTo(int target_pixels) {
return source_restrictor_->CanIncreaseResolutionTo(target_pixels);
absl::optional<VideoStreamAdapter::AdaptationTarget>
VideoStreamAdapter::GetAdaptUpTarget(
const absl::optional<EncoderSettings>& encoder_settings,
absl::optional<uint32_t> encoder_target_bitrate_bps,
VideoInputMode input_mode,
int input_pixels,
int input_fps,
AdaptationObserverInterface::AdaptReason reason) const {
// Preconditions for being able to adapt up:
if (input_mode == VideoInputMode::kNoVideo)
return absl::nullopt;
// 1. We shouldn't adapt up if we're currently waiting for a previous upgrade
// to have an effect.
// TODO(hbos): What about in the case of other degradation preferences?
bool last_adaptation_was_up =
last_adaptation_request_ &&
last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
if (last_adaptation_was_up &&
degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
input_pixels <= last_adaptation_request_->input_pixel_count_) {
return absl::nullopt;
}
// 2. We shouldn't adapt up if BalancedSettings doesn't allow it, which is
// only applicable if reason is kQuality and preference is BALANCED.
if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
EffectiveDegradationPreference(input_mode) ==
DegradationPreference::BALANCED &&
!balanced_settings_.CanAdaptUp(
GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels,
encoder_target_bitrate_bps.value_or(0))) {
return absl::nullopt;
}
// Attempt to find an allowed adaptation target.
switch (EffectiveDegradationPreference(input_mode)) {
case DegradationPreference::BALANCED: {
// Attempt to increase target frame rate.
int target_fps = balanced_settings_.MaxFps(
GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels);
if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) {
return AdaptationTarget(AdaptationAction::kIncreaseFrameRate,
target_fps);
}
// Fall-through to maybe-adapting resolution, unless |balanced_settings_|
// forbids it based on bitrate.
if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
!balanced_settings_.CanAdaptUpResolution(
GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels,
encoder_target_bitrate_bps.value_or(0))) {
return absl::nullopt;
}
// Scale up resolution.
ABSL_FALLTHROUGH_INTENDED;
}
case DegradationPreference::MAINTAIN_FRAMERATE: {
// Don't adapt resolution if CanAdaptUpResolution() forbids it based on
// bitrate and limits specified by encoder capabilities.
if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
!CanAdaptUpResolution(encoder_settings, encoder_target_bitrate_bps,
input_pixels)) {
return absl::nullopt;
}
// Attempt to increase pixel count.
int target_pixels = input_pixels;
if (source_restrictor_->adaptation_counters().resolution_adaptations ==
1) {
RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
target_pixels = std::numeric_limits<int>::max();
}
target_pixels = GetHigherResolutionThan(target_pixels);
if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kIncreaseResolution,
target_pixels);
}
case DegradationPreference::MAINTAIN_RESOLUTION: {
// Scale up framerate.
int target_fps = input_fps;
if (source_restrictor_->adaptation_counters().fps_adaptations == 1) {
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
target_fps = std::numeric_limits<int>::max();
}
target_fps = GetHigherFrameRateThan(target_fps);
if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
}
case DegradationPreference::DISABLED:
return absl::nullopt;
}
}
void VideoStreamAdapter::IncreaseResolutionTo(int target_pixels) {
source_restrictor_->IncreaseResolutionTo(target_pixels);
absl::optional<VideoStreamAdapter::AdaptationTarget>
VideoStreamAdapter::GetAdaptDownTarget(
const absl::optional<EncoderSettings>& encoder_settings,
VideoInputMode input_mode,
int input_pixels,
int input_fps,
int min_pixels_per_frame,
VideoStreamEncoderObserver* encoder_stats_observer) const {
// Preconditions for being able to adapt down:
if (input_mode == VideoInputMode::kNoVideo)
return absl::nullopt;
// 1. We are not disabled.
// TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
// causes scaling due to bandwidth constraints (QualityScalerResource) to be
// ignored, not just CPU signals. This is not a use case we want to support
// long-term; remove this enum value.
if (degradation_preference_ == DegradationPreference::DISABLED)
return absl::nullopt;
bool last_adaptation_was_down =
last_adaptation_request_ &&
last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
// 2. We shouldn't adapt down if our frame rate is below the minimum or if its
// currently unknown.
if (EffectiveDegradationPreference(input_mode) ==
DegradationPreference::MAINTAIN_RESOLUTION) {
// TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
// - delete it.
if (input_fps <= 0 ||
(last_adaptation_was_down && input_fps < kMinFramerateFps)) {
return absl::nullopt;
}
}
// 3. We shouldn't adapt down if we're currently waiting for a previous
// downgrade to have an effect.
// TODO(hbos): What about in the case of other degradation preferences?
if (last_adaptation_was_down &&
degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
input_pixels >= last_adaptation_request_->input_pixel_count_) {
return absl::nullopt;
}
// Attempt to find an allowed adaptation target.
switch (EffectiveDegradationPreference(input_mode)) {
case DegradationPreference::BALANCED: {
// Try scale down framerate, if lower.
int target_fps = balanced_settings_.MinFps(
GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels);
if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) {
return AdaptationTarget(AdaptationAction::kDecreaseFrameRate,
target_fps);
}
// Scale down resolution.
ABSL_FALLTHROUGH_INTENDED;
}
case DegradationPreference::MAINTAIN_FRAMERATE: {
// Scale down resolution.
int target_pixels = GetLowerResolutionThan(input_pixels);
// TODO(https://crbug.com/webrtc/11393): Move this logic to
// ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
// target is available should not have side-effects.
if (target_pixels < min_pixels_per_frame)
encoder_stats_observer->OnMinPixelLimitReached();
if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels,
min_pixels_per_frame)) {
return absl::nullopt;
}
return AdaptationTarget(AdaptationAction::kDecreaseResolution,
target_pixels);
}
case DegradationPreference::MAINTAIN_RESOLUTION: {
int target_fps = GetLowerFrameRateThan(input_fps);
if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
}
case DegradationPreference::DISABLED:
RTC_NOTREACHED();
return absl::nullopt;
}
}
bool VideoStreamAdapter::CanDecreaseFrameRateTo(int max_frame_rate) {
return source_restrictor_->CanDecreaseFrameRateTo(max_frame_rate);
}
void VideoStreamAdapter::DecreaseFrameRateTo(int max_frame_rate) {
source_restrictor_->DecreaseFrameRateTo(max_frame_rate);
}
bool VideoStreamAdapter::CanIncreaseFrameRateTo(int max_frame_rate) {
return source_restrictor_->CanIncreaseFrameRateTo(max_frame_rate);
}
void VideoStreamAdapter::IncreaseFrameRateTo(int max_frame_rate) {
source_restrictor_->IncreaseFrameRateTo(max_frame_rate);
void VideoStreamAdapter::ApplyAdaptationTarget(const AdaptationTarget& target,
VideoInputMode input_mode,
int input_pixels,
int input_fps,
int min_pixels_per_frame) {
// Remember the input pixels and fps of this adaptation. Used to avoid
// adapting again before this adaptation has had an effect.
last_adaptation_request_.emplace(AdaptationRequest{
input_pixels, input_fps,
AdaptationRequest::GetModeFromAdaptationAction(target.action)});
switch (target.action) {
case AdaptationAction::kIncreaseResolution:
source_restrictor_->IncreaseResolutionTo(target.value);
return;
case AdaptationAction::kDecreaseResolution:
source_restrictor_->DecreaseResolutionTo(target.value,
min_pixels_per_frame);
return;
case AdaptationAction::kIncreaseFrameRate:
source_restrictor_->IncreaseFrameRateTo(target.value);
// TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
// GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
// in DecrementFramerate() makes it hard to predict whether this will be
// the last step. Remove the dependency on GetConstAdaptCounter().
if (EffectiveDegradationPreference(input_mode) ==
DegradationPreference::BALANCED &&
source_restrictor_->adaptation_counters().fps_adaptations == 0 &&
target.value != std::numeric_limits<int>::max()) {
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
source_restrictor_->IncreaseFrameRateTo(
std::numeric_limits<int>::max());
}
return;
case AdaptationAction::kDecreaseFrameRate:
source_restrictor_->DecreaseFrameRateTo(target.value);
return;
}
}
} // namespace webrtc

133
video/adaptation/video_stream_adapter.h
View File

@ -13,7 +13,13 @@
#include <memory>
#include "absl/types/optional.h"
#include "api/rtp_parameters.h"
#include "api/video/video_stream_encoder_observer.h"
#include "call/adaptation/encoder_settings.h"
#include "call/adaptation/video_source_restrictions.h"
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/experiments/balanced_degradation_settings.h"
#include "video/adaptation/adaptation_counters.h"
namespace webrtc {
@ -26,39 +32,128 @@ namespace webrtc {
// 3. Modify the stream's restrictions in one of the valid ways.
class VideoStreamAdapter {
public:
static const int kMinFramerateFps;
enum class SetDegradationPreferenceResult {
kRestrictionsNotCleared,
kRestrictionsCleared,
};
static int GetLowerFrameRateThan(int fps);
static int GetHigherFrameRateThan(int fps);
static int GetLowerResolutionThan(int pixel_count);
static int GetHigherResolutionThan(int pixel_count);
enum class VideoInputMode {
kNoVideo,
kNormalVideo,
kScreenshareVideo,
};
enum class AdaptationAction {
kIncreaseResolution,
kDecreaseResolution,
kIncreaseFrameRate,
kDecreaseFrameRate,
};
// Describes an adaptation step: increasing or decreasing resolution or frame
// rate to a given value.
// TODO(https://crbug.com/webrtc/11393): Make these private implementation
// details, and expose something that allows you to inspect the
// VideoSourceRestrictions instead. The adaptation steps could be expressed as
// a graph, for instance.
struct AdaptationTarget {
AdaptationTarget(AdaptationAction action, int value);
// Which action the VideoSourceRestrictor needs to take.
const AdaptationAction action;
// Target pixel count or frame rate depending on |action|.
const int value;
// Allow this struct to be instantiated as an optional, even though it's in
// a private namespace.
friend class absl::optional<AdaptationTarget>;
};
VideoStreamAdapter();
~VideoStreamAdapter();
// TODO(hbos): Why isn't this const?
VideoSourceRestrictions source_restrictions() const;
const AdaptationCounters& adaptation_counters() const;
// TODO(hbos): Can we get rid of any external dependencies on
// BalancedDegradationPreference? How the adaptor generates possible next
// steps for adaptation should be an implementation detail. Can the relevant
// information be inferred from GetAdaptUpTarget()/GetAdaptDownTarget()?
const BalancedDegradationSettings& balanced_settings() const;
void ClearRestrictions();
// "Can adapt?" and "do adapt!" methods.
// TODO(https://crbug.com/webrtc/11393): Make the adapter responsible for
// deciding what the next step are, i.e. taking on degradation preference
// logic. Then, these can be expressed either as CanAdaptUp() and DoAdaptUp()
// or as GetNextRestrictionsUp() and ApplyRestrictions().
bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame);
void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame);
bool CanIncreaseResolutionTo(int target_pixels);
void IncreaseResolutionTo(int target_pixels);
bool CanDecreaseFrameRateTo(int max_frame_rate);
void DecreaseFrameRateTo(int max_frame_rate);
bool CanIncreaseFrameRateTo(int max_frame_rate);
void IncreaseFrameRateTo(int max_frame_rate);
// TODO(hbos): Setting the degradation preference should not clear
// restrictions! This is not defined in the spec and is unexpected, there is a
// tiny risk that people would discover and rely on this behavior.
SetDegradationPreferenceResult SetDegradationPreference(
DegradationPreference degradation_preference);
// TODO(hbos): This is only used in one place externally by
// OveruseFrameDetectorResourceAdaptationModule - can we get rid of that
// usage? This is exposing an implementation detail.
DegradationPreference EffectiveDegradationPreference(
VideoInputMode input_mode) const;
// Returns a target that we are guaranteed to be able to adapt to, or null if
// adaptation is not desired or not possible.
absl::optional<AdaptationTarget> GetAdaptUpTarget(
const absl::optional<EncoderSettings>& encoder_settings,
absl::optional<uint32_t> encoder_target_bitrate_bps,
VideoInputMode input_mode,
int input_pixels,
int input_fps,
AdaptationObserverInterface::AdaptReason reason) const;
// TODO(https://crbug.com/webrtc/11393): Remove the dependency on
// |encoder_stats_observer| - simply checking which adaptation target is
// available should not have side-effects.
absl::optional<AdaptationTarget> GetAdaptDownTarget(
const absl::optional<EncoderSettings>& encoder_settings,
VideoInputMode input_mode,
int input_pixels,
int input_fps,
int min_pixels_per_frame,
VideoStreamEncoderObserver* encoder_stats_observer) const;
// Applies the |target| to |source_restrictor_|.
void ApplyAdaptationTarget(const AdaptationTarget& target,
VideoInputMode input_mode,
int input_pixels,
int input_fps,
int min_pixels_per_frame);
private:
class VideoSourceRestrictor;
// The input frame rate and resolution at the time of an adaptation in the
// direction described by |mode_| (up or down).
// TODO(https://crbug.com/webrtc/11393): Can this be renamed? Can this be
// merged with AdaptationTarget?
struct AdaptationRequest {
// The pixel count produced by the source at the time of the adaptation.
int input_pixel_count_;
// Framerate received from the source at the time of the adaptation.
int framerate_fps_;
// Indicates if request was to adapt up or down.
enum class Mode { kAdaptUp, kAdaptDown } mode_;
// This is a static method rather than an anonymous namespace function due
// to namespace visiblity.
static Mode GetModeFromAdaptationAction(AdaptationAction action);
};
// Owner and modifier of the VideoSourceRestriction of this stream adaptor.
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
// Decides the next adaptation target in DegradationPreference::BALANCED.
const BalancedDegradationSettings balanced_settings_;
// When deciding the next target up or down, different strategies are used
// depending on the DegradationPreference.
// https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference
DegradationPreference degradation_preference_;
// The input frame rate, resolution and adaptation direction of the last
// ApplyAdaptationTarget(). Used to avoid adapting twice if a recent
// adaptation has not had an effect on the input frame rate or resolution yet.
// TODO(hbos): Can we implement a more general "cooldown" mechanism of
// resources intead? If we already have adapted it seems like we should wait
// a while before adapting again, so that we are not acting on usage
// measurements that are made obsolete/unreliable by an "ongoing" adaptation.
absl::optional<AdaptationRequest> last_adaptation_request_;
};
} // namespace webrtc

299
video/overuse_frame_detector_resource_adaptation_module.cc
View File

@ -164,10 +164,6 @@ class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper {
int initial_framedrop_;
};
OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget::
AdaptationTarget(AdaptationAction action, int value)
: action(action), value(value) {}
OveruseFrameDetectorResourceAdaptationModule::
OveruseFrameDetectorResourceAdaptationModule(
Clock* clock,
@ -181,8 +177,6 @@ OveruseFrameDetectorResourceAdaptationModule::
experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
has_input_video_(false),
degradation_preference_(DegradationPreference::DISABLED),
balanced_settings_(),
last_adaptation_request_(absl::nullopt),
stream_adapter_(std::make_unique<VideoStreamAdapter>()),
encode_usage_resource_(
std::make_unique<EncodeUsageResource>(std::move(overuse_detector))),
@ -260,17 +254,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo(
void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference(
DegradationPreference degradation_preference) {
if (degradation_preference_ != degradation_preference) {
// Reset adaptation state, so that we're not tricked into thinking there's
// an already pending request of the same type.
last_adaptation_request_.reset();
if (degradation_preference == DegradationPreference::BALANCED ||
degradation_preference_ == DegradationPreference::BALANCED) {
stream_adapter_->ClearRestrictions();
active_counts_.fill(AdaptationCounters());
}
}
degradation_preference_ = degradation_preference;
if (stream_adapter_->SetDegradationPreference(degradation_preference) ==
VideoStreamAdapter::SetDegradationPreferenceResult::
kRestrictionsCleared) {
active_counts_.fill(AdaptationCounters());
}
MaybeUpdateVideoSourceRestrictions();
}
@ -307,7 +296,6 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates(
void OveruseFrameDetectorResourceAdaptationModule::
ResetVideoSourceRestrictions() {
last_adaptation_request_.reset();
stream_adapter_->ClearRestrictions();
active_counts_.fill(AdaptationCounters());
MaybeUpdateVideoSourceRestrictions();
@ -397,7 +385,7 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler(
absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
if (quality_scaling_experiment_enabled_) {
experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
GetVideoCodecTypeOrGeneric());
GetVideoCodecTypeOrGeneric(encoder_settings_));
}
UpdateQualityScalerSettings(experimental_thresholds
? *experimental_thresholds
@ -411,8 +399,9 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler(
if (degradation_preference_ == DegradationPreference::BALANCED &&
quality_scaler_resource_->is_started()) {
absl::optional<VideoEncoder::QpThresholds> thresholds =
balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
LastInputFrameSizeOrDefault());
stream_adapter_->balanced_settings().GetQpThresholds(
GetVideoCodecTypeOrGeneric(encoder_settings_),
LastInputFrameSizeOrDefault());
if (thresholds) {
quality_scaler_resource_->SetQpThresholds(*thresholds);
}
@ -454,212 +443,51 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured(
}
}
absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget>
absl::optional<VideoStreamAdapter::AdaptationTarget>
OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget(
int input_pixels,
int input_fps,
AdaptationObserverInterface::AdaptReason reason) const {
// Preconditions for being able to adapt up:
if (!has_input_video_)
return absl::nullopt;
// 1. We can't adapt up if we're already at the highest setting.
// We can't adapt up if we're already at the highest setting.
// Note that this only includes counts relevant to the current degradation
// preference. e.g. we previously adapted resolution, now prefer adpating fps,
// only count the fps adaptations and not the previous resolution adaptations.
//
// TODO(https://crbug.com/webrtc/11394): Checking the counts for reason should
// be replaced with checking the overuse state of all resources.
// be replaced with checking the overuse state of all resources. This is
// effectively trying to infer if the the Resource specified by |reason| is OK
// with adapting up by looking at active counters. If the relevant Resources
// simply told us this directly we wouldn't have to depend on stats counters
// to abort GetAdaptUpTarget().
int num_downgrades = ApplyDegradationPreference(active_counts_[reason],
degradation_preference_)
.Total();
RTC_DCHECK_GE(num_downgrades, 0);
if (num_downgrades == 0)
return absl::nullopt;
// 2. We shouldn't adapt up if we're currently waiting for a previous upgrade
// to have an effect.
// TODO(hbos): What about in the case of other degradation preferences?
bool last_adaptation_was_up =
last_adaptation_request_ &&
last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
if (last_adaptation_was_up &&
degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
input_pixels <= last_adaptation_request_->input_pixel_count_) {
return absl::nullopt;
}
// 3. We shouldn't adapt up if BalancedSettings doesn't allow it, which is
// only applicable if reason is kQuality and preference is BALANCED.
if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
!balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(), input_pixels,
encoder_target_bitrate_bps_.value_or(0))) {
return absl::nullopt;
}
// Attempt to find an allowed adaptation target.
switch (EffectiveDegradationPreference()) {
case DegradationPreference::BALANCED: {
// Attempt to increase target frame rate.
int target_fps =
balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(), input_pixels);
if (stream_adapter_->CanIncreaseFrameRateTo(target_fps)) {
return AdaptationTarget(AdaptationAction::kIncreaseFrameRate,
target_fps);
}
// Fall-through to maybe-adapting resolution, unless |balanced_settings_|
// forbids it based on bitrate.
if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
!balanced_settings_.CanAdaptUpResolution(
GetVideoCodecTypeOrGeneric(), input_pixels,
encoder_target_bitrate_bps_.value_or(0))) {
return absl::nullopt;
}
// Scale up resolution.
ABSL_FALLTHROUGH_INTENDED;
}
case DegradationPreference::MAINTAIN_FRAMERATE: {
// Don't adapt resolution if CanAdaptUpResolution() forbids it based on
// bitrate and limits specified by encoder capabilities.
if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
!CanAdaptUpResolution(input_pixels,
encoder_target_bitrate_bps_.value_or(0))) {
return absl::nullopt;
}
// Attempt to increase pixel count.
int target_pixels = input_pixels;
if (stream_adapter_->adaptation_counters().resolution_adaptations == 1) {
RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
target_pixels = std::numeric_limits<int>::max();
}
target_pixels =
VideoStreamAdapter::GetHigherResolutionThan(target_pixels);
if (!stream_adapter_->CanIncreaseResolutionTo(target_pixels))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kIncreaseResolution,
target_pixels);
}
case DegradationPreference::MAINTAIN_RESOLUTION: {
// Scale up framerate.
int target_fps = input_fps;
if (stream_adapter_->adaptation_counters().fps_adaptations == 1) {
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
target_fps = std::numeric_limits<int>::max();
}
target_fps = VideoStreamAdapter::GetHigherFrameRateThan(target_fps);
if (!stream_adapter_->CanIncreaseFrameRateTo(target_fps))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
}
case DegradationPreference::DISABLED:
return absl::nullopt;
}
return stream_adapter_->GetAdaptUpTarget(
encoder_settings_, encoder_target_bitrate_bps_, GetVideoInputMode(),
input_pixels, input_fps, reason);
}
absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget>
absl::optional<VideoStreamAdapter::AdaptationTarget>
OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget(
int input_pixels,
int input_fps,
int min_pixels_per_frame) const {
// Preconditions for being able to adapt down:
if (!has_input_video_)
return absl::nullopt;
// 1. We are not disabled.
// TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
// causes scaling due to bandwidth constraints (QualityScalerResource) to be
// ignored, not just CPU signals. This is not a use case we want to support;
// remove the enum value.
if (degradation_preference_ == DegradationPreference::DISABLED)
return absl::nullopt;
bool last_adaptation_was_down =
last_adaptation_request_ &&
last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
// 2. We shouldn't adapt down if our frame rate is below the minimum or if its
// currently unknown.
if (EffectiveDegradationPreference() ==
DegradationPreference::MAINTAIN_RESOLUTION) {
// TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
// - delete it.
if (input_fps <= 0 || (last_adaptation_was_down &&
input_fps < VideoStreamAdapter::kMinFramerateFps)) {
return absl::nullopt;
}
}
// 3. We shouldn't adapt down if we're currently waiting for a previous
// downgrade to have an effect.
// TODO(hbos): What about in the case of other degradation preferences?
if (last_adaptation_was_down &&
degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
input_pixels >= last_adaptation_request_->input_pixel_count_) {
return absl::nullopt;
}
// Attempt to find an allowed adaptation target.
switch (EffectiveDegradationPreference()) {
case DegradationPreference::BALANCED: {
// Try scale down framerate, if lower.
int target_fps =
balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(), input_pixels);
if (stream_adapter_->CanDecreaseFrameRateTo(target_fps)) {
return AdaptationTarget(AdaptationAction::kDecreaseFrameRate,
target_fps);
}
// Scale down resolution.
ABSL_FALLTHROUGH_INTENDED;
}
case DegradationPreference::MAINTAIN_FRAMERATE: {
// Scale down resolution.
int target_pixels =
VideoStreamAdapter::GetLowerResolutionThan(input_pixels);
// TODO(https://crbug.com/webrtc/11222): Move this logic to
// ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
// target is available should not have side-effects.
if (target_pixels < min_pixels_per_frame)
encoder_stats_observer_->OnMinPixelLimitReached();
if (!stream_adapter_->CanDecreaseResolutionTo(target_pixels,
min_pixels_per_frame)) {
return absl::nullopt;
}
return AdaptationTarget(AdaptationAction::kDecreaseResolution,
target_pixels);
}
case DegradationPreference::MAINTAIN_RESOLUTION: {
int target_fps = VideoStreamAdapter::GetLowerFrameRateThan(input_fps);
if (!stream_adapter_->CanDecreaseFrameRateTo(target_fps))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
}
case DegradationPreference::DISABLED:
RTC_NOTREACHED();
return absl::nullopt;
}
return stream_adapter_->GetAdaptDownTarget(
encoder_settings_, GetVideoInputMode(), input_pixels, input_fps,
min_pixels_per_frame, encoder_stats_observer_);
}
void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget(
const AdaptationTarget& target,
int min_pixels_per_frame,
AdaptationObserverInterface::AdaptReason reason) {
switch (target.action) {
case AdaptationAction::kIncreaseResolution:
stream_adapter_->IncreaseResolutionTo(target.value);
return;
case AdaptationAction::kDecreaseResolution:
stream_adapter_->DecreaseResolutionTo(target.value, min_pixels_per_frame);
return;
case AdaptationAction::kIncreaseFrameRate:
stream_adapter_->IncreaseFrameRateTo(target.value);
// TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
// GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
// in DecrementFramerate() makes it hard to predict whether this will be
// the last step. Remove the dependency on GetConstAdaptCounter().
if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
stream_adapter_->adaptation_counters().fps_adaptations == 0 &&
target.value != std::numeric_limits<int>::max()) {
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
stream_adapter_->IncreaseFrameRateTo(std::numeric_limits<int>::max());
}
return;
case AdaptationAction::kDecreaseFrameRate:
stream_adapter_->DecreaseFrameRateTo(target.value);
return;
}
const VideoStreamAdapter::AdaptationTarget& target,
int input_pixels,
int input_fps,
int min_pixels_per_frame) {
stream_adapter_->ApplyAdaptationTarget(target, GetVideoInputMode(),
input_pixels, input_fps,
min_pixels_per_frame);
}
void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse(
@ -668,14 +496,13 @@ void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse(
int input_fps = encoder_stats_observer_->GetInputFrameRate();
int min_pixels_per_frame = MinPixelsPerFrame();
// Should we adapt, if so to what target?
absl::optional<AdaptationTarget> target =
absl::optional<VideoStreamAdapter::AdaptationTarget> target =
GetAdaptUpTarget(input_pixels, input_fps, reason);
if (!target.has_value())
return;
// Apply target.
ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason);
last_adaptation_request_.emplace(AdaptationRequest{
input_pixels, input_fps, AdaptationRequest::Mode::kAdaptUp});
ApplyAdaptationTarget(target.value(), input_pixels, input_fps,
min_pixels_per_frame);
// Update VideoSourceRestrictions based on adaptation. This also informs the
// |adaptation_listener_|.
MaybeUpdateVideoSourceRestrictions();
@ -693,14 +520,13 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse(
int input_fps = encoder_stats_observer_->GetInputFrameRate();
int min_pixels_per_frame = MinPixelsPerFrame();
// Should we adapt, if so to what target?
absl::optional<AdaptationTarget> target =
absl::optional<VideoStreamAdapter::AdaptationTarget> target =
GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame);
if (!target.has_value())
return ResourceListenerResponse::kNothing;
// Apply target.
ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason);
last_adaptation_request_.emplace(AdaptationRequest{
input_pixels, input_fps, AdaptationRequest::Mode::kAdaptDown});
ApplyAdaptationTarget(target.value(), input_pixels, input_fps,
min_pixels_per_frame);
// Update VideoSourceRestrictions based on adaptation. This also informs the
// |adaptation_listener_|.
MaybeUpdateVideoSourceRestrictions();
@ -709,9 +535,12 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse(
RTC_LOG(INFO) << ActiveCountsToString();
// In BALANCED, if requested FPS is higher or close to input FPS to the target
// we tell the QualityScaler to increase its frequency.
if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
target->action == AdaptationAction::kDecreaseFrameRate) {
absl::optional<int> min_diff = balanced_settings_.MinFpsDiff(input_pixels);
if (stream_adapter_->EffectiveDegradationPreference(GetVideoInputMode()) ==
DegradationPreference::BALANCED &&
target->action ==
VideoStreamAdapter::AdaptationAction::kDecreaseFrameRate) {
absl::optional<int> min_diff =
stream_adapter_->balanced_settings().MinFpsDiff(input_pixels);
if (min_diff && input_fps > 0) {
int fps_diff = input_fps - target->value;
if (fps_diff < min_diff.value()) {
@ -744,14 +573,6 @@ OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const {
return options;
}
VideoCodecType
OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric()
const {
return encoder_settings_.has_value()
? encoder_settings_->encoder_config().codec_type
: kVideoCodecGeneric;
}
int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault()
const {
// The dependency on this hardcoded resolution is inherited from old code,
@ -924,37 +745,17 @@ OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
return counts;
}
DegradationPreference
OveruseFrameDetectorResourceAdaptationModule::EffectiveDegradationPreference()
const {
// Balanced mode for screenshare works via automatic animation detection:
// Resolution is capped for fullscreen animated content.
// Adapatation is done only via framerate downgrade.
// Thus effective degradation preference is MAINTAIN_RESOLUTION.
VideoStreamAdapter::VideoInputMode
OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const {
if (!has_input_video_)
return VideoStreamAdapter::VideoInputMode::kNoVideo;
return (encoder_settings_.has_value() &&
encoder_settings_->encoder_config().content_type ==
VideoEncoderConfig::ContentType::kScreen &&
degradation_preference_ == DegradationPreference::BALANCED)
? DegradationPreference::MAINTAIN_RESOLUTION
: degradation_preference_;
VideoEncoderConfig::ContentType::kScreen)
? VideoStreamAdapter::VideoInputMode::kScreenshareVideo
: VideoStreamAdapter::VideoInputMode::kNormalVideo;
}
bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
int pixels,
uint32_t bitrate_bps) const {
absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
encoder_settings_.has_value()
? GetEncoderBitrateLimits(
encoder_settings_->encoder_info(),
VideoStreamAdapter::GetHigherResolutionThan(pixels))
: absl::nullopt;
if (!bitrate_limits.has_value() || bitrate_bps == 0) {
return true; // No limit configured or bitrate provided.
}
RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels);
return bitrate_bps >=
static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
}
void OveruseFrameDetectorResourceAdaptationModule::
MaybePerformQualityRampupExperiment() {
if (!quality_scaler_resource_->is_started())

55
video/overuse_frame_detector_resource_adaptation_module.h
View File

@ -128,55 +128,25 @@ class OveruseFrameDetectorResourceAdaptationModule
AdaptationCounters* other_active);
private:
class VideoSourceRestrictor;
class InitialFrameDropper;
enum class State { kStopped, kStarted };
struct AdaptationRequest {
// The pixel count produced by the source at the time of the adaptation.
int input_pixel_count_;
// Framerate received from the source at the time of the adaptation.
int framerate_fps_;
// Indicates if request was to adapt up or down.
enum class Mode { kAdaptUp, kAdaptDown } mode_;
};
enum class AdaptationAction {
kIncreaseResolution,
kDecreaseResolution,
kIncreaseFrameRate,
kDecreaseFrameRate,
};
// Describes an adaptation step: increasing or decreasing resolution or frame
// rate to a given value.
struct AdaptationTarget {
AdaptationTarget(AdaptationAction action, int value);
// Which action the VideoSourceRestrictor needs to take.
const AdaptationAction action;
// Target pixel count or frame rate depending on |action|.
const int value;
// Allow this struct to be instantiated as an optional, even though it's in
// a private namespace.
friend class absl::optional<AdaptationTarget>;
};
// Returns a target that we are guaranteed to be able to adapt to, or null if
// adaptation is not desired or not possible.
absl::optional<AdaptationTarget> GetAdaptUpTarget(
absl::optional<VideoStreamAdapter::AdaptationTarget> GetAdaptUpTarget(
int input_pixels,
int input_fps,
AdaptationObserverInterface::AdaptReason reason) const;
absl::optional<AdaptationTarget> GetAdaptDownTarget(
absl::optional<VideoStreamAdapter::AdaptationTarget> GetAdaptDownTarget(
int input_pixels,
int input_fps,
int min_pixels_per_frame) const;
// Applies the |target| to |source_restrictor_|.
void ApplyAdaptationTarget(const AdaptationTarget& target,
int min_pixels_per_frame,
AdaptationObserverInterface::AdaptReason reason);
void ApplyAdaptationTarget(const VideoStreamAdapter::AdaptationTarget& target,
int input_pixels,
int input_fps,
int min_pixels_per_frame);
// Performs the adaptation by getting the next target, applying it and
// informing listeners of the new VideoSourceRestriction and adapt counters.
@ -185,11 +155,11 @@ class OveruseFrameDetectorResourceAdaptationModule
AdaptationObserverInterface::AdaptReason reason);
CpuOveruseOptions GetCpuOveruseOptions() const;
VideoCodecType GetVideoCodecTypeOrGeneric() const;
int LastInputFrameSizeOrDefault() const;
int MinPixelsPerFrame() const;
VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
AdaptationObserverInterface::AdaptReason reason);
VideoStreamAdapter::VideoInputMode GetVideoInputMode() const;
// Makes |video_source_restrictions_| up-to-date and informs the
// |adaptation_listener_| if restrictions are changed, allowing the listener
@ -204,8 +174,6 @@ class OveruseFrameDetectorResourceAdaptationModule
absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason);
DegradationPreference EffectiveDegradationPreference() const;
bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const;
// Checks to see if we should execute the quality rampup experiment. The
// experiment resets all video restrictions at the start of the call in the
@ -224,11 +192,12 @@ class OveruseFrameDetectorResourceAdaptationModule
// The restrictions that |adaptation_listener_| is informed of.
VideoSourceRestrictions video_source_restrictions_;
bool has_input_video_;
// TODO(https://crbug.com/webrtc/11393): DegradationPreference has mostly
// moved to VideoStreamAdapter. Move it entirely and delete it from this
// class. If the responsibility of generating next steps for adaptations is
// owned by the adapter, this class has no buisness relying on implementation
// details of the adapter.
DegradationPreference degradation_preference_;
const BalancedDegradationSettings balanced_settings_;
// Stores a snapshot of the last adaptation request triggered by an AdaptUp
// or AdaptDown signal.
absl::optional<AdaptationRequest> last_adaptation_request_;
// Keeps track of source restrictions that this adaptation module outputs.
const std::unique_ptr<VideoStreamAdapter> stream_adapter_;
const std::unique_ptr<EncodeUsageResource> encode_usage_resource_;

46
video/video_stream_encoder.cc
View File

@ -171,43 +171,6 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
}
} // namespace
absl::optional<VideoEncoder::ResolutionBitrateLimits> GetEncoderBitrateLimits(
const VideoEncoder::EncoderInfo& encoder_info,
int frame_size_pixels) {
std::vector<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
encoder_info.resolution_bitrate_limits;
// Sort the list of bitrate limits by resolution.
sort(bitrate_limits.begin(), bitrate_limits.end(),
[](const VideoEncoder::ResolutionBitrateLimits& lhs,
const VideoEncoder::ResolutionBitrateLimits& rhs) {
return lhs.frame_size_pixels < rhs.frame_size_pixels;
});
for (size_t i = 0; i < bitrate_limits.size(); ++i) {
RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0);
RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0);
RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
bitrate_limits[i].min_bitrate_bps);
if (i > 0) {
// The bitrate limits aren't expected to decrease with resolution.
RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps,
bitrate_limits[i - 1].min_bitrate_bps);
RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps,
bitrate_limits[i - 1].min_start_bitrate_bps);
RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
bitrate_limits[i - 1].max_bitrate_bps);
}
if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) {
return absl::optional<VideoEncoder::ResolutionBitrateLimits>(
bitrate_limits[i]);
}
}
return absl::nullopt;
}
const int VideoStreamEncoder::kDefaultLastFrameInfoWidth = 176;
const int VideoStreamEncoder::kDefaultLastFrameInfoHeight = 144;
@ -503,9 +466,9 @@ void VideoStreamEncoder::ReconfigureEncoder() {
encoder_reset_required = true;
}
encoder_bitrate_limits_ = GetEncoderBitrateLimits(
encoder_->GetEncoderInfo(),
last_frame_info_->width * last_frame_info_->height);
encoder_bitrate_limits_ =
encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
last_frame_info_->width * last_frame_info_->height);
if (streams.size() == 1 && encoder_bitrate_limits_) {
// Bitrate limits can be set by app (in SDP or RtpEncodingParameters) or/and
@ -1630,7 +1593,8 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
}
absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
GetEncoderBitrateLimits(encoder_->GetEncoderInfo(), pixel_count);
encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
pixel_count);
if (encoder_bitrate_limits.has_value()) {
// Use bitrate limits provided by encoder.

4
video/video_stream_encoder.h
View File

@ -45,10 +45,6 @@
namespace webrtc {
absl::optional<VideoEncoder::ResolutionBitrateLimits> GetEncoderBitrateLimits(
const VideoEncoder::EncoderInfo& encoder_info,
int frame_size_pixels);
// VideoStreamEncoder represent a video encoder that accepts raw video frames as
// input and produces an encoded bit stream.
// Usage: