2fec64484f
There are now multiple ways to configure VP9 L1Tx: - Legacy API: configure legacy SVC and disable encodings, this gets interpreted as disabling spatial layers (non-standard API hack). - Standard API: configure scalability_mode. This can be done either with a single encoding or multiple encodings. As long as only one encoding is active we get a single L1Tx ssrc, same as legacy API. Due to a bug, the ApplySpatialLayerBitrateLimits() logic which tweaks bitrates was only applied in the legacy API code path, not the standard API code path, despite both code paths configuring L1Tx. The issue is that IsSimulcastOrMultipleSpatialLayers() was checking if `number_of_streams == 1`. This is true in legacy code path but not standard code path. The fix is to look at `numberOfSimulcastStreams == 1` instead, which is set to the correct value regardless of code path used. This CL adds comments documenting the difference between `number_of_streams` and `numberOfSimulcastStreams` to reduce the risk of more mistakes like this in the future. Bug: chromium:1455039, b:279161263 Change-Id: I69789b68cc5d45ef1b3becd310687c8dec8e7c87 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/308722 Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Commit-Queue: Henrik Boström <hbos@webrtc.org> Reviewed-by: Erik Språng <sprang@webrtc.org> Cr-Commit-Position: refs/heads/main@{#40287}
240 lines
10 KiB
C++
240 lines
10 KiB
C++
/*
|
|
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_
|
|
#define VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_
|
|
|
|
#include <atomic>
|
|
#include <map>
|
|
#include <memory>
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "absl/types/optional.h"
|
|
#include "api/adaptation/resource.h"
|
|
#include "api/field_trials_view.h"
|
|
#include "api/rtp_parameters.h"
|
|
#include "api/scoped_refptr.h"
|
|
#include "api/task_queue/task_queue_base.h"
|
|
#include "api/video/video_adaptation_counters.h"
|
|
#include "api/video/video_adaptation_reason.h"
|
|
#include "api/video/video_frame.h"
|
|
#include "api/video/video_source_interface.h"
|
|
#include "api/video_codecs/video_codec.h"
|
|
#include "api/video_codecs/video_encoder.h"
|
|
#include "call/adaptation/resource_adaptation_processor_interface.h"
|
|
#include "call/adaptation/video_stream_adapter.h"
|
|
#include "call/adaptation/video_stream_input_state_provider.h"
|
|
#include "rtc_base/experiments/quality_scaler_settings.h"
|
|
#include "rtc_base/ref_count.h"
|
|
#include "rtc_base/strings/string_builder.h"
|
|
#include "rtc_base/synchronization/mutex.h"
|
|
#include "rtc_base/thread_annotations.h"
|
|
#include "system_wrappers/include/clock.h"
|
|
#include "video/adaptation/balanced_constraint.h"
|
|
#include "video/adaptation/bandwidth_quality_scaler_resource.h"
|
|
#include "video/adaptation/bitrate_constraint.h"
|
|
#include "video/adaptation/encode_usage_resource.h"
|
|
#include "video/adaptation/overuse_frame_detector.h"
|
|
#include "video/adaptation/pixel_limit_resource.h"
|
|
#include "video/adaptation/quality_rampup_experiment_helper.h"
|
|
#include "video/adaptation/quality_scaler_resource.h"
|
|
#include "video/adaptation/video_stream_encoder_resource.h"
|
|
#include "video/config/video_encoder_config.h"
|
|
#include "video/video_stream_encoder_observer.h"
|
|
|
|
namespace webrtc {
|
|
|
|
// The assumed input frame size if we have not yet received a frame.
|
|
// TODO(hbos): This is 144p - why are we assuming super low quality? Seems like
|
|
// a bad heuristic.
|
|
extern const int kDefaultInputPixelsWidth;
|
|
extern const int kDefaultInputPixelsHeight;
|
|
|
|
// Owns adaptation-related Resources pertaining to a single VideoStreamEncoder
|
|
// and passes on the relevant input from the encoder to the resources. The
|
|
// resources provide resource usage states to the ResourceAdaptationProcessor
|
|
// which is responsible for reconfiguring streams in order not to overuse
|
|
// resources.
|
|
//
|
|
// The manager is also involved with various mitigations not part of the
|
|
// ResourceAdaptationProcessor code such as the initial frame dropping.
|
|
class VideoStreamEncoderResourceManager
|
|
: public VideoSourceRestrictionsListener,
|
|
public ResourceLimitationsListener,
|
|
public QualityRampUpExperimentListener {
|
|
public:
|
|
VideoStreamEncoderResourceManager(
|
|
VideoStreamInputStateProvider* input_state_provider,
|
|
VideoStreamEncoderObserver* encoder_stats_observer,
|
|
Clock* clock,
|
|
bool experiment_cpu_load_estimator,
|
|
std::unique_ptr<OveruseFrameDetector> overuse_detector,
|
|
DegradationPreferenceProvider* degradation_preference_provider,
|
|
const FieldTrialsView& field_trials);
|
|
~VideoStreamEncoderResourceManager() override;
|
|
|
|
void Initialize(TaskQueueBase* encoder_queue);
|
|
void SetAdaptationProcessor(
|
|
ResourceAdaptationProcessorInterface* adaptation_processor,
|
|
VideoStreamAdapter* stream_adapter);
|
|
|
|
// TODO(https://crbug.com/webrtc/11563): The degradation preference is a
|
|
// setting of the Processor, it does not belong to the Manager - can we get
|
|
// rid of this?
|
|
void SetDegradationPreferences(DegradationPreference degradation_preference);
|
|
DegradationPreference degradation_preference() const;
|
|
|
|
void ConfigureEncodeUsageResource();
|
|
// Initializes the pixel limit resource if the "WebRTC-PixelLimitResource"
|
|
// field trial is enabled. This can be used for testing.
|
|
void MaybeInitializePixelLimitResource();
|
|
// Stops the encode usage and quality scaler resources if not already stopped.
|
|
// If the pixel limit resource was created it is also stopped and nulled.
|
|
void StopManagedResources();
|
|
|
|
// Settings that affect the VideoStreamEncoder-specific resources.
|
|
void SetEncoderSettings(EncoderSettings encoder_settings);
|
|
void SetStartBitrate(DataRate start_bitrate);
|
|
void SetTargetBitrate(DataRate target_bitrate);
|
|
void SetEncoderRates(
|
|
const VideoEncoder::RateControlParameters& encoder_rates);
|
|
// TODO(https://crbug.com/webrtc/11338): This can be made private if we
|
|
// configure on SetDegredationPreference and SetEncoderSettings.
|
|
void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
|
|
void ConfigureBandwidthQualityScaler(
|
|
const VideoEncoder::EncoderInfo& encoder_info);
|
|
|
|
// Methods corresponding to different points in the encoding pipeline.
|
|
void OnFrameDroppedDueToSize();
|
|
void OnMaybeEncodeFrame();
|
|
void OnEncodeStarted(const VideoFrame& cropped_frame,
|
|
int64_t time_when_first_seen_us);
|
|
void OnEncodeCompleted(const EncodedImage& encoded_image,
|
|
int64_t time_sent_in_us,
|
|
absl::optional<int> encode_duration_us,
|
|
DataSize frame_size);
|
|
void OnFrameDropped(EncodedImageCallback::DropReason reason);
|
|
|
|
// Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order
|
|
// to update legacy getStats().
|
|
void AddResource(rtc::scoped_refptr<Resource> resource,
|
|
VideoAdaptationReason reason);
|
|
void RemoveResource(rtc::scoped_refptr<Resource> resource);
|
|
std::vector<AdaptationConstraint*> AdaptationConstraints() const;
|
|
// If true, the VideoStreamEncoder should execute its logic to maybe drop
|
|
// frames based on size and bitrate.
|
|
bool DropInitialFrames() const;
|
|
absl::optional<uint32_t> SingleActiveStreamPixels() const;
|
|
absl::optional<uint32_t> UseBandwidthAllocationBps() const;
|
|
|
|
// VideoSourceRestrictionsListener implementation.
|
|
// Updates `video_source_restrictions_`.
|
|
void OnVideoSourceRestrictionsUpdated(
|
|
VideoSourceRestrictions restrictions,
|
|
const VideoAdaptationCounters& adaptation_counters,
|
|
rtc::scoped_refptr<Resource> reason,
|
|
const VideoSourceRestrictions& unfiltered_restrictions) override;
|
|
void OnResourceLimitationChanged(
|
|
rtc::scoped_refptr<Resource> resource,
|
|
const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
|
|
resource_limitations) override;
|
|
|
|
// QualityRampUpExperimentListener implementation.
|
|
void OnQualityRampUp() override;
|
|
|
|
static bool IsSimulcastOrMultipleSpatialLayers(
|
|
const VideoEncoderConfig& encoder_config,
|
|
const VideoCodec& video_codec);
|
|
|
|
private:
|
|
class InitialFrameDropper;
|
|
|
|
VideoAdaptationReason GetReasonFromResource(
|
|
rtc::scoped_refptr<Resource> resource) const;
|
|
|
|
CpuOveruseOptions GetCpuOveruseOptions() const;
|
|
int LastFrameSizeOrDefault() const;
|
|
|
|
// Calculates an up-to-date value of the target frame rate and informs the
|
|
// `encode_usage_resource_` of the new value.
|
|
void MaybeUpdateTargetFrameRate();
|
|
|
|
// Use nullopt to disable quality scaling.
|
|
void UpdateQualityScalerSettings(
|
|
absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
|
|
|
|
void UpdateBandwidthQualityScalerSettings(
|
|
bool bandwidth_quality_scaling_allowed,
|
|
const std::vector<VideoEncoder::ResolutionBitrateLimits>&
|
|
resolution_bitrate_limits);
|
|
|
|
void UpdateStatsAdaptationSettings() const;
|
|
|
|
static std::string ActiveCountsToString(
|
|
const std::map<VideoAdaptationReason, VideoAdaptationCounters>&
|
|
active_counts);
|
|
|
|
const FieldTrialsView& field_trials_;
|
|
DegradationPreferenceProvider* const degradation_preference_provider_;
|
|
std::unique_ptr<BitrateConstraint> bitrate_constraint_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
const std::unique_ptr<BalancedConstraint> balanced_constraint_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;
|
|
const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
|
|
rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource_;
|
|
const rtc::scoped_refptr<BandwidthQualityScalerResource>
|
|
bandwidth_quality_scaler_resource_;
|
|
|
|
TaskQueueBase* encoder_queue_;
|
|
VideoStreamInputStateProvider* const input_state_provider_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
ResourceAdaptationProcessorInterface* adaptation_processor_;
|
|
VideoStreamAdapter* stream_adapter_ RTC_GUARDED_BY(encoder_queue_);
|
|
// Thread-safe.
|
|
VideoStreamEncoderObserver* const encoder_stats_observer_;
|
|
|
|
DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_);
|
|
VideoSourceRestrictions video_source_restrictions_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
|
|
VideoAdaptationCounters current_adaptation_counters_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
|
|
const BalancedDegradationSettings balanced_settings_;
|
|
Clock* clock_ RTC_GUARDED_BY(encoder_queue_);
|
|
const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_);
|
|
const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
const bool quality_scaling_experiment_enabled_ RTC_GUARDED_BY(encoder_queue_);
|
|
const bool pixel_limit_resource_experiment_enabled_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
absl::optional<uint32_t> encoder_target_bitrate_bps_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
absl::optional<VideoEncoder::RateControlParameters> encoder_rates_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
std::unique_ptr<QualityRampUpExperimentHelper> quality_rampup_experiment_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
absl::optional<EncoderSettings> encoder_settings_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
|
|
// Ties a resource to a reason for statistical reporting. This AdaptReason is
|
|
// also used by this module to make decisions about how to adapt up/down.
|
|
std::map<rtc::scoped_refptr<Resource>, VideoAdaptationReason> resources_
|
|
RTC_GUARDED_BY(encoder_queue_);
|
|
};
|
|
|
|
} // namespace webrtc
|
|
|
|
#endif // VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_
|