diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index 8515b93b05..4a41e60e0c 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -18,10 +18,15 @@ rtc_library("resource_adaptation") { "resource_adaptation_processor_interface.h", "video_source_restrictions.cc", "video_source_restrictions.h", + "video_stream_input_state.cc", + "video_stream_input_state.h", + "video_stream_input_state_provider.cc", + "video_stream_input_state_provider.h", ] deps = [ "../../api:rtp_parameters", "../../api/video:video_frame", + "../../api/video:video_stream_encoder", "../../api/video_codecs:video_codecs_api", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h index 04e4469069..8f46d7b9e0 100644 --- a/call/adaptation/resource_adaptation_processor_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h @@ -40,74 +40,10 @@ class ResourceAdaptationProcessorInterface { virtual void StartResourceAdaptation( ResourceAdaptationProcessorListener* adaptation_listener) = 0; virtual void StopResourceAdaptation() = 0; - // The resource must out-live the module. virtual void AddResource(Resource* resource) = 0; - - // The following methods are callable whether or not adaption is started. - - // Informs the module whether we have input video. By default, the module must - // assume the value is false. - virtual void SetHasInputVideo(bool has_input_video) = 0; virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; - virtual void SetEncoderSettings(EncoderSettings encoder_settings) = 0; - // TODO(bugs.webrtc.org/11222): This function shouldn't be needed, start - // bitrates should be apart of the constructor ideally. See the comment on - // VideoStreamEncoderInterface::SetStartBitrate. - virtual void SetStartBitrate(DataRate start_bitrate) = 0; - virtual void SetTargetBitrate(DataRate target_bitrate) = 0; - // The encoder rates are the target encoder bitrate distributed across spatial - // and temporal layers. This may be different than target bitrate depending on - // encoder configuration, e.g. if we can encode at desired quality in less - // than the allowed target bitrate or if the encoder has not been initialized - // yet. - virtual void SetEncoderRates( - const VideoEncoder::RateControlParameters& encoder_rates) = 0; - - // The following methods correspond to the pipeline that a frame goes through. - // Note that if the encoder is parallelized, multiple frames may be processed - // in parallel and methods may be invoked in unexpected orders. - // - // The implementation must not retain VideoFrames. Doing so may keep video - // frame buffers alive - this may even stall encoding. - // TODO(hbos): Can we replace VideoFrame with a different struct, maybe width - // and height is enough, and some sort of way to identify it at each step? - - // 1. A frame is delivered to the encoder, e.g. from the camera. Next up: it - // may get dropped or it may get encoded, see OnFrameDroppedDueToSize() and - // OnEncodeStarted(). - virtual void OnFrame(const VideoFrame& frame) = 0; - // 2.i) An input frame was dropped because its resolution is too big (e.g. for - // the target bitrate). This frame will not continue through the rest of the - // pipeline. The module should adapt down in resolution to avoid subsequent - // frames getting dropped for the same reason. - // TODO(hbos): If we take frame rate into account perhaps it would be valid to - // adapt down in frame rate as well. - virtual void OnFrameDroppedDueToSize() = 0; - // 2.ii) If the frame will not be dropped due to size then signal that it may - // get encoded. However the frame is not guaranteed to be encoded right away - // or ever (for example if encoding is paused). - // TODO(eshr): Try replace OnMaybeEncodeFrame and merge behaviour into - // EncodeStarted. - // TODO(eshr): Try to merge OnFrame, OnFrameDroppedDueToSize, and - // OnMaybeEncode frame into one method. - virtual void OnMaybeEncodeFrame() = 0; - // 2.iii) An input frame is about to be encoded. It may have been cropped and - // have different dimensions than what was observed at OnFrame(). Next - // up: encoding completes or fails, see OnEncodeCompleted(). There is - // currently no signal for encode failure. - virtual void OnEncodeStarted(const VideoFrame& cropped_frame, - int64_t time_when_first_seen_us) = 0; - // 3.i) The frame has successfully completed encoding. Next up: The encoded - // frame is dropped or packetized and sent over the network. There is - // currently no signal what happens beyond this point. - virtual void OnEncodeCompleted(const EncodedImage& encoded_image, - int64_t time_sent_in_us, - absl::optional encode_duration_us) = 0; - // A frame was dropped at any point in the pipeline. This may come from - // the encoder, or elsewhere, like a frame dropper or frame size check. - virtual void OnFrameDropped(EncodedImageCallback::DropReason reason) = 0; }; } // namespace webrtc diff --git a/call/adaptation/video_stream_input_state.cc b/call/adaptation/video_stream_input_state.cc new file mode 100644 index 0000000000..1827334b21 --- /dev/null +++ b/call/adaptation/video_stream_input_state.cc @@ -0,0 +1,72 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_stream_input_state.h" + +#include "api/video_codecs/video_encoder.h" + +namespace webrtc { + +VideoStreamInputState::VideoStreamInputState() + : has_input_(false), + frame_size_pixels_(absl::nullopt), + frames_per_second_(absl::nullopt), + video_codec_type_(VideoCodecType::kVideoCodecGeneric), + min_pixels_per_frame_(kDefaultMinPixelsPerFrame) {} + +void VideoStreamInputState::set_has_input(bool has_input) { + has_input_ = has_input; +} + +void VideoStreamInputState::set_frame_size_pixels( + absl::optional frame_size_pixels) { + frame_size_pixels_ = frame_size_pixels; +} + +void VideoStreamInputState::set_frames_per_second( + absl::optional frames_per_second) { + frames_per_second_ = frames_per_second; +} + +void VideoStreamInputState::set_video_codec_type( + VideoCodecType video_codec_type) { + video_codec_type_ = video_codec_type; +} + +void VideoStreamInputState::set_min_pixels_per_frame(int min_pixels_per_frame) { + min_pixels_per_frame_ = min_pixels_per_frame; +} + +bool VideoStreamInputState::has_input() const { + return has_input_; +} + +absl::optional VideoStreamInputState::frame_size_pixels() const { + return frame_size_pixels_; +} + +absl::optional VideoStreamInputState::frames_per_second() const { + return frames_per_second_; +} + +VideoCodecType VideoStreamInputState::video_codec_type() const { + return video_codec_type_; +} + +int VideoStreamInputState::min_pixels_per_frame() const { + return min_pixels_per_frame_; +} + +bool VideoStreamInputState::HasInputFrameSizeAndFramesPerSecond() const { + return has_input_ && frame_size_pixels_.has_value() && + frames_per_second_.has_value(); +} + +} // namespace webrtc diff --git a/call/adaptation/video_stream_input_state.h b/call/adaptation/video_stream_input_state.h new file mode 100644 index 0000000000..ef80405e75 --- /dev/null +++ b/call/adaptation/video_stream_input_state.h @@ -0,0 +1,49 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_H_ +#define CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_H_ + +#include "absl/types/optional.h" +#include "api/video/video_codec_type.h" + +namespace webrtc { + +// The source resolution, frame rate and other properties of a +// VideoStreamEncoder. +class VideoStreamInputState { + public: + VideoStreamInputState(); + + void set_has_input(bool has_input); + void set_frame_size_pixels(absl::optional frame_size_pixels); + void set_frames_per_second(absl::optional frames_per_second); + void set_video_codec_type(VideoCodecType video_codec_type); + void set_min_pixels_per_frame(int min_pixels_per_frame); + + bool has_input() const; + absl::optional frame_size_pixels() const; + absl::optional frames_per_second() const; + VideoCodecType video_codec_type() const; + int min_pixels_per_frame() const; + + bool HasInputFrameSizeAndFramesPerSecond() const; + + private: + bool has_input_; + absl::optional frame_size_pixels_; + absl::optional frames_per_second_; + VideoCodecType video_codec_type_; + int min_pixels_per_frame_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_H_ diff --git a/call/adaptation/video_stream_input_state_provider.cc b/call/adaptation/video_stream_input_state_provider.cc new file mode 100644 index 0000000000..2548a4802b --- /dev/null +++ b/call/adaptation/video_stream_input_state_provider.cc @@ -0,0 +1,48 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_stream_input_state_provider.h" + +namespace webrtc { + +VideoStreamInputStateProvider::VideoStreamInputStateProvider( + VideoStreamEncoderObserver* frame_rate_provider) + : frame_rate_provider_(frame_rate_provider) {} + +void VideoStreamInputStateProvider::OnHasInputChanged(bool has_input) { + rtc::CritScope lock(&crit_); + input_state_.set_has_input(has_input); +} + +void VideoStreamInputStateProvider::OnFrameSizeObserved(int frame_size_pixels) { + RTC_DCHECK_GT(frame_size_pixels, 0); + rtc::CritScope lock(&crit_); + input_state_.set_frame_size_pixels(frame_size_pixels); +} + +void VideoStreamInputStateProvider::OnEncoderSettingsChanged( + EncoderSettings encoder_settings) { + rtc::CritScope lock(&crit_); + input_state_.set_video_codec_type( + encoder_settings.encoder_config().codec_type); + input_state_.set_min_pixels_per_frame( + encoder_settings.encoder_info().scaling_settings.min_pixels_per_frame); +} + +VideoStreamInputState VideoStreamInputStateProvider::InputState() { + // GetInputFrameRate() is thread-safe. + int input_fps = frame_rate_provider_->GetInputFrameRate(); + rtc::CritScope lock(&crit_); + input_state_.set_frames_per_second( + input_fps >= 0 ? absl::optional(input_fps) : absl::nullopt); + return input_state_; +} + +} // namespace webrtc diff --git a/call/adaptation/video_stream_input_state_provider.h b/call/adaptation/video_stream_input_state_provider.h new file mode 100644 index 0000000000..7093e97fdd --- /dev/null +++ b/call/adaptation/video_stream_input_state_provider.h @@ -0,0 +1,40 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ +#define CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ + +#include "api/video/video_stream_encoder_observer.h" +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/video_stream_input_state.h" +#include "rtc_base/critical_section.h" + +namespace webrtc { + +class VideoStreamInputStateProvider { + public: + VideoStreamInputStateProvider( + VideoStreamEncoderObserver* frame_rate_provider); + + void OnHasInputChanged(bool has_input); + void OnFrameSizeObserved(int frame_size_pixels); + void OnEncoderSettingsChanged(EncoderSettings encoder_settings); + + VideoStreamInputState InputState(); + + private: + mutable rtc::CriticalSection crit_; + VideoStreamEncoderObserver* const frame_rate_provider_; + VideoStreamInputState input_state_ RTC_GUARDED_BY(crit_); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ diff --git a/video/adaptation/resource_adaptation_processor.cc b/video/adaptation/resource_adaptation_processor.cc index fd8207cad2..f0b827c7de 100644 --- a/video/adaptation/resource_adaptation_processor.cc +++ b/video/adaptation/resource_adaptation_processor.cc @@ -186,16 +186,17 @@ class ResourceAdaptationProcessor::InitialFrameDropper { }; ResourceAdaptationProcessor::ResourceAdaptationProcessor( + VideoStreamInputStateProvider* input_state_provider, Clock* clock, bool experiment_cpu_load_estimator, std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer, ResourceAdaptationProcessorListener* adaptation_listener) - : adaptation_listener_(adaptation_listener), + : input_state_provider_(input_state_provider), + adaptation_listener_(adaptation_listener), clock_(clock), state_(State::kStopped), experiment_cpu_load_estimator_(experiment_cpu_load_estimator), - has_input_video_(false), degradation_preference_(DegradationPreference::DISABLED), effective_degradation_preference_(DegradationPreference::DISABLED), stream_adapter_(std::make_unique()), @@ -205,8 +206,6 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( initial_frame_dropper_(std::make_unique( quality_scaler_resource_.get())), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), - last_input_frame_size_(absl::nullopt), - target_frame_rate_(absl::nullopt), encoder_target_bitrate_bps_(absl::nullopt), quality_rampup_done_(false), quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), @@ -262,11 +261,6 @@ void ResourceAdaptationProcessor::AddResource(Resource* resource, resources_.emplace_back(resource, reason); } -void ResourceAdaptationProcessor::SetHasInputVideo(bool has_input_video) { - // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS. - has_input_video_ = has_input_video; -} - void ResourceAdaptationProcessor::SetDegradationPreference( DegradationPreference degradation_preference) { degradation_preference_ = degradation_preference; @@ -310,10 +304,6 @@ void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { MaybeUpdateVideoSourceRestrictions(); } -void ResourceAdaptationProcessor::OnFrame(const VideoFrame& frame) { - last_input_frame_size_ = frame.size(); -} - void ResourceAdaptationProcessor::OnFrameDroppedDueToSize() { VideoAdaptationCounters counters_before = stream_adapter_->adaptation_counters(); @@ -447,10 +437,21 @@ ResourceAdaptationProcessor::OnResourceUsageStateMeasured( } } +bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation( + const VideoStreamInputState& input_state) const { + return input_state.HasInputFrameSizeAndFramesPerSecond() && + (effective_degradation_preference_ != + DegradationPreference::MAINTAIN_RESOLUTION || + input_state.frames_per_second() >= kMinFrameRateFps); +} + void ResourceAdaptationProcessor::OnResourceUnderuse( VideoAdaptationReason reason) { - if (!has_input_video_) + VideoStreamInputState input_state = input_state_provider_->InputState(); + if (effective_degradation_preference_ == DegradationPreference::DISABLED || + !HasSufficientInputForAdaptation(input_state)) { return; + } // We can't adapt up if we're already at the highest setting. // Note that this only includes counts relevant to the current degradation // preference. e.g. we previously adapted resolution, now prefer adpating fps, @@ -472,9 +473,8 @@ void ResourceAdaptationProcessor::OnResourceUnderuse( if (num_downgrades == 0) return; // Update video input states and encoder settings for accurate adaptation. - stream_adapter_->SetInput(LastInputFrameSizeOrDefault(), - encoder_stats_observer_->GetInputFrameRate(), - encoder_settings_, encoder_target_bitrate_bps_); + stream_adapter_->SetInput(input_state, encoder_settings_, + encoder_target_bitrate_bps_); // Should we adapt, and if so: how? Adaptation adaptation = stream_adapter_->GetAdaptationUp(reason); if (adaptation.status() != Adaptation::Status::kValid) @@ -491,12 +491,17 @@ void ResourceAdaptationProcessor::OnResourceUnderuse( ResourceListenerResponse ResourceAdaptationProcessor::OnResourceOveruse( VideoAdaptationReason reason) { - if (!has_input_video_) + VideoStreamInputState input_state = input_state_provider_->InputState(); + if (!input_state.has_input()) { return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; + } + if (effective_degradation_preference_ == DegradationPreference::DISABLED || + !HasSufficientInputForAdaptation(input_state)) { + return ResourceListenerResponse::kNothing; + } // Update video input states and encoder settings for accurate adaptation. - stream_adapter_->SetInput(LastInputFrameSizeOrDefault(), - encoder_stats_observer_->GetInputFrameRate(), - encoder_settings_, encoder_target_bitrate_bps_); + stream_adapter_->SetInput(input_state, encoder_settings_, + encoder_target_bitrate_bps_); // Should we adapt, and if so: how? Adaptation adaptation = stream_adapter_->GetAdaptationDown(); if (adaptation.min_pixel_limit_reached()) @@ -537,15 +542,8 @@ CpuOveruseOptions ResourceAdaptationProcessor::GetCpuOveruseOptions() const { } int ResourceAdaptationProcessor::LastInputFrameSizeOrDefault() const { - // The dependency on this hardcoded resolution is inherited from old code, - // which used this resolution as a stand-in for not knowing the resolution - // yet. - // TODO(hbos): Can we simply DCHECK has_value() before usage instead? Having a - // DCHECK passed all the tests but adding it does change the requirements of - // this class (= not being allowed to call OnResourceUnderuse() or - // OnResourceOveruse() before OnFrame()) and deserves a standalone CL. - return last_input_frame_size_.value_or(kDefaultInputPixelsWidth * - kDefaultInputPixelsHeight); + return input_state_provider_->InputState().frame_size_pixels().value_or( + kDefaultInputPixelsWidth * kDefaultInputPixelsHeight); } void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { diff --git a/video/adaptation/resource_adaptation_processor.h b/video/adaptation/resource_adaptation_processor.h index 847a556e22..b417f8b53f 100644 --- a/video/adaptation/resource_adaptation_processor.h +++ b/video/adaptation/resource_adaptation_processor.h @@ -30,6 +30,7 @@ #include "api/video_codecs/video_encoder_config.h" #include "call/adaptation/resource.h" #include "call/adaptation/resource_adaptation_processor_interface.h" +#include "call/adaptation/video_stream_input_state_provider.h" #include "rtc_base/experiments/quality_rampup_experiment.h" #include "rtc_base/experiments/quality_scaler_settings.h" #include "rtc_base/strings/string_builder.h" @@ -63,6 +64,7 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // The processor can be constructed on any sequence, but must be initialized // and used on a single sequence, e.g. the encoder queue. ResourceAdaptationProcessor( + VideoStreamInputStateProvider* input_state_provider, Clock* clock, bool experiment_cpu_load_estimator, std::unique_ptr overuse_detector, @@ -84,37 +86,32 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // Uses a default AdaptReason of kCpu. void AddResource(Resource* resource) override; void AddResource(Resource* resource, VideoAdaptationReason reason); - void SetHasInputVideo(bool has_input_video) override; void SetDegradationPreference( DegradationPreference degradation_preference) override; - void SetEncoderSettings(EncoderSettings encoder_settings) override; - void SetStartBitrate(DataRate start_bitrate) override; - void SetTargetBitrate(DataRate target_bitrate) override; - void SetEncoderRates( - const VideoEncoder::RateControlParameters& encoder_rates) override; - void OnFrame(const VideoFrame& frame) override; - void OnFrameDroppedDueToSize() override; - void OnMaybeEncodeFrame() override; + // Settings that affect the VideoStreamEncoder-specific resources. + void SetEncoderSettings(EncoderSettings encoder_settings); + void SetStartBitrate(DataRate start_bitrate); + void SetTargetBitrate(DataRate target_bitrate); + void SetEncoderRates( + const VideoEncoder::RateControlParameters& encoder_rates); + // TODO(https://crbug.com/webrtc/11338): This can be made private if we + // configure on SetDegredationPreference and SetEncoderSettings. + void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); + + // Methods corresponding to different points in the encoding pipeline. + void OnFrameDroppedDueToSize(); + void OnMaybeEncodeFrame(); void OnEncodeStarted(const VideoFrame& cropped_frame, - int64_t time_when_first_seen_us) override; + int64_t time_when_first_seen_us); void OnEncodeCompleted(const EncodedImage& encoded_image, int64_t time_sent_in_us, - absl::optional encode_duration_us) override; - void OnFrameDropped(EncodedImageCallback::DropReason reason) override; - - // TODO(hbos): Is dropping initial frames really just a special case of "don't - // encode frames right now"? Can this be part of VideoSourceRestrictions, - // which handles the output of the rest of the encoder settings? This is - // something we'll need to support for "disable video due to overuse", not - // initial frames. + absl::optional encode_duration_us); + void OnFrameDropped(EncodedImageCallback::DropReason reason); + // If true, the VideoStreamEncoder should eexecute its logic to maybe drop + // frames baseed on size and bitrate. bool DropInitialFrames() const; - // TODO(eshr): This can be made private if we configure on - // SetDegredationPreference and SetEncoderSettings. - // (https://crbug.com/webrtc/11338) - void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); - // ResourceUsageListener implementation. ResourceListenerResponse OnResourceUsageStateMeasured( const Resource& resource) override; @@ -136,6 +133,9 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, enum class State { kStopped, kStarted }; + bool HasSufficientInputForAdaptation( + const VideoStreamInputState& input_state) const; + // Performs the adaptation by getting the next target, applying it and // informing listeners of the new VideoSourceRestriction and adapt counters. void OnResourceUnderuse(VideoAdaptationReason reason); @@ -145,9 +145,13 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, int LastInputFrameSizeOrDefault() const; // Reinterprets "balanced + screenshare" as "maintain-resolution". - // TODO(hbos): Don't do this. This is not what "balanced" means. If the - // application wants to maintain resolution it should set that degradation - // preference rather than depend on non-standard behaviors. + // When screensharing, as far as ResourceAdaptationProcessor logic is + // concerned, we ALWAYS use "maintain-resolution". However, on a different + // layer we may cap the video resolution to 720p to make high fps + // screensharing feasible. This means that on the API layer the preference is + // "balanced" (allowing reduction in both resolution and frame rate) but on + // this layer (not responsible for caping to 720p) the preference is the same + // as "maintain-resolution". void MaybeUpdateEffectiveDegradationPreference(); // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener @@ -175,13 +179,13 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, void ResetActiveCounts(); std::string ActiveCountsToString() const; + VideoStreamInputStateProvider* const input_state_provider_; ResourceAdaptationProcessorListener* const adaptation_listener_; Clock* clock_; State state_; const bool experiment_cpu_load_estimator_; // The restrictions that |adaptation_listener_| is informed of. VideoSourceRestrictions video_source_restrictions_; - bool has_input_video_; DegradationPreference degradation_preference_; DegradationPreference effective_degradation_preference_; // Keeps track of source restrictions that this adaptation processor outputs. @@ -190,8 +194,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, const std::unique_ptr quality_scaler_resource_; const std::unique_ptr initial_frame_dropper_; const bool quality_scaling_experiment_enabled_; - absl::optional last_input_frame_size_; - absl::optional target_frame_rate_; // This is the last non-zero target bitrate for the encoder. absl::optional encoder_target_bitrate_bps_; absl::optional encoder_rates_; diff --git a/video/adaptation/video_stream_adapter.cc b/video/adaptation/video_stream_adapter.cc index abee5913ae..064ef0a459 100644 --- a/video/adaptation/video_stream_adapter.cc +++ b/video/adaptation/video_stream_adapter.cc @@ -27,13 +27,6 @@ const int kMinFrameRateFps = 2; namespace { -int MinPixelsPerFrame(const absl::optional& encoder_settings) { - return encoder_settings.has_value() - ? encoder_settings->encoder_info() - .scaling_settings.min_pixels_per_frame - : kDefaultMinPixelsPerFrame; -} - // Generate suggested higher and lower frame rates and resolutions, to be // applied to the VideoSourceRestrictor. These are used in "maintain-resolution" // and "maintain-framerate". The "balanced" degradation preference also makes @@ -161,10 +154,12 @@ class VideoStreamAdapter::VideoSourceRestrictor { adaptations_ = VideoAdaptationCounters(); } - void SetMinPixelsPerFrame(int min_pixels_per_frame) { + void set_min_pixels_per_frame(int min_pixels_per_frame) { min_pixels_per_frame_ = min_pixels_per_frame; } + int min_pixels_per_frame() const { return min_pixels_per_frame_; } + bool CanDecreaseResolutionTo(int target_pixels) { int max_pixels_per_frame = rtc::dchecked_cast( source_restrictions_.max_pixels_per_frame().value_or( @@ -319,8 +314,7 @@ VideoStreamAdapter::VideoStreamAdapter() balanced_settings_(), adaptation_validation_id_(0), degradation_preference_(DegradationPreference::DISABLED), - input_pixels_(0), - input_fps_(0), + input_state_(), encoder_settings_(absl::nullopt), encoder_target_bitrate_bps_(absl::nullopt), last_adaptation_request_(absl::nullopt) {} @@ -366,29 +360,30 @@ VideoStreamAdapter::SetDegradationPreference( } void VideoStreamAdapter::SetInput( - int input_pixels, - int input_fps, + VideoStreamInputState input_state, absl::optional encoder_settings, absl::optional encoder_target_bitrate_bps) { // Invalidate any previously returned Adaptation. ++adaptation_validation_id_; - input_pixels_ = input_pixels; - input_fps_ = input_fps; + input_state_ = input_state; + source_restrictor_->set_min_pixels_per_frame( + input_state_.min_pixels_per_frame()); encoder_settings_ = encoder_settings; encoder_target_bitrate_bps_ = encoder_target_bitrate_bps; - source_restrictor_->SetMinPixelsPerFrame( - MinPixelsPerFrame(encoder_settings_)); } Adaptation VideoStreamAdapter::GetAdaptationUp( VideoAdaptationReason reason) const { + RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); + RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond()); // Don't adapt if we're awaiting a previous adaptation to have an effect. bool last_adaptation_was_up = last_adaptation_request_ && last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; if (last_adaptation_was_up && degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && - input_pixels_ <= last_adaptation_request_->input_pixel_count_) { + input_state_.frame_size_pixels().value() <= + last_adaptation_request_->input_pixel_count_) { return Adaptation(adaptation_validation_id_, Adaptation::Status::kAwaitingPreviousAdaptation); } @@ -396,9 +391,9 @@ Adaptation VideoStreamAdapter::GetAdaptationUp( // exceed bitrate constraints. if (reason == VideoAdaptationReason::kQuality && degradation_preference_ == DegradationPreference::BALANCED && - !balanced_settings_.CanAdaptUp( - GetVideoCodecTypeOrGeneric(encoder_settings_), input_pixels_, - encoder_target_bitrate_bps_.value_or(0))) { + !balanced_settings_.CanAdaptUp(input_state_.video_codec_type(), + input_state_.frame_size_pixels().value(), + encoder_target_bitrate_bps_.value_or(0))) { return Adaptation(adaptation_validation_id_, Adaptation::Status::kIsBitrateConstrained); } @@ -407,8 +402,9 @@ Adaptation VideoStreamAdapter::GetAdaptationUp( switch (degradation_preference_) { case DegradationPreference::BALANCED: { // Attempt to increase target frame rate. - int target_fps = balanced_settings_.MaxFps( - GetVideoCodecTypeOrGeneric(encoder_settings_), input_pixels_); + int target_fps = + balanced_settings_.MaxFps(input_state_.video_codec_type(), + input_state_.frame_size_pixels().value()); if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) { return Adaptation( adaptation_validation_id_, @@ -419,7 +415,8 @@ Adaptation VideoStreamAdapter::GetAdaptationUp( // forbids it based on bitrate. if (reason == VideoAdaptationReason::kQuality && !balanced_settings_.CanAdaptUpResolution( - GetVideoCodecTypeOrGeneric(encoder_settings_), input_pixels_, + input_state_.video_codec_type(), + input_state_.frame_size_pixels().value(), encoder_target_bitrate_bps_.value_or(0))) { return Adaptation(adaptation_validation_id_, Adaptation::Status::kIsBitrateConstrained); @@ -432,12 +429,12 @@ Adaptation VideoStreamAdapter::GetAdaptationUp( // bitrate and limits specified by encoder capabilities. if (reason == VideoAdaptationReason::kQuality && !CanAdaptUpResolution(encoder_settings_, encoder_target_bitrate_bps_, - input_pixels_)) { + input_state_.frame_size_pixels().value())) { return Adaptation(adaptation_validation_id_, Adaptation::Status::kIsBitrateConstrained); } // Attempt to increase pixel count. - int target_pixels = input_pixels_; + int target_pixels = input_state_.frame_size_pixels().value(); if (source_restrictor_->adaptation_counters().resolution_adaptations == 1) { RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; @@ -455,7 +452,7 @@ Adaptation VideoStreamAdapter::GetAdaptationUp( } case DegradationPreference::MAINTAIN_RESOLUTION: { // Scale up framerate. - int target_fps = input_fps_; + int target_fps = input_state_.frames_per_second().value(); if (source_restrictor_->adaptation_counters().fps_adaptations == 1) { RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; target_fps = std::numeric_limits::max(); @@ -471,33 +468,24 @@ Adaptation VideoStreamAdapter::GetAdaptationUp( target_fps)); } case DegradationPreference::DISABLED: + RTC_NOTREACHED(); return Adaptation(adaptation_validation_id_, - Adaptation::Status::kAdaptationDisabled); + Adaptation::Status::kLimitReached); } } Adaptation VideoStreamAdapter::GetAdaptationDown() const { + RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); + RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond()); // Don't adapt adaptation is disabled. - if (degradation_preference_ == DegradationPreference::DISABLED) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kAdaptationDisabled); - } bool last_adaptation_was_down = last_adaptation_request_ && last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; - if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) { - // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake - // - delete it. - if (input_fps_ <= 0 || - (last_adaptation_was_down && input_fps_ < kMinFrameRateFps)) { - return Adaptation(adaptation_validation_id_, - Adaptation::Status::kInsufficientInput); - } - } // Don't adapt if we're awaiting a previous adaptation to have an effect. if (last_adaptation_was_down && degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && - input_pixels_ >= last_adaptation_request_->input_pixel_count_) { + input_state_.frame_size_pixels().value() >= + last_adaptation_request_->input_pixel_count_) { return Adaptation(adaptation_validation_id_, Adaptation::Status::kAwaitingPreviousAdaptation); } @@ -506,8 +494,9 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() const { switch (degradation_preference_) { case DegradationPreference::BALANCED: { // Try scale down framerate, if lower. - int target_fps = balanced_settings_.MinFps( - GetVideoCodecTypeOrGeneric(encoder_settings_), input_pixels_); + int target_fps = + balanced_settings_.MinFps(input_state_.video_codec_type(), + input_state_.frame_size_pixels().value()); if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { return Adaptation( adaptation_validation_id_, @@ -519,9 +508,10 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() const { } case DegradationPreference::MAINTAIN_FRAMERATE: { // Scale down resolution. - int target_pixels = GetLowerResolutionThan(input_pixels_); + int target_pixels = + GetLowerResolutionThan(input_state_.frame_size_pixels().value()); bool min_pixel_limit_reached = - target_pixels < MinPixelsPerFrame(encoder_settings_); + target_pixels < source_restrictor_->min_pixels_per_frame(); if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels)) { return Adaptation(adaptation_validation_id_, Adaptation::Status::kLimitReached, @@ -534,7 +524,8 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() const { min_pixel_limit_reached); } case DegradationPreference::MAINTAIN_RESOLUTION: { - int target_fps = GetLowerFrameRateThan(input_fps_); + int target_fps = + GetLowerFrameRateThan(input_state_.frames_per_second().value()); if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) { return Adaptation(adaptation_validation_id_, Adaptation::Status::kLimitReached); @@ -547,7 +538,7 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() const { case DegradationPreference::DISABLED: RTC_NOTREACHED(); return Adaptation(adaptation_validation_id_, - Adaptation::Status::kAdaptationDisabled); + Adaptation::Status::kLimitReached); } } @@ -571,7 +562,8 @@ ResourceListenerResponse VideoStreamAdapter::ApplyAdaptation( // Remember the input pixels and fps of this adaptation. Used to avoid // adapting again before this adaptation has had an effect. last_adaptation_request_.emplace(AdaptationRequest{ - input_pixels_, input_fps_, + input_state_.frame_size_pixels().value(), + input_state_.frames_per_second().value(), AdaptationRequest::GetModeFromAdaptationAction(adaptation.step().type)}); // Adapt! source_restrictor_->ApplyAdaptationStep(adaptation.step(), @@ -584,9 +576,11 @@ ResourceListenerResponse VideoStreamAdapter::ApplyAdaptation( // instead. if (degradation_preference_ == DegradationPreference::BALANCED && adaptation.step().type == Adaptation::StepType::kDecreaseFrameRate) { - absl::optional min_diff = balanced_settings_.MinFpsDiff(input_pixels_); - if (min_diff && input_fps_ > 0) { - int fps_diff = input_fps_ - adaptation.step().target; + absl::optional min_diff = + balanced_settings_.MinFpsDiff(input_state_.frame_size_pixels().value()); + if (min_diff && input_state_.frames_per_second().value() > 0) { + int fps_diff = + input_state_.frames_per_second().value() - adaptation.step().target; if (fps_diff < min_diff.value()) { return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; } diff --git a/video/adaptation/video_stream_adapter.h b/video/adaptation/video_stream_adapter.h index ca6cdb591b..481d4fac5f 100644 --- a/video/adaptation/video_stream_adapter.h +++ b/video/adaptation/video_stream_adapter.h @@ -20,6 +20,7 @@ #include "call/adaptation/encoder_settings.h" #include "call/adaptation/resource.h" #include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/experiments/balanced_degradation_settings.h" @@ -38,14 +39,6 @@ class Adaptation final { // Applying this adaptation will have an effect. All other Status codes // indicate that adaptation is not possible and why. kValid, - // Cannot adapt. DegradationPreference is DISABLED. - // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it - // causes all adaptation to be ignored, even QP-scaling. - kAdaptationDisabled, - // Cannot adapt. Adaptation is refused because we are attempting to adapt - // down while the input frame rate is either not known yet or is less than - // the minimum. - kInsufficientInput, // Cannot adapt. The minimum or maximum adaptation has already been reached. // There are no more steps to take. kLimitReached, @@ -147,8 +140,7 @@ class VideoStreamAdapter { SetDegradationPreferenceResult SetDegradationPreference( DegradationPreference degradation_preference); // The adaptaiton logic depends on these inputs. - void SetInput(int input_pixels, - int input_fps, + void SetInput(VideoStreamInputState input_state, absl::optional encoder_settings, absl::optional encoder_target_bitrate_bps); @@ -196,8 +188,7 @@ class VideoStreamAdapter { // depending on the DegradationPreference. // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference DegradationPreference degradation_preference_; - int input_pixels_; - int input_fps_; + VideoStreamInputState input_state_; absl::optional encoder_settings_; absl::optional encoder_target_bitrate_bps_; // The input frame rate, resolution and adaptation direction of the last diff --git a/video/adaptation/video_stream_adapter_unittest.cc b/video/adaptation/video_stream_adapter_unittest.cc index 48e231e7d7..3c371685eb 100644 --- a/video/adaptation/video_stream_adapter_unittest.cc +++ b/video/adaptation/video_stream_adapter_unittest.cc @@ -53,6 +53,23 @@ std::string BalancedFieldTrialConfig() { rtc::ToString(kBalancedHighFrameRateFps) + "/"; } +VideoStreamInputState InputState( + int input_pixels, + int input_fps, + absl::optional encoder_settings) { + VideoStreamInputState input_state; + input_state.set_has_input(true); + input_state.set_frame_size_pixels(input_pixels); + input_state.set_frames_per_second(input_fps); + if (encoder_settings.has_value()) { + input_state.set_video_codec_type( + encoder_settings->encoder_config().codec_type); + input_state.set_min_pixels_per_frame( + encoder_settings->encoder_info().scaling_settings.min_pixels_per_frame); + } + return input_state; +} + // Responsible for adjusting the inputs to VideoStreamAdapter (SetInput), such // as pixels and frame rate, according to the most recent source restrictions. // This helps tests that apply adaptations multiple times: if the input is not @@ -70,8 +87,8 @@ class FakeVideoStream { input_fps_(input_fps), encoder_settings_(std::move(encoder_settings)), encoder_target_bitrate_bps_(std::move(encoder_target_bitrate_bps)) { - adapter_->SetInput(input_pixels_, input_fps_, encoder_settings_, - encoder_target_bitrate_bps_); + adapter_->SetInput(InputState(input_pixels_, input_fps_, encoder_settings_), + encoder_settings_, encoder_target_bitrate_bps_); } int input_pixels() const { return input_pixels_; } @@ -94,8 +111,8 @@ class FakeVideoStream { if (restrictions.max_frame_rate().has_value()) { input_fps_ = restrictions.max_frame_rate().value(); } - adapter_->SetInput(input_pixels_, input_fps_, encoder_settings_, - encoder_target_bitrate_bps_); + adapter_->SetInput(InputState(input_pixels_, input_fps_, encoder_settings_), + encoder_settings_, encoder_target_bitrate_bps_); } private: @@ -140,7 +157,8 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_DecreasesPixelsToThreeFifths) { const int kInputPixels = 1280 * 720; VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - adapter.SetInput(kInputPixels, 30, absl::nullopt, absl::nullopt); + adapter.SetInput(InputState(kInputPixels, 30, absl::nullopt), absl::nullopt, + absl::nullopt); Adaptation adaptation = adapter.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); EXPECT_FALSE(adaptation.min_pixel_limit_reached()); @@ -157,9 +175,10 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_DecreasesPixelsToLimitReached) { const int kMinPixelsPerFrame = 640 * 480; VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - adapter.SetInput(kMinPixelsPerFrame + 1, 30, - EncoderSettingsWithMinPixelsPerFrame(kMinPixelsPerFrame), - absl::nullopt); + auto encoder_settings = + EncoderSettingsWithMinPixelsPerFrame(kMinPixelsPerFrame); + adapter.SetInput(InputState(kMinPixelsPerFrame + 1, 30, encoder_settings), + encoder_settings, absl::nullopt); // Even though we are above kMinPixelsPerFrame, because adapting down would // have exceeded the limit, we are said to have reached the limit already. // This differs from the frame rate adaptation logic, which would have clamped @@ -211,7 +230,8 @@ TEST(VideoStreamAdapterTest, MaintainResolution_DecreasesFpsToTwoThirds) { const int kInputFps = 30; VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - adapter.SetInput(1280 * 720, kInputFps, absl::nullopt, absl::nullopt); + adapter.SetInput(InputState(1280 * 720, kInputFps, absl::nullopt), + absl::nullopt, absl::nullopt); Adaptation adaptation = adapter.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); adapter.ApplyAdaptation(adaptation); @@ -286,7 +306,8 @@ TEST(VideoStreamAdapterTest, Balanced_DecreaseFrameRate) { BalancedFieldTrialConfig()); VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::BALANCED); - adapter.SetInput(kBalancedMediumResolutionPixels, kBalancedHighFrameRateFps, + adapter.SetInput(InputState(kBalancedMediumResolutionPixels, + kBalancedHighFrameRateFps, absl::nullopt), absl::nullopt, absl::nullopt); // If our frame rate is higher than the frame rate associated with our // resolution we should try to adapt to the frame rate associated with our @@ -531,30 +552,12 @@ TEST(VideoStreamAdapterTest, Balanced_LimitReached) { EXPECT_EQ(1, adapter.adaptation_counters().fps_adaptations); } -TEST(VideoStreamAdapterTest, AdaptationDisabled) { - VideoStreamAdapter adapter; - adapter.SetDegradationPreference(DegradationPreference::DISABLED); - adapter.SetInput(1280 * 720, 30, absl::nullopt, absl::nullopt); - EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, - adapter.GetAdaptationDown().status()); - EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, - adapter.GetAdaptationUp(kReasonDontCare).status()); -} - -TEST(VideoStreamAdapterTest, InsufficientInput) { - VideoStreamAdapter adapter; - adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - // No frame rate is insufficient when going down. - adapter.SetInput(1280 * 720, 0, absl::nullopt, absl::nullopt); - EXPECT_EQ(Adaptation::Status::kInsufficientInput, - adapter.GetAdaptationDown().status()); -} - // kAwaitingPreviousAdaptation is only supported in "maintain-framerate". TEST(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationDown) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - adapter.SetInput(1280 * 720, 30, absl::nullopt, absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 30, absl::nullopt), absl::nullopt, + absl::nullopt); // Adapt down once, but don't update the input. adapter.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations); @@ -655,7 +658,8 @@ TEST(VideoStreamAdapterTest, kRestrictionsNotCleared, adapter.SetDegradationPreference( DegradationPreference::MAINTAIN_FRAMERATE)); - adapter.SetInput(1280 * 720, 30, absl::nullopt, absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 30, absl::nullopt), absl::nullopt, + absl::nullopt); adapter.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_NE(VideoSourceRestrictions(), adapter.source_restrictions()); EXPECT_NE(0, adapter.adaptation_counters().Total()); @@ -687,7 +691,8 @@ TEST(VideoStreamAdapterDeathTest, SetDegradationPreferenceInvalidatesAdaptations) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); - adapter.SetInput(1280 * 720, 30, absl::nullopt, absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 30, absl::nullopt), absl::nullopt, + absl::nullopt); Adaptation adaptation = adapter.GetAdaptationDown(); adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); EXPECT_DEATH(adapter.ApplyAdaptation(adaptation), ""); @@ -696,9 +701,11 @@ TEST(VideoStreamAdapterDeathTest, TEST(VideoStreamAdapterDeathTest, SetInputInvalidatesAdaptations) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - adapter.SetInput(1280 * 720, 30, absl::nullopt, absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 30, absl::nullopt), absl::nullopt, + absl::nullopt); Adaptation adaptation = adapter.GetAdaptationDown(); - adapter.SetInput(1280 * 720, 31, absl::nullopt, absl::nullopt); + adapter.SetInput(InputState(1280 * 720, 31, absl::nullopt), absl::nullopt, + absl::nullopt); EXPECT_DEATH(adapter.PeekNextRestrictions(adaptation), ""); } diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 38665d8609..157bf661b5 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -258,8 +258,11 @@ VideoStreamEncoder::VideoStreamEncoder( video_source_sink_controller_(std::make_unique( /*sink=*/this, /*source=*/nullptr)), + input_state_provider_(std::make_unique( + encoder_stats_observer)), resource_adaptation_processor_( std::make_unique( + input_state_provider_.get(), clock_, settings_.experiment_cpu_load_estimator, std::move(overuse_detector), @@ -325,7 +328,7 @@ void VideoStreamEncoder::SetSource( video_source_sink_controller_->SetSource(source); encoder_queue_.PostTask([this, source, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); - resource_adaptation_processor_->SetHasInputVideo(source); + input_state_provider_->OnHasInputChanged(source); resource_adaptation_processor_->SetDegradationPreference( degradation_preference); if (encoder_) @@ -636,8 +639,8 @@ void VideoStreamEncoder::ReconfigureEncoder() { was_encode_called_since_last_initialization_ = false; } - resource_adaptation_processor_->SetEncoderSettings(EncoderSettings( - encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_)); + // Inform dependents of updated encoder settings. + OnEncoderSettingsChanged(); if (success) { next_frame_types_.clear(); @@ -731,6 +734,13 @@ void VideoStreamEncoder::ReconfigureEncoder() { resource_adaptation_processor_->ConfigureQualityScaler(info); } +void VideoStreamEncoder::OnEncoderSettingsChanged() { + EncoderSettings encoder_settings(encoder_->GetEncoderInfo(), + encoder_config_.Copy(), send_codec_); + input_state_provider_->OnEncoderSettingsChanged(encoder_settings); + resource_adaptation_processor_->SetEncoderSettings(encoder_settings); +} + void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); VideoFrame incoming_frame = video_frame; @@ -972,7 +982,7 @@ void VideoStreamEncoder::SetEncoderRates( void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, int64_t time_when_posted_us) { RTC_DCHECK_RUN_ON(&encoder_queue_); - resource_adaptation_processor_->OnFrame(video_frame); + input_state_provider_->OnFrameSizeObserved(video_frame.size()); if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || video_frame.height() != last_frame_info_->height || @@ -1119,8 +1129,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, } if (encoder_info_ != info) { - resource_adaptation_processor_->SetEncoderSettings(EncoderSettings( - encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_)); + OnEncoderSettingsChanged(); RTC_LOG(LS_INFO) << "Encoder settings changed from " << encoder_info_.ToString() << " to " << info.ToString(); } diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 1d2dda1149..8e36f517d4 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -28,6 +28,7 @@ #include "api/video_codecs/video_encoder.h" #include "call/adaptation/resource_adaptation_processor_interface.h" #include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state_provider.h" #include "modules/video_coding/utility/frame_dropper.h" #include "rtc_base/critical_section.h" #include "rtc_base/event.h" @@ -148,6 +149,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, }; void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_); + void OnEncoderSettingsChanged() RTC_RUN_ON(&encoder_queue_); // Implements VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; @@ -406,6 +408,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // VideoSourceSinkController can be made single-threaded, and its lock can be // replaced with a sequence checker. std::unique_ptr video_source_sink_controller_; + std::unique_ptr input_state_provider_ + RTC_GUARDED_BY(&encoder_queue_); std::unique_ptr resource_adaptation_processor_ RTC_GUARDED_BY(&encoder_queue_);