diff --git a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc index a26c0c366f..574bc6fd5a 100644 --- a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc +++ b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc @@ -48,7 +48,6 @@ const int kHeight = 240; const int kNumCores = 2; const uint32_t kFramerate = 30; const size_t kMaxPayloadSize = 800; -const int kDefaultMinPixelsPerFrame = 320 * 180; const int kLowThreshold = 10; const int kHighThreshold = 20; diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h index 061bdb8b45..34502c8ab0 100644 --- a/api/video_codecs/video_encoder.h +++ b/api/video_codecs/video_encoder.h @@ -34,6 +34,8 @@ class RTPFragmentationHeader; // TODO(pbos): Expose these through a public (root) header or change these APIs. struct CodecSpecificInfo; +constexpr int kDefaultMinPixelsPerFrame = 320 * 180; + class EncodedImageCallback { public: virtual ~EncodedImageCallback() {} @@ -115,7 +117,7 @@ class RTC_EXPORT VideoEncoder { // TODO(kthelgason): Lower this limit when better testing // on MediaCodec and fallback implementations are in place. // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206 - int min_pixels_per_frame = 320 * 180; + int min_pixels_per_frame = kDefaultMinPixelsPerFrame; private: // Private constructor; to get an object without thresholds, use diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn index ef1c6e6ee7..99b3f161cd 100644 --- a/call/adaptation/BUILD.gn +++ b/call/adaptation/BUILD.gn @@ -25,6 +25,7 @@ rtc_library("resource_adaptation") { ] deps = [ "../../api:rtp_parameters", + "../../api/video_codecs:video_codecs_api", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", "//third_party/abseil-cpp/absl/types:optional", diff --git a/call/adaptation/resource_adaptation_module_interface.cc b/call/adaptation/resource_adaptation_module_interface.cc index e89d1eff2c..63cfb7279f 100644 --- a/call/adaptation/resource_adaptation_module_interface.cc +++ b/call/adaptation/resource_adaptation_module_interface.cc @@ -10,8 +10,29 @@ #include "call/adaptation/resource_adaptation_module_interface.h" +#include + namespace webrtc { +EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info, + VideoEncoderConfig encoder_config, + VideoCodec video_codec) + : encoder_info_(std::move(encoder_info)), + encoder_config_(std::move(encoder_config)), + video_codec_(std::move(video_codec)) {} + +const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const { + return encoder_info_; +} + +const VideoEncoderConfig& EncoderSettings::encoder_config() const { + return encoder_config_; +} + +const VideoCodec& EncoderSettings::video_codec() const { + return video_codec_; +} + ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {} ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {} diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h index d71ffe8e2d..bc64b8e26e 100644 --- a/call/adaptation/resource_adaptation_module_interface.h +++ b/call/adaptation/resource_adaptation_module_interface.h @@ -12,10 +12,32 @@ #define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_ #include "api/rtp_parameters.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_config.h" #include "call/adaptation/video_source_restrictions.h" namespace webrtc { +// Information about an encoder available when reconfiguring the encoder. +class EncoderSettings { + public: + EncoderSettings(VideoEncoder::EncoderInfo encoder_info, + VideoEncoderConfig encoder_config, + VideoCodec video_codec); + + // Encoder capabilities, implementation info, etc. + const VideoEncoder::EncoderInfo& encoder_info() const; + // Configuration parameters, ultimately coming from the API and negotiation. + const VideoEncoderConfig& encoder_config() const; + // Lower level config, heavily based on the VideoEncoderConfig. + const VideoCodec& video_codec() const; + + private: + VideoEncoder::EncoderInfo encoder_info_; + VideoEncoderConfig encoder_config_; + VideoCodec video_codec_; +}; + // The listener is responsible for carrying out the reconfiguration of the video // source such that the VideoSourceRestrictions are fulfilled. class ResourceAdaptationModuleListener { @@ -61,6 +83,7 @@ class ResourceAdaptationModuleInterface { virtual void SetHasInputVideo(bool has_input_video) = 0; virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; + virtual void SetEncoderSettings(EncoderSettings encoder_settings) = 0; // Removes all restrictions; the module will need to adapt all over again. // TODO(hbos): It's not clear why anybody should be able to tell the module to // reset like this; can we get rid of this method? diff --git a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index 4c80c05e3b..a597dc40b9 100644 --- a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -50,7 +50,6 @@ constexpr int64_t kTestNtpTimeMs = 456; constexpr int64_t kInitialTimestampMs = 789; constexpr int kNumCores = 1; constexpr size_t kMaxPayloadSize = 1440; -constexpr int kDefaultMinPixelsPerFrame = 320 * 180; constexpr int kWidth = 172; constexpr int kHeight = 144; constexpr float kFramerateFps = 30; diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index 2ca3435114..bd27eda3e1 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -356,12 +356,10 @@ OveruseFrameDetectorResourceAdaptationModule:: source_restrictor_(std::make_unique()), overuse_detector_(std::move(overuse_detector)), overuse_detector_is_started_(false), - codec_max_frame_rate_(absl::nullopt), target_frame_rate_(absl::nullopt), encoder_start_bitrate_bps_(0), is_quality_scaler_enabled_(false), - encoder_config_(), - encoder_(nullptr), + encoder_settings_(absl::nullopt), encoder_stats_observer_(encoder_stats_observer) { RTC_DCHECK(adaptation_listener_); RTC_DCHECK(video_stream_encoder_); @@ -372,14 +370,9 @@ OveruseFrameDetectorResourceAdaptationModule:: OveruseFrameDetectorResourceAdaptationModule:: ~OveruseFrameDetectorResourceAdaptationModule() {} -void OveruseFrameDetectorResourceAdaptationModule::SetEncoder( - VideoEncoder* encoder) { - encoder_ = encoder; -} - void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( ResourceAdaptationModuleListener* adaptation_listener) { - RTC_DCHECK(encoder_); + RTC_DCHECK(encoder_settings_.has_value()); RTC_DCHECK(!overuse_detector_is_started_); // TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside // the interval between StartCheckForOveruse() and StopCheckForOveruse(), @@ -427,6 +420,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( MaybeUpdateVideoSourceRestrictions(); } +void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings( + EncoderSettings encoder_settings) { + encoder_settings_ = std::move(encoder_settings); + MaybeUpdateTargetFrameRate(); +} + void OveruseFrameDetectorResourceAdaptationModule:: ResetVideoSourceRestrictions() { last_adaptation_request_.reset(); @@ -473,19 +472,6 @@ void OveruseFrameDetectorResourceAdaptationModule::SetLastFramePixelCount( last_frame_pixel_count_ = last_frame_pixel_count; } -void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig( - VideoEncoderConfig encoder_config) { - encoder_config_ = std::move(encoder_config); -} - -void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFrameRate( - absl::optional codec_max_frame_rate) { - RTC_DCHECK(!codec_max_frame_rate.has_value() || - codec_max_frame_rate.value() > 0.0); - codec_max_frame_rate_ = codec_max_frame_rate; - MaybeUpdateTargetFrameRate(); -} - void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps( uint32_t encoder_start_bitrate_bps) { encoder_start_bitrate_bps_ = encoder_start_bitrate_bps; @@ -528,13 +514,13 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { case DegradationPreference::BALANCED: { // Check if quality should be increased based on bitrate. if (reason == kQuality && - !balanced_settings_.CanAdaptUp(encoder_config_.codec_type, + !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(), *last_frame_pixel_count_, encoder_start_bitrate_bps_)) { return; } // Try scale up framerate, if higher. - int fps = balanced_settings_.MaxFps(encoder_config_.codec_type, + int fps = balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(), *last_frame_pixel_count_); if (source_restrictor_->IncreaseFramerate(fps)) { GetAdaptCounter().DecrementFramerate(reason, fps); @@ -550,7 +536,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { // Check if resolution should be increased based on bitrate. if (reason == kQuality && !balanced_settings_.CanAdaptUpResolution( - encoder_config_.codec_type, *last_frame_pixel_count_, + GetVideoCodecTypeOrGeneric(), *last_frame_pixel_count_, encoder_start_bitrate_bps_)) { return; } @@ -654,7 +640,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( switch (EffectiveDegradataionPreference()) { case DegradationPreference::BALANCED: { // Try scale down framerate, if lower. - int fps = balanced_settings_.MinFps(encoder_config_.codec_type, + int fps = balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(), *last_frame_pixel_count_); if (source_restrictor_->RestrictFramerate(fps)) { GetAdaptCounter().IncrementFramerate(reason); @@ -677,7 +663,10 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( bool min_pixels_reached = false; if (!source_restrictor_->RequestResolutionLowerThan( adaptation_request.input_pixel_count_, - encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame, + encoder_settings_.has_value() + ? encoder_settings_->encoder_info() + .scaling_settings.min_pixels_per_frame + : kDefaultMinPixelsPerFrame, &min_pixels_reached)) { if (min_pixels_reached) encoder_stats_observer_->OnMinPixelLimitReached(); @@ -712,6 +701,14 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( return did_adapt; } +VideoCodecType +OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric() + const { + return encoder_settings_.has_value() + ? encoder_settings_->encoder_config().codec_type + : kVideoCodecGeneric; +} + void OveruseFrameDetectorResourceAdaptationModule:: MaybeUpdateVideoSourceRestrictions() { VideoSourceRestrictions new_restrictions = ApplyDegradationPreference( @@ -726,6 +723,11 @@ void OveruseFrameDetectorResourceAdaptationModule:: void OveruseFrameDetectorResourceAdaptationModule:: MaybeUpdateTargetFrameRate() { + absl::optional codec_max_frame_rate = + encoder_settings_.has_value() + ? absl::optional( + encoder_settings_->video_codec().maxFramerate) + : absl::nullopt; // The current target framerate is the maximum frame rate as specified by // the current codec configuration or any limit imposed by the adaptation // module. This is used to make sure overuse detection doesn't needlessly @@ -735,9 +737,9 @@ void OveruseFrameDetectorResourceAdaptationModule:: degradation_preference_) .max_frame_rate(); if (!target_frame_rate.has_value() || - (codec_max_frame_rate_.has_value() && - codec_max_frame_rate_.value() < target_frame_rate.value())) { - target_frame_rate = codec_max_frame_rate_; + (codec_max_frame_rate.has_value() && + codec_max_frame_rate.value() < target_frame_rate.value())) { + target_frame_rate = codec_max_frame_rate; } if (target_frame_rate != target_frame_rate_) { target_frame_rate_ = target_frame_rate; @@ -799,7 +801,8 @@ DegradationPreference OveruseFrameDetectorResourceAdaptationModule:: // Resolution is capped for fullscreen animated content. // Adapatation is done only via framerate downgrade. // Thus effective degradation preference is MAINTAIN_RESOLUTION. - return (encoder_config_.content_type == + return (encoder_settings_.has_value() && + encoder_settings_->encoder_config().content_type == VideoEncoderConfig::ContentType::kScreen && degradation_preference_ == DegradationPreference::BALANCED) ? DegradationPreference::MAINTAIN_RESOLUTION @@ -819,7 +822,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() { absl::optional OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const { RTC_DCHECK(last_frame_pixel_count_.has_value()); - return balanced_settings_.GetQpThresholds(encoder_config_.codec_type, + return balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(), last_frame_pixel_count_.value()); } @@ -827,9 +830,11 @@ bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution( int pixels, uint32_t bitrate_bps) const { absl::optional bitrate_limits = - GetEncoderBitrateLimits( - encoder_->GetEncoderInfo(), - source_restrictor_->GetHigherResolutionThan(pixels)); + encoder_settings_.has_value() + ? GetEncoderBitrateLimits( + encoder_settings_->encoder_info(), + source_restrictor_->GetHigherResolutionThan(pixels)) + : absl::nullopt; if (!bitrate_limits.has_value() || bitrate_bps == 0) { return true; // No limit configured or bitrate provided. } diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index 798d386191..efc2ec8364 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -22,6 +22,7 @@ #include "api/video/video_frame.h" #include "api/video/video_source_interface.h" #include "api/video/video_stream_encoder_observer.h" +#include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_config.h" #include "call/adaptation/resource_adaptation_module_interface.h" @@ -60,12 +61,6 @@ class OveruseFrameDetectorResourceAdaptationModule ResourceAdaptationModuleListener* adaptation_listener); ~OveruseFrameDetectorResourceAdaptationModule() override; - // Sets the encoder to reconfigure based on overuse. - // TODO(hbos): Don't reconfigure the encoder directly. Instead, define the - // output of a resource adaptation module as a struct and let the - // VideoStreamEncoder handle the interaction with the actual encoder. - void SetEncoder(VideoEncoder* encoder); - DegradationPreference degradation_preference() const { return degradation_preference_; } @@ -77,6 +72,7 @@ class OveruseFrameDetectorResourceAdaptationModule void SetHasInputVideo(bool has_input_video) override; void SetDegradationPreference( DegradationPreference degradation_preference) override; + void SetEncoderSettings(EncoderSettings encoder_settings) override; void ResetVideoSourceRestrictions() override; // Input to the OveruseFrameDetector, which are required for this module to @@ -96,8 +92,6 @@ class OveruseFrameDetectorResourceAdaptationModule // resource adaptation module. Unify code paths where possible. Do we really // need this many public methods? void SetLastFramePixelCount(absl::optional last_frame_pixel_count); - void SetEncoderConfig(VideoEncoderConfig encoder_config); - void SetCodecMaxFrameRate(absl::optional codec_max_frame_rate); void SetEncoderStartBitrateBps(uint32_t encoder_start_bitrate_bps); // Inform the detector whether or not the quality scaler is enabled. This // helps GetActiveCounts() return absl::nullopt when appropriate. @@ -182,6 +176,8 @@ class OveruseFrameDetectorResourceAdaptationModule enum class Mode { kAdaptUp, kAdaptDown } mode_; }; + VideoCodecType GetVideoCodecTypeOrGeneric() const; + // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener // to reconfigure the source accordingly. @@ -218,12 +214,10 @@ class OveruseFrameDetectorResourceAdaptationModule const std::unique_ptr source_restrictor_; const std::unique_ptr overuse_detector_; bool overuse_detector_is_started_; - absl::optional codec_max_frame_rate_; absl::optional target_frame_rate_; uint32_t encoder_start_bitrate_bps_; bool is_quality_scaler_enabled_; - VideoEncoderConfig encoder_config_; - VideoEncoder* encoder_; + absl::optional encoder_settings_; VideoStreamEncoderObserver* const encoder_stats_observer_; }; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 96d0d4d3e1..fa1191830a 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -23,6 +23,7 @@ #include "api/video/video_bitrate_allocator_factory.h" #include "api/video/video_codec_constants.h" #include "api/video_codecs/video_encoder.h" +#include "call/adaptation/resource_adaptation_module_interface.h" #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" #include "modules/video_coding/include/video_codec_initializer.h" #include "modules/video_coding/utility/default_video_bitrate_allocator.h" @@ -424,7 +425,6 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, (!encoder_ || encoder_config_.video_format != config.video_format || max_data_payload_length_ != max_data_payload_length); encoder_config_ = std::move(config); - resource_adaptation_module_->SetEncoderConfig(encoder_config_.Copy()); max_data_payload_length_ = max_data_payload_length; pending_encoder_reconfiguration_ = true; @@ -498,7 +498,6 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_ = settings_.encoder_factory->CreateVideoEncoder( encoder_config_.video_format); - resource_adaptation_module_->SetEncoder(encoder_.get()); // TODO(nisse): What to do if creating the encoder fails? Crash, // or just discard incoming frames? RTC_CHECK(encoder_); @@ -588,7 +587,6 @@ void VideoStreamEncoder::ReconfigureEncoder() { // Make sure the start bit rate is sane... RTC_DCHECK_LE(codec.startBitrate, 1000000); max_framerate_ = codec.maxFramerate; - resource_adaptation_module_->SetCodecMaxFrameRate(max_framerate_); // Inform source about max configured framerate. int max_framerate = 0; @@ -632,6 +630,9 @@ void VideoStreamEncoder::ReconfigureEncoder() { } send_codec_ = codec; + resource_adaptation_module_->SetEncoderSettings(EncoderSettings( + encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_)); + encoder_switch_experiment_.SetCodec(send_codec_.codecType); quality_rampup_experiment_.SetMaxBitrate( last_frame_info_->width * last_frame_info_->height, codec.maxBitrate); @@ -1180,6 +1181,8 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, } if (encoder_info_ != info) { + resource_adaptation_module_->SetEncoderSettings(EncoderSettings( + encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_)); RTC_LOG(LS_INFO) << "Encoder settings changed from " << encoder_info_.ToString() << " to " << info.ToString(); }