diff --git a/webrtc/api/android/jni/androidmediaencoder_jni.cc b/webrtc/api/android/jni/androidmediaencoder_jni.cc index f6d15b7449..8a7a724665 100644 --- a/webrtc/api/android/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/android/jni/androidmediaencoder_jni.cc @@ -37,6 +37,7 @@ #include "webrtc/modules/video_coding/utility/vp8_header_parser.h" #include "webrtc/system_wrappers/include/field_trial.h" #include "webrtc/system_wrappers/include/logcat_trace_context.h" +#include "webrtc/video_encoder.h" using rtc::Bind; using rtc::Thread; @@ -120,8 +121,6 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // rtc::MessageHandler implementation. void OnMessage(rtc::Message* msg) override; - void OnDroppedFrame() override; - bool SupportsNativeHandle() const override { return egl_context_ != nullptr; } const char* ImplementationName() const override; @@ -168,7 +167,6 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, webrtc::EncodedImageCallback* callback); int32_t ReleaseOnCodecThread(); int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate); - void OnDroppedFrameOnCodecThread(); // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members. int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info); @@ -184,6 +182,8 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // Search for H.264 start codes. int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size); + VideoEncoder::ScalingSettings GetScalingSettings() const override; + // Displays encoder statistics. void LogStatistics(bool force_log); @@ -269,13 +269,9 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, // True only when between a callback_->OnEncodedImage() call return a positive // value and the next Encode() call being ignored. bool drop_next_input_frame_; + bool scale_; // Global references; must be deleted in Release(). std::vector input_buffers_; - QualityScaler quality_scaler_; - // Dynamic resolution change, off by default. - bool scale_; - - // H264 bitstream parser, used to extract QP from encoded bitstreams. webrtc::H264BitstreamParser h264_bitstream_parser_; // VP9 variables to populate codec specific structure. @@ -417,23 +413,6 @@ int32_t MediaCodecVideoEncoder::InitEncode( ALOGD << "InitEncode request: " << init_width << " x " << init_height; ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); - if (scale_) { - if (codec_type == kVideoCodecVP8 || codec_type == kVideoCodecH264) { - quality_scaler_.Init(codec_type, codec_settings->startBitrate, - codec_settings->width, codec_settings->height, - codec_settings->maxFramerate); - } else { - // When adding codec support to additional hardware codecs, also configure - // their QP thresholds for scaling. - RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds."; - scale_ = false; - } - QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); - init_width = res.width; - init_height = res.height; - ALOGD << "Scaled resolution: " << init_width << " x " << init_height; - } - return codec_thread_->Invoke( RTC_FROM_HERE, Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, this, init_width, @@ -715,7 +694,6 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( drop_next_input_frame_ = false; current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; frames_dropped_media_encoder_++; - OnDroppedFrameOnCodecThread(); return WEBRTC_VIDEO_CODEC_OK; } @@ -737,32 +715,12 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( return ProcessHWErrorOnEncodeOnCodecThread(); } frames_dropped_media_encoder_++; - OnDroppedFrameOnCodecThread(); return WEBRTC_VIDEO_CODEC_OK; } consecutive_full_queue_frame_drops_ = 0; rtc::scoped_refptr input_buffer( frame.video_frame_buffer()); - if (scale_) { - // Check framerate before spatial resolution change. - quality_scaler_.OnEncodeFrame(frame.width(), frame.height()); - const webrtc::QualityScaler::Resolution scaled_resolution = - quality_scaler_.GetScaledResolution(); - if (scaled_resolution.width != frame.width() || - scaled_resolution.height != frame.height()) { - if (input_buffer->native_handle() != nullptr) { - input_buffer = static_cast(input_buffer.get()) - ->CropScaleAndRotate(frame.width(), frame.height(), - 0, 0, - scaled_resolution.width, - scaled_resolution.height, - webrtc::kVideoRotation_0); - } else { - input_buffer = quality_scaler_.GetScaledBuffer(input_buffer); - } - } - } VideoFrame input_frame(input_buffer, frame.timestamp(), frame.render_time_ms(), frame.rotation()); @@ -788,7 +746,6 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( if (frames_received_ > 1) { current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; frames_dropped_media_encoder_++; - OnDroppedFrameOnCodecThread(); } else { // Input buffers are not ready after codec initialization, HW is still // allocating thme - this is expected and should not result in drop @@ -969,9 +926,6 @@ int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, last_set_fps_ == frame_rate) { return WEBRTC_VIDEO_CODEC_OK; } - if (scale_) { - quality_scaler_.ReportFramerate(frame_rate); - } JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); if (new_bit_rate > 0) { @@ -1086,9 +1040,6 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { image->_frameType = (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta); image->_completeFrame = true; - image->adapt_reason_.quality_resolution_downscales = - scale_ ? quality_scaler_.downscale_shift() : -1; - webrtc::CodecSpecificInfo info; memset(&info, 0, sizeof(info)); info.codecType = codec_type; @@ -1135,23 +1086,19 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { header.fragmentationLength[0] = image->_length; header.fragmentationPlType[0] = 0; header.fragmentationTimeDiff[0] = 0; - if (codec_type == kVideoCodecVP8 && scale_) { + if (codec_type == kVideoCodecVP8) { int qp; if (webrtc::vp8::GetQp(payload, payload_size, &qp)) { current_acc_qp_ += qp; - quality_scaler_.ReportQP(qp); image->qp_ = qp; } } } else if (codec_type == kVideoCodecH264) { - if (scale_) { - h264_bitstream_parser_.ParseBitstream(payload, payload_size); - int qp; - if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { - current_acc_qp_ += qp; - quality_scaler_.ReportQP(qp); - image->qp_ = qp; - } + h264_bitstream_parser_.ParseBitstream(payload, payload_size); + int qp; + if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { + current_acc_qp_ += qp; + image->qp_ = qp; } // For H.264 search for start codes. int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {}; @@ -1252,6 +1199,11 @@ void MediaCodecVideoEncoder::LogStatistics(bool force_log) { } } +webrtc::VideoEncoder::ScalingSettings +MediaCodecVideoEncoder::GetScalingSettings() const { + return VideoEncoder::ScalingSettings(scale_); +} + int32_t MediaCodecVideoEncoder::NextNaluPosition( uint8_t *buffer, size_t buffer_size) { if (buffer_size < H264_SC_LENGTH) { @@ -1284,22 +1236,6 @@ int32_t MediaCodecVideoEncoder::NextNaluPosition( return -1; } -void MediaCodecVideoEncoder::OnDroppedFrame() { - // Methods running on the codec thread should call OnDroppedFrameOnCodecThread - // directly. - RTC_DCHECK(!codec_thread_checker_.CalledOnValidThread()); - codec_thread_->Invoke( - RTC_FROM_HERE, - Bind(&MediaCodecVideoEncoder::OnDroppedFrameOnCodecThread, this)); -} - -void MediaCodecVideoEncoder::OnDroppedFrameOnCodecThread() { - RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); - // Report dropped frame to quality_scaler_. - if (scale_) - quality_scaler_.ReportDroppedFrame(); -} - const char* MediaCodecVideoEncoder::ImplementationName() const { return "MediaCodec"; } diff --git a/webrtc/api/peerconnection_unittest.cc b/webrtc/api/peerconnection_unittest.cc index 18718c31a6..05dddfa085 100644 --- a/webrtc/api/peerconnection_unittest.cc +++ b/webrtc/api/peerconnection_unittest.cc @@ -1465,7 +1465,13 @@ class P2PTestConductor : public testing::Test { FakeConstraints setup_constraints; setup_constraints.AddMandatory(MediaConstraintsInterface::kEnableDtlsSrtp, true); - ASSERT_TRUE(CreateTestClients(&setup_constraints, &setup_constraints)); + // Disable resolution adaptation, we don't want it interfering with the + // test results. + webrtc::PeerConnectionInterface::RTCConfiguration rtc_config; + rtc_config.set_cpu_adaptation(false); + + ASSERT_TRUE(CreateTestClients(&setup_constraints, nullptr, &rtc_config, + &setup_constraints, nullptr, &rtc_config)); LocalP2PTest(); VerifyRenderedSize(640, 480); } @@ -1474,6 +1480,10 @@ class P2PTestConductor : public testing::Test { FakeConstraints setup_constraints; setup_constraints.AddMandatory(MediaConstraintsInterface::kEnableDtlsSrtp, true); + // Disable resolution adaptation, we don't want it interfering with the + // test results. + webrtc::PeerConnectionInterface::RTCConfiguration rtc_config; + rtc_config.set_cpu_adaptation(false); std::unique_ptr cert_generator( rtc::SSLStreamAdapter::HaveDtlsSrtp() ? @@ -1482,7 +1492,7 @@ class P2PTestConductor : public testing::Test { // Make sure the new client is using a different certificate. return PeerConnectionTestClient::CreateClientWithDtlsIdentityStore( - "New Peer: ", &setup_constraints, nullptr, nullptr, + "New Peer: ", &setup_constraints, nullptr, &rtc_config, std::move(cert_generator), prefer_constraint_apis_, network_thread_.get(), worker_thread_.get()); } diff --git a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc index 0de15b3d8f..a4d865e80d 100644 --- a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc +++ b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc @@ -159,16 +159,15 @@ int32_t VideoEncoderSoftwareFallbackWrapper::SetRateAllocation( return ret; } -void VideoEncoderSoftwareFallbackWrapper::OnDroppedFrame() { - if (fallback_encoder_) - return fallback_encoder_->OnDroppedFrame(); - return encoder_->OnDroppedFrame(); -} - bool VideoEncoderSoftwareFallbackWrapper::SupportsNativeHandle() const { if (fallback_encoder_) return fallback_encoder_->SupportsNativeHandle(); return encoder_->SupportsNativeHandle(); } +VideoEncoder::ScalingSettings +VideoEncoderSoftwareFallbackWrapper::GetScalingSettings() const { + return encoder_->GetScalingSettings(); +} + } // namespace webrtc diff --git a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h index 05e815df46..3d53090a27 100644 --- a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h +++ b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h @@ -42,8 +42,8 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, uint32_t framerate) override; - void OnDroppedFrame() override; bool SupportsNativeHandle() const override; + ScalingSettings GetScalingSettings() const override; private: bool InitFallbackEncoder(); diff --git a/webrtc/media/engine/videoencodersoftwarefallbackwrapper_unittest.cc b/webrtc/media/engine/videoencodersoftwarefallbackwrapper_unittest.cc index ca06e9256d..096552bbfa 100644 --- a/webrtc/media/engine/videoencodersoftwarefallbackwrapper_unittest.cc +++ b/webrtc/media/engine/videoencodersoftwarefallbackwrapper_unittest.cc @@ -74,8 +74,6 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { return WEBRTC_VIDEO_CODEC_OK; } - void OnDroppedFrame() override { ++on_dropped_frame_count_; } - bool SupportsNativeHandle() const override { ++supports_native_handle_count_; return false; @@ -93,7 +91,6 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { int release_count_ = 0; int set_channel_parameters_count_ = 0; int set_rates_count_ = 0; - int on_dropped_frame_count_ = 0; mutable int supports_native_handle_count_ = 0; }; @@ -270,20 +267,6 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); } -TEST_F(VideoEncoderSoftwareFallbackWrapperTest, - OnDroppedFrameForwardedWithoutFallback) { - fallback_wrapper_.OnDroppedFrame(); - EXPECT_EQ(1, fake_encoder_.on_dropped_frame_count_); -} - -TEST_F(VideoEncoderSoftwareFallbackWrapperTest, - OnDroppedFrameNotForwardedDuringFallback) { - UtilizeFallbackEncoder(); - fallback_wrapper_.OnDroppedFrame(); - EXPECT_EQ(0, fake_encoder_.on_dropped_frame_count_); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); -} - TEST_F(VideoEncoderSoftwareFallbackWrapperTest, SupportsNativeHandleForwardedWithoutFallback) { fallback_wrapper_.SupportsNativeHandle(); diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc index 5332a3c2e0..d7df12209d 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -235,9 +235,6 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings, return WEBRTC_VIDEO_CODEC_ERROR; } // TODO(pbos): Base init params on these values before submitting. - quality_scaler_.Init(codec_settings->codecType, codec_settings->startBitrate, - codec_settings->width, codec_settings->height, - codec_settings->maxFramerate); int video_format = EVideoFormatType::videoFormatI420; openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format); @@ -279,7 +276,6 @@ int32_t H264EncoderImpl::SetRateAllocation( target_bps_ = bitrate_allocation.get_sum_bps(); max_frame_rate_ = static_cast(framerate); - quality_scaler_.ReportFramerate(framerate); SBitrateInfo target_bitrate; memset(&target_bitrate, 0, sizeof(SBitrateInfo)); @@ -309,20 +305,6 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame, return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - quality_scaler_.OnEncodeFrame(input_frame.width(), input_frame.height()); - rtc::scoped_refptr frame_buffer = - quality_scaler_.GetScaledBuffer(input_frame.video_frame_buffer()); - if (frame_buffer->width() != width_ || frame_buffer->height() != height_) { - LOG(LS_INFO) << "Encoder reinitialized from " << width_ << "x" << height_ - << " to " << frame_buffer->width() << "x" - << frame_buffer->height(); - width_ = frame_buffer->width(); - height_ = frame_buffer->height(); - SEncParamExt encoder_params = CreateEncoderParams(); - openh264_encoder_->SetOption(ENCODER_OPTION_SVC_ENCODE_PARAM_EXT, - &encoder_params); - } - bool force_key_frame = false; if (frame_types != nullptr) { // We only support a single stream. @@ -340,7 +322,8 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame, // (If every frame is a key frame we get lag/delays.) openh264_encoder_->ForceIntraFrame(true); } - + rtc::scoped_refptr frame_buffer = + input_frame.video_frame_buffer(); // EncodeFrame input. SSourcePicture picture; memset(&picture, 0, sizeof(SSourcePicture)); @@ -384,22 +367,16 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame, // Encoder can skip frames to save bandwidth in which case // |encoded_image_._length| == 0. if (encoded_image_._length > 0) { - // Parse and report QP. - h264_bitstream_parser_.ParseBitstream(encoded_image_._buffer, - encoded_image_._length); - int qp = -1; - if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { - quality_scaler_.ReportQP(qp); - encoded_image_.qp_ = qp; - } - // Deliver encoded image. CodecSpecificInfo codec_specific; codec_specific.codecType = kVideoCodecH264; encoded_image_callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_header); - } else { - quality_scaler_.ReportDroppedFrame(); + + // Parse and report QP. + h264_bitstream_parser_.ParseBitstream(encoded_image_._buffer, + encoded_image_._length); + h264_bitstream_parser_.GetLastSliceQp(&encoded_image_.qp_); } return WEBRTC_VIDEO_CODEC_OK; } @@ -500,8 +477,8 @@ int32_t H264EncoderImpl::SetPeriodicKeyFrames(bool enable) { return WEBRTC_VIDEO_CODEC_OK; } -void H264EncoderImpl::OnDroppedFrame() { - quality_scaler_.ReportDroppedFrame(); +VideoEncoder::ScalingSettings H264EncoderImpl::GetScalingSettings() const { + return VideoEncoder::ScalingSettings(true); } } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h index ca28eb3250..aab16acf36 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h @@ -55,17 +55,17 @@ class H264EncoderImpl : public H264Encoder { const char* ImplementationName() const override; + VideoEncoder::ScalingSettings GetScalingSettings() const override; + // Unsupported / Do nothing. int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; int32_t SetPeriodicKeyFrames(bool enable) override; - void OnDroppedFrame() override; private: bool IsInitialized() const; SEncParamExt CreateEncoderParams() const; webrtc::H264BitstreamParser h264_bitstream_parser_; - QualityScaler quality_scaler_; // Reports statistics with histograms. void ReportInit(); void ReportError(); diff --git a/webrtc/modules/video_coding/codecs/i420/include/i420.h b/webrtc/modules/video_coding/codecs/i420/include/i420.h index 38dcafd0c3..0a8051f9c5 100644 --- a/webrtc/modules/video_coding/codecs/i420/include/i420.h +++ b/webrtc/modules/video_coding/codecs/i420/include/i420.h @@ -70,8 +70,6 @@ class I420Encoder : public VideoEncoder { return WEBRTC_VIDEO_CODEC_OK; } - void OnDroppedFrame() override {} - private: static uint8_t* InsertHeader(uint8_t* buffer, uint16_t width, diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index fbb464d18a..9b06e69975 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -977,38 +977,6 @@ TEST_F(VideoProcessorIntegrationTest, rc_metrics); } -// Run with no packet loss, at low bitrate. During this time we should've -// resized once. Expect 2 key frames generated (first and one for resize). -// Too slow to finish before timeout on iOS. See webrtc:4755. -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_ProcessNoLossSpatialResizeFrameDropVP8 \ - DISABLED_ProcessNoLossSpatialResizeFrameDropVP8 -#else -#define MAYBE_ProcessNoLossSpatialResizeFrameDropVP8 \ - ProcessNoLossSpatialResizeFrameDropVP8 -#endif -TEST_F(VideoProcessorIntegrationTest, - MAYBE_ProcessNoLossSpatialResizeFrameDropVP8) { - config_.networking_config.packet_loss_probability = 0; - // Bitrate and frame rate profile. - RateProfile rate_profile; - SetRateProfilePars(&rate_profile, 0, 50, 30, 0); - rate_profile.frame_index_rate_update[1] = kNbrFramesLong + 1; - rate_profile.num_frames = kNbrFramesLong; - // Codec/network settings. - CodecConfigPars process_settings; - SetCodecParameters(&process_settings, kVideoCodecVP8, 0.0f, -1, 1, false, - true, true, true); - // Metrics for expected quality. - QualityMetrics quality_metrics; - SetQualityMetrics(&quality_metrics, 25.0, 15.0, 0.70, 0.40); - // Metrics for rate control. - RateControlMetrics rc_metrics[1]; - SetRateControlMetrics(rc_metrics, 0, 160, 80, 120, 20, 70, 1, 2); - ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, - rc_metrics); -} - // VP8: Run with no packet loss, with 3 temporal layers, with a rate update in // the middle of the sequence. The max values for the frame size mismatch and // encoding rate mismatch are applied to each layer. diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc index 32472750f5..6026cb31d2 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc @@ -451,10 +451,6 @@ bool SimulcastEncoderAdapter::Initialized() const { return !streaminfos_.empty(); } -void SimulcastEncoderAdapter::OnDroppedFrame() { - streaminfos_[0].encoder->OnDroppedFrame(); -} - bool SimulcastEncoderAdapter::SupportsNativeHandle() const { // We should not be calling this method before streaminfos_ are configured. RTC_DCHECK(!streaminfos_.empty()); @@ -465,6 +461,14 @@ bool SimulcastEncoderAdapter::SupportsNativeHandle() const { return true; } +VideoEncoder::ScalingSettings SimulcastEncoderAdapter::GetScalingSettings() + const { + // Turn off quality scaling for simulcast. + if (NumberOfStreams(codec_) != 1) + return VideoEncoder::ScalingSettings(false); + return streaminfos_[0].encoder->GetScalingSettings(); +} + const char* SimulcastEncoderAdapter::ImplementationName() const { return implementation_name_.c_str(); } diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h index b76e7f4ce9..2be8779d62 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h @@ -60,7 +60,7 @@ class SimulcastEncoderAdapter : public VP8Encoder { const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation); - void OnDroppedFrame() override; + VideoEncoder::ScalingSettings GetScalingSettings() const override; bool SupportsNativeHandle() const override; const char* ImplementationName() const override; diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 148140eccf..9fd5f6587a 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -123,8 +123,7 @@ VP8EncoderImpl::VP8EncoderImpl() token_partitions_(VP8_ONE_TOKENPARTITION), down_scale_requested_(false), down_scale_bitrate_(0), - key_frame_request_(kMaxSimulcastStreams, false), - quality_scaler_enabled_(false) { + key_frame_request_(kMaxSimulcastStreams, false) { uint32_t seed = rtc::Time32(); srand(seed); @@ -253,15 +252,9 @@ int VP8EncoderImpl::SetRateAllocation(const BitrateAllocation& bitrate, return WEBRTC_VIDEO_CODEC_ERROR; } } - quality_scaler_.ReportFramerate(new_framerate); return WEBRTC_VIDEO_CODEC_OK; } -void VP8EncoderImpl::OnDroppedFrame() { - if (quality_scaler_enabled_) - quality_scaler_.ReportDroppedFrame(); -} - const char* VP8EncoderImpl::ImplementationName() const { return "libvpx"; } @@ -530,15 +523,6 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, } rps_.Init(); - quality_scaler_.Init(codec_.codecType, codec_.startBitrate, codec_.width, - codec_.height, codec_.maxFramerate); - - // Only apply scaling to improve for single-layer streams. The scaling metrics - // use frame drops as a signal and is only applicable when we drop frames. - quality_scaler_enabled_ = encoders_.size() == 1 && - configurations_[0].rc_dropframe_thresh > 0 && - codec_.VP8()->automaticResizeOn; - return InitAndSetControlSettings(); } @@ -671,6 +655,9 @@ uint32_t VP8EncoderImpl::MaxIntraTarget(uint32_t optimalBuffersize) { int VP8EncoderImpl::Encode(const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) { + RTC_DCHECK_EQ(frame.width(), codec_.width); + RTC_DCHECK_EQ(frame.height(), codec_.height); + if (!inited_) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; if (frame.IsZeroSize()) @@ -679,20 +666,6 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, return WEBRTC_VIDEO_CODEC_UNINITIALIZED; rtc::scoped_refptr input_image = frame.video_frame_buffer(); - - if (quality_scaler_enabled_) { - quality_scaler_.OnEncodeFrame(frame.width(), frame.height()); - input_image = quality_scaler_.GetScaledBuffer(input_image); - - if (input_image->width() != codec_.width || - input_image->height() != codec_.height) { - int ret = - UpdateCodecFrameSize(input_image->width(), input_image->height()); - if (ret < 0) - return ret; - } - } - // Since we are extracting raw pointers from |input_image| to // |raw_images_[0]|, the resolution of these frames must match. Note that // |input_image| might be scaled from |frame|. In that case, the resolution of @@ -989,9 +962,6 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image, codec_.simulcastStream[stream_idx].height; encoded_images_[encoder_idx]._encodedWidth = codec_.simulcastStream[stream_idx].width; - encoded_images_[encoder_idx] - .adapt_reason_.quality_resolution_downscales = - quality_scaler_enabled_ ? quality_scaler_.downscale_shift() : -1; // Report once per frame (lowest stream always sent). encoded_images_[encoder_idx].adapt_reason_.bw_resolutions_disabled = (stream_idx == 0) ? bw_resolutions_disabled : -1; @@ -1006,18 +976,16 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image, } } } - if (encoders_.size() == 1 && send_stream_[0]) { - if (encoded_images_[0]._length > 0) { - int qp_128; - vpx_codec_control(&encoders_[0], VP8E_GET_LAST_QUANTIZER, &qp_128); - quality_scaler_.ReportQP(qp_128); - } else { - quality_scaler_.ReportDroppedFrame(); - } - } return result; } +VideoEncoder::ScalingSettings VP8EncoderImpl::GetScalingSettings() const { + const bool enable_scaling = encoders_.size() == 1 && + configurations_[0].rc_dropframe_thresh > 0 && + codec_.VP8().automaticResizeOn; + return VideoEncoder::ScalingSettings(enable_scaling); +} + int VP8EncoderImpl::SetChannelParameters(uint32_t packetLoss, int64_t rtt) { rps_.SetRtt(rtt); return WEBRTC_VIDEO_CODEC_OK; diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h index 3b3a78a7e7..10977b1d8a 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h @@ -57,7 +57,7 @@ class VP8EncoderImpl : public VP8Encoder { int SetRateAllocation(const BitrateAllocation& bitrate, uint32_t new_framerate) override; - void OnDroppedFrame() override; + ScalingSettings GetScalingSettings() const override; const char* ImplementationName() const override; @@ -116,8 +116,6 @@ class VP8EncoderImpl : public VP8Encoder { std::vector encoders_; std::vector configurations_; std::vector downsampling_factors_; - QualityScaler quality_scaler_; - bool quality_scaler_enabled_; }; // end of VP8EncoderImpl class class VP8DecoderImpl : public VP8Decoder { diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h index 4eb31a4256..7078c50a70 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h @@ -49,8 +49,6 @@ class VP9EncoderImpl : public VP9Encoder { int SetRateAllocation(const BitrateAllocation& bitrate_allocation, uint32_t frame_rate) override; - void OnDroppedFrame() override {} - const char* ImplementationName() const override; struct LayerFrameRefSettings { diff --git a/webrtc/modules/video_coding/generic_encoder.cc b/webrtc/modules/video_coding/generic_encoder.cc index 80e48d1911..75bdc7c4f3 100644 --- a/webrtc/modules/video_coding/generic_encoder.cc +++ b/webrtc/modules/video_coding/generic_encoder.cc @@ -131,11 +131,6 @@ bool VCMGenericEncoder::InternalSource() const { return internal_source_; } -void VCMGenericEncoder::OnDroppedFrame() { - RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); - encoder_->OnDroppedFrame(); -} - bool VCMGenericEncoder::SupportsNativeHandle() const { RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); return encoder_->SupportsNativeHandle(); diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index 0f63edca7d..ba1d9789c7 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -13,8 +13,11 @@ #include #include +#include #include "webrtc/base/checks.h" +#include "webrtc/base/logging.h" +#include "webrtc/base/task_queue.h" // TODO(kthelgason): Some versions of Android have issues with log2. // See https://code.google.com/p/android/issues/detail?id=212634 for details @@ -26,21 +29,9 @@ namespace webrtc { namespace { // Threshold constant used until first downscale (to permit fast rampup). -static const int kMeasureSecondsFastUpscale = 2; -static const int kMeasureSecondsUpscale = 5; -static const int kMeasureSecondsDownscale = 5; +static const int kMeasureMs = 2000; +static const float kSamplePeriodScaleFactor = 2.5; static const int kFramedropPercentThreshold = 60; -// Min width/height to downscale to, set to not go below QVGA, but with some -// margin to permit "almost-QVGA" resolutions, such as QCIF. -static const int kMinDownscaleDimension = 140; -// Initial resolutions corresponding to a bitrate. Aa bit above their actual -// values to permit near-VGA and near-QVGA resolutions to use the same -// mechanism. -static const int kVgaBitrateThresholdKbps = 500; -static const int kVgaNumPixels = 700 * 500; // 640x480 -static const int kQvgaBitrateThresholdKbps = 250; -static const int kQvgaNumPixels = 400 * 300; // 320x240 - // QP scaling threshold defaults: static const int kLowH264QpThreshold = 24; static const int kHighH264QpThreshold = 37; @@ -48,20 +39,13 @@ static const int kHighH264QpThreshold = 37; // bitstream range of [0, 127] and not the user-level range of [0,63]. static const int kLowVp8QpThreshold = 29; static const int kHighVp8QpThreshold = 95; -} // namespace +const ScalingObserverInterface::ScaleReason scale_reason_ = + ScalingObserverInterface::ScaleReason::kQuality; -// Default values. Should immediately get set to something more sensible. -QualityScaler::QualityScaler() - : average_qp_(kMeasureSecondsUpscale * 30), - framedrop_percent_(kMeasureSecondsUpscale * 30), - low_qp_threshold_(-1) {} - -void QualityScaler::Init(VideoCodecType codec_type, - int initial_bitrate_kbps, - int width, - int height, - int fps) { - int low = -1, high = -1; +static VideoEncoder::QpThresholds CodecTypeToDefaultThresholds( + VideoCodecType codec_type) { + int low = -1; + int high = -1; switch (codec_type) { case kVideoCodecH264: low = kLowH264QpThreshold; @@ -74,138 +58,132 @@ void QualityScaler::Init(VideoCodecType codec_type, default: RTC_NOTREACHED() << "Invalid codec type for QualityScaler."; } - Init(low, high, initial_bitrate_kbps, width, height, fps); + return VideoEncoder::QpThresholds(low, high); } +} // namespace -void QualityScaler::Init(int low_qp_threshold, - int high_qp_threshold, - int initial_bitrate_kbps, - int width, - int height, - int fps) { - ClearSamples(); - low_qp_threshold_ = low_qp_threshold; - high_qp_threshold_ = high_qp_threshold; - downscale_shift_ = 0; - fast_rampup_ = true; - - const int init_width = width; - const int init_height = height; - if (initial_bitrate_kbps > 0) { - int init_num_pixels = width * height; - if (initial_bitrate_kbps < kVgaBitrateThresholdKbps) - init_num_pixels = kVgaNumPixels; - if (initial_bitrate_kbps < kQvgaBitrateThresholdKbps) - init_num_pixels = kQvgaNumPixels; - while (width * height > init_num_pixels) { - ++downscale_shift_; - width /= 2; - height /= 2; - } +class QualityScaler::CheckQPTask : public rtc::QueuedTask { + public: + explicit CheckQPTask(QualityScaler* scaler) : scaler_(scaler) { + LOG(LS_INFO) << "Created CheckQPTask. Scheduling on queue..."; + rtc::TaskQueue::Current()->PostDelayedTask( + std::unique_ptr(this), scaler_->GetSamplingPeriodMs()); } - UpdateTargetResolution(init_width, init_height); - ReportFramerate(fps); + void Stop() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + LOG(LS_INFO) << "Stopping QP Check task."; + stop_ = true; + } + + private: + bool Run() override { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + if (stop_) + return true; // TaskQueue will free this task. + scaler_->CheckQP(); + rtc::TaskQueue::Current()->PostDelayedTask( + std::unique_ptr(this), scaler_->GetSamplingPeriodMs()); + return false; // Retain the task in order to reuse it. + } + + QualityScaler* const scaler_; + bool stop_ = false; + rtc::SequencedTaskChecker task_checker_; +}; + +QualityScaler::QualityScaler(ScalingObserverInterface* observer, + VideoCodecType codec_type) + : QualityScaler(observer, CodecTypeToDefaultThresholds(codec_type)) {} + +QualityScaler::QualityScaler(ScalingObserverInterface* observer, + VideoEncoder::QpThresholds thresholds) + : QualityScaler(observer, thresholds, kMeasureMs) {} + +// Protected ctor, should not be called directly. +QualityScaler::QualityScaler(ScalingObserverInterface* observer, + VideoEncoder::QpThresholds thresholds, + int64_t sampling_period) + : check_qp_task_(nullptr), + observer_(observer), + sampling_period_ms_(sampling_period), + fast_rampup_(true), + // Arbitrarily choose size based on 30 fps for 5 seconds. + average_qp_(5 * 30), + framedrop_percent_(5 * 30), + thresholds_(thresholds) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + RTC_DCHECK(observer_ != nullptr); + check_qp_task_ = new CheckQPTask(this); } -// Report framerate(fps) to estimate # of samples. -void QualityScaler::ReportFramerate(int framerate) { - // Use a faster window for upscaling initially. - // This enables faster initial rampups without risking strong up-down - // behavior later. - num_samples_upscale_ = framerate * (fast_rampup_ ? kMeasureSecondsFastUpscale - : kMeasureSecondsUpscale); - num_samples_downscale_ = framerate * kMeasureSecondsDownscale; +QualityScaler::~QualityScaler() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + check_qp_task_->Stop(); +} + +int64_t QualityScaler::GetSamplingPeriodMs() const { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + return fast_rampup_ ? sampling_period_ms_ + : (sampling_period_ms_ * kSamplePeriodScaleFactor); +} + +void QualityScaler::ReportDroppedFrame() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + framedrop_percent_.AddSample(100); } void QualityScaler::ReportQP(int qp) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); framedrop_percent_.AddSample(0); average_qp_.AddSample(qp); } -void QualityScaler::ReportDroppedFrame() { - framedrop_percent_.AddSample(100); -} - -void QualityScaler::OnEncodeFrame(int width, int height) { +void QualityScaler::CheckQP() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); // Should be set through InitEncode -> Should be set by now. - RTC_DCHECK_GE(low_qp_threshold_, 0); - if (target_res_.width != width || target_res_.height != height) { - UpdateTargetResolution(width, height); - } - + RTC_DCHECK_GE(thresholds_.low, 0); + LOG(LS_INFO) << "Checking if average QP exceeds threshold"; // Check if we should scale down due to high frame drop. - const auto drop_rate = framedrop_percent_.GetAverage(num_samples_downscale_); + const rtc::Optional drop_rate = framedrop_percent_.GetAverage(); if (drop_rate && *drop_rate >= kFramedropPercentThreshold) { - ScaleDown(); + ReportQPHigh(); return; } // Check if we should scale up or down based on QP. - const auto avg_qp_down = average_qp_.GetAverage(num_samples_downscale_); - if (avg_qp_down && *avg_qp_down > high_qp_threshold_) { - ScaleDown(); + const rtc::Optional avg_qp = average_qp_.GetAverage(); + if (avg_qp && *avg_qp > thresholds_.high) { + ReportQPHigh(); return; } - const auto avg_qp_up = average_qp_.GetAverage(num_samples_upscale_); - if (avg_qp_up && *avg_qp_up <= low_qp_threshold_) { + if (avg_qp && *avg_qp <= thresholds_.low) { // QP has been low. We want to try a higher resolution. - ScaleUp(); + ReportQPLow(); return; } } -void QualityScaler::ScaleUp() { - downscale_shift_ = std::max(0, downscale_shift_ - 1); +void QualityScaler::ReportQPLow() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + LOG(LS_INFO) << "QP has been low, asking for higher resolution."; ClearSamples(); + observer_->ScaleUp(scale_reason_); } -void QualityScaler::ScaleDown() { - downscale_shift_ = std::min(maximum_shift_, downscale_shift_ + 1); +void QualityScaler::ReportQPHigh() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); + LOG(LS_INFO) << "QP has been high , asking for lower resolution."; ClearSamples(); + observer_->ScaleDown(scale_reason_); // If we've scaled down, wait longer before scaling up again. if (fast_rampup_) { fast_rampup_ = false; - num_samples_upscale_ = (num_samples_upscale_ / kMeasureSecondsFastUpscale) * - kMeasureSecondsUpscale; } } -QualityScaler::Resolution QualityScaler::GetScaledResolution() const { - const int frame_width = target_res_.width >> downscale_shift_; - const int frame_height = target_res_.height >> downscale_shift_; - return Resolution{frame_width, frame_height}; -} - -rtc::scoped_refptr QualityScaler::GetScaledBuffer( - const rtc::scoped_refptr& frame) { - Resolution res = GetScaledResolution(); - const int src_width = frame->width(); - const int src_height = frame->height(); - - if (res.width == src_width && res.height == src_height) - return frame; - rtc::scoped_refptr scaled_buffer = - pool_.CreateBuffer(res.width, res.height); - - scaled_buffer->ScaleFrom(*frame); - - return scaled_buffer; -} - -void QualityScaler::UpdateTargetResolution(int width, int height) { - if (width < kMinDownscaleDimension || height < kMinDownscaleDimension) { - maximum_shift_ = 0; - } else { - maximum_shift_ = static_cast( - log2(std::min(width, height) / kMinDownscaleDimension)); - } - target_res_ = Resolution{width, height}; -} - void QualityScaler::ClearSamples() { + RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_); framedrop_percent_.Reset(); average_qp_.Reset(); } - - } // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index c0f94409ce..5734e6647e 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -11,61 +11,73 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ +#include + #include "webrtc/common_types.h" -#include "webrtc/common_video/include/i420_buffer_pool.h" +#include "webrtc/video_encoder.h" +#include "webrtc/base/optional.h" +#include "webrtc/base/sequenced_task_checker.h" #include "webrtc/modules/video_coding/utility/moving_average.h" namespace webrtc { -class QualityScaler { + +// An interface for a class that receives scale up/down requests. +class ScalingObserverInterface { public: - struct Resolution { - int width; - int height; - }; + enum ScaleReason : size_t { kQuality = 0, kCpu = 1 }; + static const size_t kScaleReasonSize = 2; + // Called to signal that we can handle larger frames. + virtual void ScaleUp(ScaleReason reason) = 0; + // Called to signal that encoder to scale down. + virtual void ScaleDown(ScaleReason reason) = 0; - QualityScaler(); - void Init(VideoCodecType codec_type, - int initial_bitrate_kbps, - int width, - int height, - int fps); - void Init(int low_qp_threshold, - int high_qp_threshold, - int initial_bitrate_kbps, - int width, - int height, - int fps); - void ReportFramerate(int framerate); - void ReportQP(int qp); - void ReportDroppedFrame(); - void OnEncodeFrame(int width, int height); - Resolution GetScaledResolution() const; - rtc::scoped_refptr GetScaledBuffer( - const rtc::scoped_refptr& frame); - int downscale_shift() const { return downscale_shift_; } - - private: - void ClearSamples(); - void ScaleUp(); - void ScaleDown(); - void UpdateTargetResolution(int width, int height); - - I420BufferPool pool_; - - size_t num_samples_downscale_; - size_t num_samples_upscale_; - bool fast_rampup_; - MovingAverage average_qp_; - MovingAverage framedrop_percent_; - - int low_qp_threshold_; - int high_qp_threshold_; - Resolution target_res_; - - int downscale_shift_; - int maximum_shift_; + protected: + virtual ~ScalingObserverInterface() {} }; +// QualityScaler runs asynchronously and monitors QP values of encoded frames. +// It holds a reference to a ScalingObserverInterface implementation to signal +// an intent to scale up or down. +class QualityScaler { + public: + // Construct a QualityScaler with a given |observer|. + // This starts the quality scaler periodically checking what the average QP + // has been recently. + QualityScaler(ScalingObserverInterface* observer, VideoCodecType codec_type); + // If specific thresholds are desired these can be supplied as |thresholds|. + QualityScaler(ScalingObserverInterface* observer, + VideoEncoder::QpThresholds thresholds); + virtual ~QualityScaler(); + // Should be called each time the encoder drops a frame + void ReportDroppedFrame(); + // Inform the QualityScaler of the last seen QP. + void ReportQP(int qp); + + // The following members declared protected for testing purposes + protected: + QualityScaler(ScalingObserverInterface* observer, + VideoEncoder::QpThresholds thresholds, + int64_t sampling_period); + + private: + class CheckQPTask; + void CheckQP(); + void ClearSamples(); + void ReportQPLow(); + void ReportQPHigh(); + int64_t GetSamplingPeriodMs() const; + + CheckQPTask* check_qp_task_ GUARDED_BY(&task_checker_); + ScalingObserverInterface* const observer_ GUARDED_BY(&task_checker_); + rtc::SequencedTaskChecker task_checker_; + + const int64_t sampling_period_ms_; + bool fast_rampup_ GUARDED_BY(&task_checker_); + MovingAverage average_qp_ GUARDED_BY(&task_checker_); + MovingAverage framedrop_percent_ GUARDED_BY(&task_checker_); + + VideoEncoder::QpThresholds thresholds_ GUARDED_BY(&task_checker_); +}; } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index 24d5c48d38..babe14e002 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -10,28 +10,49 @@ #include "webrtc/modules/video_coding/utility/quality_scaler.h" +#include + +#include "webrtc/base/event.h" +#include "webrtc/base/task_queue.h" +#include "webrtc/test/gmock.h" #include "webrtc/test/gtest.h" namespace webrtc { namespace { -static const int kNumSeconds = 10; -static const int kWidth = 1920; -static const int kHeight = 1080; static const int kFramerate = 30; static const int kLowQp = 15; -static const int kNormalQp = 30; static const int kLowQpThreshold = 18; static const int kHighQp = 40; -static const int kDisabledBadQpThreshold = 64; -static const int kLowInitialBitrateKbps = 300; -// These values need to be in sync with corresponding constants -// in quality_scaler.cc -static const int kMeasureSecondsFastUpscale = 2; -static const int kMeasureSecondsUpscale = 5; -static const int kMeasureSecondsDownscale = 5; -static const int kMinDownscaleDimension = 140; +static const size_t kDefaultTimeoutMs = 1000; } // namespace +class MockScaleObserver : public ScalingObserverInterface { + public: + MockScaleObserver() : event(false, false) {} + virtual ~MockScaleObserver() {} + + void ScaleUp(ScaleReason r) override { + scaled_up++; + event.Set(); + } + void ScaleDown(ScaleReason r) override { + scaled_down++; + event.Set(); + } + + rtc::Event event; + int scaled_up = 0; + int scaled_down = 0; +}; + +// Pass a lower sampling period to speed up the tests. +class QualityScalerUnderTest : public QualityScaler { + public: + explicit QualityScalerUnderTest(ScalingObserverInterface* observer, + VideoEncoder::QpThresholds thresholds) + : QualityScaler(observer, thresholds, 5) {} +}; + class QualityScalerTest : public ::testing::Test { protected: enum ScaleDirection { @@ -41,346 +62,116 @@ class QualityScalerTest : public ::testing::Test { kScaleUp }; - QualityScalerTest() { - input_frame_ = I420Buffer::Create(kWidth, kHeight); - qs_.Init(kLowQpThreshold, kHighQp, 0, kWidth, kHeight, kFramerate); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); + QualityScalerTest() + : q_(new rtc::TaskQueue("QualityScalerTestQueue")), + observer_(new MockScaleObserver()) { + rtc::Event event(false, false); + q_->PostTask([this, &event] { + qs_ = std::unique_ptr(new QualityScalerUnderTest( + observer_.get(), + VideoEncoder::QpThresholds(kLowQpThreshold, kHighQp))); + event.Set(); + }); + EXPECT_TRUE(event.Wait(kDefaultTimeoutMs)); } - bool TriggerScale(ScaleDirection scale_direction) { - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - int initial_width = qs_.GetScaledResolution().width; - for (int i = 0; i < kFramerate * kNumSeconds; ++i) { + ~QualityScalerTest() { + rtc::Event event(false, false); + q_->PostTask([this, &event] { + qs_.reset(nullptr); + event.Set(); + }); + EXPECT_TRUE(event.Wait(kDefaultTimeoutMs)); + } + + void TriggerScale(ScaleDirection scale_direction) { + for (int i = 0; i < kFramerate * 5; ++i) { switch (scale_direction) { case kScaleUp: - qs_.ReportQP(kLowQp); + qs_->ReportQP(kLowQp); break; case kScaleDown: - qs_.ReportDroppedFrame(); + qs_->ReportDroppedFrame(); break; case kKeepScaleAtHighQp: - qs_.ReportQP(kHighQp); + qs_->ReportQP(kHighQp); break; case kScaleDownAboveHighQp: - qs_.ReportQP(kHighQp + 1); + qs_->ReportQP(kHighQp + 1); break; } - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - if (qs_.GetScaledResolution().width != initial_width) - return true; } - - return false; } - void ExpectOriginalFrame() { - EXPECT_EQ(input_frame_, qs_.GetScaledBuffer(input_frame_)) - << "Using scaled frame instead of original input."; - } - - void ExpectScaleUsingReportedResolution() { - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - QualityScaler::Resolution res = qs_.GetScaledResolution(); - rtc::scoped_refptr scaled_frame = - qs_.GetScaledBuffer(input_frame_); - EXPECT_EQ(res.width, scaled_frame->width()); - EXPECT_EQ(res.height, scaled_frame->height()); - } - - void ContinuouslyDownscalesByHalfDimensionsAndBackUp(); - - void DoesNotDownscaleFrameDimensions(int width, int height); - - void DownscaleEndsAt(int input_width, - int input_height, - int end_width, - int end_height); - - QualityScaler qs_; - rtc::scoped_refptr input_frame_; + std::unique_ptr q_; + std::unique_ptr qs_; + std::unique_ptr observer_; }; -TEST_F(QualityScalerTest, UsesOriginalFrameInitially) { - ExpectOriginalFrame(); -} - -TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) { - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - QualityScaler::Resolution res = qs_.GetScaledResolution(); - EXPECT_EQ(input_frame_->width(), res.width); - EXPECT_EQ(input_frame_->height(), res.height); -} - TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) { - EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds - << " seconds."; - QualityScaler::Resolution res = qs_.GetScaledResolution(); - EXPECT_LT(res.width, input_frame_->width()); - EXPECT_LT(res.height, input_frame_->height()); + q_->PostTask([this] { TriggerScale(kScaleDown); }); + EXPECT_TRUE(observer_->event.Wait(50)); + EXPECT_EQ(1, observer_->scaled_down); } TEST_F(QualityScalerTest, KeepsScaleAtHighQp) { - EXPECT_FALSE(TriggerScale(kKeepScaleAtHighQp)) - << "Downscale at high threshold which should keep scale."; - QualityScaler::Resolution res = qs_.GetScaledResolution(); - EXPECT_EQ(res.width, input_frame_->width()); - EXPECT_EQ(res.height, input_frame_->height()); + q_->PostTask([this] { TriggerScale(kKeepScaleAtHighQp); }); + EXPECT_FALSE(observer_->event.Wait(50)); + EXPECT_EQ(0, observer_->scaled_down); + EXPECT_EQ(0, observer_->scaled_up); } TEST_F(QualityScalerTest, DownscalesAboveHighQp) { - EXPECT_TRUE(TriggerScale(kScaleDownAboveHighQp)) - << "No downscale within " << kNumSeconds << " seconds."; - QualityScaler::Resolution res = qs_.GetScaledResolution(); - EXPECT_LT(res.width, input_frame_->width()); - EXPECT_LT(res.height, input_frame_->height()); + q_->PostTask([this] { TriggerScale(kScaleDownAboveHighQp); }); + EXPECT_TRUE(observer_->event.Wait(50)); + EXPECT_EQ(1, observer_->scaled_down); + EXPECT_EQ(0, observer_->scaled_up); } TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { - for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) { - qs_.ReportQP(kNormalQp); - qs_.ReportDroppedFrame(); - qs_.ReportDroppedFrame(); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - if (qs_.GetScaledResolution().width < input_frame_->width()) - return; - } - - FAIL() << "No downscale within " << kNumSeconds << " seconds."; + q_->PostTask([this] { + qs_->ReportDroppedFrame(); + qs_->ReportDroppedFrame(); + qs_->ReportQP(kHighQp); + }); + EXPECT_TRUE(observer_->event.Wait(50)); + EXPECT_EQ(1, observer_->scaled_down); + EXPECT_EQ(0, observer_->scaled_up); } TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) { - for (int i = 0; i < kFramerate * kNumSeconds; ++i) { - qs_.ReportQP(kNormalQp); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - ASSERT_EQ(input_frame_->width(), qs_.GetScaledResolution().width) - << "Unexpected scale on half framedrop."; - } + q_->PostTask([this] { TriggerScale(kScaleDownAboveHighQp); }); + EXPECT_TRUE(observer_->event.Wait(50)); + EXPECT_EQ(1, observer_->scaled_down); + EXPECT_EQ(0, observer_->scaled_up); } TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) { - for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) { - qs_.ReportQP(kNormalQp); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - ASSERT_EQ(input_frame_->width(), qs_.GetScaledResolution().width) - << "Unexpected scale on half framedrop."; - - qs_.ReportDroppedFrame(); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - ASSERT_EQ(input_frame_->width(), qs_.GetScaledResolution().width) - << "Unexpected scale on half framedrop."; - } + q_->PostTask([this] { + qs_->ReportDroppedFrame(); + qs_->ReportQP(kHighQp); + }); + EXPECT_FALSE(observer_->event.Wait(50)); + EXPECT_EQ(0, observer_->scaled_down); + EXPECT_EQ(0, observer_->scaled_up); } -void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() { - const int initial_min_dimension = - input_frame_->width() < input_frame_->height() ? input_frame_->width() - : input_frame_->height(); - int min_dimension = initial_min_dimension; - int current_shift = 0; - // Drop all frames to force-trigger downscaling. - while (min_dimension >= 2 * kMinDownscaleDimension) { - EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " - << kNumSeconds << " seconds."; - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - QualityScaler::Resolution res = qs_.GetScaledResolution(); - min_dimension = res.width < res.height ? res.width : res.height; - ++current_shift; - ASSERT_EQ(input_frame_->width() >> current_shift, res.width); - ASSERT_EQ(input_frame_->height() >> current_shift, res.height); - ExpectScaleUsingReportedResolution(); - } - - // Make sure we can scale back with good-quality frames. - while (min_dimension < initial_min_dimension) { - EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds - << " seconds."; - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - QualityScaler::Resolution res = qs_.GetScaledResolution(); - min_dimension = res.width < res.height ? res.width : res.height; - --current_shift; - ASSERT_EQ(input_frame_->width() >> current_shift, res.width); - ASSERT_EQ(input_frame_->height() >> current_shift, res.height); - ExpectScaleUsingReportedResolution(); - } - - // Verify we don't start upscaling after further low use. - for (int i = 0; i < kFramerate * kNumSeconds; ++i) { - qs_.ReportQP(kLowQp); - ExpectOriginalFrame(); - } +TEST_F(QualityScalerTest, UpscalesAfterLowQp) { + q_->PostTask([this] { TriggerScale(kScaleUp); }); + EXPECT_TRUE(observer_->event.Wait(50)); + EXPECT_EQ(0, observer_->scaled_down); + EXPECT_EQ(1, observer_->scaled_up); } -TEST_F(QualityScalerTest, ContinuouslyDownscalesByHalfDimensionsAndBackUp) { - ContinuouslyDownscalesByHalfDimensionsAndBackUp(); -} - -TEST_F(QualityScalerTest, - ContinuouslyDownscalesOddResolutionsByHalfDimensionsAndBackUp) { - const int kOddWidth = 517; - const int kOddHeight = 1239; - input_frame_ = I420Buffer::Create(kOddWidth, kOddHeight); - ContinuouslyDownscalesByHalfDimensionsAndBackUp(); -} - -void QualityScalerTest::DoesNotDownscaleFrameDimensions(int width, int height) { - input_frame_ = I420Buffer::Create(width, height); - - for (int i = 0; i < kFramerate * kNumSeconds; ++i) { - qs_.ReportDroppedFrame(); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - ASSERT_EQ(input_frame_->width(), qs_.GetScaledResolution().width) - << "Unexpected scale of minimal-size frame."; - } -} - -TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxWidth) { - DoesNotDownscaleFrameDimensions(1, kHeight); -} - -TEST_F(QualityScalerTest, DoesNotDownscaleFrom1PxHeight) { - DoesNotDownscaleFrameDimensions(kWidth, 1); -} - -TEST_F(QualityScalerTest, DoesNotDownscaleFrom1Px) { - DoesNotDownscaleFrameDimensions(1, 1); -} - -TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) { - DoesNotDownscaleFrameDimensions( - 2 * kMinDownscaleDimension - 1, 1000); -} - -TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsHeight) { - DoesNotDownscaleFrameDimensions( - 1000, 2 * kMinDownscaleDimension - 1); -} - -TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) { - static const int kWidth720p = 1280; - static const int kHeight720p = 720; - static const int kInitialBitrateKbps = 300; - input_frame_ = I420Buffer::Create(kWidth720p, kHeight720p); - qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, kInitialBitrateKbps, - kWidth720p, kHeight720p, kFramerate); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - int init_width = qs_.GetScaledResolution().width; - int init_height = qs_.GetScaledResolution().height; - EXPECT_EQ(640, init_width); - EXPECT_EQ(360, init_height); -} - -TEST_F(QualityScalerTest, DownscaleToQvgaOnLowerInitialBitrate) { - static const int kWidth720p = 1280; - static const int kHeight720p = 720; - static const int kInitialBitrateKbps = 200; - input_frame_ = I420Buffer::Create(kWidth720p, kHeight720p); - qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, kInitialBitrateKbps, - kWidth720p, kHeight720p, kFramerate); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - int init_width = qs_.GetScaledResolution().width; - int init_height = qs_.GetScaledResolution().height; - EXPECT_EQ(320, init_width); - EXPECT_EQ(180, init_height); -} - -TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) { - qs_.Init(kLowQpThreshold, kHighQp, 0, kWidth, kHeight, kFramerate); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - QualityScaler::Resolution initial_res = qs_.GetScaledResolution(); - - // Should not downscale if less than kMeasureSecondsDownscale seconds passed. - for (int i = 0; i < kFramerate * kMeasureSecondsDownscale - 1; ++i) { - qs_.ReportQP(kHighQp + 1); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - } - EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); - - // Should downscale if more than kMeasureSecondsDownscale seconds passed (add - // last frame). - qs_.ReportQP(kHighQp + 1); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height); - - // Should not upscale if less than kMeasureSecondsUpscale seconds passed since - // we saw issues initially (have already gone down). - for (int i = 0; i < kFramerate * kMeasureSecondsUpscale - 1; ++i) { - qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - } - EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height); - - // Should upscale (back to initial) if kMeasureSecondsUpscale seconds passed - // (add last frame). - qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); -} - -TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) { - qs_.Init(kLowQpThreshold, kHighQp, kLowInitialBitrateKbps, kWidth, kHeight, - kFramerate); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - QualityScaler::Resolution initial_res = qs_.GetScaledResolution(); - - // Should not upscale if less than kMeasureSecondsFastUpscale seconds passed. - for (int i = 0; i < kFramerate * kMeasureSecondsFastUpscale - 1; ++i) { - qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - } - EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height); - - // Should upscale if kMeasureSecondsFastUpscale seconds passed (add last - // frame). - qs_.ReportQP(kLowQp); - qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); - EXPECT_LT(initial_res.width, qs_.GetScaledResolution().width); - EXPECT_LT(initial_res.height, qs_.GetScaledResolution().height); -} - -void QualityScalerTest::DownscaleEndsAt(int input_width, - int input_height, - int end_width, - int end_height) { - // Create a frame with 2x expected end width/height to verify that we can - // scale down to expected end width/height. - input_frame_ = I420Buffer::Create(input_width, input_height); - - int last_width = input_width; - int last_height = input_height; - // Drop all frames to force-trigger downscaling. - while (true) { - TriggerScale(kScaleDown); - QualityScaler::Resolution res = qs_.GetScaledResolution(); - if (last_width == res.width) { - EXPECT_EQ(last_height, res.height); - EXPECT_EQ(end_width, res.width); - EXPECT_EQ(end_height, res.height); - break; - } - last_width = res.width; - last_height = res.height; - } -} - -TEST_F(QualityScalerTest, DownscalesTo320x180) { - DownscaleEndsAt(640, 360, 320, 180); -} - -TEST_F(QualityScalerTest, DownscalesTo180x320) { - DownscaleEndsAt(360, 640, 180, 320); -} - -TEST_F(QualityScalerTest, DownscalesFrom1280x720To320x180) { - DownscaleEndsAt(1280, 720, 320, 180); -} - -TEST_F(QualityScalerTest, DoesntDownscaleInitialQvga) { - DownscaleEndsAt(320, 180, 320, 180); +TEST_F(QualityScalerTest, ScalesDownAndBackUp) { + q_->PostTask([this] { TriggerScale(kScaleDown); }); + EXPECT_TRUE(observer_->event.Wait(50)); + EXPECT_EQ(1, observer_->scaled_down); + EXPECT_EQ(0, observer_->scaled_up); + q_->PostTask([this] { TriggerScale(kScaleUp); }); + EXPECT_TRUE(observer_->event.Wait(50)); + EXPECT_EQ(1, observer_->scaled_down); + EXPECT_EQ(1, observer_->scaled_up); } } // namespace webrtc diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h index b76c61753a..f6e53063fb 100644 --- a/webrtc/modules/video_coding/video_coding_impl.h +++ b/webrtc/modules/video_coding/video_coding_impl.h @@ -115,6 +115,7 @@ class VideoSender : public Module { VCMGenericEncoder* _encoder; media_optimization::MediaOptimization _mediaOpt; VCMEncodedFrameCallback _encodedFrameCallback GUARDED_BY(encoder_crit_); + EncodedImageCallback* const post_encode_callback_; VCMSendStatisticsCallback* const send_stats_callback_; VCMCodecDataBase _codecDataBase GUARDED_BY(encoder_crit_); bool frame_dropper_enabled_ GUARDED_BY(encoder_crit_); diff --git a/webrtc/modules/video_coding/video_sender.cc b/webrtc/modules/video_coding/video_sender.cc index 148d2374db..d754c56550 100644 --- a/webrtc/modules/video_coding/video_sender.cc +++ b/webrtc/modules/video_coding/video_sender.cc @@ -34,6 +34,7 @@ VideoSender::VideoSender(Clock* clock, _encoder(nullptr), _mediaOpt(clock_), _encodedFrameCallback(post_encode_callback, &_mediaOpt), + post_encode_callback_(post_encode_callback), send_stats_callback_(send_stats_callback), _codecDataBase(&_encodedFrameCallback), frame_dropper_enabled_(true), @@ -310,7 +311,7 @@ int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, << " loss rate " << encoder_params.loss_rate << " rtt " << encoder_params.rtt << " input frame rate " << encoder_params.input_frame_rate; - _encoder->OnDroppedFrame(); + post_encode_callback_->OnDroppedFrame(); return VCM_OK; } // TODO(pbos): Make sure setting send codec is synchronized with video diff --git a/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.h b/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.h index 2413f773b8..05887227e6 100644 --- a/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.h +++ b/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.h @@ -46,7 +46,6 @@ class H264VideoToolboxEncoder : public H264Encoder { int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; - void OnDroppedFrame() override; int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) override; @@ -67,6 +66,8 @@ class H264VideoToolboxEncoder : public H264Encoder { uint32_t timestamp, VideoRotation rotation); + ScalingSettings GetScalingSettings() const override; + private: int ResetCompressionSession(); void ConfigureCompressionSession(); @@ -85,10 +86,7 @@ class H264VideoToolboxEncoder : public H264Encoder { int32_t height_; const CFStringRef profile_; - rtc::CriticalSection quality_scaler_crit_; - QualityScaler quality_scaler_ GUARDED_BY(quality_scaler_crit_); H264BitstreamParser h264_bitstream_parser_; - bool enable_scaling_; std::vector nv12_scale_buffer_; }; // H264VideoToolboxEncoder diff --git a/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm b/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm index 9f435b2e1a..01f6d1172f 100644 --- a/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm +++ b/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm @@ -360,18 +360,9 @@ int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, size_t max_payload_size) { RTC_DCHECK(codec_settings); RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); - { - rtc::CritScope lock(&quality_scaler_crit_); - quality_scaler_.Init(internal::kLowH264QpThreshold, - internal::kHighH264QpThreshold, - codec_settings->startBitrate, codec_settings->width, - codec_settings->height, codec_settings->maxFramerate); - QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); - // TODO(tkchin): We may need to enforce width/height dimension restrictions - // to match what the encoder supports. - width_ = res.width; - height_ = res.height; - } + + width_ = codec_settings->width; + height_ = codec_settings->height; // We can only set average bitrate on the HW encoder. target_bitrate_bps_ = codec_settings->startBitrate; bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); @@ -386,6 +377,9 @@ int H264VideoToolboxEncoder::Encode( const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) { + // |input_frame| size should always match codec settings. + RTC_DCHECK_EQ(frame.width(), width_); + RTC_DCHECK_EQ(frame.height(), height_); RTC_DCHECK(!frame.IsZeroSize()); if (!callback_ || !compression_session_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; @@ -399,18 +393,6 @@ int H264VideoToolboxEncoder::Encode( #endif bool is_keyframe_required = false; - quality_scaler_.OnEncodeFrame(frame.width(), frame.height()); - const QualityScaler::Resolution scaled_res = - quality_scaler_.GetScaledResolution(); - - if (scaled_res.width != width_ || scaled_res.height != height_) { - width_ = scaled_res.width; - height_ = scaled_res.height; - int ret = ResetCompressionSession(); - if (ret < 0) - return ret; - } - // Get a pixel buffer from the pool and copy frame data over. CVPixelBufferPoolRef pixel_buffer_pool = VTCompressionSessionGetPixelBufferPool(compression_session_); @@ -457,11 +439,8 @@ int H264VideoToolboxEncoder::Encode( if (!pixel_buffer) { return WEBRTC_VIDEO_CODEC_ERROR; } - // TODO(magjed): Optimize by merging scaling and NV12 pixel buffer - // conversion once libyuv::MergeUVPlanes is available. - rtc::scoped_refptr scaled_i420_buffer = - quality_scaler_.GetScaledBuffer(frame.video_frame_buffer()); - if (!internal::CopyVideoFrameToPixelBuffer(scaled_i420_buffer, + RTC_DCHECK(pixel_buffer); + if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer(), pixel_buffer)) { LOG(LS_ERROR) << "Failed to copy frame data."; CVBufferRelease(pixel_buffer); @@ -517,11 +496,6 @@ int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( return WEBRTC_VIDEO_CODEC_OK; } -void H264VideoToolboxEncoder::OnDroppedFrame() { - rtc::CritScope lock(&quality_scaler_crit_); - quality_scaler_.ReportDroppedFrame(); -} - int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, int64_t rtt) { // Encoder doesn't know anything about packet loss or rtt so just return. @@ -533,10 +507,6 @@ int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, target_bitrate_bps_ = 1000 * new_bitrate_kbit; bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); - - rtc::CritScope lock(&quality_scaler_crit_); - quality_scaler_.ReportFramerate(frame_rate); - return WEBRTC_VIDEO_CODEC_OK; } @@ -709,8 +679,6 @@ void H264VideoToolboxEncoder::OnEncodedFrame( } if (info_flags & kVTEncodeInfo_FrameDropped) { LOG(LS_INFO) << "H264 encode dropped frame."; - rtc::CritScope lock(&quality_scaler_crit_); - quality_scaler_.ReportDroppedFrame(); return; } @@ -752,20 +720,20 @@ void H264VideoToolboxEncoder::OnEncodedFrame( frame.rotation_ = rotation; h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); - int qp; - if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { - rtc::CritScope lock(&quality_scaler_crit_); - quality_scaler_.ReportQP(qp); - frame.qp_ = qp; - } + h264_bitstream_parser_.GetLastSliceQp(&frame.qp_); - EncodedImageCallback::Result result = + EncodedImageCallback::Result res = callback_->OnEncodedImage(frame, &codec_specific_info, header.get()); - if (result.error != EncodedImageCallback::Result::OK) { - LOG(LS_ERROR) << "Encode callback failed: " << result.error; + if (res.error != EncodedImageCallback::Result::OK) { + LOG(LS_ERROR) << "Encode callback failed: " << res.error; return; } bitrate_adjuster_.Update(frame._size); } +VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings() + const { + return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold, + internal::kHighH264QpThreshold); +} } // namespace webrtc diff --git a/webrtc/video/overuse_frame_detector.cc b/webrtc/video/overuse_frame_detector.cc index 1670dc3aaa..f4a9c866e0 100644 --- a/webrtc/video/overuse_frame_detector.cc +++ b/webrtc/video/overuse_frame_detector.cc @@ -49,6 +49,7 @@ const int kMaxOverusesBeforeApplyRampupDelay = 4; const float kSampleDiffMs = 33.0f; const float kMaxExp = 7.0f; +const auto kScaleReasonCpu = ScalingObserverInterface::ScaleReason::kCpu; } // namespace CpuOveruseOptions::CpuOveruseOptions() @@ -204,7 +205,7 @@ class OveruseFrameDetector::CheckOveruseTask : public rtc::QueuedTask { OveruseFrameDetector::OveruseFrameDetector( Clock* clock, const CpuOveruseOptions& options, - CpuOveruseObserver* observer, + ScalingObserverInterface* observer, EncodedFrameObserver* encoder_timing, CpuOveruseMetricsObserver* metrics_observer) : check_overuse_task_(nullptr), @@ -370,13 +371,13 @@ void OveruseFrameDetector::CheckForOveruse() { ++num_overuse_detections_; if (observer_) - observer_->OveruseDetected(); + observer_->ScaleDown(kScaleReasonCpu); } else if (IsUnderusing(*metrics_, now)) { last_rampup_time_ms_ = now; in_quick_rampup_ = true; if (observer_) - observer_->NormalUsage(); + observer_->ScaleUp(kScaleReasonCpu); } int rampup_delay = diff --git a/webrtc/video/overuse_frame_detector.h b/webrtc/video/overuse_frame_detector.h index f1a99d74bf..70219843b4 100644 --- a/webrtc/video/overuse_frame_detector.h +++ b/webrtc/video/overuse_frame_detector.h @@ -20,6 +20,7 @@ #include "webrtc/base/sequenced_task_checker.h" #include "webrtc/base/task_queue.h" #include "webrtc/base/thread_annotations.h" +#include "webrtc/modules/video_coding/utility/quality_scaler.h" namespace webrtc { @@ -27,19 +28,6 @@ class Clock; class EncodedFrameObserver; class VideoFrame; -// CpuOveruseObserver is called when a system overuse is detected and -// VideoEngine cannot keep up the encoding frequency. -class CpuOveruseObserver { - public: - // Called as soon as an overuse is detected. - virtual void OveruseDetected() = 0; - // Called periodically when the system is not overused any longer. - virtual void NormalUsage() = 0; - - protected: - virtual ~CpuOveruseObserver() {} -}; - struct CpuOveruseOptions { CpuOveruseOptions(); @@ -79,7 +67,7 @@ class OveruseFrameDetector { public: OveruseFrameDetector(Clock* clock, const CpuOveruseOptions& options, - CpuOveruseObserver* overuse_observer, + ScalingObserverInterface* overuse_observer, EncodedFrameObserver* encoder_timing_, CpuOveruseMetricsObserver* metrics_observer); ~OveruseFrameDetector(); @@ -131,7 +119,7 @@ class OveruseFrameDetector { const CpuOveruseOptions options_; // Observer getting overuse reports. - CpuOveruseObserver* const observer_; + ScalingObserverInterface* const observer_; EncodedFrameObserver* const encoder_timing_; // Stats metrics. diff --git a/webrtc/video/overuse_frame_detector_unittest.cc b/webrtc/video/overuse_frame_detector_unittest.cc index f00b32df6e..6768591f01 100644 --- a/webrtc/video/overuse_frame_detector_unittest.cc +++ b/webrtc/video/overuse_frame_detector_unittest.cc @@ -16,10 +16,11 @@ #include "webrtc/test/gtest.h" #include "webrtc/video/overuse_frame_detector.h" #include "webrtc/video_frame.h" +#include "webrtc/modules/video_coding/utility/quality_scaler.h" namespace webrtc { -using ::testing::Invoke; +using ::testing::InvokeWithoutArgs; namespace { const int kWidth = 640; @@ -29,24 +30,24 @@ namespace { const int kProcessTime5ms = 5; } // namespace -class MockCpuOveruseObserver : public CpuOveruseObserver { +class MockCpuOveruseObserver : public ScalingObserverInterface { public: MockCpuOveruseObserver() {} virtual ~MockCpuOveruseObserver() {} - MOCK_METHOD0(OveruseDetected, void()); - MOCK_METHOD0(NormalUsage, void()); + MOCK_METHOD1(ScaleUp, void(ScaleReason)); + MOCK_METHOD1(ScaleDown, void(ScaleReason)); }; -class CpuOveruseObserverImpl : public CpuOveruseObserver { +class CpuOveruseObserverImpl : public ScalingObserverInterface { public: CpuOveruseObserverImpl() : overuse_(0), normaluse_(0) {} virtual ~CpuOveruseObserverImpl() {} - void OveruseDetected() { ++overuse_; } - void NormalUsage() { ++normaluse_; } + void ScaleDown(ScaleReason) { ++overuse_; } + void ScaleUp(ScaleReason) { ++normaluse_; } int overuse_; int normaluse_; @@ -56,7 +57,7 @@ class OveruseFrameDetectorUnderTest : public OveruseFrameDetector { public: OveruseFrameDetectorUnderTest(Clock* clock, const CpuOveruseOptions& options, - CpuOveruseObserver* overuse_observer, + ScalingObserverInterface* overuse_observer, EncodedFrameObserver* encoder_timing, CpuOveruseMetricsObserver* metrics_observer) : OveruseFrameDetector(clock, @@ -145,6 +146,8 @@ class OveruseFrameDetectorTest : public ::testing::Test, std::unique_ptr observer_; std::unique_ptr overuse_detector_; CpuOveruseMetrics metrics_; + + static const auto reason_ = ScalingObserverInterface::ScaleReason::kCpu; }; @@ -152,33 +155,33 @@ class OveruseFrameDetectorTest : public ::testing::Test, // UsagePercent() < low_encode_usage_threshold_percent => underuse. TEST_F(OveruseFrameDetectorTest, TriggerOveruse) { // usage > high => overuse - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(1); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)).Times(1); TriggerOveruse(options_.high_threshold_consecutive_count); } TEST_F(OveruseFrameDetectorTest, OveruseAndRecover) { // usage > high => overuse - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(1); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)).Times(1); TriggerOveruse(options_.high_threshold_consecutive_count); // usage < low => underuse - EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(testing::AtLeast(1)); + EXPECT_CALL(*(observer_.get()), ScaleUp(reason_)).Times(testing::AtLeast(1)); TriggerUnderuse(); } TEST_F(OveruseFrameDetectorTest, OveruseAndRecoverWithNoObserver) { overuse_detector_.reset(new OveruseFrameDetectorUnderTest( clock_.get(), options_, nullptr, nullptr, this)); - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(0); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)).Times(0); TriggerOveruse(options_.high_threshold_consecutive_count); - EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(0); + EXPECT_CALL(*(observer_.get()), ScaleUp(reason_)).Times(0); TriggerUnderuse(); } TEST_F(OveruseFrameDetectorTest, DoubleOveruseAndRecover) { - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(2); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)).Times(2); TriggerOveruse(options_.high_threshold_consecutive_count); TriggerOveruse(options_.high_threshold_consecutive_count); - EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(testing::AtLeast(1)); + EXPECT_CALL(*(observer_.get()), ScaleUp(reason_)).Times(testing::AtLeast(1)); TriggerUnderuse(); } @@ -197,22 +200,22 @@ TEST_F(OveruseFrameDetectorTest, TriggerUnderuseWithMinProcessCount) { } TEST_F(OveruseFrameDetectorTest, ConstantOveruseGivesNoNormalUsage) { - EXPECT_CALL(*(observer_.get()), NormalUsage()).Times(0); - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(64); + EXPECT_CALL(*(observer_.get()), ScaleUp(reason_)).Times(0); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)).Times(64); for (size_t i = 0; i < 64; ++i) { TriggerOveruse(options_.high_threshold_consecutive_count); } } TEST_F(OveruseFrameDetectorTest, ConsecutiveCountTriggersOveruse) { - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(1); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)).Times(1); options_.high_threshold_consecutive_count = 2; ReinitializeOveruseDetector(); TriggerOveruse(2); } TEST_F(OveruseFrameDetectorTest, IncorrectConsecutiveCountTriggersNoOveruse) { - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(0); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)).Times(0); options_.high_threshold_consecutive_count = 2; ReinitializeOveruseDetector(); TriggerOveruse(1); @@ -278,7 +281,8 @@ TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) { } TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(testing::AtLeast(1)); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)) + .Times(testing::AtLeast(1)); static const int kIntervalMs = 33; static const size_t kNumFramesEncodingDelay = 3; VideoFrame frame(I420Buffer::Create(kWidth, kHeight), @@ -299,7 +303,8 @@ TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) { // >85% encoding time should trigger overuse. - EXPECT_CALL(*(observer_.get()), OveruseDetected()).Times(testing::AtLeast(1)); + EXPECT_CALL(*(observer_.get()), ScaleDown(reason_)) + .Times(testing::AtLeast(1)); static const int kIntervalMs = 33; static const int kDelayMs = 30; VideoFrame frame(I420Buffer::Create(kWidth, kHeight), @@ -332,8 +337,8 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { // Expect NormalUsage(). When called, stop the |overuse_detector_| and then // set |event| to end the test. - EXPECT_CALL(*(observer_.get()), NormalUsage()) - .WillOnce(Invoke([this, &event] { + EXPECT_CALL(*(observer_.get()), ScaleUp(reason_)) + .WillOnce(InvokeWithoutArgs([this, &event] { overuse_detector_->StopCheckForOveruse(); event.Set(); })); diff --git a/webrtc/video/send_statistics_proxy.cc b/webrtc/video/send_statistics_proxy.cc index 8c4dae57b5..82d7f0dc75 100644 --- a/webrtc/video/send_statistics_proxy.cc +++ b/webrtc/video/send_statistics_proxy.cc @@ -505,9 +505,8 @@ void SendStatisticsProxy::OnSendEncodedImage( uma_container_->key_frame_counter_.Add(encoded_image._frameType == kVideoFrameKey); - stats_.bw_limited_resolution = - encoded_image.adapt_reason_.quality_resolution_downscales > 0 || + stats_.bw_limited_resolution || encoded_image.adapt_reason_.bw_resolutions_disabled > 0; if (encoded_image.adapt_reason_.quality_resolution_downscales != -1) { @@ -588,10 +587,11 @@ void SendStatisticsProxy::OnIncomingFrame(int width, int height) { uma_container_->cpu_limited_frame_counter_.Add(stats_.cpu_limited_resolution); } -void SendStatisticsProxy::SetCpuRestrictedResolution( - bool cpu_restricted_resolution) { +void SendStatisticsProxy::SetResolutionRestrictionStats(bool bandwidth, + bool cpu) { rtc::CritScope lock(&crit_); - stats_.cpu_limited_resolution = cpu_restricted_resolution; + stats_.bw_limited_resolution = bandwidth; + stats_.cpu_limited_resolution = cpu; } void SendStatisticsProxy::OnCpuRestrictedResolutionChanged( @@ -601,6 +601,13 @@ void SendStatisticsProxy::OnCpuRestrictedResolutionChanged( ++stats_.number_of_cpu_adapt_changes; } +void SendStatisticsProxy::OnQualityRestrictedResolutionChanged( + bool restricted) { + rtc::CritScope lock(&crit_); + uma_container_->quality_downscales_counter_.Add(restricted); + stats_.bw_limited_resolution = restricted; +} + void SendStatisticsProxy::RtcpPacketTypesCounterUpdated( uint32_t ssrc, const RtcpPacketTypeCounter& packet_counter) { diff --git a/webrtc/video/send_statistics_proxy.h b/webrtc/video/send_statistics_proxy.h index 803fb61f14..934dab2f93 100644 --- a/webrtc/video/send_statistics_proxy.h +++ b/webrtc/video/send_statistics_proxy.h @@ -57,12 +57,9 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver, // Used to update incoming frame rate. void OnIncomingFrame(int width, int height); - // Used to indicate that the current input frame resolution is restricted due - // to cpu usage. - void SetCpuRestrictedResolution(bool cpu_restricted); - // Used to update the number of times the input frame resolution has changed - // due to cpu adaptation. void OnCpuRestrictedResolutionChanged(bool cpu_restricted_resolution); + void OnQualityRestrictedResolutionChanged(bool restricted); + void SetResolutionRestrictionStats(bool bandwidth, bool cpu); void OnEncoderStatsUpdate(uint32_t framerate, uint32_t bitrate); void OnSuspendChange(bool is_suspended); diff --git a/webrtc/video/send_statistics_proxy_unittest.cc b/webrtc/video/send_statistics_proxy_unittest.cc index aa7c14f1a1..4cee1046dd 100644 --- a/webrtc/video/send_statistics_proxy_unittest.cc +++ b/webrtc/video/send_statistics_proxy_unittest.cc @@ -722,22 +722,10 @@ TEST_F(SendStatisticsProxyTest, GetStatsReportsBandwidthLimitedResolution) { EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); // Resolution not scaled. encoded_image.adapt_reason_.bw_resolutions_disabled = 0; - encoded_image.adapt_reason_.quality_resolution_downscales = 0; statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); // Resolution scaled due to bandwidth. encoded_image.adapt_reason_.bw_resolutions_disabled = 1; - encoded_image.adapt_reason_.quality_resolution_downscales = 0; - statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); - EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution); - // Resolution not scaled. - encoded_image.adapt_reason_.bw_resolutions_disabled = 0; - encoded_image.adapt_reason_.quality_resolution_downscales = 0; - statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); - EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution); - // Resolution scaled due to quality. - encoded_image.adapt_reason_.bw_resolutions_disabled = 0; - encoded_image.adapt_reason_.quality_resolution_downscales = 1; statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr); EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution); } diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index c2598f48a9..5722d06c1e 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -257,9 +257,6 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, has_received_rpsi_(false), picture_id_rpsi_(0), clock_(Clock::GetRealTimeClock()), - degradation_preference_( - VideoSendStream::DegradationPreference::kBalanced), - cpu_restricted_counter_(0), last_frame_width_(0), last_frame_height_(0), last_captured_timestamp_(0), @@ -292,6 +289,7 @@ void ViEEncoder::Stop() { rate_allocator_.reset(); video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); + quality_scaler_ = nullptr; shutdown_event_.Set(); }); @@ -318,17 +316,12 @@ void ViEEncoder::SetSource( source_proxy_->SetSource(source, degradation_preference); encoder_queue_.PostTask([this, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); - degradation_preference_ = degradation_preference; - // Set the stats for if we are currently CPU restricted. We are CPU - // restricted depending on degradation preference and - // if the overusedetector has currently detected overuse which is counted in - // |cpu_restricted_counter_| - // We do this on the encoder task queue to avoid a race with the stats set - // in ViEEncoder::NormalUsage and ViEEncoder::OveruseDetected. - stats_proxy_->SetCpuRestrictedResolution( - degradation_preference_ != - VideoSendStream::DegradationPreference::kMaintainResolution && - cpu_restricted_counter_ != 0); + scaling_enabled_ = + (degradation_preference != + VideoSendStream::DegradationPreference::kMaintainResolution); + stats_proxy_->SetResolutionRestrictionStats( + scaling_enabled_ && scale_counter_[kQuality] > 0, + scaling_enabled_ && scale_counter_[kCpu] > 0); }); } @@ -420,6 +413,18 @@ void ViEEncoder::ReconfigureEncoder() { sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); + + const auto scaling_settings = settings_.encoder->GetScalingSettings(); + if (scaling_settings.enabled && scaling_enabled_) { + if (scaling_settings.thresholds) { + quality_scaler_.reset( + new QualityScaler(this, *(scaling_settings.thresholds))); + } else { + quality_scaler_.reset(new QualityScaler(this, codec_type_)); + } + } else { + quality_scaler_.reset(nullptr); + } } void ViEEncoder::OnFrame(const VideoFrame& video_frame) { @@ -497,6 +502,7 @@ void ViEEncoder::TraceFrameDropEnd() { void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, int64_t time_when_posted_in_ms) { RTC_DCHECK_RUN_ON(&encoder_queue_); + if (pre_encode_callback_) pre_encode_callback_->OnFrame(video_frame); @@ -574,15 +580,25 @@ EncodedImageCallback::Result ViEEncoder::OnEncodedImage( int64_t time_sent = clock_->TimeInMilliseconds(); uint32_t timestamp = encoded_image._timeStamp; - - encoder_queue_.PostTask([this, timestamp, time_sent] { + const int qp = encoded_image.qp_; + encoder_queue_.PostTask([this, timestamp, time_sent, qp] { RTC_DCHECK_RUN_ON(&encoder_queue_); overuse_detector_.FrameSent(timestamp, time_sent); + if (quality_scaler_) + quality_scaler_->ReportQP(qp); }); return result; } +void ViEEncoder::OnDroppedFrame() { + encoder_queue_.PostTask([this] { + RTC_DCHECK_RUN_ON(&encoder_queue_); + if (quality_scaler_) + quality_scaler_->ReportDroppedFrame(); + }); +} + void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) { RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); if (stats_proxy_) @@ -654,47 +670,67 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, } } -void ViEEncoder::OveruseDetected() { +void ViEEncoder::ScaleDown(ScaleReason reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); - if (degradation_preference_ == - VideoSendStream::DegradationPreference::kMaintainResolution || - cpu_restricted_counter_ >= kMaxCpuDowngrades) { + if (!scaling_enabled_) return; - } - LOG(LS_INFO) << "CPU overuse detected. Requesting lower resolution."; // Request lower resolution if the current resolution is lower than last time // we asked for the resolution to be lowered. - // Update stats accordingly. int current_pixel_count = last_frame_height_ * last_frame_width_; - if (!max_pixel_count_ || current_pixel_count < *max_pixel_count_) { - max_pixel_count_ = rtc::Optional(current_pixel_count); - max_pixel_count_step_up_ = rtc::Optional(); - stats_proxy_->OnCpuRestrictedResolutionChanged(true); - ++cpu_restricted_counter_; - source_proxy_->RequestResolutionLowerThan(current_pixel_count); + if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) + return; + switch (reason) { + case kQuality: + if (scale_counter_[reason] >= kMaxQualityDowngrades) + return; + stats_proxy_->OnQualityRestrictedResolutionChanged(true); + break; + case kCpu: + if (scale_counter_[reason] >= kMaxCpuDowngrades) + return; + // Update stats accordingly. + stats_proxy_->OnCpuRestrictedResolutionChanged(true); + break; + } + max_pixel_count_ = rtc::Optional(current_pixel_count); + max_pixel_count_step_up_ = rtc::Optional(); + ++scale_counter_[reason]; + source_proxy_->RequestResolutionLowerThan(current_pixel_count); + LOG(LS_INFO) << "Scaling down resolution."; + for (size_t i = 0; i < kScaleReasonSize; ++i) { + LOG(LS_INFO) << "Scaled " << scale_counter_[i] + << " times for reason: " << (i ? "quality" : "cpu"); } } -void ViEEncoder::NormalUsage() { +void ViEEncoder::ScaleUp(ScaleReason reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); - if (degradation_preference_ == - VideoSendStream::DegradationPreference::kMaintainResolution || - cpu_restricted_counter_ == 0) { + if (scale_counter_[reason] == 0 || !scaling_enabled_) return; - } - - LOG(LS_INFO) << "CPU underuse detected. Requesting higher resolution."; + // Only scale if resolution is higher than last time + // we requested higher resolution. int current_pixel_count = last_frame_height_ * last_frame_width_; - // Request higher resolution if we are CPU restricted and the the current - // resolution is higher than last time we requested higher resolution. - // Update stats accordingly. - if (!max_pixel_count_step_up_ || - current_pixel_count > *max_pixel_count_step_up_) { - max_pixel_count_ = rtc::Optional(); - max_pixel_count_step_up_ = rtc::Optional(current_pixel_count); - --cpu_restricted_counter_; - stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0); - source_proxy_->RequestHigherResolutionThan(current_pixel_count); + if (current_pixel_count <= max_pixel_count_step_up_.value_or(0)) + return; + switch (reason) { + case kQuality: + stats_proxy_->OnQualityRestrictedResolutionChanged( + scale_counter_[reason] > 1); + break; + case kCpu: + // Update stats accordingly. + stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] > + 1); + break; + } + max_pixel_count_ = rtc::Optional(); + max_pixel_count_step_up_ = rtc::Optional(current_pixel_count); + --scale_counter_[reason]; + source_proxy_->RequestHigherResolutionThan(current_pixel_count); + LOG(LS_INFO) << "Scaling up resolution."; + for (size_t i = 0; i < kScaleReasonSize; ++i) { + LOG(LS_INFO) << "Scaled " << scale_counter_[i] + << " times for reason: " << (i ? "quality" : "cpu"); } } diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 8373a3f51d..12669622d1 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -25,6 +25,7 @@ #include "webrtc/common_video/rotation.h" #include "webrtc/media/base/videosinkinterface.h" #include "webrtc/modules/video_coding/include/video_coding_defines.h" +#include "webrtc/modules/video_coding/utility/quality_scaler.h" #include "webrtc/modules/video_coding/video_coding_impl.h" #include "webrtc/modules/video_processing/include/video_processing.h" #include "webrtc/system_wrappers/include/atomic32.h" @@ -49,7 +50,7 @@ class SendStatisticsProxy; class ViEEncoder : public rtc::VideoSinkInterface, public EncodedImageCallback, public VCMSendStatisticsCallback, - public CpuOveruseObserver { + public ScalingObserverInterface { public: // Interface for receiving encoded video frames and notifications about // configuration changes. @@ -60,8 +61,10 @@ class ViEEncoder : public rtc::VideoSinkInterface, int min_transmit_bitrate_bps) = 0; }; - // Down grade resolution at most 2 times for CPU reasons. + // Downscale resolution at most 2 times for CPU reasons. static const int kMaxCpuDowngrades = 2; + // Downscale resolution at most 2 times for low-quality reasons. + static const int kMaxQualityDowngrades = 2; ViEEncoder(uint32_t number_of_cores, SendStatisticsProxy* stats_proxy, @@ -111,14 +114,14 @@ class ViEEncoder : public rtc::VideoSinkInterface, int64_t round_trip_time_ms); protected: - // Used for testing. For example the |CpuOveruseObserver| methods must be - // called on |encoder_queue_|. + // Used for testing. For example the |ScalingObserverInterface| methods must + // be called on |encoder_queue_|. rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } - // webrtc::CpuOveruseObserver implementation. + // webrtc::ScalingObserverInterface implementation. // These methods are protected for easier testing. - void OveruseDetected() override; - void NormalUsage() override; + void ScaleUp(ScaleReason reason) override; + void ScaleDown(ScaleReason reason) override; private: class ConfigureEncoderTask; @@ -161,6 +164,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) override; + void OnDroppedFrame() override; + bool EncoderPaused() const; void TraceFrameDropStart(); void TraceFrameDropEnd(); @@ -176,6 +181,7 @@ class ViEEncoder : public rtc::VideoSinkInterface, vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); OveruseFrameDetector overuse_detector_ ACCESS_ON(&encoder_queue_); + std::unique_ptr quality_scaler_ ACCESS_ON(&encoder_queue_); SendStatisticsProxy* const stats_proxy_; rtc::VideoSinkInterface* const pre_encode_callback_; @@ -203,12 +209,11 @@ class ViEEncoder : public rtc::VideoSinkInterface, bool has_received_rpsi_ ACCESS_ON(&encoder_queue_); uint64_t picture_id_rpsi_ ACCESS_ON(&encoder_queue_); Clock* const clock_; - - VideoSendStream::DegradationPreference degradation_preference_ - ACCESS_ON(&encoder_queue_); - // Counter used for deciding if the video resolution is currently - // restricted by CPU usage. - int cpu_restricted_counter_ ACCESS_ON(&encoder_queue_); + // Counters used for deciding if the video resolution is currently + // restricted, and if so, why. + int scale_counter_[kScaleReasonSize] ACCESS_ON(&encoder_queue_) = {0}; + // Set depending on degradation preferences + bool scaling_enabled_ ACCESS_ON(&encoder_queue_) = false; int last_frame_width_ ACCESS_ON(&encoder_queue_); int last_frame_height_ ACCESS_ON(&encoder_queue_); diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc index 71e68d3ae5..6cc25cfb80 100644 --- a/webrtc/video/vie_encoder_unittest.cc +++ b/webrtc/video/vie_encoder_unittest.cc @@ -22,6 +22,9 @@ namespace webrtc { +using DegredationPreference = VideoSendStream::DegradationPreference; +using ScaleReason = ScalingObserverInterface::ScaleReason; + namespace { const size_t kMaxPayloadLength = 1440; const int kTargetBitrateBps = 100000; @@ -42,32 +45,30 @@ class TestBuffer : public webrtc::I420Buffer { class ViEEncoderUnderTest : public ViEEncoder { public: - ViEEncoderUnderTest( - SendStatisticsProxy* stats_proxy, - const webrtc::VideoSendStream::Config::EncoderSettings& settings) + ViEEncoderUnderTest(SendStatisticsProxy* stats_proxy, + const VideoSendStream::Config::EncoderSettings& settings) : ViEEncoder(1 /* number_of_cores */, stats_proxy, settings, nullptr /* pre_encode_callback */, nullptr /* encoder_timing */) {} - void TriggerCpuOveruse() { + void PostTaskAndWait(bool down, ScaleReason reason) { rtc::Event event(false, false); - encoder_queue()->PostTask([this, &event] { - OveruseDetected(); + encoder_queue()->PostTask([this, &event, reason, down] { + down ? ScaleDown(reason) : ScaleUp(reason); event.Set(); }); - event.Wait(rtc::Event::kForever); + RTC_DCHECK(event.Wait(5000)); } - void TriggerCpuNormalUsage() { - rtc::Event event(false, false); - encoder_queue()->PostTask([this, &event] { - NormalUsage(); - event.Set(); - }); - event.Wait(rtc::Event::kForever); - } + void TriggerCpuOveruse() { PostTaskAndWait(true, ScaleReason::kCpu); } + + void TriggerCpuNormalUsage() { PostTaskAndWait(false, ScaleReason::kCpu); } + + void TriggerQualityLow() { PostTaskAndWait(true, ScaleReason::kQuality); } + + void TriggerQualityHigh() { PostTaskAndWait(false, ScaleReason::kQuality); } }; class VideoStreamFactory @@ -183,6 +184,10 @@ class ViEEncoderTest : public ::testing::Test { block_next_encode_ = true; } + VideoEncoder::ScalingSettings GetScalingSettings() const override { + return VideoEncoder::ScalingSettings(true, 1, 2); + } + void ContinueEncode() { continue_encode_event_.Set(); } void CheckLastTimeStampsMatch(int64_t ntp_time_ms, @@ -549,7 +554,7 @@ TEST_F(ViEEncoderTest, SinkWantsFromOveruseDetector) { frame_height /= 2; } - // Trigger CPU overuse a one more time. This should not trigger request for + // Trigger CPU overuse one more time. This should not trigger a request for // lower resolution. rtc::VideoSinkWants current_wants = video_source_.sink_wants(); video_source_.IncomingCapturedFrame(CreateFrame( @@ -653,6 +658,138 @@ TEST_F(ViEEncoderTest, StatsTracksAdaptationStats) { vie_encoder_->Stop(); } +TEST_F(ViEEncoderTest, SwitchingSourceKeepsCpuAdaptation) { + const int kTargetBitrateBps = 100000; + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + + int frame_width = 1280; + int frame_height = 720; + video_source_.IncomingCapturedFrame( + CreateFrame(1, frame_width, frame_height)); + sink_.WaitForEncodedFrame(1); + + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_EQ(0, stats.number_of_cpu_adapt_changes); + + vie_encoder_->TriggerCpuOveruse(); + + video_source_.IncomingCapturedFrame( + CreateFrame(2, frame_width, frame_height)); + sink_.WaitForEncodedFrame(2); + stats = stats_proxy_->GetStats(); + EXPECT_TRUE(stats.cpu_limited_resolution); + EXPECT_EQ(1, stats.number_of_cpu_adapt_changes); + + // Set new source with adaptation still enabled. + test::FrameForwarder new_video_source; + vie_encoder_->SetSource(&new_video_source, + VideoSendStream::DegradationPreference::kBalanced); + + new_video_source.IncomingCapturedFrame( + CreateFrame(3, frame_width, frame_height)); + sink_.WaitForEncodedFrame(3); + stats = stats_proxy_->GetStats(); + EXPECT_TRUE(stats.cpu_limited_resolution); + EXPECT_EQ(1, stats.number_of_cpu_adapt_changes); + + // Set adaptation disabled. + vie_encoder_->SetSource( + &new_video_source, + VideoSendStream::DegradationPreference::kMaintainResolution); + + new_video_source.IncomingCapturedFrame( + CreateFrame(4, frame_width, frame_height)); + sink_.WaitForEncodedFrame(4); + stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_EQ(1, stats.number_of_cpu_adapt_changes); + + // Set adaptation back to enabled. + vie_encoder_->SetSource(&new_video_source, + VideoSendStream::DegradationPreference::kBalanced); + + new_video_source.IncomingCapturedFrame( + CreateFrame(5, frame_width, frame_height)); + sink_.WaitForEncodedFrame(5); + stats = stats_proxy_->GetStats(); + EXPECT_TRUE(stats.cpu_limited_resolution); + EXPECT_EQ(1, stats.number_of_cpu_adapt_changes); + + vie_encoder_->TriggerCpuNormalUsage(); + + new_video_source.IncomingCapturedFrame( + CreateFrame(6, frame_width, frame_height)); + sink_.WaitForEncodedFrame(6); + stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_EQ(2, stats.number_of_cpu_adapt_changes); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, SwitchingSourceKeepsQualityAdaptation) { + const int kTargetBitrateBps = 100000; + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + + int frame_width = 1280; + int frame_height = 720; + video_source_.IncomingCapturedFrame( + CreateFrame(1, frame_width, frame_height)); + sink_.WaitForEncodedFrame(1); + + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.bw_limited_resolution); + EXPECT_EQ(0, stats.number_of_cpu_adapt_changes); + + // Set new source with adaptation still enabled. + test::FrameForwarder new_video_source; + vie_encoder_->SetSource(&new_video_source, + VideoSendStream::DegradationPreference::kBalanced); + + new_video_source.IncomingCapturedFrame( + CreateFrame(2, frame_width, frame_height)); + sink_.WaitForEncodedFrame(2); + stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.bw_limited_resolution); + EXPECT_EQ(0, stats.number_of_cpu_adapt_changes); + + vie_encoder_->TriggerQualityLow(); + + new_video_source.IncomingCapturedFrame( + CreateFrame(3, frame_width, frame_height)); + sink_.WaitForEncodedFrame(3); + stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_TRUE(stats.bw_limited_resolution); + + vie_encoder_->SetSource(&new_video_source, + VideoSendStream::DegradationPreference::kBalanced); + + new_video_source.IncomingCapturedFrame( + CreateFrame(4, frame_width, frame_height)); + sink_.WaitForEncodedFrame(4); + stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_TRUE(stats.bw_limited_resolution); + + // Set adaptation disabled. + vie_encoder_->SetSource( + &new_video_source, + VideoSendStream::DegradationPreference::kMaintainResolution); + + new_video_source.IncomingCapturedFrame( + CreateFrame(5, frame_width, frame_height)); + sink_.WaitForEncodedFrame(5); + stats = stats_proxy_->GetStats(); + EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.bw_limited_resolution); + + vie_encoder_->Stop(); +} + TEST_F(ViEEncoderTest, StatsTracksAdaptationStatsWhenSwitchingSource) { vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); @@ -727,6 +864,54 @@ TEST_F(ViEEncoderTest, StatsTracksPreferredBitrate) { vie_encoder_->Stop(); } +TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) { + const int kTargetBitrateBps = 100000; + int frame_width = 1280; + int frame_height = 720; + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + + // Expect no scaling to begin with + EXPECT_FALSE(video_source_.sink_wants().max_pixel_count); + EXPECT_FALSE(video_source_.sink_wants().max_pixel_count_step_up); + + // Trigger scale down + vie_encoder_->TriggerQualityLow(); + video_source_.IncomingCapturedFrame( + CreateFrame(1, frame_width, frame_height)); + sink_.WaitForEncodedFrame(1); + + // Expect a scale down. + EXPECT_TRUE(video_source_.sink_wants().max_pixel_count); + EXPECT_LT(*video_source_.sink_wants().max_pixel_count, + frame_width * frame_height); + + // Set adaptation disabled. + test::FrameForwarder new_video_source; + vie_encoder_->SetSource( + &new_video_source, + VideoSendStream::DegradationPreference::kMaintainResolution); + + // Trigger scale down + vie_encoder_->TriggerQualityLow(); + new_video_source.IncomingCapturedFrame( + CreateFrame(2, frame_width, frame_height)); + sink_.WaitForEncodedFrame(2); + + // Expect no scaling + EXPECT_FALSE(new_video_source.sink_wants().max_pixel_count); + + // Trigger scale up + vie_encoder_->TriggerQualityHigh(); + new_video_source.IncomingCapturedFrame( + CreateFrame(3, frame_width, frame_height)); + sink_.WaitForEncodedFrame(3); + + // Expect nothing to change, still no scaling + EXPECT_FALSE(new_video_source.sink_wants().max_pixel_count); + + vie_encoder_->Stop(); +} + TEST_F(ViEEncoderTest, UMACpuLimitedResolutionInPercent) { vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h index f8a04f2e74..0a21fa345a 100644 --- a/webrtc/video_encoder.h +++ b/webrtc/video_encoder.h @@ -19,6 +19,7 @@ #include "webrtc/common_types.h" #include "webrtc/typedefs.h" #include "webrtc/video_frame.h" +#include "webrtc/base/optional.h" namespace webrtc { @@ -59,10 +60,38 @@ class EncodedImageCallback { const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) = 0; + + virtual void OnDroppedFrame() {} }; class VideoEncoder { public: + enum EncoderType { + kH264, + kVp8, + kVp9, + kUnsupportedCodec, + }; + struct QpThresholds { + QpThresholds(int l, int h) : low(l), high(h) {} + QpThresholds() : low(-1), high(-1) {} + int low; + int high; + }; + struct ScalingSettings { + ScalingSettings(bool on, int low, int high) + : enabled(on), + thresholds(rtc::Optional(QpThresholds(low, high))) {} + explicit ScalingSettings(bool on) : enabled(on) {} + const bool enabled; + const rtc::Optional thresholds; + }; + static VideoEncoder* Create(EncoderType codec_type); + // Returns true if this type of encoder can be created using + // VideoEncoder::Create. + static bool IsSupportedSoftware(EncoderType codec_type); + static EncoderType CodecToEncoderType(VideoCodecType codec_type); + static VideoCodecVP8 GetDefaultVp8Settings(); static VideoCodecVP9 GetDefaultVp9Settings(); static VideoCodecH264 GetDefaultH264Settings(); @@ -148,8 +177,13 @@ class VideoEncoder { return SetRates(allocation.get_sum_kbps(), framerate); } + // Any encoder implementation wishing to use the WebRTC provided + // quality scaler must implement this method. + virtual ScalingSettings GetScalingSettings() const { + return ScalingSettings(false); + } + virtual int32_t SetPeriodicKeyFrames(bool enable) { return -1; } - virtual void OnDroppedFrame() {} virtual bool SupportsNativeHandle() const { return false; } virtual const char* ImplementationName() const { return "unknown"; } };