diff --git a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc index 4db089cd6e..25ffe91586 100644 --- a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc +++ b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc @@ -100,6 +100,8 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, void OnDroppedFrame() override; + int GetTargetFramerate() override; + private: // CHECK-fail if not running on |codec_thread_|. void CheckOnCodecThread(); @@ -199,6 +201,7 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, scoped_ptr quality_scaler_; // Dynamic resolution change, off by default. bool scale_; + int updated_framerate_; }; MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { @@ -294,11 +297,12 @@ int32_t MediaCodecVideoEncoder::InitEncode( ALOGD("InitEncode request"); scale_ = false; - if (codecType_ == kVideoCodecVP8) { - quality_scaler_->Init(kMaxQP / kLowQpThresholdDenominator); + if (scale_ && codecType_ == kVideoCodecVP8) { + quality_scaler_->Init(kMaxQP / kLowQpThresholdDenominator, true); quality_scaler_->SetMinResolution(kMinWidth, kMinHeight); quality_scaler_->ReportFramerate(codec_settings->maxFramerate); } + updated_framerate_ = codec_settings->maxFramerate; return codec_thread_->Invoke( Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, this, @@ -337,8 +341,11 @@ int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */, int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, uint32_t frame_rate) { - if (codecType_ == kVideoCodecVP8) + if (scale_ && codecType_ == kVideoCodecVP8) { quality_scaler_->ReportFramerate(frame_rate); + } else { + updated_framerate_ = frame_rate; + } return codec_thread_->Invoke( Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread, this, @@ -493,9 +500,13 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( } CHECK(frame_types->size() == 1) << "Unexpected stream count"; - const VideoFrame& input_frame = (scale_ && codecType_ == kVideoCodecVP8) - ? quality_scaler_->GetScaledFrame(frame) - : frame; + // Check framerate before spatial resolution change. + if (scale_ && codecType_ == kVideoCodecVP8) { + quality_scaler_->OnEncodeFrame(frame); + updated_framerate_ = quality_scaler_->GetTargetFramerate(); + } + const VideoFrame& input_frame = (scale_ && codecType_ == kVideoCodecVP8) ? + quality_scaler_->GetScaledFrame(frame) : frame; if (input_frame.width() != width_ || input_frame.height() != height_) { ALOGD("Frame resolution change from %d x %d to %d x %d", @@ -506,8 +517,6 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( return WEBRTC_VIDEO_CODEC_OK; } - bool key_frame = frame_types->front() != webrtc::kDeltaFrame; - // Check if we accumulated too many frames in encoder input buffers // or the encoder latency exceeds 70 ms and drop frame if so. if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) { @@ -566,6 +575,7 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( render_times_ms_.push_back(input_frame.render_time_ms()); frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); + bool key_frame = frame_types->front() != webrtc::kDeltaFrame; bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, j_encode_method_, key_frame, @@ -712,7 +722,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { last_input_timestamp_ms_ - last_output_timestamp_ms_, frame_encoding_time_ms); - if (payload_size && codecType_ == kVideoCodecVP8) + if (payload_size && scale_ && codecType_ == kVideoCodecVP8) quality_scaler_->ReportQP(webrtc::vp8::GetQP(payload)); // Calculate and print encoding statistics - every 3 seconds. @@ -860,10 +870,14 @@ int32_t MediaCodecVideoEncoder::NextNaluPosition( } void MediaCodecVideoEncoder::OnDroppedFrame() { - if (codecType_ == kVideoCodecVP8) + if (scale_ && codecType_ == kVideoCodecVP8) quality_scaler_->ReportDroppedFrame(); } +int MediaCodecVideoEncoder::GetTargetFramerate() { + return updated_framerate_; +} + MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc index ee7fd859b5..7c4164c8ce 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc @@ -504,4 +504,8 @@ void SimulcastEncoderAdapter::OnDroppedFrame() { streaminfos_[0].encoder->OnDroppedFrame(); } +int SimulcastEncoderAdapter::GetTargetFramerate() { + return streaminfos_[0].encoder->GetTargetFramerate(); +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h index cd782342b5..bca1e00a71 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h @@ -57,6 +57,8 @@ class SimulcastEncoderAdapter : public VP8Encoder { void OnDroppedFrame() override; + int GetTargetFramerate() override; + private: struct StreamInfo { StreamInfo() diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 05e0799b52..31b92ea960 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -580,7 +580,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, } rps_.Init(); - quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator); + quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator, + false); quality_scaler_.ReportFramerate(codec_.maxFramerate); return InitAndSetControlSettings(); @@ -709,6 +710,8 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, const bool use_quality_scaler = encoders_.size() == 1 && configurations_[0].rc_dropframe_thresh > 0 && codec_.codecSpecific.VP8.automaticResizeOn; + if (use_quality_scaler) + quality_scaler_.OnEncodeFrame(frame); const VideoFrame& input_image = use_quality_scaler ? quality_scaler_.GetScaledFrame(frame) : frame; diff --git a/webrtc/modules/video_coding/main/interface/video_coding_defines.h b/webrtc/modules/video_coding/main/interface/video_coding_defines.h index 259e75e6a8..d40023c2b1 100644 --- a/webrtc/modules/video_coding/main/interface/video_coding_defines.h +++ b/webrtc/modules/video_coding/main/interface/video_coding_defines.h @@ -182,6 +182,8 @@ class VCMQMSettingsCallback { const uint32_t width, const uint32_t height) = 0; + virtual void SetTargetFramerate(int frame_rate) = 0; + protected: virtual ~VCMQMSettingsCallback() { } diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc index f2cdd599ae..63e3976ca1 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc @@ -214,6 +214,10 @@ bool VCMGenericEncoder::SupportsNativeHandle() const { return encoder_->SupportsNativeHandle(); } +int VCMGenericEncoder::GetTargetFramerate() { + return encoder_->GetTargetFramerate(); +} + /*************************** * Callback Implementation ***************************/ diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.h b/webrtc/modules/video_coding/main/source/generic_encoder.h index 862c06ba20..25235b6b46 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.h +++ b/webrtc/modules/video_coding/main/source/generic_encoder.h @@ -140,6 +140,8 @@ public: bool SupportsNativeHandle() const; + int GetTargetFramerate(); + private: VideoEncoder* const encoder_; VideoEncoderRateObserver* const rate_observer_; diff --git a/webrtc/modules/video_coding/main/source/video_sender.cc b/webrtc/modules/video_coding/main/source/video_sender.cc index fb04b0e871..93d82084f3 100644 --- a/webrtc/modules/video_coding/main/source/video_sender.cc +++ b/webrtc/modules/video_coding/main/source/video_sender.cc @@ -363,6 +363,8 @@ int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, for (size_t i = 0; i < _nextFrameTypes.size(); ++i) { _nextFrameTypes[i] = kVideoFrameDelta; // Default frame type. } + if (qm_settings_callback_) + qm_settings_callback_->SetTargetFramerate(_encoder->GetTargetFramerate()); return VCM_OK; } diff --git a/webrtc/modules/video_coding/utility/include/quality_scaler.h b/webrtc/modules/video_coding/utility/include/quality_scaler.h index 1d6c917ec4..8535171f65 100644 --- a/webrtc/modules/video_coding/utility/include/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/include/quality_scaler.h @@ -25,14 +25,16 @@ class QualityScaler { }; QualityScaler(); - void Init(int low_qp_threshold); + void Init(int low_qp_threshold, bool use_framerate_reduction); void SetMinResolution(int min_width, int min_height); void ReportFramerate(int framerate); void ReportQP(int qp); void ReportDroppedFrame(); void Reset(int framerate, int bitrate, int width, int height); - Resolution GetScaledResolution(const VideoFrame& frame); + void OnEncodeFrame(const VideoFrame& frame); + Resolution GetScaledResolution() const; const VideoFrame& GetScaledFrame(const VideoFrame& frame); + int GetTargetFramerate() const; private: void AdjustScale(bool up); @@ -42,11 +44,16 @@ class QualityScaler { VideoFrame scaled_frame_; size_t num_samples_; + int framerate_; + int target_framerate_; int low_qp_threshold_; MovingAverage framedrop_percent_; MovingAverage average_qp_; + Resolution res_; int downscale_shift_; + int framerate_down_; + bool use_framerate_reduction_; int min_width_; int min_height_; }; diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index 7a2a9c0261..f7d4651511 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -24,13 +24,16 @@ QualityScaler::QualityScaler() : num_samples_(0), low_qp_threshold_(-1), downscale_shift_(0), + framerate_down_(false), min_width_(kDefaultMinDownscaleDimension), min_height_(kDefaultMinDownscaleDimension) { } -void QualityScaler::Init(int low_qp_threshold) { +void QualityScaler::Init(int low_qp_threshold, bool use_framerate_reduction) { ClearSamples(); low_qp_threshold_ = low_qp_threshold; + use_framerate_reduction_ = use_framerate_reduction; + target_framerate_ = -1; } void QualityScaler::SetMinResolution(int min_width, int min_height) { @@ -42,6 +45,7 @@ void QualityScaler::SetMinResolution(int min_width, int min_height) { void QualityScaler::ReportFramerate(int framerate) { num_samples_ = static_cast( kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate)); + framerate_ = framerate; } void QualityScaler::ReportQP(int qp) { @@ -53,41 +57,65 @@ void QualityScaler::ReportDroppedFrame() { framedrop_percent_.AddSample(100); } -QualityScaler::Resolution QualityScaler::GetScaledResolution( - const VideoFrame& frame) { +void QualityScaler::OnEncodeFrame(const VideoFrame& frame) { // Should be set through InitEncode -> Should be set by now. assert(low_qp_threshold_ >= 0); assert(num_samples_ > 0); - - Resolution res; - res.width = frame.width(); - res.height = frame.height(); + res_.width = frame.width(); + res_.height = frame.height(); // Update scale factor. int avg_drop = 0; int avg_qp = 0; + + // When encoder consistently overshoots, framerate reduction and spatial + // resizing will be triggered to get a smoother video. if (framedrop_percent_.GetAverage(num_samples_, &avg_drop) && avg_drop >= kFramedropPercentThreshold) { - AdjustScale(false); + // Reducing frame rate before spatial resolution change. + // Reduce frame rate only when it is above a certain number. + // Only one reduction is allowed for now. + // TODO(jackychen): Allow more than one framerate reduction. + if (use_framerate_reduction_ && !framerate_down_ && framerate_ >= 20) { + target_framerate_ = framerate_ / 2; + framerate_down_ = true; + // If frame rate has been updated, clear the buffer. We don't want + // spatial resolution to change right after frame rate change. + ClearSamples(); + } else { + AdjustScale(false); + } } else if (average_qp_.GetAverage(num_samples_, &avg_qp) && avg_qp <= low_qp_threshold_) { - AdjustScale(true); + if (use_framerate_reduction_ && framerate_down_) { + target_framerate_ = -1; + framerate_down_ = false; + ClearSamples(); + } else { + AdjustScale(true); + } } assert(downscale_shift_ >= 0); for (int shift = downscale_shift_; - shift > 0 && (res.width >> 1 >= min_width_) && - (res.height >> 1 >= min_height_); + shift > 0 && (res_.width / 2 >= min_width_) && + (res_.height / 2 >= min_height_); --shift) { - res.width >>= 1; - res.height >>= 1; + res_.width /= 2; + res_.height /= 2; } +} - return res; +QualityScaler::Resolution QualityScaler::GetScaledResolution() const { + return res_; +} + +int QualityScaler::GetTargetFramerate() const { + return target_framerate_; } const VideoFrame& QualityScaler::GetScaledFrame(const VideoFrame& frame) { - Resolution res = GetScaledResolution(frame); + Resolution res = GetScaledResolution(); if (res.width == frame.width()) return frame; diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index c09dffb828..d1710b6dd7 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -25,18 +25,28 @@ static const int kMaxQp = 56; } // namespace class QualityScalerTest : public ::testing::Test { + public: + // Temporal and spatial resolution. + struct Resolution { + int framerate; + int width; + int height; + }; protected: enum ScaleDirection { kScaleDown, kScaleUp }; + enum BadQualityMetric { kDropFrame, kReportLowQP }; QualityScalerTest() { input_frame_.CreateEmptyFrame( kWidth, kHeight, kWidth, kHalfWidth, kHalfWidth); - qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator); + qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, false); qs_.ReportFramerate(kFramerate); + qs_.OnEncodeFrame(input_frame_); } bool TriggerScale(ScaleDirection scale_direction) { - int initial_width = qs_.GetScaledResolution(input_frame_).width; + qs_.OnEncodeFrame(input_frame_); + int initial_width = qs_.GetScaledResolution().width; for (int i = 0; i < kFramerate * kNumSeconds; ++i) { switch (scale_direction) { case kScaleUp: @@ -46,8 +56,8 @@ class QualityScalerTest : public ::testing::Test { qs_.ReportDroppedFrame(); break; } - - if (qs_.GetScaledResolution(input_frame_).width != initial_width) + qs_.OnEncodeFrame(input_frame_); + if (qs_.GetScaledResolution().width != initial_width) return true; } @@ -60,7 +70,8 @@ class QualityScalerTest : public ::testing::Test { } void ExpectScaleUsingReportedResolution() { - QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_); + qs_.OnEncodeFrame(input_frame_); + QualityScaler::Resolution res = qs_.GetScaledResolution(); const VideoFrame& scaled_frame = qs_.GetScaledFrame(input_frame_); EXPECT_EQ(res.width, scaled_frame.width()); EXPECT_EQ(res.height, scaled_frame.height()); @@ -70,6 +81,14 @@ class QualityScalerTest : public ::testing::Test { void DoesNotDownscaleFrameDimensions(int width, int height); + Resolution TriggerResolutionChange(BadQualityMetric dropframe_lowqp, + int num_second, + int initial_framerate); + + void VerifyQualityAdaptation(int initial_framerate, int seconds, + bool expect_spatial_resize, + bool expect_framerate_reduction); + void DownscaleEndsAt(int input_width, int input_height, int end_width, @@ -84,7 +103,8 @@ TEST_F(QualityScalerTest, UsesOriginalFrameInitially) { } TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) { - QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_); + qs_.OnEncodeFrame(input_frame_); + QualityScaler::Resolution res = qs_.GetScaledResolution(); EXPECT_EQ(input_frame_.width(), res.width); EXPECT_EQ(input_frame_.height(), res.height); } @@ -92,7 +112,7 @@ TEST_F(QualityScalerTest, ReportsOriginalResolutionInitially) { TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) { EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds << " seconds."; - QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_); + QualityScaler::Resolution res = qs_.GetScaledResolution(); EXPECT_LT(res.width, input_frame_.width()); EXPECT_LT(res.height, input_frame_.height()); } @@ -102,7 +122,8 @@ TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { qs_.ReportQP(kNormalQp); qs_.ReportDroppedFrame(); qs_.ReportDroppedFrame(); - if (qs_.GetScaledResolution(input_frame_).width < input_frame_.width()) + qs_.OnEncodeFrame(input_frame_); + if (qs_.GetScaledResolution().width < input_frame_.width()) return; } @@ -112,7 +133,8 @@ TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) { for (int i = 0; i < kFramerate * kNumSeconds; ++i) { qs_.ReportQP(kNormalQp); - ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width) + qs_.OnEncodeFrame(input_frame_); + ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) << "Unexpected scale on half framedrop."; } } @@ -120,11 +142,13 @@ TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) { TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) { for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) { qs_.ReportQP(kNormalQp); - ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width) + qs_.OnEncodeFrame(input_frame_); + ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) << "Unexpected scale on half framedrop."; qs_.ReportDroppedFrame(); - ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width) + qs_.OnEncodeFrame(input_frame_); + ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) << "Unexpected scale on half framedrop."; } } @@ -139,7 +163,8 @@ void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() { while (min_dimension >= 2 * QualityScaler::kDefaultMinDownscaleDimension) { EXPECT_TRUE(TriggerScale(kScaleDown)) << "No downscale within " << kNumSeconds << " seconds."; - QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_); + qs_.OnEncodeFrame(input_frame_); + QualityScaler::Resolution res = qs_.GetScaledResolution(); min_dimension = res.width < res.height ? res.width : res.height; ++current_shift; ASSERT_EQ(input_frame_.width() >> current_shift, res.width); @@ -151,7 +176,8 @@ void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() { while (min_dimension < initial_min_dimension) { EXPECT_TRUE(TriggerScale(kScaleUp)) << "No upscale within " << kNumSeconds << " seconds."; - QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_); + qs_.OnEncodeFrame(input_frame_); + QualityScaler::Resolution res = qs_.GetScaledResolution(); min_dimension = res.width < res.height ? res.width : res.height; --current_shift; ASSERT_EQ(input_frame_.width() >> current_shift, res.width); @@ -186,7 +212,8 @@ void QualityScalerTest::DoesNotDownscaleFrameDimensions(int width, int height) { for (int i = 0; i < kFramerate * kNumSeconds; ++i) { qs_.ReportDroppedFrame(); - ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width) + qs_.OnEncodeFrame(input_frame_); + ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution().width) << "Unexpected scale of minimal-size frame."; } } @@ -203,6 +230,94 @@ TEST_F(QualityScalerTest, DoesNotDownscaleFrom1Px) { DoesNotDownscaleFrameDimensions(1, 1); } +QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange( + BadQualityMetric dropframe_lowqp, int num_second, int initial_framerate) { + QualityScalerTest::Resolution res; + res.framerate = initial_framerate; + qs_.OnEncodeFrame(input_frame_); + res.width = qs_.GetScaledResolution().width; + res.height = qs_.GetScaledResolution().height; + for (int i = 0; i < kFramerate * num_second; ++i) { + switch (dropframe_lowqp) { + case kReportLowQP: + qs_.ReportQP(kLowQp); + break; + case kDropFrame: + qs_.ReportDroppedFrame(); + break; + } + qs_.OnEncodeFrame(input_frame_); + // Simulate the case when SetRates is called right after reducing + // framerate. + qs_.ReportFramerate(initial_framerate); + res.framerate = qs_.GetTargetFramerate(); + if (res.framerate != -1) + qs_.ReportFramerate(res.framerate); + res.width = qs_.GetScaledResolution().width; + res.height = qs_.GetScaledResolution().height; + } + return res; +} + +void QualityScalerTest::VerifyQualityAdaptation( + int initial_framerate, int seconds, bool expect_spatial_resize, + bool expect_framerate_reduction) { + qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, true); + qs_.OnEncodeFrame(input_frame_); + int init_width = qs_.GetScaledResolution().width; + int init_height = qs_.GetScaledResolution().height; + + // Test reducing framerate by dropping frame continuously. + QualityScalerTest::Resolution res = TriggerResolutionChange( + kDropFrame, seconds, initial_framerate); + + if (expect_framerate_reduction) { + EXPECT_LT(res.framerate, initial_framerate); + } else { + // No framerate reduction, video decimator should be disabled. + EXPECT_EQ(-1, res.framerate); + } + + if (expect_spatial_resize) { + EXPECT_LT(res.width, init_width); + EXPECT_LT(res.height, init_height); + } else { + EXPECT_EQ(init_width, res.width); + EXPECT_EQ(init_height, res.height); + } + + // The "seconds * 1.5" is to ensure spatial resolution to recover. + // For example, in 10 seconds test, framerate reduction happens in the first + // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the + // original one. Then it will take only 75 samples to downscale (twice in 150 + // samples). So to recover the resolution changes, we need more than 10 + // seconds (i.e, seconds * 1.5). This is because the framerate increases + // before spatial size recovers, so it will take 150 samples to recover + // spatial size (300 for twice). + res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate); + EXPECT_EQ(-1, res.framerate); + EXPECT_EQ(init_width, res.width); + EXPECT_EQ(init_height, res.height); +} + +// In 5 seconds test, only framerate adjusting should happen. +TEST_F(QualityScalerTest, ChangeFramerateOnly) { + VerifyQualityAdaptation(kFramerate, 5, false, true); +} + +// In 10 seconds test, framerate adjusting and scaling are both +// triggered, it shows that scaling would happen after framerate +// adjusting. +TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) { + VerifyQualityAdaptation(kFramerate, 10, true, true); +} + +// When starting from a low framerate, only spatial size will be changed. +TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) { + qs_.ReportFramerate(kFramerate >> 1); + VerifyQualityAdaptation(kFramerate >> 1, 10, true, false); +} + TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) { DoesNotDownscaleFrameDimensions( 2 * QualityScaler::kDefaultMinDownscaleDimension - 1, 1000); @@ -227,7 +342,7 @@ void QualityScalerTest::DownscaleEndsAt(int input_width, // Drop all frames to force-trigger downscaling. while (true) { TriggerScale(kScaleDown); - QualityScaler::Resolution res = qs_.GetScaledResolution(input_frame_); + QualityScaler::Resolution res = qs_.GetScaledResolution(); if (last_width == res.width) { EXPECT_EQ(last_height, res.height); EXPECT_EQ(end_width, res.width); diff --git a/webrtc/modules/video_processing/main/interface/video_processing.h b/webrtc/modules/video_processing/main/interface/video_processing.h index 0cab9fcc58..08b9db7779 100644 --- a/webrtc/modules/video_processing/main/interface/video_processing.h +++ b/webrtc/modules/video_processing/main/interface/video_processing.h @@ -214,6 +214,8 @@ class VideoProcessingModule : public Module { uint32_t height, uint32_t frame_rate) = 0; + virtual void SetTargetFramerate(int frame_rate) {} + /** Get decimated(target) frame rate */ diff --git a/webrtc/modules/video_processing/main/source/frame_preprocessor.cc b/webrtc/modules/video_processing/main/source/frame_preprocessor.cc index cf4bc8113e..a9d77c2e0c 100644 --- a/webrtc/modules/video_processing/main/source/frame_preprocessor.cc +++ b/webrtc/modules/video_processing/main/source/frame_preprocessor.cc @@ -38,7 +38,6 @@ void VPMFramePreprocessor::Reset() { frame_cnt_ = 0; } - void VPMFramePreprocessor::EnableTemporalDecimation(bool enable) { vd_->EnableTemporalDecimation(enable); } @@ -62,12 +61,19 @@ int32_t VPMFramePreprocessor::SetTargetResolution( if (ret_val < 0) return ret_val; - ret_val = vd_->SetTargetFramerate(frame_rate); - if (ret_val < 0) return ret_val; - + vd_->SetTargetFramerate(frame_rate); return VPM_OK; } +void VPMFramePreprocessor::SetTargetFramerate(int frame_rate) { + if (frame_rate == -1) { + vd_->EnableTemporalDecimation(false); + } else { + vd_->EnableTemporalDecimation(true); + vd_->SetTargetFramerate(frame_rate); + } +} + void VPMFramePreprocessor::UpdateIncomingframe_rate() { vd_->UpdateIncomingframe_rate(); } diff --git a/webrtc/modules/video_processing/main/source/frame_preprocessor.h b/webrtc/modules/video_processing/main/source/frame_preprocessor.h index 81e92ed9eb..895e457cc6 100644 --- a/webrtc/modules/video_processing/main/source/frame_preprocessor.h +++ b/webrtc/modules/video_processing/main/source/frame_preprocessor.h @@ -41,6 +41,9 @@ class VPMFramePreprocessor { int32_t SetTargetResolution(uint32_t width, uint32_t height, uint32_t frame_rate); + // Set target frame rate. + void SetTargetFramerate(int frame_rate); + // Update incoming frame rate/dimension. void UpdateIncomingframe_rate(); diff --git a/webrtc/modules/video_processing/main/source/video_decimator.cc b/webrtc/modules/video_processing/main/source/video_decimator.cc index bf05bd7154..449c3bd870 100644 --- a/webrtc/modules/video_processing/main/source/video_decimator.cc +++ b/webrtc/modules/video_processing/main/source/video_decimator.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "webrtc/base/checks.h" #include "webrtc/modules/video_processing/main/interface/video_processing.h" #include "webrtc/modules/video_processing/main/source/video_decimator.h" #include "webrtc/system_wrappers/interface/tick_util.h" @@ -36,11 +37,9 @@ void VPMVideoDecimator::EnableTemporalDecimation(bool enable) { enable_temporal_decimation_ = enable; } -int32_t VPMVideoDecimator::SetTargetFramerate(uint32_t frame_rate) { - if (frame_rate == 0) return VPM_PARAMETER_ERROR; - +void VPMVideoDecimator::SetTargetFramerate(int frame_rate) { + DCHECK(frame_rate); target_frame_rate_ = frame_rate; - return VPM_OK; } bool VPMVideoDecimator::DropFrame() { diff --git a/webrtc/modules/video_processing/main/source/video_decimator.h b/webrtc/modules/video_processing/main/source/video_decimator.h index fca74aeae1..3d4573caf8 100644 --- a/webrtc/modules/video_processing/main/source/video_decimator.h +++ b/webrtc/modules/video_processing/main/source/video_decimator.h @@ -25,7 +25,7 @@ class VPMVideoDecimator { void EnableTemporalDecimation(bool enable); - int32_t SetTargetFramerate(uint32_t frame_rate); + void SetTargetFramerate(int frame_rate); bool DropFrame(); diff --git a/webrtc/modules/video_processing/main/source/video_processing_impl.cc b/webrtc/modules/video_processing/main/source/video_processing_impl.cc index afb8290dc9..80419e85dd 100644 --- a/webrtc/modules/video_processing/main/source/video_processing_impl.cc +++ b/webrtc/modules/video_processing/main/source/video_processing_impl.cc @@ -148,6 +148,11 @@ int32_t VideoProcessingModuleImpl::SetTargetResolution(uint32_t width, return frame_pre_processor_.SetTargetResolution(width, height, frame_rate); } +void VideoProcessingModuleImpl::SetTargetFramerate(int frame_rate) { + CriticalSectionScoped cs(&mutex_); + frame_pre_processor_.SetTargetFramerate(frame_rate); +} + uint32_t VideoProcessingModuleImpl::Decimatedframe_rate() { CriticalSectionScoped cs(&mutex_); return frame_pre_processor_.Decimatedframe_rate(); diff --git a/webrtc/modules/video_processing/main/source/video_processing_impl.h b/webrtc/modules/video_processing/main/source/video_processing_impl.h index b78c019fdb..437d1d6358 100644 --- a/webrtc/modules/video_processing/main/source/video_processing_impl.h +++ b/webrtc/modules/video_processing/main/source/video_processing_impl.h @@ -48,6 +48,8 @@ class VideoProcessingModuleImpl : public VideoProcessingModule { uint32_t height, uint32_t frame_rate) override; + void SetTargetFramerate(int frame_rate) override; + // Get decimated values: frame rate/dimension uint32_t Decimatedframe_rate() override; uint32_t DecimatedWidth() const override; diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc index fd213f8df2..8847a1072e 100644 --- a/webrtc/video/video_encoder.cc +++ b/webrtc/video/video_encoder.cc @@ -134,4 +134,10 @@ bool VideoEncoderSoftwareFallbackWrapper::SupportsNativeHandle() const { return encoder_->SupportsNativeHandle(); } +int VideoEncoderSoftwareFallbackWrapper::GetTargetFramerate() { + if (fallback_encoder_) + return fallback_encoder_->GetTargetFramerate(); + return encoder_->GetTargetFramerate(); +} + } // namespace webrtc diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h index 776b22bba5..a512c98917 100644 --- a/webrtc/video_encoder.h +++ b/webrtc/video_encoder.h @@ -125,6 +125,7 @@ class VideoEncoder { return -1; } virtual void OnDroppedFrame() {} + virtual int GetTargetFramerate() { return -1; } virtual bool SupportsNativeHandle() const { return false; } }; @@ -151,6 +152,7 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; void OnDroppedFrame() override; + int GetTargetFramerate() override; bool SupportsNativeHandle() const override; private: diff --git a/webrtc/video_engine/vie_encoder.cc b/webrtc/video_engine/vie_encoder.cc index 7ed207fd54..0eea5f5506 100644 --- a/webrtc/video_engine/vie_encoder.cc +++ b/webrtc/video_engine/vie_encoder.cc @@ -78,6 +78,9 @@ class QMVideoSettingsCallback : public VCMQMSettingsCallback { const uint32_t width, const uint32_t height); + // Update target frame rate. + void SetTargetFramerate(int frame_rate); + private: VideoProcessingModule* vpm_; }; @@ -868,4 +871,8 @@ int32_t QMVideoSettingsCallback::SetVideoQMSettings( return vpm_->SetTargetResolution(width, height, frame_rate); } +void QMVideoSettingsCallback::SetTargetFramerate(int frame_rate) { + vpm_->SetTargetFramerate(frame_rate); +} + } // namespace webrtc