From 83399caec5762d2dad038b8e9d86163e92c18c9f Mon Sep 17 00:00:00 2001 From: kthelgason Date: Wed, 1 Feb 2017 01:31:52 -0800 Subject: [PATCH] Drop frames until specified bitrate is achieved. This CL fixes a regression introduced with the new quality scaler where the video would no longer start in a scaled mode. This CL adds code that compares incoming captured frames to the target bitrate, and if they are found to be too large, they are dropped and sinkWants set to a lower resolution. The number of dropped frames should be low (0-4 in most cases) and should not introduce a noticeable delay, or at least should be preferrable to having the first 2-4 seconds of video have very low quality. BUG=webrtc:6953 Review-Url: https://codereview.webrtc.org/2630333002 Cr-Commit-Position: refs/heads/master@{#16391} --- webrtc/api/video/video_frame.cc | 4 + webrtc/api/video/video_frame.h | 3 +- .../engine/webrtcvideoengine2_unittest.cc | 41 +++++---- .../default_video_bitrate_allocator.cc | 1 - webrtc/video/vie_encoder.cc | 40 ++++++++- webrtc/video/vie_encoder.h | 4 + webrtc/video/vie_encoder_unittest.cc | 90 +++++++++++++++++-- 7 files changed, 151 insertions(+), 32 deletions(-) diff --git a/webrtc/api/video/video_frame.cc b/webrtc/api/video/video_frame.cc index 0e3efb2be1..b2647461c7 100644 --- a/webrtc/api/video/video_frame.cc +++ b/webrtc/api/video/video_frame.cc @@ -51,6 +51,10 @@ int VideoFrame::height() const { return video_frame_buffer_ ? video_frame_buffer_->height() : 0; } +uint32_t VideoFrame::size() const { + return width() * height(); +} + rtc::scoped_refptr VideoFrame::video_frame_buffer() const { return video_frame_buffer_; } diff --git a/webrtc/api/video/video_frame.h b/webrtc/api/video/video_frame.h index 5c57213f01..8840782cad 100644 --- a/webrtc/api/video/video_frame.h +++ b/webrtc/api/video/video_frame.h @@ -48,9 +48,10 @@ class VideoFrame { // Get frame width. int width() const; - // Get frame height. int height() const; + // Get frame size in pixels. + uint32_t size() const; // System monotonic clock, same timebase as rtc::TimeMicros(). int64_t timestamp_us() const { return timestamp_us_; } diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc index ef77e9e927..22606a6446 100644 --- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc @@ -101,6 +101,12 @@ void VerifySendStreamHasRtxTypes(const webrtc::VideoSendStream::Config& config, it->second == config.rtp.ulpfec.red_rtx_payload_type); } } + +cricket::MediaConfig GetMediaConfig() { + cricket::MediaConfig media_config; + media_config.video.enable_cpu_overuse_detection = false; + return media_config; +} } // namespace namespace cricket { @@ -329,7 +335,7 @@ TEST_F(WebRtcVideoEngine2Test, CVOSetHeaderExtensionAfterCapturer) { TEST_F(WebRtcVideoEngine2Test, SetSendFailsBeforeSettingCodecs) { engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(123))); @@ -342,7 +348,7 @@ TEST_F(WebRtcVideoEngine2Test, SetSendFailsBeforeSettingCodecs) { TEST_F(WebRtcVideoEngine2Test, GetStatsWithoutSendCodecsSetDoesNotCrash) { engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(123))); VideoMediaInfo info; channel->GetStats(&info); @@ -437,7 +443,7 @@ void WebRtcVideoEngine2Test::TestExtendedEncoderOveruse( } else { engine_.Init(); channel.reset( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); } ASSERT_TRUE( channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); @@ -547,7 +553,7 @@ VideoMediaChannel* WebRtcVideoEngine2Test::SetUpForExternalEncoderFactory( engine_.Init(); VideoMediaChannel* channel = - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions()); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions()); cricket::VideoSendParameters parameters; // We need to look up the codec in the engine to get the correct payload type. for (const VideoCodec& codec : encoder_factory->supported_codecs()) @@ -565,7 +571,7 @@ VideoMediaChannel* WebRtcVideoEngine2Test::SetUpForExternalDecoderFactory( engine_.Init(); VideoMediaChannel* channel = - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions()); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions()); cricket::VideoRecvParameters parameters; parameters.codecs = codecs; EXPECT_TRUE(channel->SetRecvParameters(parameters)); @@ -638,7 +644,7 @@ TEST_F(WebRtcVideoEngine2Test, engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); cricket::VideoSendParameters parameters; parameters.codecs.push_back(GetEngineCodec("VP8")); EXPECT_TRUE(channel->SetSendParameters(parameters)); @@ -659,7 +665,7 @@ TEST_F(WebRtcVideoEngine2Test, engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); cricket::VideoSendParameters parameters; parameters.codecs.push_back(GetEngineCodec("VP8")); EXPECT_TRUE(channel->SetSendParameters(parameters)); @@ -699,7 +705,7 @@ TEST_F(WebRtcVideoEngine2Test, engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); cricket::VideoSendParameters parameters; parameters.codecs.push_back(GetEngineCodec("H264")); EXPECT_TRUE(channel->SetSendParameters(parameters)); @@ -962,8 +968,8 @@ class WebRtcVideoChannel2Test : public WebRtcVideoEngine2Test { void SetUp() override { fake_call_.reset(new FakeCall(webrtc::Call::Config(&event_log_))); engine_.Init(); - channel_.reset( - engine_.CreateChannel(fake_call_.get(), MediaConfig(), VideoOptions())); + channel_.reset(engine_.CreateChannel(fake_call_.get(), GetMediaConfig(), + VideoOptions())); channel_->OnReadyToSend(true); last_ssrc_ = 123; send_parameters_.codecs = engine_.codecs(); @@ -1771,7 +1777,7 @@ TEST_F(WebRtcVideoChannel2Test, SuspendBelowMinBitrateDisabledByDefault) { } TEST_F(WebRtcVideoChannel2Test, SetMediaConfigSuspendBelowMinBitrate) { - MediaConfig media_config = MediaConfig(); + MediaConfig media_config = GetMediaConfig(); media_config.video.suspend_below_min_bitrate = true; channel_.reset( @@ -2068,7 +2074,7 @@ TEST_F(WebRtcVideoChannel2Test, AdaptsOnOveruseAndChangeResolution) { cricket::VideoSendParameters parameters; parameters.codecs.push_back(codec); - MediaConfig media_config = MediaConfig(); + MediaConfig media_config = GetMediaConfig(); channel_.reset( engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions())); channel_->OnReadyToSend(true); @@ -2143,7 +2149,8 @@ TEST_F(WebRtcVideoChannel2Test, PreviousAdaptationDoesNotApplyToScreenshare) { cricket::VideoSendParameters parameters; parameters.codecs.push_back(codec); - MediaConfig media_config = MediaConfig(); + MediaConfig media_config = GetMediaConfig(); + media_config.video.enable_cpu_overuse_detection = true; channel_.reset( engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions())); channel_->OnReadyToSend(true); @@ -2208,9 +2215,9 @@ void WebRtcVideoChannel2Test::TestCpuAdaptation(bool enable_overuse, cricket::VideoSendParameters parameters; parameters.codecs.push_back(codec); - MediaConfig media_config = MediaConfig(); - if (!enable_overuse) { - media_config.video.enable_cpu_overuse_detection = false; + MediaConfig media_config = GetMediaConfig(); + if (enable_overuse) { + media_config.video.enable_cpu_overuse_detection = true; } channel_.reset( engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions())); @@ -4041,7 +4048,7 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test { void SetUp() override { engine_.Init(); channel_.reset( - engine_.CreateChannel(&fake_call_, MediaConfig(), VideoOptions())); + engine_.CreateChannel(&fake_call_, GetMediaConfig(), VideoOptions())); channel_->OnReadyToSend(true); last_ssrc_ = 123; } diff --git a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc index 482c8ec3cf..a914e8a44a 100644 --- a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc +++ b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc @@ -35,7 +35,6 @@ BitrateAllocation DefaultVideoBitrateAllocator::GetAllocation( } else { allocation.SetBitrate(0, 0, total_bitrate_bps); } - return allocation; } diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 01e64feebd..5c312bc667 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -42,6 +42,10 @@ const int kMinPixelsPerFrame = 320 * 180; const int kMinPixelsPerFrame = 120 * 90; #endif +// The maximum number of frames to drop at beginning of stream +// to try and achieve desired bitrate. +const int kMaxInitialFramedrop = 4; + // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle // pipelining encoders better (multiple input frames before something comes // out). This should effectively turn off CPU adaptations for systems that @@ -55,6 +59,17 @@ CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { return options; } +uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) { + if (kbps > 0) { + if (kbps < 300 /* qvga */) { + return 320 * 240; + } else if (kbps < 500 /* vga */) { + return 640 * 480; + } + } + return std::numeric_limits::max(); +} + } // namespace class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask { @@ -244,6 +259,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, EncodedFrameObserver* encoder_timing) : shutdown_event_(true /* manual_reset */, false), number_of_cores_(number_of_cores), + initial_rampup_(0), source_proxy_(new VideoSourceProxy(this)), sink_(nullptr), settings_(settings), @@ -340,8 +356,8 @@ void ViEEncoder::SetSource( RTC_DCHECK_RUN_ON(&encoder_queue_); scaling_enabled_ = (degradation_preference != VideoSendStream::DegradationPreference::kMaintainResolution); - stats_proxy_->SetResolutionRestrictionStats( - scaling_enabled_, scale_counter_[kCpu] > 0, scale_counter_[kQuality]); + initial_rampup_ = scaling_enabled_ ? 0 : kMaxInitialFramedrop; + ConfigureQualityScaler(); }); } @@ -436,8 +452,14 @@ void ViEEncoder::ReconfigureEncoder() { sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); + ConfigureQualityScaler(); +} + +void ViEEncoder::ConfigureQualityScaler() { + RTC_DCHECK_RUN_ON(&encoder_queue_); const auto scaling_settings = settings_.encoder->GetScalingSettings(); if (scaling_enabled_ && scaling_settings.enabled) { + // Drop frames and scale down until desired quality is achieved. if (scaling_settings.thresholds) { quality_scaler_.reset( new QualityScaler(this, *(scaling_settings.thresholds))); @@ -446,9 +468,9 @@ void ViEEncoder::ReconfigureEncoder() { } } else { quality_scaler_.reset(nullptr); - stats_proxy_->SetResolutionRestrictionStats( - false, scale_counter_[kCpu] > 0, scale_counter_[kQuality]); } + stats_proxy_->SetResolutionRestrictionStats( + scaling_enabled_, scale_counter_[kCpu] > 0, scale_counter_[kQuality]); } void ViEEncoder::OnFrame(const VideoFrame& video_frame) { @@ -547,6 +569,16 @@ void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, << ", texture=" << last_frame_info_->is_texture; } + if (initial_rampup_ < kMaxInitialFramedrop && + video_frame.size() > + MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000)) { + LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; + ScaleDown(kQuality); + ++initial_rampup_; + return; + } + initial_rampup_ = kMaxInitialFramedrop; + int64_t now_ms = clock_->TimeInMilliseconds(); if (pending_encoder_reconfiguration_) { ReconfigureEncoder(); diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 4a2ca1f301..ed514c20ed 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -150,6 +150,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, bool nack_enabled); void ReconfigureEncoder(); + void ConfigureQualityScaler(); + // Implements VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; @@ -175,6 +177,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, rtc::Event shutdown_event_; const uint32_t number_of_cores_; + // Counts how many frames we've dropped in the initial rampup phase. + int initial_rampup_; const std::unique_ptr source_proxy_; EncoderSink* sink_; diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc index dae172dea3..5dee314683 100644 --- a/webrtc/video/vie_encoder_unittest.cc +++ b/webrtc/video/vie_encoder_unittest.cc @@ -43,7 +43,9 @@ using ::testing::Return; namespace { const size_t kMaxPayloadLength = 1440; -const int kTargetBitrateBps = 100000; +const int kTargetBitrateBps = 1000000; +const int kLowTargetBitrateBps = kTargetBitrateBps / 10; +const int kMaxInitialFramedrop = 4; class TestBuffer : public webrtc::I420Buffer { public: @@ -149,7 +151,7 @@ class ViEEncoderTest : public ::testing::Test { vie_encoder_->SetSink(&sink_, false /* rotation_applied */); vie_encoder_->SetSource(&video_source_, VideoSendStream::DegradationPreference::kBalanced); - vie_encoder_->SetStartBitrate(10000); + vie_encoder_->SetStartBitrate(kTargetBitrateBps); vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), kMaxPayloadLength, nack_enabled); } @@ -162,7 +164,7 @@ class ViEEncoderTest : public ::testing::Test { VideoEncoderConfig video_encoder_config; video_encoder_config.number_of_streams = num_streams; - video_encoder_config.max_bitrate_bps = 1000000; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; video_encoder_config.video_stream_factory = new rtc::RefCountedObject(num_temporal_layers); ConfigureEncoder(std::move(video_encoder_config), nack_enabled); @@ -263,6 +265,8 @@ class ViEEncoderTest : public ::testing::Test { test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp); } + void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(20)); } + void SetExpectNoFrames() { rtc::CritScope lock(&crit_); expect_frames_ = false; @@ -679,7 +683,6 @@ TEST_F(ViEEncoderTest, StatsTracksAdaptationStats) { } TEST_F(ViEEncoderTest, SwitchingSourceKeepsCpuAdaptation) { - const int kTargetBitrateBps = 100000; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); int frame_width = 1280; @@ -749,7 +752,6 @@ TEST_F(ViEEncoderTest, SwitchingSourceKeepsCpuAdaptation) { } TEST_F(ViEEncoderTest, SwitchingSourceKeepsQualityAdaptation) { - const int kTargetBitrateBps = 100000; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); int frame_width = 1280; @@ -885,7 +887,6 @@ TEST_F(ViEEncoderTest, StatsTracksPreferredBitrate) { } TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) { - const int kTargetBitrateBps = 100000; int frame_width = 1280; int frame_height = 720; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); @@ -938,7 +939,6 @@ TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) { } TEST_F(ViEEncoderTest, DoesNotScaleBelowSetLimit) { - const int kTargetBitrateBps = 100000; int frame_width = 1280; int frame_height = 720; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); @@ -995,12 +995,12 @@ TEST_F(ViEEncoderTest, CallsBitrateObserver) { const int kDefaultFps = 30; const BitrateAllocation expected_bitrate = DefaultVideoBitrateAllocator(fake_encoder_.codec_config()) - .GetAllocation(kTargetBitrateBps, kDefaultFps); + .GetAllocation(kLowTargetBitrateBps, kDefaultFps); // First called on bitrate updated, then again on first frame. EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate)) .Times(2); - vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + vie_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0); const int64_t kStartTimeMs = 1; video_source_.IncomingCapturedFrame( @@ -1029,4 +1029,76 @@ TEST_F(ViEEncoderTest, CallsBitrateObserver) { vie_encoder_->Stop(); } +TEST_F(ViEEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) { + vie_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0); + int frame_width = 640; + int frame_height = 360; + + video_source_.IncomingCapturedFrame( + CreateFrame(1, frame_width, frame_height)); + + // Expect to drop this frame, the wait should time out. + sink_.ExpectDroppedFrame(); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_TRUE(video_source_.sink_wants().max_pixel_count); + EXPECT_LT(*video_source_.sink_wants().max_pixel_count, 1000 * 1000); + + int last_pixel_count = *video_source_.sink_wants().max_pixel_count; + + // Next frame is scaled + video_source_.IncomingCapturedFrame( + CreateFrame(2, frame_width * 3 / 4, frame_height * 3 / 4)); + + // Expect to drop this frame, the wait should time out. + sink_.ExpectDroppedFrame(); + + EXPECT_LT(*video_source_.sink_wants().max_pixel_count, last_pixel_count); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, NrOfDroppedFramesLimited) { + // 1kbps. This can never be achieved. + vie_encoder_->OnBitrateUpdated(1000, 0, 0); + int frame_width = 640; + int frame_height = 360; + + // We expect the n initial frames to get dropped. + int i; + for (i = 1; i <= kMaxInitialFramedrop; ++i) { + video_source_.IncomingCapturedFrame( + CreateFrame(i, frame_width, frame_height)); + sink_.ExpectDroppedFrame(); + } + // The n+1th frame should not be dropped, even though it's size is too large. + video_source_.IncomingCapturedFrame( + CreateFrame(i, frame_width, frame_height)); + sink_.WaitForEncodedFrame(i); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_TRUE(video_source_.sink_wants().max_pixel_count); + EXPECT_LT(*video_source_.sink_wants().max_pixel_count, 1000 * 1000); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, InitialFrameDropOffWithMaintainResolutionPreference) { + int frame_width = 640; + int frame_height = 360; + vie_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0); + + // Set degradation preference. + vie_encoder_->SetSource( + &video_source_, + VideoSendStream::DegradationPreference::kMaintainResolution); + + video_source_.IncomingCapturedFrame( + CreateFrame(1, frame_width, frame_height)); + // Frame should not be dropped, even if it's too large. + sink_.WaitForEncodedFrame(1); + + vie_encoder_->Stop(); +} + } // namespace webrtc