diff --git a/webrtc/api/video/video_frame.cc b/webrtc/api/video/video_frame.cc index 0e3efb2be1..b2647461c7 100644 --- a/webrtc/api/video/video_frame.cc +++ b/webrtc/api/video/video_frame.cc @@ -51,6 +51,10 @@ int VideoFrame::height() const { return video_frame_buffer_ ? video_frame_buffer_->height() : 0; } +uint32_t VideoFrame::size() const { + return width() * height(); +} + rtc::scoped_refptr VideoFrame::video_frame_buffer() const { return video_frame_buffer_; } diff --git a/webrtc/api/video/video_frame.h b/webrtc/api/video/video_frame.h index 5c57213f01..8840782cad 100644 --- a/webrtc/api/video/video_frame.h +++ b/webrtc/api/video/video_frame.h @@ -48,9 +48,10 @@ class VideoFrame { // Get frame width. int width() const; - // Get frame height. int height() const; + // Get frame size in pixels. + uint32_t size() const; // System monotonic clock, same timebase as rtc::TimeMicros(). int64_t timestamp_us() const { return timestamp_us_; } diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc index 4b73faa92a..06973e6f45 100644 --- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc @@ -102,6 +102,12 @@ void VerifySendStreamHasRtxTypes(const webrtc::VideoSendStream::Config& config, it->second == config.rtp.ulpfec.red_rtx_payload_type); } } + +cricket::MediaConfig GetMediaConfig() { + cricket::MediaConfig media_config; + media_config.video.enable_cpu_overuse_detection = false; + return media_config; +} } // namespace namespace cricket { @@ -330,7 +336,7 @@ TEST_F(WebRtcVideoEngine2Test, CVOSetHeaderExtensionAfterCapturer) { TEST_F(WebRtcVideoEngine2Test, SetSendFailsBeforeSettingCodecs) { engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(123))); @@ -343,7 +349,7 @@ TEST_F(WebRtcVideoEngine2Test, SetSendFailsBeforeSettingCodecs) { TEST_F(WebRtcVideoEngine2Test, GetStatsWithoutSendCodecsSetDoesNotCrash) { engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); EXPECT_TRUE(channel->AddSendStream(StreamParams::CreateLegacy(123))); VideoMediaInfo info; channel->GetStats(&info); @@ -438,7 +444,7 @@ void WebRtcVideoEngine2Test::TestExtendedEncoderOveruse( } else { engine_.Init(); channel.reset( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); } ASSERT_TRUE( channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); @@ -548,7 +554,7 @@ VideoMediaChannel* WebRtcVideoEngine2Test::SetUpForExternalEncoderFactory( engine_.Init(); VideoMediaChannel* channel = - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions()); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions()); cricket::VideoSendParameters parameters; // We need to look up the codec in the engine to get the correct payload type. for (const VideoCodec& codec : encoder_factory->supported_codecs()) @@ -566,7 +572,7 @@ VideoMediaChannel* WebRtcVideoEngine2Test::SetUpForExternalDecoderFactory( engine_.Init(); VideoMediaChannel* channel = - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions()); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions()); cricket::VideoRecvParameters parameters; parameters.codecs = codecs; EXPECT_TRUE(channel->SetRecvParameters(parameters)); @@ -639,7 +645,7 @@ TEST_F(WebRtcVideoEngine2Test, engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); cricket::VideoSendParameters parameters; parameters.codecs.push_back(GetEngineCodec("VP8")); EXPECT_TRUE(channel->SetSendParameters(parameters)); @@ -660,7 +666,7 @@ TEST_F(WebRtcVideoEngine2Test, engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); cricket::VideoSendParameters parameters; parameters.codecs.push_back(GetEngineCodec("VP8")); EXPECT_TRUE(channel->SetSendParameters(parameters)); @@ -700,7 +706,7 @@ TEST_F(WebRtcVideoEngine2Test, engine_.Init(); std::unique_ptr channel( - engine_.CreateChannel(call_.get(), MediaConfig(), VideoOptions())); + engine_.CreateChannel(call_.get(), GetMediaConfig(), VideoOptions())); cricket::VideoSendParameters parameters; parameters.codecs.push_back(GetEngineCodec("H264")); EXPECT_TRUE(channel->SetSendParameters(parameters)); @@ -963,8 +969,8 @@ class WebRtcVideoChannel2Test : public WebRtcVideoEngine2Test { void SetUp() override { fake_call_.reset(new FakeCall(webrtc::Call::Config(&event_log_))); engine_.Init(); - channel_.reset( - engine_.CreateChannel(fake_call_.get(), MediaConfig(), VideoOptions())); + channel_.reset(engine_.CreateChannel(fake_call_.get(), GetMediaConfig(), + VideoOptions())); channel_->OnReadyToSend(true); last_ssrc_ = 123; send_parameters_.codecs = engine_.codecs(); @@ -1773,7 +1779,7 @@ TEST_F(WebRtcVideoChannel2Test, SuspendBelowMinBitrateDisabledByDefault) { } TEST_F(WebRtcVideoChannel2Test, SetMediaConfigSuspendBelowMinBitrate) { - MediaConfig media_config = MediaConfig(); + MediaConfig media_config = GetMediaConfig(); media_config.video.suspend_below_min_bitrate = true; channel_.reset( @@ -2070,7 +2076,7 @@ TEST_F(WebRtcVideoChannel2Test, AdaptsOnOveruseAndChangeResolution) { cricket::VideoSendParameters parameters; parameters.codecs.push_back(codec); - MediaConfig media_config = MediaConfig(); + MediaConfig media_config = GetMediaConfig(); channel_.reset( engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions())); channel_->OnReadyToSend(true); @@ -2145,7 +2151,8 @@ TEST_F(WebRtcVideoChannel2Test, PreviousAdaptationDoesNotApplyToScreenshare) { cricket::VideoSendParameters parameters; parameters.codecs.push_back(codec); - MediaConfig media_config = MediaConfig(); + MediaConfig media_config = GetMediaConfig(); + media_config.video.enable_cpu_overuse_detection = true; channel_.reset( engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions())); channel_->OnReadyToSend(true); @@ -2210,9 +2217,9 @@ void WebRtcVideoChannel2Test::TestCpuAdaptation(bool enable_overuse, cricket::VideoSendParameters parameters; parameters.codecs.push_back(codec); - MediaConfig media_config = MediaConfig(); - if (!enable_overuse) { - media_config.video.enable_cpu_overuse_detection = false; + MediaConfig media_config = GetMediaConfig(); + if (enable_overuse) { + media_config.video.enable_cpu_overuse_detection = true; } channel_.reset( engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions())); @@ -4061,7 +4068,7 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test { void SetUp() override { engine_.Init(); channel_.reset( - engine_.CreateChannel(&fake_call_, MediaConfig(), VideoOptions())); + engine_.CreateChannel(&fake_call_, GetMediaConfig(), VideoOptions())); channel_->OnReadyToSend(true); last_ssrc_ = 123; } diff --git a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc index 482c8ec3cf..a914e8a44a 100644 --- a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc +++ b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc @@ -35,7 +35,6 @@ BitrateAllocation DefaultVideoBitrateAllocator::GetAllocation( } else { allocation.SetBitrate(0, 0, total_bitrate_bps); } - return allocation; } diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index a419195c07..9c734604ec 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -329,12 +329,10 @@ class VideoAnalyzer : public PacketReceiver, // No previous frame rendered, this one was dropped after sending but // before rendering. ++dropped_frames_before_rendering_; - frames_.pop_front(); - RTC_CHECK(!frames_.empty()); - continue; + } else { + AddFrameComparison(frames_.front(), *last_rendered_frame_, true, + render_time_ms); } - AddFrameComparison(frames_.front(), *last_rendered_frame_, true, - render_time_ms); frames_.pop_front(); RTC_DCHECK(!frames_.empty()); } @@ -1196,6 +1194,8 @@ void VideoQualityTest::SetupScreenshare() { // Fill out codec settings. video_encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen; + degradation_preference_ = + VideoSendStream::DegradationPreference::kMaintainResolution; if (params_.video.codec == "VP8") { VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); vp8_settings.denoisingOn = false; @@ -1360,9 +1360,9 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { analyzer.SetSendStream(video_send_stream_); if (video_receive_streams_.size() == 1) analyzer.SetReceiveStream(video_receive_streams_[0]); - video_send_stream_->SetSource( - analyzer.OutputInterface(), - VideoSendStream::DegradationPreference::kBalanced); + + video_send_stream_->SetSource(analyzer.OutputInterface(), + degradation_preference_); CreateCapturer(); rtc::VideoSinkWants wants; @@ -1504,9 +1504,8 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { video_receive_stream = call->CreateVideoReceiveStream( video_receive_configs_[stream_id].Copy()); CreateCapturer(); - video_send_stream_->SetSource( - video_capturer_.get(), - VideoSendStream::DegradationPreference::kBalanced); + video_send_stream_->SetSource(video_capturer_.get(), + degradation_preference_); } AudioReceiveStream* audio_receive_stream = nullptr; diff --git a/webrtc/video/video_quality_test.h b/webrtc/video/video_quality_test.h index b741ad331a..89432fb538 100644 --- a/webrtc/video/video_quality_test.h +++ b/webrtc/video/video_quality_test.h @@ -131,6 +131,8 @@ class VideoQualityTest : public test::CallTest { int receive_logs_; int send_logs_; + VideoSendStream::DegradationPreference degradation_preference_ = + VideoSendStream::DegradationPreference::kBalanced; Params params_; }; diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 4ee6b8f127..a8d0500eb5 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -44,6 +44,10 @@ const int kMinPixelsPerFrame = 320 * 180; const int kMinPixelsPerFrame = 120 * 90; #endif +// The maximum number of frames to drop at beginning of stream +// to try and achieve desired bitrate. +const int kMaxInitialFramedrop = 4; + // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle // pipelining encoders better (multiple input frames before something comes // out). This should effectively turn off CPU adaptations for systems that @@ -57,6 +61,17 @@ CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { return options; } +uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) { + if (kbps > 0) { + if (kbps < 300 /* qvga */) { + return 320 * 240; + } else if (kbps < 500 /* vga */) { + return 640 * 480; + } + } + return std::numeric_limits::max(); +} + } // namespace class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask { @@ -242,6 +257,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, EncodedFrameObserver* encoder_timing) : shutdown_event_(true /* manual_reset */, false), number_of_cores_(number_of_cores), + initial_rampup_(0), source_proxy_(new VideoSourceProxy(this)), sink_(nullptr), settings_(settings), @@ -339,10 +355,11 @@ void ViEEncoder::SetSource( RTC_DCHECK_RUN_ON(&encoder_queue_); degradation_preference_ = degradation_preference; - stats_proxy_->SetResolutionRestrictionStats( - degradation_preference != - VideoSendStream::DegradationPreference::kMaintainResolution, - scale_counter_[kCpu] > 0, scale_counter_[kQuality]); + initial_rampup_ = + degradation_preference_ != DegradationPreference::kMaintainResolution + ? 0 + : kMaxInitialFramedrop; + ConfigureQualityScaler(); }); } @@ -437,9 +454,16 @@ void ViEEncoder::ReconfigureEncoder() { sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); + ConfigureQualityScaler(); +} + +void ViEEncoder::ConfigureQualityScaler() { + RTC_DCHECK_RUN_ON(&encoder_queue_); const auto scaling_settings = settings_.encoder->GetScalingSettings(); - if (degradation_preference_ != DegradationPreference::kMaintainResolution && - scaling_settings.enabled) { + const bool degradation_preference_allows_scaling = + degradation_preference_ != DegradationPreference::kMaintainResolution; + if (degradation_preference_allows_scaling && scaling_settings.enabled) { + // Drop frames and scale down until desired quality is achieved. if (scaling_settings.thresholds) { quality_scaler_.reset( new QualityScaler(this, *(scaling_settings.thresholds))); @@ -448,9 +472,10 @@ void ViEEncoder::ReconfigureEncoder() { } } else { quality_scaler_.reset(nullptr); - stats_proxy_->SetResolutionRestrictionStats( - false, scale_counter_[kCpu] > 0, scale_counter_[kQuality]); } + stats_proxy_->SetResolutionRestrictionStats( + degradation_preference_allows_scaling, scale_counter_[kCpu] > 0, + scale_counter_[kQuality]); } void ViEEncoder::OnFrame(const VideoFrame& video_frame) { @@ -549,6 +574,16 @@ void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, << ", texture=" << last_frame_info_->is_texture; } + if (initial_rampup_ < kMaxInitialFramedrop && + video_frame.size() > + MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000)) { + LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; + AdaptDown(kQuality); + ++initial_rampup_; + return; + } + initial_rampup_ = kMaxInitialFramedrop; + int64_t now_ms = clock_->TimeInMilliseconds(); if (pending_encoder_reconfiguration_) { ReconfigureEncoder(); diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 0c564724d4..f4c3e6f53e 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -150,6 +150,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, bool nack_enabled); void ReconfigureEncoder(); + void ConfigureQualityScaler(); + // Implements VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; @@ -175,6 +177,8 @@ class ViEEncoder : public rtc::VideoSinkInterface, rtc::Event shutdown_event_; const uint32_t number_of_cores_; + // Counts how many frames we've dropped in the initial rampup phase. + int initial_rampup_; const std::unique_ptr source_proxy_; EncoderSink* sink_; diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc index 427796ed5e..09d9a97d8c 100644 --- a/webrtc/video/vie_encoder_unittest.cc +++ b/webrtc/video/vie_encoder_unittest.cc @@ -44,7 +44,9 @@ using ::testing::Return; namespace { const size_t kMaxPayloadLength = 1440; -const int kTargetBitrateBps = 100000; +const int kTargetBitrateBps = 1000000; +const int kLowTargetBitrateBps = kTargetBitrateBps / 10; +const int kMaxInitialFramedrop = 4; class TestBuffer : public webrtc::I420Buffer { public: @@ -196,7 +198,7 @@ class ViEEncoderTest : public ::testing::Test { vie_encoder_->SetSink(&sink_, false /* rotation_applied */); vie_encoder_->SetSource(&video_source_, VideoSendStream::DegradationPreference::kBalanced); - vie_encoder_->SetStartBitrate(10000); + vie_encoder_->SetStartBitrate(kTargetBitrateBps); vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), kMaxPayloadLength, nack_enabled); } @@ -209,7 +211,7 @@ class ViEEncoderTest : public ::testing::Test { VideoEncoderConfig video_encoder_config; video_encoder_config.number_of_streams = num_streams; - video_encoder_config.max_bitrate_bps = 1000000; + video_encoder_config.max_bitrate_bps = kTargetBitrateBps; video_encoder_config.video_stream_factory = new rtc::RefCountedObject(num_temporal_layers); ConfigureEncoder(std::move(video_encoder_config), nack_enabled); @@ -324,6 +326,8 @@ class ViEEncoderTest : public ::testing::Test { EXPECT_EQ(expected_width, width); } + void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(20)); } + void SetExpectNoFrames() { rtc::CritScope lock(&crit_); expect_frames_ = false; @@ -744,7 +748,6 @@ TEST_F(ViEEncoderTest, StatsTracksAdaptationStats) { } TEST_F(ViEEncoderTest, SwitchingSourceKeepsCpuAdaptation) { - const int kTargetBitrateBps = 100000; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); int frame_width = 1280; @@ -814,7 +817,6 @@ TEST_F(ViEEncoderTest, SwitchingSourceKeepsCpuAdaptation) { } TEST_F(ViEEncoderTest, SwitchingSourceKeepsQualityAdaptation) { - const int kTargetBitrateBps = 100000; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); int frame_width = 1280; @@ -950,7 +952,6 @@ TEST_F(ViEEncoderTest, StatsTracksPreferredBitrate) { } TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) { - const int kTargetBitrateBps = 100000; int frame_width = 1280; int frame_height = 720; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); @@ -1003,7 +1004,6 @@ TEST_F(ViEEncoderTest, ScalingUpAndDownDoesNothingWithMaintainResolution) { } TEST_F(ViEEncoderTest, DoesNotScaleBelowSetLimit) { - const int kTargetBitrateBps = 100000; int frame_width = 1280; int frame_height = 720; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); @@ -1060,12 +1060,12 @@ TEST_F(ViEEncoderTest, CallsBitrateObserver) { const int kDefaultFps = 30; const BitrateAllocation expected_bitrate = DefaultVideoBitrateAllocator(fake_encoder_.codec_config()) - .GetAllocation(kTargetBitrateBps, kDefaultFps); + .GetAllocation(kLowTargetBitrateBps, kDefaultFps); // First called on bitrate updated, then again on first frame. EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate)) .Times(2); - vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + vie_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0); const int64_t kStartTimeMs = 1; video_source_.IncomingCapturedFrame( @@ -1094,6 +1094,78 @@ TEST_F(ViEEncoderTest, CallsBitrateObserver) { vie_encoder_->Stop(); } +TEST_F(ViEEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) { + vie_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0); + int frame_width = 640; + int frame_height = 360; + + video_source_.IncomingCapturedFrame( + CreateFrame(1, frame_width, frame_height)); + + // Expect to drop this frame, the wait should time out. + sink_.ExpectDroppedFrame(); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_TRUE(video_source_.sink_wants().max_pixel_count); + EXPECT_LT(*video_source_.sink_wants().max_pixel_count, 1000 * 1000); + + int last_pixel_count = *video_source_.sink_wants().max_pixel_count; + + // Next frame is scaled + video_source_.IncomingCapturedFrame( + CreateFrame(2, frame_width * 3 / 4, frame_height * 3 / 4)); + + // Expect to drop this frame, the wait should time out. + sink_.ExpectDroppedFrame(); + + EXPECT_LT(*video_source_.sink_wants().max_pixel_count, last_pixel_count); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, NrOfDroppedFramesLimited) { + // 1kbps. This can never be achieved. + vie_encoder_->OnBitrateUpdated(1000, 0, 0); + int frame_width = 640; + int frame_height = 360; + + // We expect the n initial frames to get dropped. + int i; + for (i = 1; i <= kMaxInitialFramedrop; ++i) { + video_source_.IncomingCapturedFrame( + CreateFrame(i, frame_width, frame_height)); + sink_.ExpectDroppedFrame(); + } + // The n+1th frame should not be dropped, even though it's size is too large. + video_source_.IncomingCapturedFrame( + CreateFrame(i, frame_width, frame_height)); + sink_.WaitForEncodedFrame(i); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_TRUE(video_source_.sink_wants().max_pixel_count); + EXPECT_LT(*video_source_.sink_wants().max_pixel_count, 1000 * 1000); + + vie_encoder_->Stop(); +} + +TEST_F(ViEEncoderTest, InitialFrameDropOffWithMaintainResolutionPreference) { + int frame_width = 640; + int frame_height = 360; + vie_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0); + + // Set degradation preference. + vie_encoder_->SetSource( + &video_source_, + VideoSendStream::DegradationPreference::kMaintainResolution); + + video_source_.IncomingCapturedFrame( + CreateFrame(1, frame_width, frame_height)); + // Frame should not be dropped, even if it's too large. + sink_.WaitForEncodedFrame(1); + + vie_encoder_->Stop(); +} + // TODO(sprang): Extend this with fps throttling and any "balanced" extensions. TEST_F(ViEEncoderTest, AdaptsResolutionOnOveruse) { vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);