From 1387476dc684f54cbf8115113e510633eea269a3 Mon Sep 17 00:00:00 2001 From: asapersson Date: Wed, 7 Jun 2017 00:01:02 -0700 Subject: [PATCH] Compare adapt up/down request with sink_wants_ in VideoSourceProxy methods to make sure it is higher/lower than last request. Add methods RestrictFramerate, IncreaseFramerate. To be used by kBalanced mode. This CL is split from: https://codereview.webrtc.org/2887303003/ BUG=webrtc:7607 Review-Url: https://codereview.webrtc.org/2903563002 Cr-Commit-Position: refs/heads/master@{#18463} --- webrtc/video/vie_encoder.cc | 153 +++++++++++++++------------ webrtc/video/vie_encoder_unittest.cc | 20 ++-- 2 files changed, 100 insertions(+), 73 deletions(-) diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index b56bbf5153..203d1c405a 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -230,54 +230,50 @@ class ViEEncoder::VideoSourceProxy { bool RequestResolutionLowerThan(int pixel_count) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); - if (!IsResolutionScalingEnabled(degradation_preference_)) { + if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { // This can happen since |degradation_preference_| is set on libjingle's // worker thread but the adaptation is done on the encoder task queue. return false; } - // The input video frame size will have a resolution with less than or - // equal to |max_pixel_count| depending on how the source can scale the - // input frame size. + // The input video frame size will have a resolution less than or equal to + // |max_pixel_count| depending on how the source can scale the frame size. const int pixels_wanted = (pixel_count * 3) / 5; - if (pixels_wanted < kMinPixelsPerFrame) + if (pixels_wanted < kMinPixelsPerFrame || + pixels_wanted >= sink_wants_.max_pixel_count) { return false; - + } + LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted; sink_wants_.max_pixel_count = pixels_wanted; sink_wants_.target_pixel_count = rtc::Optional(); - if (source_) - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); return true; } - void RequestFramerateLowerThan(int framerate_fps) { + bool RequestFramerateLowerThan(int fps) { // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!IsFramerateScalingEnabled(degradation_preference_)) { - // This can happen since |degradation_preference_| is set on libjingle's - // worker thread but the adaptation is done on the encoder task queue. - return; - } - // The input video frame rate will be scaled down to 2/3 of input fps, - // rounding down. - const int framerate_wanted = - std::max(kMinFramerateFps, (framerate_fps * 2) / 3); - sink_wants_.max_framerate_fps = framerate_wanted; - if (source_) - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + // The input video frame rate will be scaled down to 2/3, rounding down. + return RestrictFramerate((fps * 2) / 3); } - void RequestHigherResolutionThan(int pixel_count) { + bool RequestHigherResolutionThan(int pixel_count) { + // Called on the encoder task queue. rtc::CritScope lock(&crit_); - if (!IsResolutionScalingEnabled(degradation_preference_)) { + if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { // This can happen since |degradation_preference_| is set on libjingle's // worker thread but the adaptation is done on the encoder task queue. - return; + return false; } + int max_pixels_wanted = pixel_count; + if (max_pixels_wanted != std::numeric_limits::max()) + max_pixels_wanted = pixel_count * 4; - if (pixel_count == std::numeric_limits::max()) { + if (max_pixels_wanted <= sink_wants_.max_pixel_count) + return false; + + sink_wants_.max_pixel_count = max_pixels_wanted; + if (max_pixels_wanted == std::numeric_limits::max()) { // Remove any constraints. sink_wants_.target_pixel_count.reset(); - sink_wants_.max_pixel_count = std::numeric_limits::max(); } else { // On step down we request at most 3/5 the pixel count of the previous // resolution, so in order to take "one step up" we request a resolution @@ -287,31 +283,52 @@ class ViEEncoder::VideoSourceProxy { // most four time the current number of pixels. sink_wants_.target_pixel_count = rtc::Optional((pixel_count * 5) / 3); - sink_wants_.max_pixel_count = pixel_count * 4; } - if (source_) - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + return true; } - void RequestHigherFramerateThan(int framerate_fps) { + bool RequestHigherFramerateThan(int fps) { + // Called on the encoder task queue. + // The input frame rate will be scaled up to the last step, with rounding. + int framerate_wanted = fps; + if (fps != std::numeric_limits::max()) + framerate_wanted = (fps * 3) / 2; + + return IncreaseFramerate(framerate_wanted); + } + + bool RestrictFramerate(int fps) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); - if (!IsFramerateScalingEnabled(degradation_preference_)) { - // This can happen since |degradation_preference_| is set on libjingle's - // worker thread but the adaptation is done on the encoder task queue. - return; - } - if (framerate_fps == std::numeric_limits::max()) { - // Remove any restrains. - sink_wants_.max_framerate_fps = std::numeric_limits::max(); - } else { - // The input video frame rate will be scaled up to the last step, with - // rounding. - const int framerate_wanted = (framerate_fps * 3) / 2; - sink_wants_.max_framerate_fps = framerate_wanted; - } - if (source_) - source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) + return false; + + const int fps_wanted = std::max(kMinFramerateFps, fps); + if (fps_wanted >= sink_wants_.max_framerate_fps) + return false; + + LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; + sink_wants_.max_framerate_fps = fps_wanted; + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + return true; + } + + bool IncreaseFramerate(int fps) { + // Called on the encoder task queue. + rtc::CritScope lock(&crit_); + if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) + return false; + + const int fps_wanted = std::max(kMinFramerateFps, fps); + if (fps_wanted <= sink_wants_.max_framerate_fps) + return false; + + LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; + sink_wants_.max_framerate_fps = fps_wanted; + source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants()); + return true; } private: @@ -827,17 +844,19 @@ void ViEEncoder::AdaptDown(AdaptReason reason) { case VideoSendStream::DegradationPreference::kBalanced: FALLTHROUGH(); case VideoSendStream::DegradationPreference::kMaintainFramerate: + // Scale down resolution. if (!source_proxy_->RequestResolutionLowerThan( adaptation_request.input_pixel_count_)) { return; } - LOG(LS_INFO) << "Scaling down resolution."; GetAdaptCounter().IncrementResolution(reason, 1); break; case VideoSendStream::DegradationPreference::kMaintainResolution: - source_proxy_->RequestFramerateLowerThan( - adaptation_request.framerate_fps_); - LOG(LS_INFO) << "Scaling down framerate."; + // Scale down framerate. + if (!source_proxy_->RequestFramerateLowerThan( + adaptation_request.framerate_fps_)) { + return; + } GetAdaptCounter().IncrementFramerate(reason, 1); break; case VideoSendStream::DegradationPreference::kDegradationDisabled: @@ -892,30 +911,30 @@ void ViEEncoder::AdaptUp(AdaptReason reason) { switch (degradation_preference_) { case VideoSendStream::DegradationPreference::kBalanced: FALLTHROUGH(); - case VideoSendStream::DegradationPreference::kMaintainFramerate: - if (adapt_counter.TotalCount() == 1) { + case VideoSendStream::DegradationPreference::kMaintainFramerate: { + // Scale up resolution. + int pixel_count = adaptation_request.input_pixel_count_; + if (adapt_counter.ResolutionCount() == 1) { LOG(LS_INFO) << "Removing resolution down-scaling setting."; - source_proxy_->RequestHigherResolutionThan( - std::numeric_limits::max()); - } else { - source_proxy_->RequestHigherResolutionThan( - adaptation_request.input_pixel_count_); - LOG(LS_INFO) << "Scaling up resolution."; + pixel_count = std::numeric_limits::max(); } + if (!source_proxy_->RequestHigherResolutionThan(pixel_count)) + return; GetAdaptCounter().IncrementResolution(reason, -1); break; - case VideoSendStream::DegradationPreference::kMaintainResolution: - if (adapt_counter.TotalCount() == 1) { + } + case VideoSendStream::DegradationPreference::kMaintainResolution: { + // Scale up framerate. + int fps = adaptation_request.framerate_fps_; + if (adapt_counter.FramerateCount() == 1) { LOG(LS_INFO) << "Removing framerate down-scaling setting."; - source_proxy_->RequestHigherFramerateThan( - std::numeric_limits::max()); - } else { - source_proxy_->RequestHigherFramerateThan( - adaptation_request.framerate_fps_); - LOG(LS_INFO) << "Scaling up framerate."; + fps = std::numeric_limits::max(); } + if (!source_proxy_->RequestHigherFramerateThan(fps)) + return; GetAdaptCounter().IncrementFramerate(reason, -1); break; + } case VideoSendStream::DegradationPreference::kDegradationDisabled: RTC_NOTREACHED(); } diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc index eb39bf7c4d..7a54fbe925 100644 --- a/webrtc/video/vie_encoder_unittest.cc +++ b/webrtc/video/vie_encoder_unittest.cc @@ -1048,8 +1048,8 @@ TEST_F(ViEEncoderTest, SwitchingSourceKeepsQualityAdaptation) { video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); sink_.WaitForEncodedFrame(1); VideoSendStream::Stats stats = stats_proxy_->GetStats(); - EXPECT_FALSE(stats.cpu_limited_resolution); EXPECT_FALSE(stats.bw_limited_resolution); + EXPECT_FALSE(stats.bw_limited_framerate); EXPECT_EQ(0, stats.number_of_quality_adapt_changes); // Set new source with adaptation still enabled. @@ -1060,8 +1060,8 @@ TEST_F(ViEEncoderTest, SwitchingSourceKeepsQualityAdaptation) { new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); sink_.WaitForEncodedFrame(2); stats = stats_proxy_->GetStats(); - EXPECT_FALSE(stats.cpu_limited_resolution); EXPECT_FALSE(stats.bw_limited_resolution); + EXPECT_FALSE(stats.bw_limited_framerate); EXPECT_EQ(0, stats.number_of_quality_adapt_changes); // Trigger adapt down. @@ -1069,8 +1069,8 @@ TEST_F(ViEEncoderTest, SwitchingSourceKeepsQualityAdaptation) { new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight)); sink_.WaitForEncodedFrame(3); stats = stats_proxy_->GetStats(); - EXPECT_FALSE(stats.cpu_limited_resolution); EXPECT_TRUE(stats.bw_limited_resolution); + EXPECT_FALSE(stats.bw_limited_framerate); EXPECT_EQ(1, stats.number_of_quality_adapt_changes); // Set new source with adaptation still enabled. @@ -1080,8 +1080,8 @@ TEST_F(ViEEncoderTest, SwitchingSourceKeepsQualityAdaptation) { new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight)); sink_.WaitForEncodedFrame(4); stats = stats_proxy_->GetStats(); - EXPECT_FALSE(stats.cpu_limited_resolution); EXPECT_TRUE(stats.bw_limited_resolution); + EXPECT_FALSE(stats.bw_limited_framerate); EXPECT_EQ(1, stats.number_of_quality_adapt_changes); // Disable resolution scaling. @@ -1092,8 +1092,8 @@ TEST_F(ViEEncoderTest, SwitchingSourceKeepsQualityAdaptation) { new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight)); sink_.WaitForEncodedFrame(5); stats = stats_proxy_->GetStats(); - EXPECT_FALSE(stats.cpu_limited_resolution); EXPECT_FALSE(stats.bw_limited_resolution); + EXPECT_FALSE(stats.bw_limited_framerate); EXPECT_EQ(1, stats.number_of_quality_adapt_changes); EXPECT_EQ(0, stats.number_of_cpu_adapt_changes); @@ -1154,6 +1154,7 @@ TEST_F(ViEEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { sink_.WaitForEncodedFrame(sequence++); VideoSendStream::Stats stats = stats_proxy_->GetStats(); EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); EXPECT_EQ(0, stats.number_of_cpu_adapt_changes); // Trigger CPU overuse, should now adapt down. @@ -1162,6 +1163,7 @@ TEST_F(ViEEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { sink_.WaitForEncodedFrame(sequence++); stats = stats_proxy_->GetStats(); EXPECT_TRUE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); EXPECT_EQ(1, stats.number_of_cpu_adapt_changes); // Set new source with adaptation still enabled. @@ -1219,6 +1221,7 @@ TEST_F(ViEEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { stats = stats_proxy_->GetStats(); EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); EXPECT_EQ(2, stats.number_of_cpu_adapt_changes); // Try to trigger overuse. Should not succeed. @@ -1228,6 +1231,7 @@ TEST_F(ViEEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { stats = stats_proxy_->GetStats(); EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); EXPECT_EQ(2, stats.number_of_cpu_adapt_changes); // Switch back the source with resolution adaptation enabled. @@ -1238,6 +1242,7 @@ TEST_F(ViEEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { sink_.WaitForEncodedFrame(sequence++); stats = stats_proxy_->GetStats(); EXPECT_TRUE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); EXPECT_EQ(2, stats.number_of_cpu_adapt_changes); // Trigger CPU normal usage. @@ -1246,6 +1251,7 @@ TEST_F(ViEEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { sink_.WaitForEncodedFrame(sequence++); stats = stats_proxy_->GetStats(); EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); EXPECT_EQ(3, stats.number_of_cpu_adapt_changes); // Back to the source with adaptation off, set it back to maintain-resolution. @@ -1256,8 +1262,9 @@ TEST_F(ViEEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { CreateFrame(sequence, kWidth, kHeight)); sink_.WaitForEncodedFrame(sequence++); stats = stats_proxy_->GetStats(); - // Disabled, since we previously switched the source too disabled. + // Disabled, since we previously switched the source to disabled. EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_TRUE(stats.cpu_limited_framerate); EXPECT_EQ(3, stats.number_of_cpu_adapt_changes); // Trigger CPU normal usage. @@ -1267,6 +1274,7 @@ TEST_F(ViEEncoderTest, StatsTracksCpuAdaptationStatsWhenSwitchingSource) { sink_.WaitForEncodedFrame(sequence++); stats = stats_proxy_->GetStats(); EXPECT_FALSE(stats.cpu_limited_resolution); + EXPECT_FALSE(stats.cpu_limited_framerate); EXPECT_EQ(4, stats.number_of_cpu_adapt_changes); EXPECT_EQ(0, stats.number_of_quality_adapt_changes);