From 3f7e0ede1e71f37a5d093c5ee98f6a54b70d4e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Fri, 18 Oct 2019 15:03:13 +0200 Subject: [PATCH] Add option to make first scale factor depend on input resolution. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scale factors are 3/4, 2/3, 3/4, 2/3, ... Adds possibly to start with: - 2/3 (if width/height multiple of 3) - 2/3, 2/3 (if width/height multiple of 9) Bug: none Change-Id: Idbeddfec4baea893c240bbb897d01ac1cff3b435 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/157105 Commit-Queue: Åsa Persson Reviewed-by: Niels Moller Reviewed-by: Sergey Silkin Cr-Commit-Position: refs/heads/master@{#29538} --- media/BUILD.gn | 1 + media/base/video_adapter.cc | 50 +++++++++- media/base/video_adapter.h | 1 + media/base/video_adapter_unittest.cc | 137 ++++++++++++++++++++++++++- 4 files changed, 183 insertions(+), 6 deletions(-) diff --git a/media/BUILD.gn b/media/BUILD.gn index 59c2bfc600..371f6cd772 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -106,6 +106,7 @@ rtc_library("rtc_media_base") { "../rtc_base/system:file_wrapper", "../rtc_base/system:rtc_export", "../rtc_base/third_party/sigslot", + "../system_wrappers:field_trial", "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", diff --git a/media/base/video_adapter.cc b/media/base/video_adapter.cc index 4f1181e65a..75c1a47b57 100644 --- a/media/base/video_adapter.cc +++ b/media/base/video_adapter.cc @@ -21,12 +21,31 @@ #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" +#include "system_wrappers/include/field_trial.h" namespace { +int Gcd(int a, int b) { + RTC_DCHECK_GE(a, 0); + RTC_DCHECK_GT(b, 0); + int c = a % b; + while (c != 0) { + a = b; + b = c; + c = a % b; + } + return b; +} + struct Fraction { int numerator; int denominator; + void DivideByGcd() { + int g = Gcd(numerator, denominator); + numerator /= g; + denominator /= g; + } + // Determines number of output pixels if both width and height of an input of // |input_pixels| pixels is scaled with the fraction numerator / denominator. int scale_pixel_count(int input_pixels) { @@ -45,18 +64,37 @@ int roundUp(int value_to_round, int multiple, int max_value) { // Generates a scale factor that makes |input_pixels| close to |target_pixels|, // but no higher than |max_pixels|. -Fraction FindScale(int input_pixels, int target_pixels, int max_pixels) { +Fraction FindScale(int input_width, + int input_height, + int target_pixels, + int max_pixels, + bool variable_start_scale_factor) { // This function only makes sense for a positive target. RTC_DCHECK_GT(target_pixels, 0); RTC_DCHECK_GT(max_pixels, 0); RTC_DCHECK_GE(max_pixels, target_pixels); + const int input_pixels = input_width * input_height; + // Don't scale up original. if (target_pixels >= input_pixels) return Fraction{1, 1}; Fraction current_scale = Fraction{1, 1}; Fraction best_scale = Fraction{1, 1}; + + if (variable_start_scale_factor) { + // Start scaling down by 2/3 depending on |input_width| and |input_height|. + if (input_width % 3 == 0 && input_height % 3 == 0) { + // 2/3 (then alternates 3/4, 2/3, 3/4,...). + current_scale = Fraction{6, 6}; + } + if (input_width % 9 == 0 && input_height % 9 == 0) { + // 2/3, 2/3 (then alternates 3/4, 2/3, 3/4,...). + current_scale = Fraction{36, 36}; + } + } + // The minimum (absolute) difference between the number of output pixels and // the target pixel count. int min_pixel_diff = std::numeric_limits::max(); @@ -65,7 +103,7 @@ Fraction FindScale(int input_pixels, int target_pixels, int max_pixels) { min_pixel_diff = std::abs(input_pixels - target_pixels); } - // Alternately scale down by 2/3 and 3/4. This results in fractions which are + // Alternately scale down by 3/4 and 2/3. This results in fractions which are // effectively scalable. For instance, starting at 1280x720 will result in // the series (3/4) => 960x540, (1/2) => 640x360, (3/8) => 480x270, // (1/4) => 320x180, (3/16) => 240x125, (1/8) => 160x90. @@ -90,6 +128,7 @@ Fraction FindScale(int input_pixels, int target_pixels, int max_pixels) { } } } + best_scale.DivideByGcd(); return best_scale; } @@ -104,6 +143,8 @@ VideoAdapter::VideoAdapter(int required_resolution_alignment) adaption_changes_(0), previous_width_(0), previous_height_(0), + variable_start_scale_factor_(webrtc::field_trial::IsEnabled( + "WebRTC-Video-VariableStartScaleFactor")), required_resolution_alignment_(required_resolution_alignment), resolution_request_target_pixel_count_(std::numeric_limits::max()), resolution_request_max_pixel_count_(std::numeric_limits::max()), @@ -217,8 +258,9 @@ bool VideoAdapter::AdaptFrameResolution(int in_width, *cropped_height = std::min(in_height, static_cast(in_width / requested_aspect)); } - const Fraction scale = FindScale((*cropped_width) * (*cropped_height), - target_pixel_count, max_pixel_count); + const Fraction scale = + FindScale(*cropped_width, *cropped_height, target_pixel_count, + max_pixel_count, variable_start_scale_factor_); // Adjust cropping slightly to get even integer output size and a perfect // scale factor. Make sure the resulting dimensions are aligned correctly // to be nice to hardware encoders. diff --git a/media/base/video_adapter.h b/media/base/video_adapter.h index bbe08862dc..a846cc00b1 100644 --- a/media/base/video_adapter.h +++ b/media/base/video_adapter.h @@ -105,6 +105,7 @@ class VideoAdapter { int adaption_changes_; // Number of changes in scale factor. int previous_width_; // Previous adapter output width. int previous_height_; // Previous adapter output height. + const bool variable_start_scale_factor_; // Resolution must be divisible by this factor. const int required_resolution_alignment_; // The target timestamp for the next frame based on requested format. diff --git a/media/base/video_adapter_unittest.cc b/media/base/video_adapter_unittest.cc index 2bfe8037a3..6529ee7713 100644 --- a/media/base/video_adapter_unittest.cc +++ b/media/base/video_adapter_unittest.cc @@ -12,11 +12,14 @@ #include #include +#include #include #include "api/video/video_frame.h" #include "media/base/fake_frame_source.h" +#include "rtc_base/arraysize.h" #include "rtc_base/time_utils.h" +#include "test/field_trial.h" #include "test/gtest.h" namespace cricket { @@ -29,8 +32,10 @@ const int kDefaultFps = 30; class VideoAdapterTest : public ::testing::Test, public ::testing::WithParamInterface { public: - VideoAdapterTest() - : frame_source_(std::make_unique( + VideoAdapterTest() : VideoAdapterTest("") {} + explicit VideoAdapterTest(const std::string& field_trials) + : override_field_trials_(field_trials), + frame_source_(std::make_unique( kWidth, kHeight, VideoFormat::FpsToInterval(kDefaultFps) / @@ -115,6 +120,7 @@ class VideoAdapterTest : public ::testing::Test, cricket::FOURCC_I420)); } + webrtc::test::ScopedFieldTrials override_field_trials_; const std::unique_ptr frame_source_; VideoAdapter adapter_; int cropped_width_; @@ -125,10 +131,20 @@ class VideoAdapterTest : public ::testing::Test, const bool use_new_format_request_; }; +class VideoAdapterTestVariableStartScale : public VideoAdapterTest { + public: + VideoAdapterTestVariableStartScale() + : VideoAdapterTest("WebRTC-Video-VariableStartScaleFactor/Enabled/") {} +}; + INSTANTIATE_TEST_SUITE_P(OnOutputFormatRequests, VideoAdapterTest, ::testing::Values(true, false)); +INSTANTIATE_TEST_SUITE_P(OnOutputFormatRequests, + VideoAdapterTestVariableStartScale, + ::testing::Values(true, false)); + // Do not adapt the frame rate or the resolution. Expect no frame drop, no // cropping, and no resolution change. TEST_P(VideoAdapterTest, AdaptNothing) { @@ -1159,4 +1175,121 @@ TEST(VideoAdapterTestMultipleOrientation, TestForcePortrait) { EXPECT_EQ(640, out_height); } +TEST_P(VideoAdapterTest, AdaptResolutionInSteps) { + const int kWidth = 1280; + const int kHeight = 720; + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); // 16:9 aspect. + + // Scale factors: 3/4, 2/3, 3/4, 2/3, ... + // Scale : 3/4, 1/2, 3/8, 1/4, 3/16, 1/8. + const int kExpectedWidths[] = {960, 640, 480, 320, 240, 160}; + const int kExpectedHeights[] = {540, 360, 270, 180, 135, 90}; + + int request_width = kWidth; + int request_height = kHeight; + + for (size_t i = 0; i < arraysize(kExpectedWidths); ++i) { + // Adapt down one step. + adapter_.OnResolutionFramerateRequest(absl::nullopt, + request_width * request_height - 1, + std::numeric_limits::max()); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + EXPECT_EQ(kExpectedWidths[i], out_width_); + EXPECT_EQ(kExpectedHeights[i], out_height_); + request_width = out_width_; + request_height = out_height_; + } +} + +// Scale factors are 3/4, 2/3, 3/4, 2/3, ... (see test above). +// In VideoAdapterTestVariableStartScale, first scale factor depends on +// resolution. May start with: +// - 2/3 (if width/height multiple of 3) or +// - 2/3, 2/3 (if width/height multiple of 9). +TEST_P(VideoAdapterTestVariableStartScale, AdaptResolutionInStepsFirst3_4) { + const int kWidth = 1280; + const int kHeight = 720; + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); // 16:9 aspect. + + // Scale factors: 3/4, 2/3, 3/4, 2/3, ... + // Scale : 3/4, 1/2, 3/8, 1/4, 3/16, 1/8. + const int kExpectedWidths[] = {960, 640, 480, 320, 240, 160}; + const int kExpectedHeights[] = {540, 360, 270, 180, 135, 90}; + + int request_width = kWidth; + int request_height = kHeight; + + for (size_t i = 0; i < arraysize(kExpectedWidths); ++i) { + // Adapt down one step. + adapter_.OnResolutionFramerateRequest(absl::nullopt, + request_width * request_height - 1, + std::numeric_limits::max()); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + EXPECT_EQ(kExpectedWidths[i], out_width_); + EXPECT_EQ(kExpectedHeights[i], out_height_); + request_width = out_width_; + request_height = out_height_; + } +} + +TEST_P(VideoAdapterTestVariableStartScale, AdaptResolutionInStepsFirst2_3) { + const int kWidth = 1920; + const int kHeight = 1080; + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); // 16:9 aspect. + + // Scale factors: 2/3, 3/4, 2/3, 3/4, ... + // Scale: 2/3, 1/2, 1/3, 1/4, 1/6, 1/8, 1/12. + const int kExpectedWidths[] = {1280, 960, 640, 480, 320, 240, 160}; + const int kExpectedHeights[] = {720, 540, 360, 270, 180, 135, 90}; + + int request_width = kWidth; + int request_height = kHeight; + + for (size_t i = 0; i < arraysize(kExpectedWidths); ++i) { + // Adapt down one step. + adapter_.OnResolutionFramerateRequest(absl::nullopt, + request_width * request_height - 1, + std::numeric_limits::max()); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + EXPECT_EQ(kExpectedWidths[i], out_width_); + EXPECT_EQ(kExpectedHeights[i], out_height_); + request_width = out_width_; + request_height = out_height_; + } +} + +TEST_P(VideoAdapterTestVariableStartScale, AdaptResolutionInStepsFirst2x2_3) { + const int kWidth = 1440; + const int kHeight = 1080; + OnOutputFormatRequest(kWidth, kHeight, absl::nullopt); // 4:3 aspect. + + // Scale factors: 2/3, 2/3, 3/4, 2/3, 3/4, ... + // Scale : 2/3, 4/9, 1/3, 2/9, 1/6, 1/9, 1/12, 1/18, 1/24, 1/36. + const int kExpectedWidths[] = {960, 640, 480, 320, 240, 160, 120, 80, 60, 40}; + const int kExpectedHeights[] = {720, 480, 360, 240, 180, 120, 90, 60, 45, 30}; + + int request_width = kWidth; + int request_height = kHeight; + + for (size_t i = 0; i < arraysize(kExpectedWidths); ++i) { + // Adapt down one step. + adapter_.OnResolutionFramerateRequest(absl::nullopt, + request_width * request_height - 1, + std::numeric_limits::max()); + EXPECT_TRUE(adapter_.AdaptFrameResolution(kWidth, kHeight, 0, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_)); + EXPECT_EQ(kExpectedWidths[i], out_width_); + EXPECT_EQ(kExpectedHeights[i], out_height_); + request_width = out_width_; + request_height = out_height_; + } +} + } // namespace cricket