From 710c335d785b104bda4a912bd7909e4d27f9b04f Mon Sep 17 00:00:00 2001 From: kthelgason Date: Thu, 8 Dec 2016 02:12:31 -0800 Subject: [PATCH] Add ability to scale to arbitrary factors This CL adds a fallback for the case when no optimized scale factor produces a low enough resolution for what was requested. It also ensures that all resolutions provided by the video adapter are divisible by four. This is required by some hardware implementations. BUG=webrtc:6837 Review-Url: https://codereview.webrtc.org/2555483005 Cr-Commit-Position: refs/heads/master@{#15469} --- webrtc/media/base/adaptedvideotracksource.cc | 5 + webrtc/media/base/adaptedvideotracksource.h | 3 + webrtc/media/base/videoadapter.cc | 93 ++++++++++++++----- webrtc/media/base/videoadapter.h | 3 + .../src/jni/androidvideotracksource.cc | 8 +- 5 files changed, 86 insertions(+), 26 deletions(-) diff --git a/webrtc/media/base/adaptedvideotracksource.cc b/webrtc/media/base/adaptedvideotracksource.cc index acc147cc89..5900439e12 100644 --- a/webrtc/media/base/adaptedvideotracksource.cc +++ b/webrtc/media/base/adaptedvideotracksource.cc @@ -16,6 +16,11 @@ AdaptedVideoTrackSource::AdaptedVideoTrackSource() { thread_checker_.DetachFromThread(); } +AdaptedVideoTrackSource::AdaptedVideoTrackSource(int required_alignment) + : video_adapter_(required_alignment) { + thread_checker_.DetachFromThread(); +} + bool AdaptedVideoTrackSource::GetStats(Stats* stats) { rtc::CritScope lock(&stats_crit_); diff --git a/webrtc/media/base/adaptedvideotracksource.h b/webrtc/media/base/adaptedvideotracksource.h index 5b420aeb9e..9d51c692c2 100644 --- a/webrtc/media/base/adaptedvideotracksource.h +++ b/webrtc/media/base/adaptedvideotracksource.h @@ -28,6 +28,9 @@ class AdaptedVideoTrackSource AdaptedVideoTrackSource(); protected: + // Allows derived classes to initialize |video_adapter_| with a custom + // alignment. + AdaptedVideoTrackSource(int required_alignment); // Checks the apply_rotation() flag. If the frame needs rotation, and it is a // plain memory frame, it is rotated. Subclasses producing native frames must // handle apply_rotation() themselves. diff --git a/webrtc/media/base/videoadapter.cc b/webrtc/media/base/videoadapter.cc index acb0e2c1a1..8bc9b0a50f 100644 --- a/webrtc/media/base/videoadapter.cc +++ b/webrtc/media/base/videoadapter.cc @@ -11,16 +11,18 @@ #include "webrtc/media/base/videoadapter.h" #include +#include #include #include +#include "webrtc/base/arraysize.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/base/optional.h" #include "webrtc/media/base/mediaconstants.h" #include "webrtc/media/base/videocommon.h" namespace { - struct Fraction { int numerator; int denominator; @@ -37,17 +39,42 @@ const Fraction kScaleFractions[] = { {3, 16}, }; -// Round |valueToRound| to a multiple of |multiple|. Prefer rounding upwards, -// but never more than |maxValue|. -int roundUp(int valueToRound, int multiple, int maxValue) { - const int roundedValue = (valueToRound + multiple - 1) / multiple * multiple; - return roundedValue <= maxValue ? roundedValue - : (maxValue / multiple * multiple); +// Round |value_to_round| to a multiple of |multiple|. Prefer rounding upwards, +// but never more than |max_value|. +int roundUp(int value_to_round, int multiple, int max_value) { + const int rounded_value = + (value_to_round + multiple - 1) / multiple * multiple; + return rounded_value <= max_value ? rounded_value + : (max_value / multiple * multiple); } +// Generates a scale factor that makes |input_num_pixels| smaller than +// |target_num_pixels|. This should only be used after making sure none +// of the optimized factors are small enough. Fraction FindScaleLessThanOrEqual(int input_num_pixels, int target_num_pixels) { + // Start searching from the last of the optimal fractions; + Fraction best_scale = kScaleFractions[arraysize(kScaleFractions) - 1]; + const float target_scale = + sqrt(target_num_pixels / static_cast(input_num_pixels)); + do { + if (best_scale.numerator % 3 == 0 && best_scale.denominator % 2 == 0) { + // Multiply by 2/3 + best_scale.numerator /= 3; + best_scale.denominator /= 2; + } else { + // Multiply by 3/4 + best_scale.numerator *= 3; + best_scale.denominator *= 4; + } + } while (best_scale.numerator > (target_scale * best_scale.denominator)); + return best_scale; +} + +rtc::Optional FindOptimizedScaleLessThanOrEqual( + int input_num_pixels, + int target_num_pixels) { float best_distance = std::numeric_limits::max(); - Fraction best_scale = {0, 1}; // Default to 0 if nothing matches. + rtc::Optional best_scale; for (const auto& fraction : kScaleFractions) { const float scale = fraction.numerator / static_cast(fraction.denominator); @@ -58,7 +85,7 @@ Fraction FindScaleLessThanOrEqual(int input_num_pixels, int target_num_pixels) { } if (diff < best_distance) { best_distance = diff; - best_scale = fraction; + best_scale = rtc::Optional(fraction); if (best_distance == 0) { // Found exact match. break; } @@ -67,9 +94,9 @@ Fraction FindScaleLessThanOrEqual(int input_num_pixels, int target_num_pixels) { return best_scale; } -Fraction FindScaleLargerThan(int input_num_pixels, - int target_num_pixels, - int* resulting_number_of_pixels) { +Fraction FindOptimizedScaleLargerThan(int input_num_pixels, + int target_num_pixels, + int* resulting_number_of_pixels) { float best_distance = std::numeric_limits::max(); Fraction best_scale = {1, 1}; // Default to unscaled if nothing matches. // Default to input number of pixels. @@ -93,35 +120,47 @@ Fraction FindScaleLargerThan(int input_num_pixels, return best_scale; } -Fraction FindScale(int input_num_pixels, - int max_pixel_count_step_up, - int max_pixel_count) { +rtc::Optional FindOptimizedScale(int input_num_pixels, + int max_pixel_count_step_up, + int max_pixel_count) { // Try scale just above |max_pixel_count_step_up_|. if (max_pixel_count_step_up > 0) { int resulting_pixel_count; - const Fraction scale = FindScaleLargerThan( + const Fraction scale = FindOptimizedScaleLargerThan( input_num_pixels, max_pixel_count_step_up, &resulting_pixel_count); if (resulting_pixel_count <= max_pixel_count) - return scale; + return rtc::Optional(scale); } // Return largest scale below |max_pixel_count|. - return FindScaleLessThanOrEqual(input_num_pixels, max_pixel_count); + return FindOptimizedScaleLessThanOrEqual(input_num_pixels, max_pixel_count); } +Fraction FindScale(int input_num_pixels, + int max_pixel_count_step_up, + int max_pixel_count) { + const rtc::Optional optimized_scale = FindOptimizedScale( + input_num_pixels, max_pixel_count_step_up, max_pixel_count); + if (optimized_scale) + return *optimized_scale; + return FindScaleLessThanOrEqual(input_num_pixels, max_pixel_count); +} } // namespace namespace cricket { -VideoAdapter::VideoAdapter() +VideoAdapter::VideoAdapter(int required_resolution_alignment) : frames_in_(0), frames_out_(0), frames_scaled_(0), adaption_changes_(0), previous_width_(0), previous_height_(0), + required_resolution_alignment_(required_resolution_alignment), resolution_request_max_pixel_count_(std::numeric_limits::max()), resolution_request_max_pixel_count_step_up_(0) {} +VideoAdapter::VideoAdapter() : VideoAdapter(1) {} + VideoAdapter::~VideoAdapter() {} bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) { @@ -211,22 +250,26 @@ bool VideoAdapter::AdaptFrameResolution(int in_width, *cropped_height = std::min(in_height, static_cast(in_width / requested_aspect)); } - - // Find best scale factor. const Fraction scale = FindScale(*cropped_width * *cropped_height, resolution_request_max_pixel_count_step_up_, max_pixel_count); - // Adjust cropping slightly to get even integer output size and a perfect - // scale factor. - *cropped_width = roundUp(*cropped_width, scale.denominator, in_width); - *cropped_height = roundUp(*cropped_height, scale.denominator, in_height); + // scale factor. Make sure the resulting dimensions are aligned correctly + // to be nice to hardware encoders. + *cropped_width = + roundUp(*cropped_width, + scale.denominator * required_resolution_alignment_, in_width); + *cropped_height = + roundUp(*cropped_height, + scale.denominator * required_resolution_alignment_, in_height); RTC_DCHECK_EQ(0, *cropped_width % scale.denominator); RTC_DCHECK_EQ(0, *cropped_height % scale.denominator); // Calculate final output size. *out_width = *cropped_width / scale.denominator * scale.numerator; *out_height = *cropped_height / scale.denominator * scale.numerator; + RTC_DCHECK_EQ(0, *out_height % required_resolution_alignment_); + RTC_DCHECK_EQ(0, *out_height % required_resolution_alignment_); ++frames_out_; if (scale.numerator != scale.denominator) diff --git a/webrtc/media/base/videoadapter.h b/webrtc/media/base/videoadapter.h index 9d17f5ce15..9ea649df25 100644 --- a/webrtc/media/base/videoadapter.h +++ b/webrtc/media/base/videoadapter.h @@ -25,6 +25,7 @@ namespace cricket { class VideoAdapter { public: VideoAdapter(); + VideoAdapter(int required_resolution_alignment); virtual ~VideoAdapter(); // Return the adapted resolution and cropping parameters given the @@ -63,6 +64,8 @@ class VideoAdapter { int adaption_changes_; // Number of changes in scale factor. int previous_width_; // Previous adapter output width. int previous_height_; // Previous adapter output height. + // Resolution must be divisible by this factor. + const int required_resolution_alignment_; // The target timestamp for the next frame based on requested format. rtc::Optional next_frame_timestamp_ns_ GUARDED_BY(critical_section_); diff --git a/webrtc/sdk/android/src/jni/androidvideotracksource.cc b/webrtc/sdk/android/src/jni/androidvideotracksource.cc index 57bbc4a312..1a4a276ae2 100644 --- a/webrtc/sdk/android/src/jni/androidvideotracksource.cc +++ b/webrtc/sdk/android/src/jni/androidvideotracksource.cc @@ -12,13 +12,19 @@ #include +namespace { +// MediaCodec wants resolution to be divisible by 2. +const int kRequiredResolutionAlignment = 2; +} + namespace webrtc { AndroidVideoTrackSource::AndroidVideoTrackSource(rtc::Thread* signaling_thread, JNIEnv* jni, jobject j_egl_context, bool is_screencast) - : signaling_thread_(signaling_thread), + : AdaptedVideoTrackSource(kRequiredResolutionAlignment), + signaling_thread_(signaling_thread), surface_texture_helper_(webrtc_jni::SurfaceTextureHelper::create( jni, "Camera SurfaceTextureHelper",