diff --git a/webrtc/media/base/videoadapter.cc b/webrtc/media/base/videoadapter.cc index 797a876f2b..9c3837c572 100644 --- a/webrtc/media/base/videoadapter.cc +++ b/webrtc/media/base/videoadapter.cc @@ -13,69 +13,77 @@ #include #include +#include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/media/base/mediaconstants.h" #include "webrtc/media/base/videocommon.h" namespace { -// Scale factors optimized for in libYUV that we accept. -// Must be sorted in decreasing scale factors for FindScaleLargerThan to work. -const float kScaleFactors[] = { - 1.f / 1.f, // Full size. - 3.f / 4.f, // 3/4 scale. - 1.f / 2.f, // 1/2 scale. - 3.f / 8.f, // 3/8 scale. - 1.f / 4.f, // 1/4 scale. - 3.f / 16.f, // 3/16 scale. +struct Fraction { + int numerator; + int denominator; }; -float FindScaleLessThanOrEqual(int width, - int height, - int target_num_pixels, - int* resulting_number_of_pixels) { +// Scale factors optimized for in libYUV that we accept. +// Must be sorted in decreasing scale factors for FindScaleLargerThan to work. +const Fraction kScaleFractions[] = { + {1, 1}, + {3, 4}, + {1, 2}, + {3, 8}, + {1, 4}, + {3, 16}, +}; + +// Round |valueToRound| to a multiple of |multiple|. Prefer rounding upwards, +// but never more than |maxValue|. +int roundUp(int valueToRound, int multiple, int maxValue) { + const int roundedValue = (valueToRound + multiple - 1) / multiple * multiple; + return roundedValue <= maxValue ? roundedValue + : (maxValue / multiple * multiple); +} + +Fraction FindScaleLessThanOrEqual(int input_num_pixels, int target_num_pixels) { float best_distance = std::numeric_limits::max(); - float best_scale = 0.0f; // Default to 0 if nothing matches. - float pixels = width * height; - float best_number_of_pixels = 0.0f; - for (const auto& scale : kScaleFactors) { - float test_num_pixels = pixels * scale * scale; + Fraction best_scale = {0, 1}; // Default to 0 if nothing matches. + for (const auto& fraction : kScaleFractions) { + const float scale = + fraction.numerator / static_cast(fraction.denominator); + float test_num_pixels = input_num_pixels * scale * scale; float diff = target_num_pixels - test_num_pixels; if (diff < 0) { continue; } if (diff < best_distance) { best_distance = diff; - best_scale = scale; - best_number_of_pixels = test_num_pixels; + best_scale = fraction; if (best_distance == 0) { // Found exact match. break; } } } - if (resulting_number_of_pixels) { - *resulting_number_of_pixels = static_cast(best_number_of_pixels + .5f); - } return best_scale; } -float FindScaleLargerThan(int width, - int height, - int target_num_pixels, - int* resulting_number_of_pixels) { +Fraction FindScaleLargerThan(int input_num_pixels, + int target_num_pixels, + int* resulting_number_of_pixels) { float best_distance = std::numeric_limits::max(); - float best_scale = 1.f; // Default to unscaled if nothing matches. - float pixels = width * height; - float best_number_of_pixels = pixels; // Default to input number of pixels. - for (const auto& scale : kScaleFactors) { - float test_num_pixels = pixels * scale * scale; + Fraction best_scale = {1, 1}; // Default to unscaled if nothing matches. + // Default to input number of pixels. + float best_number_of_pixels = input_num_pixels; + for (const auto& fraction : kScaleFractions) { + const float scale = + fraction.numerator / static_cast(fraction.denominator); + float test_num_pixels = input_num_pixels * scale * scale; float diff = test_num_pixels - target_num_pixels; if (diff <= 0) { break; } if (diff < best_distance) { best_distance = diff; - best_scale = scale; + best_scale = fraction; best_number_of_pixels = test_num_pixels; } } @@ -84,21 +92,36 @@ float FindScaleLargerThan(int width, return best_scale; } +Fraction FindScale(int input_num_pixels, + int max_pixel_count_step_up, + int max_pixel_count) { + // Try scale just above |max_pixel_count_step_up_|. + if (max_pixel_count_step_up > 0) { + int resulting_pixel_count; + const Fraction scale = FindScaleLargerThan( + input_num_pixels, max_pixel_count_step_up, &resulting_pixel_count); + if (resulting_pixel_count <= max_pixel_count) + return scale; + } + // Return largest scale below |max_pixel_count|. + return FindScaleLessThanOrEqual(input_num_pixels, max_pixel_count); +} + } // namespace namespace cricket { VideoAdapter::VideoAdapter() - : output_num_pixels_(std::numeric_limits::max()), - frames_in_(0), + : frames_in_(0), frames_out_(0), frames_scaled_(0), adaption_changes_(0), previous_width_(0), previous_height_(0), + input_interval_(0), interval_next_frame_(0), - format_request_max_pixel_count_(std::numeric_limits::max()), - resolution_request_max_pixel_count_(std::numeric_limits::max()) {} + resolution_request_max_pixel_count_(std::numeric_limits::max()), + resolution_request_max_pixel_count_step_up_(0) {} VideoAdapter::~VideoAdapter() {} @@ -106,57 +129,43 @@ void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) { // TODO(perkj): Consider measuring input frame rate instead. // Frame rate typically varies depending on lighting. rtc::CritScope cs(&critical_section_); - input_format_.interval = interval; + input_interval_ = interval; } -void VideoAdapter::SetInputFormat(const VideoFormat& format) { - bool is_resolution_change = (input_format().width != format.width || - input_format().height != format.height); - int64_t old_input_interval = input_format_.interval; - input_format_ = format; - output_format_.interval = - std::max(output_format_.interval, input_format_.interval); - if (old_input_interval != input_format_.interval) { - LOG(LS_INFO) << "VAdapt input interval changed from " - << old_input_interval << " to " << input_format_.interval; - } - if (is_resolution_change) { - // Trigger the adaptation logic again, to potentially reset the adaptation - // state for things like view requests that may not longer be capping - // output (or may now cap output). - Adapt(std::min(format_request_max_pixel_count_, - resolution_request_max_pixel_count_), - 0); - } -} - -const VideoFormat& VideoAdapter::input_format() const { - rtc::CritScope cs(&critical_section_); - return input_format_; -} - -VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { +void VideoAdapter::AdaptFrameResolution(int in_width, + int in_height, + int* cropped_width, + int* cropped_height, + int* out_width, + int* out_height) { rtc::CritScope cs(&critical_section_); ++frames_in_; - SetInputFormat(VideoFormat( - in_width, in_height, input_format_.interval, input_format_.fourcc)); + // The max output pixel count is the minimum of the requests from + // OnOutputFormatRequest and OnResolutionRequest. + int max_pixel_count = resolution_request_max_pixel_count_; + if (requested_format_) { + max_pixel_count = std::min( + max_pixel_count, requested_format_->width * requested_format_->height); + } // Drop the input frame if necessary. bool should_drop = false; - if (!output_num_pixels_) { + if (max_pixel_count == 0) { // Drop all frames as the output format is 0x0. should_drop = true; - } else { + } else if (requested_format_ && requested_format_->interval > 0) { // Drop some frames based on input fps and output fps. // Normally output fps is less than input fps. - interval_next_frame_ += input_format_.interval; - if (output_format_.interval > 0) { - if (interval_next_frame_ >= output_format_.interval) { - interval_next_frame_ %= output_format_.interval; - } else { - should_drop = true; - } + interval_next_frame_ += input_interval_; + if (interval_next_frame_ >= requested_format_->interval) { + interval_next_frame_ -= requested_format_->interval; + // Reset |interval_next_frame_| if it accumulates too much to avoid + // "catching up" behaviour. + if (interval_next_frame_ >= requested_format_->interval) + interval_next_frame_ = 0; + } else { + should_drop = true; } } if (should_drop) { @@ -170,48 +179,79 @@ VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { << " Changes: " << adaption_changes_ << " Input: " << in_width << "x" << in_height - << " i" << input_format_.interval - << " Output: i" << output_format_.interval; + << " i" << input_interval_ + << " Output: i" + << (requested_format_ ? requested_format_->interval : 0); } - return VideoFormat(); // Drop frame. + // Drop frame. + *cropped_width = 0; + *cropped_height = 0; + *out_width = 0; + *out_height = 0; + return; } - const float scale = FindScaleLessThanOrEqual(in_width, in_height, - output_num_pixels_, nullptr); - const int output_width = static_cast(in_width * scale + .5f); - const int output_height = static_cast(in_height * scale + .5f); + // Calculate how the input should be cropped. + if (!requested_format_ || + requested_format_->width == 0 || requested_format_->height == 0) { + *cropped_width = in_width; + *cropped_height = in_height; + } else { + // Adjust |requested_format_| orientation to match input. + if ((in_width > in_height) != + (requested_format_->width > requested_format_->height)) { + std::swap(requested_format_->width, requested_format_->height); + } + const float requested_aspect = + requested_format_->width / + static_cast(requested_format_->height); + *cropped_width = + std::min(in_width, static_cast(in_height * requested_aspect)); + *cropped_height = + std::min(in_height, static_cast(in_width / requested_aspect)); + } + + // Find best scale factor. + const Fraction scale = + FindScale(*cropped_width * *cropped_height, + resolution_request_max_pixel_count_step_up_, max_pixel_count); + + // Adjust cropping slightly to get even integer output size and a perfect + // scale factor. + *cropped_width = roundUp(*cropped_width, scale.denominator, in_width); + *cropped_height = roundUp(*cropped_height, scale.denominator, in_height); + RTC_DCHECK_EQ(0, *cropped_width % scale.denominator); + RTC_DCHECK_EQ(0, *cropped_height % scale.denominator); + + // Calculate final output size. + *out_width = *cropped_width / scale.denominator * scale.numerator; + *out_height = *cropped_height / scale.denominator * scale.numerator; ++frames_out_; - if (scale != 1) + if (scale.numerator != scale.denominator) ++frames_scaled_; - if (previous_width_ && (previous_width_ != output_width || - previous_height_ != output_height)) { + if (previous_width_ && (previous_width_ != *out_width || + previous_height_ != *out_height)) { ++adaption_changes_; LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out " << frames_out_ << " / in " << frames_in_ << " Changes: " << adaption_changes_ << " Input: " << in_width - << "x" << in_height << " i" << input_format_.interval - << " Scale: " << scale << " Output: " << output_width << "x" - << output_height << " i" << output_format_.interval; + << "x" << in_height << " i" << input_interval_ + << " Scale: " << scale.numerator << "/" << scale.denominator + << " Output: " << *out_width << "x" << *out_height << " i" + << (requested_format_ ? requested_format_->interval : 0); } - output_format_.width = output_width; - output_format_.height = output_height; - previous_width_ = output_width; - previous_height_ = output_height; - - return output_format_; + previous_width_ = *out_width; + previous_height_ = *out_height; } void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { rtc::CritScope cs(&critical_section_); - format_request_max_pixel_count_ = format.width * format.height; - output_format_.interval = format.interval; - Adapt(std::min(format_request_max_pixel_count_, - resolution_request_max_pixel_count_), - 0); + requested_format_ = rtc::Optional(format); + interval_next_frame_ = 0; } void VideoAdapter::OnResolutionRequest( @@ -220,44 +260,8 @@ void VideoAdapter::OnResolutionRequest( rtc::CritScope cs(&critical_section_); resolution_request_max_pixel_count_ = max_pixel_count.value_or(std::numeric_limits::max()); - Adapt(std::min(format_request_max_pixel_count_, - resolution_request_max_pixel_count_), - max_pixel_count_step_up.value_or(0)); -} - -bool VideoAdapter::Adapt(int max_num_pixels, int max_pixel_count_step_up) { - float scale_lower = - FindScaleLessThanOrEqual(input_format_.width, input_format_.height, - max_num_pixels, &max_num_pixels); - float scale_upper = - max_pixel_count_step_up > 0 - ? FindScaleLargerThan(input_format_.width, input_format_.height, - max_pixel_count_step_up, - &max_pixel_count_step_up) - : 1.f; - - bool use_max_pixel_count_step_up = - max_pixel_count_step_up > 0 && max_num_pixels > max_pixel_count_step_up; - - int old_num_pixels = output_num_pixels_; - output_num_pixels_ = - use_max_pixel_count_step_up ? max_pixel_count_step_up : max_num_pixels; - // Log the new size. - float scale = use_max_pixel_count_step_up ? scale_upper : scale_lower; - int new_width = static_cast(input_format_.width * scale + .5f); - int new_height = static_cast(input_format_.height * scale + .5f); - - bool changed = output_num_pixels_ != old_num_pixels; - LOG(LS_INFO) << "OnResolutionRequest: " - << " Max pixels: " << max_num_pixels - << " Max pixels step up: " << max_pixel_count_step_up - << " Output Pixels: " << output_num_pixels_ - << " Input: " << input_format_.width << "x" - << input_format_.height << " Scale: " << scale - << " Resolution: " << new_width << "x" << new_height - << " Changed: " << (changed ? "true" : "false"); - - return changed; + resolution_request_max_pixel_count_step_up_ = + max_pixel_count_step_up.value_or(0); } } // namespace cricket diff --git a/webrtc/media/base/videoadapter.h b/webrtc/media/base/videoadapter.h index 50b7d4c4be..2db0ada271 100644 --- a/webrtc/media/base/videoadapter.h +++ b/webrtc/media/base/videoadapter.h @@ -32,12 +32,23 @@ class VideoAdapter { // interval. void SetExpectedInputFrameInterval(int64_t interval); - // Return the adapted resolution given the input resolution. The returned - // resolution will be 0x0 if the frame should be dropped. - VideoFormat AdaptFrameResolution(int in_width, int in_height); + // Return the adapted resolution given the input resolution. The input + // resolution should first be cropped to the specified resolution, and then + // scaled to the final output resolution. The output resolution will be 0x0 if + // the frame should be dropped. + void AdaptFrameResolution(int in_width, + int in_height, + int* cropped_width, + int* cropped_height, + int* out_width, + int* out_height); // Requests the output frame size and frame interval from - // |AdaptFrameResolution| to not be larger than |format|. + // |AdaptFrameResolution| to not be larger than |format|. Also, the input + // frame size will be cropped to match the requested aspect ratio. The + // requested aspect ratio is orientation agnostic and will be adjusted to + // maintain the input orientation, so it doesn't matter if e.g. 1280x720 or + // 720x1280 is requested. void OnOutputFormatRequest(const VideoFormat& format); // Requests the output frame size from |AdaptFrameResolution| to not have @@ -46,28 +57,22 @@ class VideoAdapter { void OnResolutionRequest(rtc::Optional max_pixel_count, rtc::Optional max_pixel_count_step_up); - const VideoFormat& input_format() const; - private: - void SetInputFormat(const VideoFormat& format); - bool Adapt(int max_num_pixels, int max_pixel_count_step_up); - - VideoFormat input_format_; - VideoFormat output_format_; - int output_num_pixels_; int frames_in_; // Number of input frames. int frames_out_; // Number of output frames. int frames_scaled_; // Number of frames scaled. int adaption_changes_; // Number of changes in scale factor. int previous_width_; // Previous adapter output width. int previous_height_; // Previous adapter output height. - int64_t interval_next_frame_; + int input_interval_ GUARDED_BY(critical_section_); + int64_t interval_next_frame_ GUARDED_BY(critical_section_); // Max number of pixels requested via calls to OnOutputFormatRequest, // OnResolutionRequest respectively. // The adapted output format is the minimum of these. - int format_request_max_pixel_count_; - int resolution_request_max_pixel_count_; + rtc::Optional requested_format_ GUARDED_BY(critical_section_); + int resolution_request_max_pixel_count_ GUARDED_BY(critical_section_); + int resolution_request_max_pixel_count_step_up_ GUARDED_BY(critical_section_); // The critical section to protect the above variables. rtc::CriticalSection critical_section_; diff --git a/webrtc/media/base/videoadapter_unittest.cc b/webrtc/media/base/videoadapter_unittest.cc index d793d566dd..92b4a558b9 100644 --- a/webrtc/media/base/videoadapter_unittest.cc +++ b/webrtc/media/base/videoadapter_unittest.cc @@ -50,8 +50,10 @@ class VideoAdapterTest : public testing::Test { int dropped_frames; bool last_adapt_was_no_op; - int adapted_width; - int adapted_height; + int cropped_width; + int cropped_height; + int out_width; + int out_height; }; explicit VideoCapturerListener(VideoAdapter* adapter) @@ -66,12 +68,21 @@ class VideoAdapterTest : public testing::Test { rtc::CritScope lock(&crit_); const int in_width = captured_frame->width; const int in_height = abs(captured_frame->height); - const VideoFormat adapted_format = - video_adapter_->AdaptFrameResolution(in_width, in_height); - if (!adapted_format.IsSize0x0()) { - adapted_format_ = adapted_format; - last_adapt_was_no_op_ = (in_width == adapted_format.width && - in_height == adapted_format.height); + int cropped_width; + int cropped_height; + int out_width; + int out_height; + video_adapter_->AdaptFrameResolution(in_width, in_height, + &cropped_width, &cropped_height, + &out_width, &out_height); + if (out_width != 0 && out_height != 0) { + cropped_width_ = cropped_width; + cropped_height_ = cropped_height; + out_width_ = out_width; + out_height_ = out_height; + last_adapt_was_no_op_ = + (in_width == cropped_width && in_height == cropped_height && + in_width == out_width && in_height == out_height); } else { ++dropped_frames_; } @@ -84,20 +95,20 @@ class VideoAdapterTest : public testing::Test { stats.captured_frames = captured_frames_; stats.dropped_frames = dropped_frames_; stats.last_adapt_was_no_op = last_adapt_was_no_op_; - if (!adapted_format_.IsSize0x0()) { - stats.adapted_width = adapted_format_.width; - stats.adapted_height = adapted_format_.height; - } else { - stats.adapted_width = stats.adapted_height = -1; - } - + stats.cropped_width = cropped_width_; + stats.cropped_height = cropped_height_; + stats.out_width = out_width_; + stats.out_height = out_height_; return stats; } private: rtc::CriticalSection crit_; VideoAdapter* video_adapter_; - VideoFormat adapted_format_; + int cropped_width_; + int cropped_height_; + int out_width_; + int out_height_; int captured_frames_; int dropped_frames_; bool last_adapt_was_no_op_; @@ -105,20 +116,28 @@ class VideoAdapterTest : public testing::Test { void VerifyAdaptedResolution(const VideoCapturerListener::Stats& stats, - int width, - int height) { - EXPECT_EQ(width, stats.adapted_width); - EXPECT_EQ(height, stats.adapted_height); + int cropped_width, + int cropped_height, + int out_width, + int out_height) { + EXPECT_EQ(cropped_width, stats.cropped_width); + EXPECT_EQ(cropped_height, stats.cropped_height); + EXPECT_EQ(out_width, stats.out_width); + EXPECT_EQ(out_height, stats.out_height); } std::unique_ptr capturer_; VideoAdapter adapter_; + int cropped_width_; + int cropped_height_; + int out_width_; + int out_height_; std::unique_ptr listener_; VideoFormat capture_format_; }; -// Do not adapt the frame rate or the resolution. Expect no frame drop and no -// resolution change. +// Do not adapt the frame rate or the resolution. Expect no frame drop, no +// cropping, and no resolution change. TEST_F(VideoAdapterTest, AdaptNothing) { EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) @@ -128,7 +147,8 @@ TEST_F(VideoAdapterTest, AdaptNothing) { VideoCapturerListener::Stats stats = listener_->GetStats(); EXPECT_GE(stats.captured_frames, 10); EXPECT_EQ(0, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height); + VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, + capture_format_.width, capture_format_.height); EXPECT_TRUE(stats.last_adapt_was_no_op); } @@ -144,7 +164,8 @@ TEST_F(VideoAdapterTest, AdaptZeroInterval) { VideoCapturerListener::Stats stats = listener_->GetStats(); EXPECT_GE(stats.captured_frames, 10); EXPECT_EQ(0, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height); + VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, + capture_format_.width, capture_format_.height); } // Adapt the frame rate to be half of the capture rate at the beginning. Expect @@ -161,7 +182,8 @@ TEST_F(VideoAdapterTest, AdaptFramerate) { VideoCapturerListener::Stats stats = listener_->GetStats(); EXPECT_GE(stats.captured_frames, 10); EXPECT_EQ(stats.captured_frames / 2, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height); + VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, + capture_format_.width, capture_format_.height); } // Adapt the frame rate to be half of the capture rate at the beginning. Expect @@ -179,7 +201,8 @@ TEST_F(VideoAdapterTest, AdaptFramerateVariable) { EXPECT_GE(stats.captured_frames, 30); // Verify 2 / 3 kept (20) and 1 / 3 dropped (10). EXPECT_EQ(stats.captured_frames * 1 / 3, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height); + VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, + capture_format_.width, capture_format_.height); } // Adapt the frame rate to be half of the capture rate after capturing no less @@ -206,39 +229,49 @@ TEST_F(VideoAdapterTest, AdaptFramerateOntheFly) { EXPECT_GT(listener_->GetStats().dropped_frames, 0); } -// Set a very high output pixel resolution. Expect no resolution change. +// Set a very high output pixel resolution. Expect no cropping or resolution +// change. TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) { VideoFormat output_format = capture_format_; - output_format.width = 2560; - output_format.height = 2560; + output_format.width *= 10; + output_format.height *= 10; adapter_.OnOutputFormatRequest(output_format); - VideoFormat adapted_format = adapter_.AdaptFrameResolution( - capture_format_.width, capture_format_.height); - EXPECT_EQ(capture_format_.width, adapted_format.width); - EXPECT_EQ(capture_format_.height, adapted_format.height); + adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(capture_format_.width, cropped_width_); + EXPECT_EQ(capture_format_.height, cropped_height_); + EXPECT_EQ(capture_format_.width, out_width_); + EXPECT_EQ(capture_format_.height, out_height_); } // Adapt the frame resolution to be the same as capture resolution. Expect no -// resolution change. +// cropping or resolution change. TEST_F(VideoAdapterTest, AdaptFrameResolutionIdentical) { adapter_.OnOutputFormatRequest(capture_format_); - const VideoFormat adapted_format = adapter_.AdaptFrameResolution( - capture_format_.width, capture_format_.height); - EXPECT_EQ(capture_format_.width, adapted_format.width); - EXPECT_EQ(capture_format_.height, adapted_format.height); + adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(capture_format_.width, cropped_width_); + EXPECT_EQ(capture_format_.height, cropped_height_); + EXPECT_EQ(capture_format_.width, out_width_); + EXPECT_EQ(capture_format_.height, out_height_); } // Adapt the frame resolution to be a quarter of the capture resolution. Expect -// resolution change. +// no cropping, but a resolution change. TEST_F(VideoAdapterTest, AdaptFrameResolutionQuarter) { VideoFormat request_format = capture_format_; request_format.width /= 2; request_format.height /= 2; adapter_.OnOutputFormatRequest(request_format); - const VideoFormat adapted_format = adapter_.AdaptFrameResolution( - request_format.width, request_format.height); - EXPECT_EQ(request_format.width, adapted_format.width); - EXPECT_EQ(request_format.height, adapted_format.height); + adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(capture_format_.width, cropped_width_); + EXPECT_EQ(capture_format_.height, cropped_height_); + EXPECT_EQ(request_format.width, out_width_); + EXPECT_EQ(request_format.height, out_height_); } // Adapt the pixel resolution to 0. Expect frame drop. @@ -247,14 +280,15 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionDrop) { output_format.width = 0; output_format.height = 0; adapter_.OnOutputFormatRequest(output_format); - EXPECT_TRUE( - adapter_ - .AdaptFrameResolution(capture_format_.width, capture_format_.height) - .IsSize0x0()); + adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(0, out_width_); + EXPECT_EQ(0, out_height_); } // Adapt the frame resolution to be a quarter of the capture resolution at the -// beginning. Expect resolution change. +// beginning. Expect no cropping but a resolution change. TEST_F(VideoAdapterTest, AdaptResolution) { VideoFormat request_format = capture_format_; request_format.width /= 2; @@ -264,10 +298,11 @@ TEST_F(VideoAdapterTest, AdaptResolution) { for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); - // Verify no frame drop and resolution change. + // Verify no frame drop, no cropping, and resolution change. VideoCapturerListener::Stats stats = listener_->GetStats(); EXPECT_EQ(0, stats.dropped_frames); - VerifyAdaptedResolution(stats, request_format.width, request_format.height); + VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, + request_format.width, request_format.height); } // Adapt the frame resolution to be a quarter of the capture resolution after @@ -281,8 +316,9 @@ TEST_F(VideoAdapterTest, AdaptResolutionOnTheFly) { capturer_->CaptureFrame(); // Verify no resolution change before adaptation. - VerifyAdaptedResolution( - listener_->GetStats(), request_format.width, request_format.height); + VerifyAdaptedResolution(listener_->GetStats(), + capture_format_.width, capture_format_.height, + request_format.width, request_format.height); // Adapt the frame resolution. request_format.width /= 2; @@ -292,8 +328,9 @@ TEST_F(VideoAdapterTest, AdaptResolutionOnTheFly) { capturer_->CaptureFrame(); // Verify resolution change after adaptation. - VerifyAdaptedResolution( - listener_->GetStats(), request_format.width, request_format.height); + VerifyAdaptedResolution(listener_->GetStats(), + capture_format_.width, capture_format_.height, + request_format.width, request_format.height); } // Drop all frames. @@ -313,104 +350,155 @@ TEST_F(VideoAdapterTest, DropAllFrames) { TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), 0); adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); - VideoFormat out_format = - adapter_.AdaptFrameResolution(format.width, format.height); - EXPECT_EQ(format, adapter_.input_format()); - EXPECT_EQ(format, out_format); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(400, out_height_); // Format request 640x400. format.height = 400; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 400); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(400, out_format.height); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(400, out_height_); - // Request 1280x720, higher than input. Adapt nothing. + // Request 1280x720, higher than input, but aspect 16:9. Expect cropping but + // no scaling. format.width = 1280; format.height = 720; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 400); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(400, out_format.height); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); // Request 0x0. format.width = 0; format.height = 0; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 400); - EXPECT_TRUE(out_format.IsSize0x0()); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(0, out_width_); + EXPECT_EQ(0, out_height_); - // Request 320x200. + // Request 320x200. Expect scaling, but no cropping. format.width = 320; format.height = 200; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 400); - EXPECT_EQ(320, out_format.width); - EXPECT_EQ(200, out_format.height); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(320, out_width_); + EXPECT_EQ(200, out_height_); - // Request resolution of 2 / 3. Expect adapt down. Scaling to 1/3 is not - // optimized and not allowed. - format.width = (640 * 2 + 1) / 3; - format.height = (400 * 2 + 1) / 3; + // Request resolution close to 2/3 scale. Expect adapt down. Scaling to 2/3 + // is not optimized and not allowed, therefore 1/2 scaling will be used + // instead. + format.width = 424; + format.height = 265; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 400); - EXPECT_EQ(320, out_format.width); - EXPECT_EQ(200, out_format.height); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(320, out_width_); + EXPECT_EQ(200, out_height_); // Request resolution of 3 / 8. Expect adapt down. format.width = 640 * 3 / 8; format.height = 400 * 3 / 8; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 400); - EXPECT_EQ(640 * 3 / 8, out_format.width); - EXPECT_EQ(400 * 3 / 8, out_format.height); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(640 * 3 / 8, out_width_); + EXPECT_EQ(400 * 3 / 8, out_height_); // Switch back up. Expect adapt. format.width = 320; format.height = 200; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 400); - EXPECT_EQ(320, out_format.width); - EXPECT_EQ(200, out_format.height); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(320, out_width_); + EXPECT_EQ(200, out_height_); // Format request 480x300. format.width = 480; format.height = 300; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 400); - EXPECT_EQ(480, out_format.width); - EXPECT_EQ(300, out_format.height); + adapter_.AdaptFrameResolution(640, 400, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(400, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(300, out_height_); } TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { // Start at HD. VideoFormat format(1280, 720, VideoFormat::FpsToInterval(30), 0); adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); - VideoFormat out_format = - adapter_.AdaptFrameResolution(format.width, format.height); - EXPECT_EQ(format, adapter_.input_format()); - EXPECT_EQ(out_format, adapter_.input_format()); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); // Format request for VGA. format.width = 640; format.height = 360; adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); // Now, the camera reopens at VGA. // Both the frame and the output format should be 640x360. - out_format = adapter_.AdaptFrameResolution(640, 360); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(640, 360, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); // And another view request comes in for 640x360, which should have no // real impact. adapter_.OnOutputFormatRequest(format); - out_format = adapter_.AdaptFrameResolution(640, 360); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(640, 360, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); } TEST_F(VideoAdapterTest, TestVGAWidth) { @@ -419,130 +507,305 @@ TEST_F(VideoAdapterTest, TestVGAWidth) { adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); adapter_.OnOutputFormatRequest(format); - VideoFormat out_format = adapter_.AdaptFrameResolution(640, 480); - // At this point, we have to adapt down to something lower. - EXPECT_EQ(480, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + // Expect cropping. + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); // But if frames come in at 640x360, we shouldn't adapt them down. - out_format = adapter_.AdaptFrameResolution(640, 360); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(640, 360, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); - out_format = adapter_.AdaptFrameResolution(640, 480); - EXPECT_EQ(480, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); } TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { - VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(1280, out_format.width); - EXPECT_EQ(720, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); // Adapt down one step. adapter_.OnResolutionRequest(rtc::Optional(1280 * 720 - 1), rtc::Optional()); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(960, out_format.width); - EXPECT_EQ(540, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(960, out_width_); + EXPECT_EQ(540, out_height_); // Adapt down one step more. adapter_.OnResolutionRequest(rtc::Optional(960 * 540 - 1), rtc::Optional()); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); // Adapt down one step more. adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(480, out_format.width); - EXPECT_EQ(270, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); // Adapt up one step. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(480 * 270)); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); // Adapt up one step more. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(640 * 360)); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(960, out_format.width); - EXPECT_EQ(540, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(960, out_width_); + EXPECT_EQ(540, out_height_); // Adapt up one step more. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(1280, out_format.width); - EXPECT_EQ(720, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); } TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) { - VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(1280, out_format.width); - EXPECT_EQ(720, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); adapter_.OnResolutionRequest(rtc::Optional(0), rtc::Optional()); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(0, out_format.width); - EXPECT_EQ(0, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(0, out_width_); + EXPECT_EQ(0, out_height_); } TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(480, out_format.width); - EXPECT_EQ(270, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(1280, out_format.width); - EXPECT_EQ(720, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); } TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(480, out_format.width); - EXPECT_EQ(270, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); VideoFormat new_format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); adapter_.OnOutputFormatRequest(new_format); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(480, out_format.width); - EXPECT_EQ(270, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(640, out_format.width); - EXPECT_EQ(360, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); } TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { - VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(1280, out_format.width); - EXPECT_EQ(720, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(480, out_format.width); - EXPECT_EQ(270, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional()); - out_format = adapter_.AdaptFrameResolution(1280, 720); - EXPECT_EQ(1280, out_format.width); - EXPECT_EQ(720, out_format.height); + adapter_.AdaptFrameResolution(1280, 720, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(1280, cropped_width_); + EXPECT_EQ(720, cropped_height_); + EXPECT_EQ(1280, out_width_); + EXPECT_EQ(720, out_height_); +} + +TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { + // Ask for 640x360 (16:9 aspect). + adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + adapter_.OnOutputFormatRequest( + VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420)); + // Send 640x480 (4:3 aspect). + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + // Expect cropping to 16:9 format and no scaling. + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // Adapt down one step. + adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), + rtc::Optional()); + // Expect cropping to 16:9 format and 3/4 scaling. + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + // Adapt down one step more. + adapter_.OnResolutionRequest(rtc::Optional(480 * 270 - 1), + rtc::Optional()); + // Expect cropping to 16:9 format and 1/2 scaling. + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(320, out_width_); + EXPECT_EQ(180, out_height_); + + // Adapt up one step. + adapter_.OnResolutionRequest(rtc::Optional(), + rtc::Optional(320 * 180)); + // Expect cropping to 16:9 format and 3/4 scaling. + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(480, out_width_); + EXPECT_EQ(270, out_height_); + + // Adapt up one step more. + adapter_.OnResolutionRequest(rtc::Optional(), + rtc::Optional(480 * 270)); + // Expect cropping to 16:9 format and no scaling. + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); + + // Try to adapt up one step more. + adapter_.OnResolutionRequest(rtc::Optional(), + rtc::Optional(640 * 360)); + // Expect cropping to 16:9 format and no scaling. + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(360, cropped_height_); + EXPECT_EQ(640, out_width_); + EXPECT_EQ(360, out_height_); +} + +TEST_F(VideoAdapterTest, TestCroppingOddResolution) { + // Ask for 640x360 (16:9 aspect), with 3/16 scaling. + adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + adapter_.OnOutputFormatRequest( + VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420)); + adapter_.OnResolutionRequest(rtc::Optional(640 * 360 * 3 / 16 * 3 / 16), + rtc::Optional()); + + // Send 640x480 (4:3 aspect). + adapter_.AdaptFrameResolution(640, 480, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + + // Instead of getting the exact aspect ratio with cropped resolution 640x360, + // the resolution should be adjusted to get a perfect scale factor instead. + EXPECT_EQ(640, cropped_width_); + EXPECT_EQ(368, cropped_height_); + EXPECT_EQ(120, out_width_); + EXPECT_EQ(69, out_height_); } } // namespace cricket diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc index b00cbfe8e3..084a9b913c 100644 --- a/webrtc/media/base/videocapturer.cc +++ b/webrtc/media/base/videocapturer.cc @@ -220,17 +220,19 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*, return; } - int adapted_width = captured_frame->width; - int adapted_height = captured_frame->height; + int cropped_width = captured_frame->width; + int cropped_height = captured_frame->height; + int out_width = captured_frame->width; + int out_height = captured_frame->height; if (enable_video_adapter_ && !IsScreencast()) { - const VideoFormat adapted_format = - video_adapter_.AdaptFrameResolution(adapted_width, adapted_height); - if (adapted_format.IsSize0x0()) { + video_adapter_.AdaptFrameResolution( + captured_frame->width, captured_frame->height, + &cropped_width, &cropped_height, + &out_width, &out_height); + if (out_width == 0 || out_height == 0) { // VideoAdapter dropped the frame. return; } - adapted_width = adapted_format.width; - adapted_height = adapted_format.height; } if (!frame_factory_) { @@ -238,17 +240,15 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*, return; } - // TODO(nisse): Reorganize frame factory methods, deleting crop - // support there too. + // TODO(nisse): Reorganize frame factory methods. std::unique_ptr adapted_frame(frame_factory_->CreateAliasedFrame( - captured_frame, captured_frame->width, captured_frame->height, - adapted_width, adapted_height)); + captured_frame, cropped_width, cropped_height, out_width, out_height)); if (!adapted_frame) { // TODO(fbarchard): LOG more information about captured frame attributes. LOG(LS_ERROR) << "Couldn't convert to I420! " << "From " << ToString(captured_frame) << " To " - << adapted_width << " x " << adapted_height; + << out_width << " x " << out_height; return; }