admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

VideoAdapter: Add cropping based on OnOutputFormatRequest()

Browse Source 
If OnOutputFormatRequest() is called, VideoAdapter will crop to the same
aspect ratio as the requested format. The output from
VideoAdapter.AdaptFrameResolution() now contains both how to crop the
input frame, and how to scale the cropped frame to the final adapted
resolution.

BUG=b/28622232

Review-Url: https://codereview.webrtc.org/1966273002
Cr-Commit-Position: refs/heads/master@{#12732}
This commit is contained in:
magjed 2016-05-13 10:26:00 -07:00 committed by Commit bot
parent c8d848b104
commit 709f73c04e
4 changed files with 598 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

280
webrtc/media/base/videoadapter.cc
View File

@ -13,69 +13,77 @@
#include <algorithm>
#include <limits>
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/media/base/mediaconstants.h"
#include "webrtc/media/base/videocommon.h"
namespace {
// Scale factors optimized for in libYUV that we accept.
// Must be sorted in decreasing scale factors for FindScaleLargerThan to work.
const float kScaleFactors[] = {
1.f / 1.f, // Full size.
3.f / 4.f, // 3/4 scale.
1.f / 2.f, // 1/2 scale.
3.f / 8.f, // 3/8 scale.
1.f / 4.f, // 1/4 scale.
3.f / 16.f, // 3/16 scale.
struct Fraction {
int numerator;
int denominator;
};
float FindScaleLessThanOrEqual(int width,
int height,
int target_num_pixels,
int* resulting_number_of_pixels) {
// Scale factors optimized for in libYUV that we accept.
// Must be sorted in decreasing scale factors for FindScaleLargerThan to work.
const Fraction kScaleFractions[] = {
{1, 1},
{3, 4},
{1, 2},
{3, 8},
{1, 4},
{3, 16},
};
// Round |valueToRound| to a multiple of |multiple|. Prefer rounding upwards,
// but never more than |maxValue|.
int roundUp(int valueToRound, int multiple, int maxValue) {
const int roundedValue = (valueToRound + multiple - 1) / multiple * multiple;
return roundedValue <= maxValue ? roundedValue
: (maxValue / multiple * multiple);
}
Fraction FindScaleLessThanOrEqual(int input_num_pixels, int target_num_pixels) {
float best_distance = std::numeric_limits<float>::max();
float best_scale = 0.0f; // Default to 0 if nothing matches.
float pixels = width * height;
float best_number_of_pixels = 0.0f;
for (const auto& scale : kScaleFactors) {
float test_num_pixels = pixels * scale * scale;
Fraction best_scale = {0, 1}; // Default to 0 if nothing matches.
for (const auto& fraction : kScaleFractions) {
const float scale =
fraction.numerator / static_cast<float>(fraction.denominator);
float test_num_pixels = input_num_pixels * scale * scale;
float diff = target_num_pixels - test_num_pixels;
if (diff < 0) {
continue;
}
if (diff < best_distance) {
best_distance = diff;
best_scale = scale;
best_number_of_pixels = test_num_pixels;
best_scale = fraction;
if (best_distance == 0) { // Found exact match.
break;
}
}
}
if (resulting_number_of_pixels) {
*resulting_number_of_pixels = static_cast<int>(best_number_of_pixels + .5f);
}
return best_scale;
}
float FindScaleLargerThan(int width,
int height,
int target_num_pixels,
int* resulting_number_of_pixels) {
Fraction FindScaleLargerThan(int input_num_pixels,
int target_num_pixels,
int* resulting_number_of_pixels) {
float best_distance = std::numeric_limits<float>::max();
float best_scale = 1.f; // Default to unscaled if nothing matches.
float pixels = width * height;
float best_number_of_pixels = pixels; // Default to input number of pixels.
for (const auto& scale : kScaleFactors) {
float test_num_pixels = pixels * scale * scale;
Fraction best_scale = {1, 1}; // Default to unscaled if nothing matches.
// Default to input number of pixels.
float best_number_of_pixels = input_num_pixels;
for (const auto& fraction : kScaleFractions) {
const float scale =
fraction.numerator / static_cast<float>(fraction.denominator);
float test_num_pixels = input_num_pixels * scale * scale;
float diff = test_num_pixels - target_num_pixels;
if (diff <= 0) {
break;
}
if (diff < best_distance) {
best_distance = diff;
best_scale = scale;
best_scale = fraction;
best_number_of_pixels = test_num_pixels;
}
}
@ -84,21 +92,36 @@ float FindScaleLargerThan(int width,
return best_scale;
}
Fraction FindScale(int input_num_pixels,
int max_pixel_count_step_up,
int max_pixel_count) {
// Try scale just above |max_pixel_count_step_up_|.
if (max_pixel_count_step_up > 0) {
int resulting_pixel_count;
const Fraction scale = FindScaleLargerThan(
input_num_pixels, max_pixel_count_step_up, &resulting_pixel_count);
if (resulting_pixel_count <= max_pixel_count)
return scale;
}
// Return largest scale below |max_pixel_count|.
return FindScaleLessThanOrEqual(input_num_pixels, max_pixel_count);
}
} // namespace
namespace cricket {
VideoAdapter::VideoAdapter()
: output_num_pixels_(std::numeric_limits<int>::max()),
frames_in_(0),
: frames_in_(0),
frames_out_(0),
frames_scaled_(0),
adaption_changes_(0),
previous_width_(0),
previous_height_(0),
input_interval_(0),
interval_next_frame_(0),
format_request_max_pixel_count_(std::numeric_limits<int>::max()),
resolution_request_max_pixel_count_(std::numeric_limits<int>::max()) {}
resolution_request_max_pixel_count_(std::numeric_limits<int>::max()),
resolution_request_max_pixel_count_step_up_(0) {}
VideoAdapter::~VideoAdapter() {}
@ -106,57 +129,43 @@ void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) {
// TODO(perkj): Consider measuring input frame rate instead.
// Frame rate typically varies depending on lighting.
rtc::CritScope cs(&critical_section_);
input_format_.interval = interval;
input_interval_ = interval;
}
void VideoAdapter::SetInputFormat(const VideoFormat& format) {
bool is_resolution_change = (input_format().width != format.width ||
input_format().height != format.height);
int64_t old_input_interval = input_format_.interval;
input_format_ = format;
output_format_.interval =
std::max(output_format_.interval, input_format_.interval);
if (old_input_interval != input_format_.interval) {
LOG(LS_INFO) << "VAdapt input interval changed from "
<< old_input_interval << " to " << input_format_.interval;
}
if (is_resolution_change) {
// Trigger the adaptation logic again, to potentially reset the adaptation
// state for things like view requests that may not longer be capping
// output (or may now cap output).
Adapt(std::min(format_request_max_pixel_count_,
resolution_request_max_pixel_count_),
0);
}
}
const VideoFormat& VideoAdapter::input_format() const {
rtc::CritScope cs(&critical_section_);
return input_format_;
}
VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) {
void VideoAdapter::AdaptFrameResolution(int in_width,
int in_height,
int* cropped_width,
int* cropped_height,
int* out_width,
int* out_height) {
rtc::CritScope cs(&critical_section_);
++frames_in_;
SetInputFormat(VideoFormat(
in_width, in_height, input_format_.interval, input_format_.fourcc));
// The max output pixel count is the minimum of the requests from
// OnOutputFormatRequest and OnResolutionRequest.
int max_pixel_count = resolution_request_max_pixel_count_;
if (requested_format_) {
max_pixel_count = std::min(
max_pixel_count, requested_format_->width * requested_format_->height);
}
// Drop the input frame if necessary.
bool should_drop = false;
if (!output_num_pixels_) {
if (max_pixel_count == 0) {
// Drop all frames as the output format is 0x0.
should_drop = true;
} else {
} else if (requested_format_ && requested_format_->interval > 0) {
// Drop some frames based on input fps and output fps.
// Normally output fps is less than input fps.
interval_next_frame_ += input_format_.interval;
if (output_format_.interval > 0) {
if (interval_next_frame_ >= output_format_.interval) {
interval_next_frame_ %= output_format_.interval;
} else {
should_drop = true;
}
interval_next_frame_ += input_interval_;
if (interval_next_frame_ >= requested_format_->interval) {
interval_next_frame_ -= requested_format_->interval;
// Reset |interval_next_frame_| if it accumulates too much to avoid
// "catching up" behaviour.
if (interval_next_frame_ >= requested_format_->interval)
interval_next_frame_ = 0;
} else {
should_drop = true;
}
}
if (should_drop) {
@ -170,48 +179,79 @@ VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) {
<< " Changes: " << adaption_changes_
<< " Input: " << in_width
<< "x" << in_height
<< " i" << input_format_.interval
<< " Output: i" << output_format_.interval;
<< " i" << input_interval_
<< " Output: i"
<< (requested_format_ ? requested_format_->interval : 0);
}
return VideoFormat(); // Drop frame.
// Drop frame.
*cropped_width = 0;
*cropped_height = 0;
*out_width = 0;
*out_height = 0;
return;
}
const float scale = FindScaleLessThanOrEqual(in_width, in_height,
output_num_pixels_, nullptr);
const int output_width = static_cast<int>(in_width * scale + .5f);
const int output_height = static_cast<int>(in_height * scale + .5f);
// Calculate how the input should be cropped.
if (!requested_format_ ||
requested_format_->width == 0 || requested_format_->height == 0) {
*cropped_width = in_width;
*cropped_height = in_height;
} else {
// Adjust |requested_format_| orientation to match input.
if ((in_width > in_height) !=
(requested_format_->width > requested_format_->height)) {
std::swap(requested_format_->width, requested_format_->height);
}
const float requested_aspect =
requested_format_->width /
static_cast<float>(requested_format_->height);
*cropped_width =
std::min(in_width, static_cast<int>(in_height * requested_aspect));
*cropped_height =
std::min(in_height, static_cast<int>(in_width / requested_aspect));
}
// Find best scale factor.
const Fraction scale =
FindScale(*cropped_width * *cropped_height,
resolution_request_max_pixel_count_step_up_, max_pixel_count);
// Adjust cropping slightly to get even integer output size and a perfect
// scale factor.
*cropped_width = roundUp(*cropped_width, scale.denominator, in_width);
*cropped_height = roundUp(*cropped_height, scale.denominator, in_height);
RTC_DCHECK_EQ(0, *cropped_width % scale.denominator);
RTC_DCHECK_EQ(0, *cropped_height % scale.denominator);
// Calculate final output size.
*out_width = *cropped_width / scale.denominator * scale.numerator;
*out_height = *cropped_height / scale.denominator * scale.numerator;
++frames_out_;
if (scale != 1)
if (scale.numerator != scale.denominator)
++frames_scaled_;
if (previous_width_ && (previous_width_ != output_width ||
previous_height_ != output_height)) {
if (previous_width_ && (previous_width_ != *out_width ||
previous_height_ != *out_height)) {
++adaption_changes_;
LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out "
<< frames_out_ << " / in " << frames_in_
<< " Changes: " << adaption_changes_ << " Input: " << in_width
<< "x" << in_height << " i" << input_format_.interval
<< " Scale: " << scale << " Output: " << output_width << "x"
<< output_height << " i" << output_format_.interval;
<< "x" << in_height << " i" << input_interval_
<< " Scale: " << scale.numerator << "/" << scale.denominator
<< " Output: " << *out_width << "x" << *out_height << " i"
<< (requested_format_ ? requested_format_->interval : 0);
}
output_format_.width = output_width;
output_format_.height = output_height;
previous_width_ = output_width;
previous_height_ = output_height;
return output_format_;
previous_width_ = *out_width;
previous_height_ = *out_height;
}
void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
rtc::CritScope cs(&critical_section_);
format_request_max_pixel_count_ = format.width * format.height;
output_format_.interval = format.interval;
Adapt(std::min(format_request_max_pixel_count_,
resolution_request_max_pixel_count_),
0);
requested_format_ = rtc::Optional<VideoFormat>(format);
interval_next_frame_ = 0;
}
void VideoAdapter::OnResolutionRequest(
@ -220,44 +260,8 @@ void VideoAdapter::OnResolutionRequest(
rtc::CritScope cs(&critical_section_);
resolution_request_max_pixel_count_ =
max_pixel_count.value_or(std::numeric_limits<int>::max());
Adapt(std::min(format_request_max_pixel_count_,
resolution_request_max_pixel_count_),
max_pixel_count_step_up.value_or(0));
}
bool VideoAdapter::Adapt(int max_num_pixels, int max_pixel_count_step_up) {
float scale_lower =
FindScaleLessThanOrEqual(input_format_.width, input_format_.height,
max_num_pixels, &max_num_pixels);
float scale_upper =
max_pixel_count_step_up > 0
? FindScaleLargerThan(input_format_.width, input_format_.height,
max_pixel_count_step_up,
&max_pixel_count_step_up)
: 1.f;
bool use_max_pixel_count_step_up =
max_pixel_count_step_up > 0 && max_num_pixels > max_pixel_count_step_up;
int old_num_pixels = output_num_pixels_;
output_num_pixels_ =
use_max_pixel_count_step_up ? max_pixel_count_step_up : max_num_pixels;
// Log the new size.
float scale = use_max_pixel_count_step_up ? scale_upper : scale_lower;
int new_width = static_cast<int>(input_format_.width * scale + .5f);
int new_height = static_cast<int>(input_format_.height * scale + .5f);
bool changed = output_num_pixels_ != old_num_pixels;
LOG(LS_INFO) << "OnResolutionRequest: "
<< " Max pixels: " << max_num_pixels
<< " Max pixels step up: " << max_pixel_count_step_up
<< " Output Pixels: " << output_num_pixels_
<< " Input: " << input_format_.width << "x"
<< input_format_.height << " Scale: " << scale
<< " Resolution: " << new_width << "x" << new_height
<< " Changed: " << (changed ? "true" : "false");
return changed;
resolution_request_max_pixel_count_step_up_ =
max_pixel_count_step_up.value_or(0);
}
} // namespace cricket

35
webrtc/media/base/videoadapter.h
View File

@ -32,12 +32,23 @@ class VideoAdapter {
// interval.
void SetExpectedInputFrameInterval(int64_t interval);
// Return the adapted resolution given the input resolution. The returned
// resolution will be 0x0 if the frame should be dropped.
VideoFormat AdaptFrameResolution(int in_width, int in_height);
// Return the adapted resolution given the input resolution. The input
// resolution should first be cropped to the specified resolution, and then
// scaled to the final output resolution. The output resolution will be 0x0 if
// the frame should be dropped.
void AdaptFrameResolution(int in_width,
int in_height,
int* cropped_width,
int* cropped_height,
int* out_width,
int* out_height);
// Requests the output frame size and frame interval from
// |AdaptFrameResolution| to not be larger than |format|.
// |AdaptFrameResolution| to not be larger than |format|. Also, the input
// frame size will be cropped to match the requested aspect ratio. The
// requested aspect ratio is orientation agnostic and will be adjusted to
// maintain the input orientation, so it doesn't matter if e.g. 1280x720 or
// 720x1280 is requested.
void OnOutputFormatRequest(const VideoFormat& format);
// Requests the output frame size from |AdaptFrameResolution| to not have
@ -46,28 +57,22 @@ class VideoAdapter {
void OnResolutionRequest(rtc::Optional<int> max_pixel_count,
rtc::Optional<int> max_pixel_count_step_up);
const VideoFormat& input_format() const;
private:
void SetInputFormat(const VideoFormat& format);
bool Adapt(int max_num_pixels, int max_pixel_count_step_up);
VideoFormat input_format_;
VideoFormat output_format_;
int output_num_pixels_;
int frames_in_; // Number of input frames.
int frames_out_; // Number of output frames.
int frames_scaled_; // Number of frames scaled.
int adaption_changes_; // Number of changes in scale factor.
int previous_width_; // Previous adapter output width.
int previous_height_; // Previous adapter output height.
int64_t interval_next_frame_;
int input_interval_ GUARDED_BY(critical_section_);
int64_t interval_next_frame_ GUARDED_BY(critical_section_);
// Max number of pixels requested via calls to OnOutputFormatRequest,
// OnResolutionRequest respectively.
// The adapted output format is the minimum of these.
int format_request_max_pixel_count_;
int resolution_request_max_pixel_count_;
rtc::Optional<VideoFormat> requested_format_ GUARDED_BY(critical_section_);
int resolution_request_max_pixel_count_ GUARDED_BY(critical_section_);
int resolution_request_max_pixel_count_step_up_ GUARDED_BY(critical_section_);
// The critical section to protect the above variables.
rtc::CriticalSection critical_section_;

585
webrtc/media/base/videoadapter_unittest.cc
View File

@ -50,8 +50,10 @@ class VideoAdapterTest : public testing::Test {
int dropped_frames;
bool last_adapt_was_no_op;
int adapted_width;
int adapted_height;
int cropped_width;
int cropped_height;
int out_width;
int out_height;
};
explicit VideoCapturerListener(VideoAdapter* adapter)
@ -66,12 +68,21 @@ class VideoAdapterTest : public testing::Test {
rtc::CritScope lock(&crit_);
const int in_width = captured_frame->width;
const int in_height = abs(captured_frame->height);
const VideoFormat adapted_format =
video_adapter_->AdaptFrameResolution(in_width, in_height);
if (!adapted_format.IsSize0x0()) {
adapted_format_ = adapted_format;
last_adapt_was_no_op_ = (in_width == adapted_format.width &&
in_height == adapted_format.height);
int cropped_width;
int cropped_height;
int out_width;
int out_height;
video_adapter_->AdaptFrameResolution(in_width, in_height,
&cropped_width, &cropped_height,
&out_width, &out_height);
if (out_width != 0 && out_height != 0) {
cropped_width_ = cropped_width;
cropped_height_ = cropped_height;
out_width_ = out_width;
out_height_ = out_height;
last_adapt_was_no_op_ =
(in_width == cropped_width && in_height == cropped_height &&
in_width == out_width && in_height == out_height);
} else {
++dropped_frames_;
}
@ -84,20 +95,20 @@ class VideoAdapterTest : public testing::Test {
stats.captured_frames = captured_frames_;
stats.dropped_frames = dropped_frames_;
stats.last_adapt_was_no_op = last_adapt_was_no_op_;
if (!adapted_format_.IsSize0x0()) {
stats.adapted_width = adapted_format_.width;
stats.adapted_height = adapted_format_.height;
} else {
stats.adapted_width = stats.adapted_height = -1;
}
stats.cropped_width = cropped_width_;
stats.cropped_height = cropped_height_;
stats.out_width = out_width_;
stats.out_height = out_height_;
return stats;
}
private:
rtc::CriticalSection crit_;
VideoAdapter* video_adapter_;
VideoFormat adapted_format_;
int cropped_width_;
int cropped_height_;
int out_width_;
int out_height_;
int captured_frames_;
int dropped_frames_;
bool last_adapt_was_no_op_;
@ -105,20 +116,28 @@ class VideoAdapterTest : public testing::Test {
void VerifyAdaptedResolution(const VideoCapturerListener::Stats& stats,
int width,
int height) {
EXPECT_EQ(width, stats.adapted_width);
EXPECT_EQ(height, stats.adapted_height);
int cropped_width,
int cropped_height,
int out_width,
int out_height) {
EXPECT_EQ(cropped_width, stats.cropped_width);
EXPECT_EQ(cropped_height, stats.cropped_height);
EXPECT_EQ(out_width, stats.out_width);
EXPECT_EQ(out_height, stats.out_height);
}
std::unique_ptr<FakeVideoCapturer> capturer_;
VideoAdapter adapter_;
int cropped_width_;
int cropped_height_;
int out_width_;
int out_height_;
std::unique_ptr<VideoCapturerListener> listener_;
VideoFormat capture_format_;
};
// Do not adapt the frame rate or the resolution. Expect no frame drop and no
// resolution change.
// Do not adapt the frame rate or the resolution. Expect no frame drop, no
// cropping, and no resolution change.
TEST_F(VideoAdapterTest, AdaptNothing) {
EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_));
for (int i = 0; i < 10; ++i)
@ -128,7 +147,8 @@ TEST_F(VideoAdapterTest, AdaptNothing) {
VideoCapturerListener::Stats stats = listener_->GetStats();
EXPECT_GE(stats.captured_frames, 10);
EXPECT_EQ(0, stats.dropped_frames);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height,
capture_format_.width, capture_format_.height);
EXPECT_TRUE(stats.last_adapt_was_no_op);
}
@ -144,7 +164,8 @@ TEST_F(VideoAdapterTest, AdaptZeroInterval) {
VideoCapturerListener::Stats stats = listener_->GetStats();
EXPECT_GE(stats.captured_frames, 10);
EXPECT_EQ(0, stats.dropped_frames);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height,
capture_format_.width, capture_format_.height);
}
// Adapt the frame rate to be half of the capture rate at the beginning. Expect
@ -161,7 +182,8 @@ TEST_F(VideoAdapterTest, AdaptFramerate) {
VideoCapturerListener::Stats stats = listener_->GetStats();
EXPECT_GE(stats.captured_frames, 10);
EXPECT_EQ(stats.captured_frames / 2, stats.dropped_frames);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height,
capture_format_.width, capture_format_.height);
}
// Adapt the frame rate to be half of the capture rate at the beginning. Expect
@ -179,7 +201,8 @@ TEST_F(VideoAdapterTest, AdaptFramerateVariable) {
EXPECT_GE(stats.captured_frames, 30);
// Verify 2 / 3 kept (20) and 1 / 3 dropped (10).
EXPECT_EQ(stats.captured_frames * 1 / 3, stats.dropped_frames);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height,
capture_format_.width, capture_format_.height);
}
// Adapt the frame rate to be half of the capture rate after capturing no less
@ -206,39 +229,49 @@ TEST_F(VideoAdapterTest, AdaptFramerateOntheFly) {
EXPECT_GT(listener_->GetStats().dropped_frames, 0);
}
// Set a very high output pixel resolution. Expect no resolution change.
// Set a very high output pixel resolution. Expect no cropping or resolution
// change.
TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) {
VideoFormat output_format = capture_format_;
output_format.width = 2560;
output_format.height = 2560;
output_format.width *= 10;
output_format.height *= 10;
adapter_.OnOutputFormatRequest(output_format);
VideoFormat adapted_format = adapter_.AdaptFrameResolution(
capture_format_.width, capture_format_.height);
EXPECT_EQ(capture_format_.width, adapted_format.width);
EXPECT_EQ(capture_format_.height, adapted_format.height);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
EXPECT_EQ(capture_format_.width, out_width_);
EXPECT_EQ(capture_format_.height, out_height_);
}
// Adapt the frame resolution to be the same as capture resolution. Expect no
// resolution change.
// cropping or resolution change.
TEST_F(VideoAdapterTest, AdaptFrameResolutionIdentical) {
adapter_.OnOutputFormatRequest(capture_format_);
const VideoFormat adapted_format = adapter_.AdaptFrameResolution(
capture_format_.width, capture_format_.height);
EXPECT_EQ(capture_format_.width, adapted_format.width);
EXPECT_EQ(capture_format_.height, adapted_format.height);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
EXPECT_EQ(capture_format_.width, out_width_);
EXPECT_EQ(capture_format_.height, out_height_);
}
// Adapt the frame resolution to be a quarter of the capture resolution. Expect
// resolution change.
// no cropping, but a resolution change.
TEST_F(VideoAdapterTest, AdaptFrameResolutionQuarter) {
VideoFormat request_format = capture_format_;
request_format.width /= 2;
request_format.height /= 2;
adapter_.OnOutputFormatRequest(request_format);
const VideoFormat adapted_format = adapter_.AdaptFrameResolution(
request_format.width, request_format.height);
EXPECT_EQ(request_format.width, adapted_format.width);
EXPECT_EQ(request_format.height, adapted_format.height);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
EXPECT_EQ(request_format.width, out_width_);
EXPECT_EQ(request_format.height, out_height_);
}
// Adapt the pixel resolution to 0. Expect frame drop.
@ -247,14 +280,15 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionDrop) {
output_format.width = 0;
output_format.height = 0;
adapter_.OnOutputFormatRequest(output_format);
EXPECT_TRUE(
adapter_
.AdaptFrameResolution(capture_format_.width, capture_format_.height)
.IsSize0x0());
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
EXPECT_EQ(0, out_height_);
}
// Adapt the frame resolution to be a quarter of the capture resolution at the
// beginning. Expect resolution change.
// beginning. Expect no cropping but a resolution change.
TEST_F(VideoAdapterTest, AdaptResolution) {
VideoFormat request_format = capture_format_;
request_format.width /= 2;
@ -264,10 +298,11 @@ TEST_F(VideoAdapterTest, AdaptResolution) {
for (int i = 0; i < 10; ++i)
capturer_->CaptureFrame();
// Verify no frame drop and resolution change.
// Verify no frame drop, no cropping, and resolution change.
VideoCapturerListener::Stats stats = listener_->GetStats();
EXPECT_EQ(0, stats.dropped_frames);
VerifyAdaptedResolution(stats, request_format.width, request_format.height);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height,
request_format.width, request_format.height);
}
// Adapt the frame resolution to be a quarter of the capture resolution after
@ -281,8 +316,9 @@ TEST_F(VideoAdapterTest, AdaptResolutionOnTheFly) {
capturer_->CaptureFrame();
// Verify no resolution change before adaptation.
VerifyAdaptedResolution(
listener_->GetStats(), request_format.width, request_format.height);
VerifyAdaptedResolution(listener_->GetStats(),
capture_format_.width, capture_format_.height,
request_format.width, request_format.height);
// Adapt the frame resolution.
request_format.width /= 2;
@ -292,8 +328,9 @@ TEST_F(VideoAdapterTest, AdaptResolutionOnTheFly) {
capturer_->CaptureFrame();
// Verify resolution change after adaptation.
VerifyAdaptedResolution(
listener_->GetStats(), request_format.width, request_format.height);
VerifyAdaptedResolution(listener_->GetStats(),
capture_format_.width, capture_format_.height,
request_format.width, request_format.height);
}
// Drop all frames.
@ -313,104 +350,155 @@ TEST_F(VideoAdapterTest, DropAllFrames) {
TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), 0);
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
VideoFormat out_format =
adapter_.AdaptFrameResolution(format.width, format.height);
EXPECT_EQ(format, adapter_.input_format());
EXPECT_EQ(format, out_format);
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(400, out_height_);
// Format request 640x400.
format.height = 400;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 400);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(400, out_format.height);
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(400, out_height_);
// Request 1280x720, higher than input. Adapt nothing.
// Request 1280x720, higher than input, but aspect 16:9. Expect cropping but
// no scaling.
format.width = 1280;
format.height = 720;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 400);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(400, out_format.height);
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
// Request 0x0.
format.width = 0;
format.height = 0;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 400);
EXPECT_TRUE(out_format.IsSize0x0());
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
EXPECT_EQ(0, out_height_);
// Request 320x200.
// Request 320x200. Expect scaling, but no cropping.
format.width = 320;
format.height = 200;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 400);
EXPECT_EQ(320, out_format.width);
EXPECT_EQ(200, out_format.height);
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(320, out_width_);
EXPECT_EQ(200, out_height_);
// Request resolution of 2 / 3. Expect adapt down. Scaling to 1/3 is not
// optimized and not allowed.
format.width = (640 * 2 + 1) / 3;
format.height = (400 * 2 + 1) / 3;
// Request resolution close to 2/3 scale. Expect adapt down. Scaling to 2/3
// is not optimized and not allowed, therefore 1/2 scaling will be used
// instead.
format.width = 424;
format.height = 265;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 400);
EXPECT_EQ(320, out_format.width);
EXPECT_EQ(200, out_format.height);
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(320, out_width_);
EXPECT_EQ(200, out_height_);
// Request resolution of 3 / 8. Expect adapt down.
format.width = 640 * 3 / 8;
format.height = 400 * 3 / 8;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 400);
EXPECT_EQ(640 * 3 / 8, out_format.width);
EXPECT_EQ(400 * 3 / 8, out_format.height);
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(640 * 3 / 8, out_width_);
EXPECT_EQ(400 * 3 / 8, out_height_);
// Switch back up. Expect adapt.
format.width = 320;
format.height = 200;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 400);
EXPECT_EQ(320, out_format.width);
EXPECT_EQ(200, out_format.height);
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(320, out_width_);
EXPECT_EQ(200, out_height_);
// Format request 480x300.
format.width = 480;
format.height = 300;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 400);
EXPECT_EQ(480, out_format.width);
EXPECT_EQ(300, out_format.height);
adapter_.AdaptFrameResolution(640, 400,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(300, out_height_);
}
TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
// Start at HD.
VideoFormat format(1280, 720, VideoFormat::FpsToInterval(30), 0);
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
VideoFormat out_format =
adapter_.AdaptFrameResolution(format.width, format.height);
EXPECT_EQ(format, adapter_.input_format());
EXPECT_EQ(out_format, adapter_.input_format());
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
EXPECT_EQ(720, out_height_);
// Format request for VGA.
format.width = 640;
format.height = 360;
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
// Now, the camera reopens at VGA.
// Both the frame and the output format should be 640x360.
out_format = adapter_.AdaptFrameResolution(640, 360);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(640, 360,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
// And another view request comes in for 640x360, which should have no
// real impact.
adapter_.OnOutputFormatRequest(format);
out_format = adapter_.AdaptFrameResolution(640, 360);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(640, 360,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
}
TEST_F(VideoAdapterTest, TestVGAWidth) {
@ -419,130 +507,305 @@ TEST_F(VideoAdapterTest, TestVGAWidth) {
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
adapter_.OnOutputFormatRequest(format);
VideoFormat out_format = adapter_.AdaptFrameResolution(640, 480);
// At this point, we have to adapt down to something lower.
EXPECT_EQ(480, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
// Expect cropping.
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
// But if frames come in at 640x360, we shouldn't adapt them down.
out_format = adapter_.AdaptFrameResolution(640, 360);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(640, 360,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
out_format = adapter_.AdaptFrameResolution(640, 480);
EXPECT_EQ(480, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(1280, out_format.width);
EXPECT_EQ(720, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
EXPECT_EQ(720, out_height_);
// Adapt down one step.
adapter_.OnResolutionRequest(rtc::Optional<int>(1280 * 720 - 1),
rtc::Optional<int>());
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(960, out_format.width);
EXPECT_EQ(540, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(960, out_width_);
EXPECT_EQ(540, out_height_);
// Adapt down one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(960 * 540 - 1),
rtc::Optional<int>());
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
// Adapt down one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(480, out_format.width);
EXPECT_EQ(270, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
// Adapt up one step.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(480 * 270));
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
// Adapt up one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(640 * 360));
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(960, out_format.width);
EXPECT_EQ(540, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(960, out_width_);
EXPECT_EQ(540, out_height_);
// Adapt up one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(1280, out_format.width);
EXPECT_EQ(720, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
EXPECT_EQ(720, out_height_);
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) {
VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(1280, out_format.width);
EXPECT_EQ(720, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
EXPECT_EQ(720, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(0), rtc::Optional<int>());
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(0, out_format.width);
EXPECT_EQ(0, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
EXPECT_EQ(0, out_height_);
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(480, out_format.width);
EXPECT_EQ(270, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(1280, out_format.width);
EXPECT_EQ(720, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
EXPECT_EQ(720, out_height_);
}
TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(480, out_format.width);
EXPECT_EQ(270, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
VideoFormat new_format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420);
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
adapter_.OnOutputFormatRequest(new_format);
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(480, out_format.width);
EXPECT_EQ(270, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(640, out_format.width);
EXPECT_EQ(360, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) {
VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(1280, out_format.width);
EXPECT_EQ(720, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
EXPECT_EQ(720, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(480, out_format.width);
EXPECT_EQ(270, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(), rtc::Optional<int>());
out_format = adapter_.AdaptFrameResolution(1280, 720);
EXPECT_EQ(1280, out_format.width);
EXPECT_EQ(720, out_format.height);
adapter_.AdaptFrameResolution(1280, 720,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
EXPECT_EQ(720, out_height_);
}
TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
// Ask for 640x360 (16:9 aspect).
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
adapter_.OnOutputFormatRequest(
VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420));
// Send 640x480 (4:3 aspect).
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
// Expect cropping to 16:9 format and no scaling.
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
// Adapt down one step.
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
// Expect cropping to 16:9 format and 3/4 scaling.
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
// Adapt down one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(480 * 270 - 1),
rtc::Optional<int>());
// Expect cropping to 16:9 format and 1/2 scaling.
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(320, out_width_);
EXPECT_EQ(180, out_height_);
// Adapt up one step.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(320 * 180));
// Expect cropping to 16:9 format and 3/4 scaling.
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
// Adapt up one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(480 * 270));
// Expect cropping to 16:9 format and no scaling.
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
// Try to adapt up one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(640 * 360));
// Expect cropping to 16:9 format and no scaling.
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
}
TEST_F(VideoAdapterTest, TestCroppingOddResolution) {
// Ask for 640x360 (16:9 aspect), with 3/16 scaling.
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
adapter_.OnOutputFormatRequest(
VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420));
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 * 3 / 16 * 3 / 16),
rtc::Optional<int>());
// Send 640x480 (4:3 aspect).
adapter_.AdaptFrameResolution(640, 480,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
// Instead of getting the exact aspect ratio with cropped resolution 640x360,
// the resolution should be adjusted to get a perfect scale factor instead.
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(368, cropped_height_);
EXPECT_EQ(120, out_width_);
EXPECT_EQ(69, out_height_);
}
} // namespace cricket

24
webrtc/media/base/videocapturer.cc
View File

@ -220,17 +220,19 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*,
return;
}
int adapted_width = captured_frame->width;
int adapted_height = captured_frame->height;
int cropped_width = captured_frame->width;
int cropped_height = captured_frame->height;
int out_width = captured_frame->width;
int out_height = captured_frame->height;
if (enable_video_adapter_ && !IsScreencast()) {
const VideoFormat adapted_format =
video_adapter_.AdaptFrameResolution(adapted_width, adapted_height);
if (adapted_format.IsSize0x0()) {
video_adapter_.AdaptFrameResolution(
captured_frame->width, captured_frame->height,
&cropped_width, &cropped_height,
&out_width, &out_height);
if (out_width == 0 || out_height == 0) {
// VideoAdapter dropped the frame.
return;
}
adapted_width = adapted_format.width;
adapted_height = adapted_format.height;
}
if (!frame_factory_) {
@ -238,17 +240,15 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*,
return;
}
// TODO(nisse): Reorganize frame factory methods, deleting crop
// support there too.
// TODO(nisse): Reorganize frame factory methods.
std::unique_ptr<VideoFrame> adapted_frame(frame_factory_->CreateAliasedFrame(
captured_frame, captured_frame->width, captured_frame->height,
adapted_width, adapted_height));
captured_frame, cropped_width, cropped_height, out_width, out_height));
if (!adapted_frame) {
// TODO(fbarchard): LOG more information about captured frame attributes.
LOG(LS_ERROR) << "Couldn't convert to I420! "
<< "From " << ToString(captured_frame) << " To "
<< adapted_width << " x " << adapted_height;
<< out_width << " x " << out_height;
return;
}