From 766ad3b9899f0d7a61e9de13ac070ca44f9dcc49 Mon Sep 17 00:00:00 2001 From: Per Date: Tue, 5 Apr 2016 15:23:49 +0200 Subject: [PATCH] This cl do a major cleanup of the VideoAdapter and make sure it does care about the VideoSinkWants.max_pixel_count and VideoSinkWants.max_pixel_count_step_up. Unit tests are updated to test that screen share is not adapted but it does not change the VideoSinkWants in WebRtcVideoEngine2::SendStream due to a switch to screen share. The reason is that it works anyway and sprang is looking into how to do adaptation based on frame rate as well and use the adapter for screen share as well. BUG=webrtc:5688, webrtc:5426 R=nisse@webrtc.org, pbos@webrtc.org, sprang@google.com Review URL: https://codereview.webrtc.org/1836043004 . Cr-Commit-Position: refs/heads/master@{#12240} --- webrtc/api/androidvideocapturer.cc | 5 +- webrtc/media/base/videoadapter.cc | 693 ++-------- webrtc/media/base/videoadapter.h | 178 +-- webrtc/media/base/videoadapter_unittest.cc | 1165 +++-------------- webrtc/media/base/videocapturer.cc | 4 +- webrtc/media/base/videocapturer.h | 7 +- webrtc/media/base/videocapturer_unittest.cc | 19 +- webrtc/media/engine/webrtcvideoengine2.cc | 25 +- webrtc/media/engine/webrtcvideoengine2.h | 9 + .../engine/webrtcvideoengine2_unittest.cc | 99 +- 10 files changed, 493 insertions(+), 1711 deletions(-) diff --git a/webrtc/api/androidvideocapturer.cc b/webrtc/api/androidvideocapturer.cc index db94289838..e98a4be573 100644 --- a/webrtc/api/androidvideocapturer.cc +++ b/webrtc/api/androidvideocapturer.cc @@ -189,9 +189,8 @@ void AndroidVideoCapturer::OnIncomingFrame( void AndroidVideoCapturer::OnOutputFormatRequest( int width, int height, int fps) { RTC_CHECK(thread_checker_.CalledOnValidThread()); - const cricket::VideoFormat& current = video_adapter()->output_format(); - cricket::VideoFormat format( - width, height, cricket::VideoFormat::FpsToInterval(fps), current.fourcc); + cricket::VideoFormat format(width, height, + cricket::VideoFormat::FpsToInterval(fps), 0); video_adapter()->OnOutputFormatRequest(format); } diff --git a/webrtc/media/base/videoadapter.cc b/webrtc/media/base/videoadapter.cc index e3ddc92241..797a876f2b 100644 --- a/webrtc/media/base/videoadapter.cc +++ b/webrtc/media/base/videoadapter.cc @@ -10,161 +10,108 @@ #include "webrtc/media/base/videoadapter.h" -#include // For INT_MAX #include +#include #include "webrtc/base/logging.h" -#include "webrtc/base/timeutils.h" #include "webrtc/media/base/mediaconstants.h" #include "webrtc/media/base/videocommon.h" -#include "webrtc/media/base/videoframe.h" -namespace cricket { +namespace { -// TODO(fbarchard): Make downgrades settable -static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU. -// The number of cpu samples to require before adapting. This value depends on -// the cpu monitor sampling frequency being 2000ms. -static const int kCpuLoadMinSamples = 3; -// The amount of weight to give to each new cpu load sample. The lower the -// value, the slower we'll adapt to changing cpu conditions. -static const float kCpuLoadWeightCoefficient = 0.4f; -// The seed value for the cpu load moving average. -static const float kCpuLoadInitialAverage = 0.5f; - -// Desktop needs 1/8 scale for HD (1280 x 720) to QQVGA (160 x 90) -static const float kScaleFactors[] = { - 1.f / 1.f, // Full size. - 3.f / 4.f, // 3/4 scale. - 1.f / 2.f, // 1/2 scale. - 3.f / 8.f, // 3/8 scale. - 1.f / 4.f, // 1/4 scale. - 3.f / 16.f, // 3/16 scale. - 1.f / 8.f, // 1/8 scale. - 0.f // End of table. +// Scale factors optimized for in libYUV that we accept. +// Must be sorted in decreasing scale factors for FindScaleLargerThan to work. +const float kScaleFactors[] = { + 1.f / 1.f, // Full size. + 3.f / 4.f, // 3/4 scale. + 1.f / 2.f, // 1/2 scale. + 3.f / 8.f, // 3/8 scale. + 1.f / 4.f, // 1/4 scale. + 3.f / 16.f, // 3/16 scale. }; -// TODO(fbarchard): Use this table (optionally) for CPU and GD as well. -static const float kViewScaleFactors[] = { - 1.f / 1.f, // Full size. - 3.f / 4.f, // 3/4 scale. - 2.f / 3.f, // 2/3 scale. // Allow 1080p to 720p. - 1.f / 2.f, // 1/2 scale. - 3.f / 8.f, // 3/8 scale. - 1.f / 3.f, // 1/3 scale. // Allow 1080p to 360p. - 1.f / 4.f, // 1/4 scale. - 3.f / 16.f, // 3/16 scale. - 1.f / 8.f, // 1/8 scale. - 0.f // End of table. -}; - -const float* VideoAdapter::GetViewScaleFactors() const { - return scale_third_ ? kViewScaleFactors : kScaleFactors; -} - -// For resolutions that would scale down a little instead of up a little, -// bias toward scaling up a little. This will tend to choose 3/4 scale instead -// of 2/3 scale, when the 2/3 is not an exact match. -static const float kUpBias = -0.9f; -// Find the scale factor that, when applied to width and height, is closest -// to num_pixels. -float VideoAdapter::FindScale(const float* scale_factors, - const float upbias, - int width, int height, - int target_num_pixels) { - const float kMinNumPixels = 160 * 90; - if (!target_num_pixels) { - return 0.f; - } - float best_distance = static_cast(INT_MAX); - float best_scale = 1.f; // Default to unscaled if nothing matches. - float pixels = static_cast(width * height); - for (int i = 0; ; ++i) { - float scale = scale_factors[i]; +float FindScaleLessThanOrEqual(int width, + int height, + int target_num_pixels, + int* resulting_number_of_pixels) { + float best_distance = std::numeric_limits::max(); + float best_scale = 0.0f; // Default to 0 if nothing matches. + float pixels = width * height; + float best_number_of_pixels = 0.0f; + for (const auto& scale : kScaleFactors) { float test_num_pixels = pixels * scale * scale; - // Do not consider scale factors that produce too small images. - // Scale factor of 0 at end of table will also exit here. - if (test_num_pixels < kMinNumPixels) { - break; - } float diff = target_num_pixels - test_num_pixels; - // If resolution is higher than desired, bias the difference based on - // preference for slightly larger for nearest, or avoid completely if - // looking for lower resolutions only. if (diff < 0) { - diff = diff * kUpBias; + continue; } if (diff < best_distance) { best_distance = diff; best_scale = scale; + best_number_of_pixels = test_num_pixels; if (best_distance == 0) { // Found exact match. break; } } } + if (resulting_number_of_pixels) { + *resulting_number_of_pixels = static_cast(best_number_of_pixels + .5f); + } return best_scale; } -// Find the closest scale factor. -float VideoAdapter::FindClosestScale(int width, int height, - int target_num_pixels) { - return FindScale(kScaleFactors, kUpBias, - width, height, target_num_pixels); +float FindScaleLargerThan(int width, + int height, + int target_num_pixels, + int* resulting_number_of_pixels) { + float best_distance = std::numeric_limits::max(); + float best_scale = 1.f; // Default to unscaled if nothing matches. + float pixels = width * height; + float best_number_of_pixels = pixels; // Default to input number of pixels. + for (const auto& scale : kScaleFactors) { + float test_num_pixels = pixels * scale * scale; + float diff = test_num_pixels - target_num_pixels; + if (diff <= 0) { + break; + } + if (diff < best_distance) { + best_distance = diff; + best_scale = scale; + best_number_of_pixels = test_num_pixels; + } + } + + *resulting_number_of_pixels = static_cast(best_number_of_pixels + .5f); + return best_scale; } -// Find the closest view scale factor. -float VideoAdapter::FindClosestViewScale(int width, int height, - int target_num_pixels) { - return FindScale(GetViewScaleFactors(), kUpBias, - width, height, target_num_pixels); -} +} // namespace -// Finds the scale factor that, when applied to width and height, produces -// fewer than num_pixels. -static const float kUpAvoidBias = -1000000000.f; -float VideoAdapter::FindLowerScale(int width, int height, - int target_num_pixels) { - return FindScale(GetViewScaleFactors(), kUpAvoidBias, - width, height, target_num_pixels); -} +namespace cricket { -// There are several frame sizes used by Adapter. This explains them -// input_format - set once by server to frame size expected from the camera. -// The input frame size is also updated in AdaptFrameResolution. -// output_format - size that output would like to be. Includes framerate. -// The output frame size is also updated in AdaptFrameResolution. -// output_num_pixels - size that output should be constrained to. Used to -// compute output_format from in_frame. -// in_frame - actual camera captured frame size, which is typically the same -// as input_format. This can also be rotated or cropped for aspect ratio. -// out_frame - actual frame output by adapter. Should be a direct scale of -// in_frame maintaining rotation and aspect ratio. -// OnOutputFormatRequest - server requests you send this resolution based on -// view requests. -// OnEncoderResolutionRequest - encoder requests you send this resolution based -// on bandwidth -// OnCpuLoadUpdated - cpu monitor requests you send this resolution based on -// cpu load. - -/////////////////////////////////////////////////////////////////////// -// Implementation of VideoAdapter VideoAdapter::VideoAdapter() - : output_num_pixels_(INT_MAX), - scale_third_(false), + : output_num_pixels_(std::numeric_limits::max()), frames_in_(0), frames_out_(0), frames_scaled_(0), adaption_changes_(0), previous_width_(0), previous_height_(0), - interval_next_frame_(0) { -} + interval_next_frame_(0), + format_request_max_pixel_count_(std::numeric_limits::max()), + resolution_request_max_pixel_count_(std::numeric_limits::max()) {} -VideoAdapter::~VideoAdapter() { +VideoAdapter::~VideoAdapter() {} + +void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) { + // TODO(perkj): Consider measuring input frame rate instead. + // Frame rate typically varies depending on lighting. + rtc::CritScope cs(&critical_section_); + input_format_.interval = interval; } void VideoAdapter::SetInputFormat(const VideoFormat& format) { - rtc::CritScope cs(&critical_section_); + bool is_resolution_change = (input_format().width != format.width || + input_format().height != format.height); int64_t old_input_interval = input_format_.interval; input_format_ = format; output_format_.interval = @@ -173,73 +120,21 @@ void VideoAdapter::SetInputFormat(const VideoFormat& format) { LOG(LS_INFO) << "VAdapt input interval changed from " << old_input_interval << " to " << input_format_.interval; } -} - -void CoordinatedVideoAdapter::SetInputFormat(const VideoFormat& format) { - int previous_width = input_format().width; - int previous_height = input_format().height; - bool is_resolution_change = previous_width > 0 && format.width > 0 && - (previous_width != format.width || - previous_height != format.height); - VideoAdapter::SetInputFormat(format); if (is_resolution_change) { - int width, height; // Trigger the adaptation logic again, to potentially reset the adaptation // state for things like view requests that may not longer be capping // output (or may now cap output). - AdaptToMinimumFormat(&width, &height); - LOG(LS_INFO) << "VAdapt Input Resolution Change: " - << "Previous input resolution: " - << previous_width << "x" << previous_height - << " New input resolution: " - << format.width << "x" << format.height - << " New output resolution: " - << width << "x" << height; + Adapt(std::min(format_request_max_pixel_count_, + resolution_request_max_pixel_count_), + 0); } } -void CoordinatedVideoAdapter::set_cpu_smoothing(bool enable) { - LOG(LS_INFO) << "CPU smoothing is now " - << (enable ? "enabled" : "disabled"); - cpu_smoothing_ = enable; -} - -void VideoAdapter::SetOutputFormat(const VideoFormat& format) { - rtc::CritScope cs(&critical_section_); - int64_t old_output_interval = output_format_.interval; - output_format_ = format; - output_num_pixels_ = output_format_.width * output_format_.height; - output_format_.interval = - std::max(output_format_.interval, input_format_.interval); - if (old_output_interval != output_format_.interval) { - LOG(LS_INFO) << "VAdapt output interval changed from " - << old_output_interval << " to " << output_format_.interval; - } -} - -const VideoFormat& VideoAdapter::input_format() { +const VideoFormat& VideoAdapter::input_format() const { rtc::CritScope cs(&critical_section_); return input_format_; } -bool VideoAdapter::drops_all_frames() const { - return output_num_pixels_ == 0; -} - -const VideoFormat& VideoAdapter::output_format() { - rtc::CritScope cs(&critical_section_); - return output_format_; -} - -// Constrain output resolution to this many pixels overall -void VideoAdapter::SetOutputNumPixels(int num_pixels) { - output_num_pixels_ = num_pixels; -} - -int VideoAdapter::GetOutputNumPixels() const { - return output_num_pixels_; -} - VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { rtc::CritScope cs(&critical_section_); ++frames_in_; @@ -255,8 +150,6 @@ VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { } else { // Drop some frames based on input fps and output fps. // Normally output fps is less than input fps. - // TODO(fbarchard): Consider adjusting interval to reflect the adjusted - // interval between frames after dropping some frames. interval_next_frame_ += input_format_.interval; if (output_format_.interval > 0) { if (interval_next_frame_ >= output_format_.interval) { @@ -284,44 +177,24 @@ VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { return VideoFormat(); // Drop frame. } - const float scale = VideoAdapter::FindClosestViewScale( - in_width, in_height, output_num_pixels_); - const size_t output_width = static_cast(in_width * scale + .5f); - const size_t output_height = static_cast(in_height * scale + .5f); + const float scale = FindScaleLessThanOrEqual(in_width, in_height, + output_num_pixels_, nullptr); + const int output_width = static_cast(in_width * scale + .5f); + const int output_height = static_cast(in_height * scale + .5f); ++frames_out_; if (scale != 1) ++frames_scaled_; - // Show VAdapt log every 90 frames output. (3 seconds) - // TODO(fbarchard): Consider GetLogSeverity() to change interval to less - // for LS_VERBOSE and more for LS_INFO. - bool show = (frames_out_) % 90 == 0; - // TODO(fbarchard): LOG the previous output resolution and track input - // resolution changes as well. Consider dropping the statistics into their - // own class which could be queried publically. - bool changed = false; if (previous_width_ && (previous_width_ != output_width || previous_height_ != output_height)) { - show = true; ++adaption_changes_; - changed = true; - } - if (show) { - // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed - // in default calls. - LOG(LS_INFO) << "VAdapt Frame: scaled " << frames_scaled_ - << " / out " << frames_out_ - << " / in " << frames_in_ - << " Changes: " << adaption_changes_ - << " Input: " << in_width - << "x" << in_height - << " i" << input_format_.interval - << " Scale: " << scale - << " Output: " << output_width - << "x" << output_height - << " i" << output_format_.interval - << " Changed: " << (changed ? "true" : "false"); + LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out " + << frames_out_ << " / in " << frames_in_ + << " Changes: " << adaption_changes_ << " Input: " << in_width + << "x" << in_height << " i" << input_format_.interval + << " Scale: " << scale << " Output: " << output_width << "x" + << output_height << " i" << output_format_.interval; } output_format_.width = output_width; @@ -332,385 +205,57 @@ VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { return output_format_; } -void VideoAdapter::set_scale_third(bool enable) { - LOG(LS_INFO) << "Video Adapter third scaling is now " - << (enable ? "enabled" : "disabled"); - scale_third_ = enable; +void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { + rtc::CritScope cs(&critical_section_); + format_request_max_pixel_count_ = format.width * format.height; + output_format_.interval = format.interval; + Adapt(std::min(format_request_max_pixel_count_, + resolution_request_max_pixel_count_), + 0); } -/////////////////////////////////////////////////////////////////////// -// Implementation of CoordinatedVideoAdapter -CoordinatedVideoAdapter::CoordinatedVideoAdapter() - : cpu_adaptation_(true), - cpu_smoothing_(false), - gd_adaptation_(true), - view_adaptation_(true), - view_switch_(false), - cpu_downgrade_count_(0), - cpu_load_min_samples_(kCpuLoadMinSamples), - cpu_load_num_samples_(0), - high_system_threshold_(kHighSystemCpuThreshold), - low_system_threshold_(kLowSystemCpuThreshold), - process_threshold_(kProcessCpuThreshold), - view_desired_num_pixels_(INT_MAX), - view_desired_interval_(0), - encoder_desired_num_pixels_(INT_MAX), - cpu_desired_num_pixels_(INT_MAX), - adapt_reason_(ADAPTREASON_NONE), - system_load_average_(kCpuLoadInitialAverage) { -} - -// Helper function to UPGRADE or DOWNGRADE a number of pixels -void CoordinatedVideoAdapter::StepPixelCount( - CoordinatedVideoAdapter::AdaptRequest request, - int* num_pixels) { - switch (request) { - case CoordinatedVideoAdapter::DOWNGRADE: - *num_pixels /= 2; - break; - - case CoordinatedVideoAdapter::UPGRADE: - *num_pixels *= 2; - break; - - default: // No change in pixel count - break; - } - return; -} - -// Find the adaptation request of the cpu based on the load. Return UPGRADE if -// the load is low, DOWNGRADE if the load is high, and KEEP otherwise. -CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest( - int current_cpus, int max_cpus, - float process_load, float system_load) { - // Downgrade if system is high and plugin is at least more than midrange. - if (system_load >= high_system_threshold_ * max_cpus && - process_load >= process_threshold_ * current_cpus) { - return CoordinatedVideoAdapter::DOWNGRADE; - // Upgrade if system is low. - } else if (system_load < low_system_threshold_ * max_cpus) { - return CoordinatedVideoAdapter::UPGRADE; - } - return CoordinatedVideoAdapter::KEEP; -} - -// A remote view request for a new resolution. -void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { - rtc::CritScope cs(&request_critical_section_); - if (!view_adaptation_) { - return; - } - // Set output for initial aspect ratio in mediachannel unittests. - int old_num_pixels = GetOutputNumPixels(); - SetOutputFormat(format); - SetOutputNumPixels(old_num_pixels); - view_desired_num_pixels_ = format.width * format.height; - view_desired_interval_ = format.interval; - int new_width, new_height; - bool changed = AdaptToMinimumFormat(&new_width, &new_height); - LOG(LS_INFO) << "VAdapt View Request: " - << format.width << "x" << format.height - << " Pixels: " << view_desired_num_pixels_ - << " Changed: " << (changed ? "true" : "false") - << " To: " << new_width << "x" << new_height; -} - -void CoordinatedVideoAdapter::set_cpu_load_min_samples( - int cpu_load_min_samples) { - if (cpu_load_min_samples_ != cpu_load_min_samples) { - LOG(LS_INFO) << "VAdapt Change Cpu Adapt Min Samples from: " - << cpu_load_min_samples_ << " to " - << cpu_load_min_samples; - cpu_load_min_samples_ = cpu_load_min_samples; - } -} - -void CoordinatedVideoAdapter::set_high_system_threshold( - float high_system_threshold) { - ASSERT(high_system_threshold <= 1.0f); - ASSERT(high_system_threshold >= 0.0f); - if (high_system_threshold_ != high_system_threshold) { - LOG(LS_INFO) << "VAdapt Change High System Threshold from: " - << high_system_threshold_ << " to " << high_system_threshold; - high_system_threshold_ = high_system_threshold; - } -} - -void CoordinatedVideoAdapter::set_low_system_threshold( - float low_system_threshold) { - ASSERT(low_system_threshold <= 1.0f); - ASSERT(low_system_threshold >= 0.0f); - if (low_system_threshold_ != low_system_threshold) { - LOG(LS_INFO) << "VAdapt Change Low System Threshold from: " - << low_system_threshold_ << " to " << low_system_threshold; - low_system_threshold_ = low_system_threshold; - } -} - -void CoordinatedVideoAdapter::set_process_threshold(float process_threshold) { - ASSERT(process_threshold <= 1.0f); - ASSERT(process_threshold >= 0.0f); - if (process_threshold_ != process_threshold) { - LOG(LS_INFO) << "VAdapt Change High Process Threshold from: " - << process_threshold_ << " to " << process_threshold; - process_threshold_ = process_threshold; - } -} - -// A Bandwidth GD request for new resolution -void CoordinatedVideoAdapter::OnEncoderResolutionRequest( - int width, int height, AdaptRequest request) { - rtc::CritScope cs(&request_critical_section_); - if (!gd_adaptation_) { - return; - } - int old_encoder_desired_num_pixels = encoder_desired_num_pixels_; - if (KEEP != request) { - int new_encoder_desired_num_pixels = width * height; - int old_num_pixels = GetOutputNumPixels(); - if (new_encoder_desired_num_pixels != old_num_pixels) { - LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored"; - } else { - // Update the encoder desired format based on the request. - encoder_desired_num_pixels_ = new_encoder_desired_num_pixels; - StepPixelCount(request, &encoder_desired_num_pixels_); - } - } - int new_width, new_height; - bool changed = AdaptToMinimumFormat(&new_width, &new_height); - - // Ignore up or keep if no change. - if (DOWNGRADE != request && view_switch_ && !changed) { - encoder_desired_num_pixels_ = old_encoder_desired_num_pixels; - LOG(LS_VERBOSE) << "VAdapt ignoring GD request."; - } - - LOG(LS_INFO) << "VAdapt GD Request: " - << (DOWNGRADE == request ? "down" : - (UPGRADE == request ? "up" : "keep")) - << " From: " << width << "x" << height - << " Pixels: " << encoder_desired_num_pixels_ - << " Changed: " << (changed ? "true" : "false") - << " To: " << new_width << "x" << new_height; -} - -void CoordinatedVideoAdapter::OnCpuResolutionRequest( +void VideoAdapter::OnResolutionRequest( rtc::Optional max_pixel_count, rtc::Optional max_pixel_count_step_up) { - rtc::CritScope cs(&request_critical_section_); - // TODO(perkj): We should support taking larger steps up and down and - // actually look at the values set in max_pixel_count and - // max_pixel_count_step_up. - if (max_pixel_count && *max_pixel_count < GetOutputNumPixels()) { - OnCpuResolutionRequest(DOWNGRADE); - } else if (max_pixel_count_step_up && - *max_pixel_count_step_up >= GetOutputNumPixels()) { - OnCpuResolutionRequest(UPGRADE); - } + rtc::CritScope cs(&critical_section_); + resolution_request_max_pixel_count_ = + max_pixel_count.value_or(std::numeric_limits::max()); + Adapt(std::min(format_request_max_pixel_count_, + resolution_request_max_pixel_count_), + max_pixel_count_step_up.value_or(0)); } -// A Bandwidth GD request for new resolution -void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) { - rtc::CritScope cs(&request_critical_section_); - if (!cpu_adaptation_) { - return; - } +bool VideoAdapter::Adapt(int max_num_pixels, int max_pixel_count_step_up) { + float scale_lower = + FindScaleLessThanOrEqual(input_format_.width, input_format_.height, + max_num_pixels, &max_num_pixels); + float scale_upper = + max_pixel_count_step_up > 0 + ? FindScaleLargerThan(input_format_.width, input_format_.height, + max_pixel_count_step_up, + &max_pixel_count_step_up) + : 1.f; - // Update how many times we have downgraded due to the cpu load. - switch (request) { - case DOWNGRADE: - // Ignore downgrades if we have downgraded the maximum times. - if (cpu_downgrade_count_ < kMaxCpuDowngrades) { - ++cpu_downgrade_count_; - } else { - LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade " - "because maximum downgrades reached"; - SignalCpuAdaptationUnable(); - } - break; - case UPGRADE: - if (cpu_downgrade_count_ > 0) { - bool is_min = IsMinimumFormat(cpu_desired_num_pixels_); - if (is_min) { - --cpu_downgrade_count_; - } else { - LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade " - "because cpu is not limiting resolution"; - } - } else { - LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade " - "because minimum downgrades reached"; - } - break; - case KEEP: - default: - break; - } - if (KEEP != request) { - // TODO(fbarchard): compute stepping up/down from OutputNumPixels but - // clamp to inputpixels / 4 (2 steps) - cpu_desired_num_pixels_ = cpu_downgrade_count_ == 0 ? INT_MAX : - static_cast(input_format().width * input_format().height >> - cpu_downgrade_count_); - } - int new_width, new_height; - bool changed = AdaptToMinimumFormat(&new_width, &new_height); - LOG(LS_INFO) << "VAdapt CPU Request: " - << (DOWNGRADE == request ? "down" : - (UPGRADE == request ? "up" : "keep")) - << " Steps: " << cpu_downgrade_count_ - << " Changed: " << (changed ? "true" : "false") - << " To: " << new_width << "x" << new_height; -} + bool use_max_pixel_count_step_up = + max_pixel_count_step_up > 0 && max_num_pixels > max_pixel_count_step_up; -// A CPU request for new resolution -// TODO(fbarchard): Move outside adapter. -void CoordinatedVideoAdapter::OnCpuLoadUpdated( - int current_cpus, int max_cpus, float process_load, float system_load) { - rtc::CritScope cs(&request_critical_section_); - if (!cpu_adaptation_) { - return; - } - // Update the moving average of system load. Even if we aren't smoothing, - // we'll still calculate this information, in case smoothing is later enabled. - system_load_average_ = kCpuLoadWeightCoefficient * system_load + - (1.0f - kCpuLoadWeightCoefficient) * system_load_average_; - ++cpu_load_num_samples_; - if (cpu_smoothing_) { - system_load = system_load_average_; - } - AdaptRequest request = FindCpuRequest(current_cpus, max_cpus, - process_load, system_load); - // Make sure we're not adapting too quickly. - if (request != KEEP) { - if (cpu_load_num_samples_ < cpu_load_min_samples_) { - LOG(LS_VERBOSE) << "VAdapt CPU load high/low but do not adapt until " - << (cpu_load_min_samples_ - cpu_load_num_samples_) - << " more samples"; - request = KEEP; - } - } + int old_num_pixels = output_num_pixels_; + output_num_pixels_ = + use_max_pixel_count_step_up ? max_pixel_count_step_up : max_num_pixels; + // Log the new size. + float scale = use_max_pixel_count_step_up ? scale_upper : scale_lower; + int new_width = static_cast(input_format_.width * scale + .5f); + int new_height = static_cast(input_format_.height * scale + .5f); - OnCpuResolutionRequest(request); -} - -// Called by cpu adapter on up requests. -bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) { - // Find closest scale factor that matches input resolution to min_num_pixels - // and set that for output resolution. This is not needed for VideoAdapter, - // but provides feedback to unittests and users on expected resolution. - // Actual resolution is based on input frame. - VideoFormat new_output = output_format(); - VideoFormat input = input_format(); - if (input_format().IsSize0x0()) { - input = new_output; - } - float scale = 1.0f; - if (!input.IsSize0x0()) { - scale = FindClosestScale(input.width, - input.height, - pixels); - } - new_output.width = static_cast(input.width * scale + .5f); - new_output.height = static_cast(input.height * scale + .5f); - int new_pixels = new_output.width * new_output.height; - int num_pixels = GetOutputNumPixels(); - return new_pixels <= num_pixels; -} - -// Called by all coordinators when there is a change. -bool CoordinatedVideoAdapter::AdaptToMinimumFormat(int* new_width, - int* new_height) { - VideoFormat new_output = output_format(); - VideoFormat input = input_format(); - if (input_format().IsSize0x0()) { - input = new_output; - } - int old_num_pixels = GetOutputNumPixels(); - int min_num_pixels = INT_MAX; - adapt_reason_ = ADAPTREASON_NONE; - - // Reduce resolution based on encoder bandwidth (GD). - if (encoder_desired_num_pixels_ && - (encoder_desired_num_pixels_ < min_num_pixels)) { - adapt_reason_ |= ADAPTREASON_BANDWIDTH; - min_num_pixels = encoder_desired_num_pixels_; - } - // Reduce resolution based on CPU. - if (cpu_adaptation_ && cpu_desired_num_pixels_ && - (cpu_desired_num_pixels_ <= min_num_pixels)) { - if (cpu_desired_num_pixels_ < min_num_pixels) { - adapt_reason_ = ADAPTREASON_CPU; - } else { - adapt_reason_ |= ADAPTREASON_CPU; - } - min_num_pixels = cpu_desired_num_pixels_; - } - // Round resolution for GD or CPU to allow 1/2 to map to 9/16. - if (!input.IsSize0x0() && min_num_pixels != INT_MAX) { - float scale = FindClosestScale(input.width, input.height, min_num_pixels); - min_num_pixels = static_cast(input.width * scale + .5f) * - static_cast(input.height * scale + .5f); - } - // Reduce resolution based on View Request. - if (view_desired_num_pixels_ <= min_num_pixels) { - if (view_desired_num_pixels_ < min_num_pixels) { - adapt_reason_ = ADAPTREASON_VIEW; - } else { - adapt_reason_ |= ADAPTREASON_VIEW; - } - min_num_pixels = view_desired_num_pixels_; - } - // Snap to a scale factor. - float scale = 1.0f; - if (!input.IsSize0x0()) { - scale = FindLowerScale(input.width, input.height, min_num_pixels); - min_num_pixels = static_cast(input.width * scale + .5f) * - static_cast(input.height * scale + .5f); - } - if (scale == 1.0f) { - adapt_reason_ = ADAPTREASON_NONE; - } - *new_width = new_output.width = static_cast(input.width * scale + .5f); - *new_height = new_output.height = static_cast(input.height * scale + - .5f); - SetOutputNumPixels(min_num_pixels); - - new_output.interval = view_desired_interval_; - SetOutputFormat(new_output); - int new_num_pixels = GetOutputNumPixels(); - bool changed = new_num_pixels != old_num_pixels; - - static const char* kReasons[8] = { - "None", - "CPU", - "BANDWIDTH", - "CPU+BANDWIDTH", - "VIEW", - "CPU+VIEW", - "BANDWIDTH+VIEW", - "CPU+BANDWIDTH+VIEW", - }; - - LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_ - << " GD: " << encoder_desired_num_pixels_ - << " CPU: " << cpu_desired_num_pixels_ - << " Pixels: " << min_num_pixels - << " Input: " << input.width - << "x" << input.height - << " Scale: " << scale - << " Resolution: " << new_output.width - << "x" << new_output.height - << " Changed: " << (changed ? "true" : "false") - << " Reason: " << kReasons[adapt_reason_]; - - if (changed) { - // When any adaptation occurs, historic CPU load levels are no longer - // accurate. Clear out our state so we can re-learn at the new normal. - cpu_load_num_samples_ = 0; - system_load_average_ = kCpuLoadInitialAverage; - } + bool changed = output_num_pixels_ != old_num_pixels; + LOG(LS_INFO) << "OnResolutionRequest: " + << " Max pixels: " << max_num_pixels + << " Max pixels step up: " << max_pixel_count_step_up + << " Output Pixels: " << output_num_pixels_ + << " Input: " << input_format_.width << "x" + << input_format_.height << " Scale: " << scale + << " Resolution: " << new_width << "x" << new_height + << " Changed: " << (changed ? "true" : "false"); return changed; } diff --git a/webrtc/media/base/videoadapter.h b/webrtc/media/base/videoadapter.h index 13a342d86e..b7aba1406a 100644 --- a/webrtc/media/base/videoadapter.h +++ b/webrtc/media/base/videoadapter.h @@ -8,188 +8,72 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT +#ifndef WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_ #define WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_ -#include "webrtc/base/common.h" // For ASSERT #include "webrtc/base/criticalsection.h" #include "webrtc/base/optional.h" -#include "webrtc/base/sigslot.h" #include "webrtc/media/base/videocommon.h" namespace cricket { -class VideoFrame; - // VideoAdapter adapts an input video frame to an output frame based on the // specified input and output formats. The adaptation includes dropping frames -// to reduce frame rate and scaling frames. VideoAdapter is thread safe. +// to reduce frame rate and scaling frames. +// VideoAdapter is thread safe. class VideoAdapter { public: VideoAdapter(); virtual ~VideoAdapter(); - virtual void SetInputFormat(const VideoFormat& format); - void SetOutputFormat(const VideoFormat& format); - // Constrain output resolution to this many pixels overall - void SetOutputNumPixels(int num_pixels); - int GetOutputNumPixels() const; - - const VideoFormat& input_format(); - // Returns true if the adapter will always return zero size from - // AdaptFrameResolution. - bool drops_all_frames() const; - const VideoFormat& output_format(); + // Sets the expected frame interval. This controls how often frames should + // be dropped if |OnOutputFormatRequest| is called with a lower frame + // interval. + void SetExpectedInputFrameInterval(int64_t interval); // Return the adapted resolution given the input resolution. The returned // resolution will be 0x0 if the frame should be dropped. VideoFormat AdaptFrameResolution(int in_width, int in_height); - void set_scale_third(bool enable); - bool scale_third() const { return scale_third_; } + // Requests the output frame size and frame interval from + // |AdaptFrameResolution| to not be larger than |format|. + void OnOutputFormatRequest(const VideoFormat& format); - int adaptation_changes() const { return adaption_changes_; } + // Requests the output frame size from |AdaptFrameResolution| to not have + // more than |max_pixel_count| pixels and have "one step" up more pixels than + // max_pixel_count_step_up. + void OnResolutionRequest(rtc::Optional max_pixel_count, + rtc::Optional max_pixel_count_step_up); - protected: - float FindClosestScale(int width, int height, int target_num_pixels); - float FindClosestViewScale(int width, int height, int target_num_pixels); - float FindLowerScale(int width, int height, int target_num_pixels); + const VideoFormat& input_format() const; private: - const float* GetViewScaleFactors() const; - float FindScale(const float* scale_factors, - const float upbias, int width, int height, - int target_num_pixels); + void SetInputFormat(const VideoFormat& format); + bool Adapt(int max_num_pixels, int max_pixel_count_step_up); VideoFormat input_format_; VideoFormat output_format_; int output_num_pixels_; - bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3. - int frames_in_; // Number of input frames. - int frames_out_; // Number of output frames. - int frames_scaled_; // Number of frames scaled. + int frames_in_; // Number of input frames. + int frames_out_; // Number of output frames. + int frames_scaled_; // Number of frames scaled. int adaption_changes_; // Number of changes in scale factor. - size_t previous_width_; // Previous adapter output width. - size_t previous_height_; // Previous adapter output height. + int previous_width_; // Previous adapter output width. + int previous_height_; // Previous adapter output height. int64_t interval_next_frame_; + + // Max number of pixels requested via calls to OnOutputFormatRequest, + // OnResolutionRequest respectively. + // The adapted output format is the minimum of these. + int format_request_max_pixel_count_; + int resolution_request_max_pixel_count_; + // The critical section to protect the above variables. rtc::CriticalSection critical_section_; RTC_DISALLOW_COPY_AND_ASSIGN(VideoAdapter); }; -// CoordinatedVideoAdapter adapts the video input to the encoder by coordinating -// the format request from the server, the resolution request from the encoder, -// and the CPU load. -class CoordinatedVideoAdapter - : public VideoAdapter, public sigslot::has_slots<> { - public: - enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE }; - enum AdaptReasonEnum { - ADAPTREASON_NONE = 0, - ADAPTREASON_CPU = 1, - ADAPTREASON_BANDWIDTH = 2, - ADAPTREASON_VIEW = 4 - }; - typedef int AdaptReason; - - CoordinatedVideoAdapter(); - virtual ~CoordinatedVideoAdapter() {} - - virtual void SetInputFormat(const VideoFormat& format); - - // Enable or disable video adaptation due to the change of the CPU load. - void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; } - bool cpu_adaptation() const { return cpu_adaptation_; } - // Enable or disable smoothing when doing CPU adaptation. When smoothing is - // enabled, system CPU load is tracked using an exponential weighted - // average. - void set_cpu_smoothing(bool enable); - bool cpu_smoothing() const { return cpu_smoothing_; } - // Enable or disable video adaptation due to the change of the GD - void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; } - bool gd_adaptation() const { return gd_adaptation_; } - // Enable or disable video adaptation due to the change of the View - void set_view_adaptation(bool enable) { view_adaptation_ = enable; } - bool view_adaptation() const { return view_adaptation_; } - // Enable or disable video adaptation to fast switch View - void set_view_switch(bool enable) { view_switch_ = enable; } - bool view_switch() const { return view_switch_; } - - CoordinatedVideoAdapter::AdaptReason adapt_reason() const { - return adapt_reason_; - } - - // When the video is decreased, set the waiting time for CPU adaptation to - // decrease video again. - void set_cpu_load_min_samples(int cpu_load_min_samples); - int cpu_load_min_samples() const { return cpu_load_min_samples_; } - // CPU system load high threshold for reducing resolution. e.g. 0.85f - void set_high_system_threshold(float high_system_threshold); - float high_system_threshold() const { return high_system_threshold_; } - // CPU system load low threshold for increasing resolution. e.g. 0.70f - void set_low_system_threshold(float low_system_threshold); - float low_system_threshold() const { return low_system_threshold_; } - // CPU process load threshold for reducing resolution. e.g. 0.10f - void set_process_threshold(float process_threshold); - float process_threshold() const { return process_threshold_; } - - // Handle the format request from the server via Jingle update message. - void OnOutputFormatRequest(const VideoFormat& format); - // Handle the resolution request from the encoder due to bandwidth changes. - void OnEncoderResolutionRequest(int width, int height, AdaptRequest request); - // Handle the resolution request for CPU overuse. - void OnCpuResolutionRequest(AdaptRequest request); - void OnCpuResolutionRequest(rtc::Optional max_pixel_count, - rtc::Optional max_pixel_count_step_up); - - // Handle the CPU load provided by a CPU monitor. - void OnCpuLoadUpdated(int current_cpus, int max_cpus, - float process_load, float system_load); - - sigslot::signal0<> SignalCpuAdaptationUnable; - - private: - // Adapt to the minimum of the formats the server requests, the CPU wants, and - // the encoder wants. Returns true if resolution changed. - bool AdaptToMinimumFormat(int* new_width, int* new_height); - bool IsMinimumFormat(int pixels); - void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request, - int* num_pixels); - CoordinatedVideoAdapter::AdaptRequest FindCpuRequest( - int current_cpus, int max_cpus, - float process_load, float system_load); - - bool cpu_adaptation_; // True if cpu adaptation is enabled. - bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation). - bool gd_adaptation_; // True if gd adaptation is enabled. - bool view_adaptation_; // True if view adaptation is enabled. - bool view_switch_; // True if view switch is enabled. - int cpu_downgrade_count_; - int cpu_load_min_samples_; - int cpu_load_num_samples_; - // cpu system load thresholds relative to max cpus. - float high_system_threshold_; - float low_system_threshold_; - // cpu process load thresholds relative to current cpus. - float process_threshold_; - // Video formats that the server view requests, the CPU wants, and the encoder - // wants respectively. The adapted output format is the minimum of these. - int view_desired_num_pixels_; - int64_t view_desired_interval_; - int encoder_desired_num_pixels_; - int cpu_desired_num_pixels_; - CoordinatedVideoAdapter::AdaptReason adapt_reason_; - // The critical section to protect handling requests. - rtc::CriticalSection request_critical_section_; - - // The weighted average of cpu load over time. It's always updated (if cpu - // adaptation is on), but only used if cpu_smoothing_ is set. - float system_load_average_; - - RTC_DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter); -}; - } // namespace cricket -#endif // WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT +#endif // WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_ diff --git a/webrtc/media/base/videoadapter_unittest.cc b/webrtc/media/base/videoadapter_unittest.cc index cc4c37aae2..1ad58d165c 100644 --- a/webrtc/media/base/videoadapter_unittest.cc +++ b/webrtc/media/base/videoadapter_unittest.cc @@ -8,17 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ -// If we don't have a WebRtcVideoFrame, just skip all of these tests. -#if defined(HAVE_WEBRTC_VIDEO) #include // For INT_MAX -#include #include #include #include "webrtc/base/gunit.h" #include "webrtc/base/logging.h" -#include "webrtc/base/sigslot.h" #include "webrtc/media/base/fakevideocapturer.h" #include "webrtc/media/base/mediachannel.h" #include "webrtc/media/base/testutils.h" @@ -26,27 +22,15 @@ namespace cricket { -namespace { -static const uint32_t kWaitTimeout = 3000U; // 3 seconds. -static const uint32_t kShortWaitTimeout = 1000U; // 1 second. - void UpdateCpuLoad(CoordinatedVideoAdapter* adapter, - int current_cpus, int max_cpus, float process_load, float system_load) { - adapter->set_cpu_load_min_samples(1); - adapter->OnCpuLoadUpdated(current_cpus, max_cpus, - process_load, system_load); - } -} - class VideoAdapterTest : public testing::Test { public: virtual void SetUp() { capturer_.reset(new FakeVideoCapturer); capture_format_ = capturer_->GetSupportedFormats()->at(0); capture_format_.interval = VideoFormat::FpsToInterval(50); - adapter_.reset(new VideoAdapter()); - adapter_->SetInputFormat(capture_format_); + adapter_.SetExpectedInputFrameInterval(capture_format_.interval); - listener_.reset(new VideoCapturerListener(adapter_.get())); + listener_.reset(new VideoCapturerListener(&adapter_)); capturer_->SignalFrameCaptured.connect( listener_.get(), &VideoCapturerListener::OnFrameCaptured); } @@ -118,14 +102,6 @@ class VideoAdapterTest : public testing::Test { bool last_adapt_was_no_op_; }; - class CpuAdapterListener: public sigslot::has_slots<> { - public: - CpuAdapterListener() : received_cpu_signal_(false) {} - void OnCpuAdaptationSignalled() { received_cpu_signal_ = true; } - bool received_cpu_signal() { return received_cpu_signal_; } - private: - bool received_cpu_signal_; - }; void VerifyAdaptedResolution(const VideoCapturerListener::Stats& stats, int width, @@ -135,40 +111,14 @@ class VideoAdapterTest : public testing::Test { } std::unique_ptr capturer_; - std::unique_ptr adapter_; + VideoAdapter adapter_; std::unique_ptr listener_; VideoFormat capture_format_; }; - -// Test adapter remembers exact pixel count -TEST_F(VideoAdapterTest, AdaptNumPixels) { - adapter_->SetOutputNumPixels(123456); - EXPECT_EQ(123456, adapter_->GetOutputNumPixels()); -} - -// Test adapter is constructed but not activated. Expect no frame drop and no -// resolution change. -TEST_F(VideoAdapterTest, AdaptInactive) { - // Output resolution is not set. - EXPECT_EQ(INT_MAX, adapter_->GetOutputNumPixels()); - - // Call Adapter with some frames. - EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); - for (int i = 0; i < 10; ++i) - capturer_->CaptureFrame(); - - // Verify no frame drop and no resolution change. - VideoCapturerListener::Stats stats = listener_->GetStats(); - EXPECT_GE(stats.captured_frames, 10); - EXPECT_EQ(0, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height); -} - // Do not adapt the frame rate or the resolution. Expect no frame drop and no // resolution change. TEST_F(VideoAdapterTest, AdaptNothing) { - adapter_->SetOutputFormat(capture_format_); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); @@ -184,8 +134,7 @@ TEST_F(VideoAdapterTest, AdaptNothing) { TEST_F(VideoAdapterTest, AdaptZeroInterval) { VideoFormat format = capturer_->GetSupportedFormats()->at(0); format.interval = 0; - adapter_->SetInputFormat(format); - adapter_->SetOutputFormat(format); + adapter_.OnOutputFormatRequest(format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); @@ -202,7 +151,7 @@ TEST_F(VideoAdapterTest, AdaptZeroInterval) { TEST_F(VideoAdapterTest, AdaptFramerate) { VideoFormat request_format = capture_format_; request_format.interval *= 2; - adapter_->SetOutputFormat(request_format); + adapter_.OnOutputFormatRequest(request_format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); @@ -219,7 +168,7 @@ TEST_F(VideoAdapterTest, AdaptFramerate) { TEST_F(VideoAdapterTest, AdaptFramerateVariable) { VideoFormat request_format = capture_format_; request_format.interval = request_format.interval * 3 / 2; - adapter_->SetOutputFormat(request_format); + adapter_.OnOutputFormatRequest(request_format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 30; ++i) capturer_->CaptureFrame(); @@ -237,7 +186,7 @@ TEST_F(VideoAdapterTest, AdaptFramerateVariable) { // after adaptation. TEST_F(VideoAdapterTest, AdaptFramerateOntheFly) { VideoFormat request_format = capture_format_; - adapter_->SetOutputFormat(request_format); + adapter_.OnOutputFormatRequest(request_format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); @@ -247,7 +196,7 @@ TEST_F(VideoAdapterTest, AdaptFramerateOntheFly) { // Adapat the frame rate. request_format.interval *= 2; - adapter_->SetOutputFormat(request_format); + adapter_.OnOutputFormatRequest(request_format); for (int i = 0; i < 20; ++i) capturer_->CaptureFrame(); @@ -258,24 +207,21 @@ TEST_F(VideoAdapterTest, AdaptFramerateOntheFly) { // Set a very high output pixel resolution. Expect no resolution change. TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) { - adapter_->SetOutputNumPixels(INT_MAX); - VideoFormat adapted_format = adapter_->AdaptFrameResolution( + VideoFormat output_format = capture_format_; + output_format.width = 2560; + output_format.height = 2560; + adapter_.OnOutputFormatRequest(output_format); + VideoFormat adapted_format = adapter_.AdaptFrameResolution( capture_format_.width, capture_format_.height); EXPECT_EQ(capture_format_.width, adapted_format.width); EXPECT_EQ(capture_format_.height, adapted_format.height); - - adapter_->SetOutputNumPixels(987654321); - adapted_format = capture_format_, - adapter_->AdaptFrameResolution(capture_format_.width, capture_format_.height); - EXPECT_EQ(capture_format_.width, adapted_format.width); - EXPECT_EQ(capture_format_.height, adapted_format.height); } // Adapt the frame resolution to be the same as capture resolution. Expect no // resolution change. TEST_F(VideoAdapterTest, AdaptFrameResolutionIdentical) { - adapter_->SetOutputFormat(capture_format_); - const VideoFormat adapted_format = adapter_->AdaptFrameResolution( + adapter_.OnOutputFormatRequest(capture_format_); + const VideoFormat adapted_format = adapter_.AdaptFrameResolution( capture_format_.width, capture_format_.height); EXPECT_EQ(capture_format_.width, adapted_format.width); EXPECT_EQ(capture_format_.height, adapted_format.height); @@ -287,8 +233,8 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionQuarter) { VideoFormat request_format = capture_format_; request_format.width /= 2; request_format.height /= 2; - adapter_->SetOutputFormat(request_format); - const VideoFormat adapted_format = adapter_->AdaptFrameResolution( + adapter_.OnOutputFormatRequest(request_format); + const VideoFormat adapted_format = adapter_.AdaptFrameResolution( request_format.width, request_format.height); EXPECT_EQ(request_format.width, adapted_format.width); EXPECT_EQ(request_format.height, adapted_format.height); @@ -296,10 +242,14 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionQuarter) { // Adapt the pixel resolution to 0. Expect frame drop. TEST_F(VideoAdapterTest, AdaptFrameResolutionDrop) { - adapter_->SetOutputNumPixels(0); + VideoFormat output_format = capture_format_; + output_format.width = 0; + output_format.height = 0; + adapter_.OnOutputFormatRequest(output_format); EXPECT_TRUE( - adapter_->AdaptFrameResolution(capture_format_.width, - capture_format_.height).IsSize0x0()); + adapter_ + .AdaptFrameResolution(capture_format_.width, capture_format_.height) + .IsSize0x0()); } // Adapt the frame resolution to be a quarter of the capture resolution at the @@ -308,7 +258,7 @@ TEST_F(VideoAdapterTest, AdaptResolution) { VideoFormat request_format = capture_format_; request_format.width /= 2; request_format.height /= 2; - adapter_->SetOutputFormat(request_format); + adapter_.OnOutputFormatRequest(request_format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); @@ -319,44 +269,12 @@ TEST_F(VideoAdapterTest, AdaptResolution) { VerifyAdaptedResolution(stats, request_format.width, request_format.height); } -// Adapt the frame resolution to half width. Expect resolution change. -TEST_F(VideoAdapterTest, AdaptResolutionNarrow) { - VideoFormat request_format = capture_format_; - request_format.width /= 2; - adapter_->set_scale_third(true); - adapter_->SetOutputFormat(request_format); - EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); - for (int i = 0; i < 10; ++i) - capturer_->CaptureFrame(); - - // Verify resolution change. - VerifyAdaptedResolution(listener_->GetStats(), - capture_format_.width * 2 / 3, - capture_format_.height * 2 / 3); -} - -// Adapt the frame resolution to half height. Expect resolution change. -TEST_F(VideoAdapterTest, AdaptResolutionWide) { - VideoFormat request_format = capture_format_; - request_format.height /= 2; - adapter_->set_scale_third(true); - adapter_->SetOutputFormat(request_format); - EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); - for (int i = 0; i < 10; ++i) - capturer_->CaptureFrame(); - - // Verify resolution change. - VerifyAdaptedResolution(listener_->GetStats(), - capture_format_.width * 2 / 3, - capture_format_.height * 2 / 3); -} - // Adapt the frame resolution to be a quarter of the capture resolution after // capturing no less than 10 frames. Expect no resolution change before // adaptation and resolution change after adaptation. TEST_F(VideoAdapterTest, AdaptResolutionOnTheFly) { VideoFormat request_format = capture_format_; - adapter_->SetOutputFormat(request_format); + adapter_.OnOutputFormatRequest(request_format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); @@ -368,7 +286,7 @@ TEST_F(VideoAdapterTest, AdaptResolutionOnTheFly) { // Adapt the frame resolution. request_format.width /= 2; request_format.height /= 2; - adapter_->SetOutputFormat(request_format); + adapter_.OnOutputFormatRequest(request_format); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); @@ -380,7 +298,7 @@ TEST_F(VideoAdapterTest, AdaptResolutionOnTheFly) { // Drop all frames. TEST_F(VideoAdapterTest, DropAllFrames) { VideoFormat format; // with resolution 0x0. - adapter_->SetOutputFormat(format); + adapter_.OnOutputFormatRequest(format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); @@ -391,902 +309,239 @@ TEST_F(VideoAdapterTest, DropAllFrames) { EXPECT_EQ(stats.captured_frames, stats.dropped_frames); } -TEST(CoordinatedVideoAdapterTest, TestCoordinatedWithoutCpuAdaptation) { - CoordinatedVideoAdapter adapter; - adapter.set_cpu_adaptation(false); +TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { + VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), 0); + adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + VideoFormat out_format = + adapter_.AdaptFrameResolution(format.width, format.height); + EXPECT_EQ(format, adapter_.input_format()); + EXPECT_EQ(format, out_format); - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - adapter.set_scale_third(true); - EXPECT_EQ(format, adapter.input_format()); - EXPECT_TRUE(adapter.output_format().IsSize0x0()); - - // Server format request 640x400. + // Format request 640x400. format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 400); + EXPECT_EQ(640, out_format.width); + EXPECT_EQ(400, out_format.height); - // Server format request 1280x720, higher than input. Adapt nothing. + // Request 1280x720, higher than input. Adapt nothing. format.width = 1280; format.height = 720; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 400); + EXPECT_EQ(640, out_format.width); + EXPECT_EQ(400, out_format.height); - // Cpu load is high, but cpu adaptation is disabled. Adapt nothing. - adapter.OnCpuLoadUpdated(1, 1, 0.99f, 0.99f); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Encoder resolution request: downgrade with different size. Adapt nothing. - adapter.OnEncoderResolutionRequest(320, 200, - CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Encoder resolution request: downgrade. - adapter.OnEncoderResolutionRequest(640, 400, - CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Encoder resolution request: downgrade. But GD off. Adapt nothing. - adapter.set_gd_adaptation(false); - adapter.OnEncoderResolutionRequest(480, 300, - CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - adapter.set_gd_adaptation(true); - - // Encoder resolution request: downgrade. - adapter.OnEncoderResolutionRequest(480, 300, - CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Encoder resolution request: keep. Adapt nothing. - adapter.OnEncoderResolutionRequest(320, 200, - CoordinatedVideoAdapter::KEEP); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Encoder resolution request: upgrade. - adapter.OnEncoderResolutionRequest(320, 200, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Server format request 0x0. + // Request 0x0. format.width = 0; format.height = 0; - adapter.OnOutputFormatRequest(format); - EXPECT_TRUE(adapter.output_format().IsSize0x0()); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 400); + EXPECT_TRUE(out_format.IsSize0x0()); - // Server format request 320x200. + // Request 320x200. format.width = 320; format.height = 200; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 400); + EXPECT_EQ(320, out_format.width); + EXPECT_EQ(200, out_format.height); - // Server format request 160x100. But view disabled. Adapt nothing. - adapter.set_view_adaptation(false); - format.width = 160; - format.height = 100; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - adapter.set_view_adaptation(true); - - // Enable View Switch. Expect adapt down. - adapter.set_view_switch(true); - format.width = 160; - format.height = 100; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(160, adapter.output_format().width); - EXPECT_EQ(100, adapter.output_format().height); - - // Encoder resolution request: upgrade. Adapt nothing. - adapter.OnEncoderResolutionRequest(160, 100, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(160, adapter.output_format().width); - EXPECT_EQ(100, adapter.output_format().height); - - // Request View of 2 / 3. Expect adapt down. - adapter.set_view_switch(true); + // Request resolution of 2 / 3. Expect adapt down. Scaling to 1/3 is not + // optimized and not allowed. format.width = (640 * 2 + 1) / 3; format.height = (400 * 2 + 1) / 3; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ((640 * 2 + 1) / 3, adapter.output_format().width); - EXPECT_EQ((400 * 2 + 1) / 3, adapter.output_format().height); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 400); + EXPECT_EQ(320, out_format.width); + EXPECT_EQ(200, out_format.height); - - // Request View of 3 / 8. Expect adapt down. - adapter.set_view_switch(true); + // Request resolution of 3 / 8. Expect adapt down. format.width = 640 * 3 / 8; format.height = 400 * 3 / 8; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640 * 3 / 8, adapter.output_format().width); - EXPECT_EQ(400 * 3 / 8, adapter.output_format().height); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 400); + EXPECT_EQ(640 * 3 / 8, out_format.width); + EXPECT_EQ(400 * 3 / 8, out_format.height); - // View Switch back up. Expect adapt. + // Switch back up. Expect adapt. format.width = 320; format.height = 200; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 400); + EXPECT_EQ(320, out_format.width); + EXPECT_EQ(200, out_format.height); - adapter.set_view_switch(false); - - // Encoder resolution request: upgrade. Constrained by server request. - adapter.OnEncoderResolutionRequest(320, 200, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Server format request 480x300. + // Format request 480x300. format.width = 480; format.height = 300; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 400); + EXPECT_EQ(480, out_format.width); + EXPECT_EQ(300, out_format.height); } -TEST(CoordinatedVideoAdapterTest, TestCoordinatedWithCpuAdaptation) { - CoordinatedVideoAdapter adapter; - adapter.set_cpu_adaptation(true); - EXPECT_FALSE(adapter.cpu_smoothing()); - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - - // Server format request 640x400. - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Process load is medium, but system load is high. Downgrade. - UpdateCpuLoad(&adapter, 1, 1, 0.55f, 0.98f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // CPU high, but cpu adaptation disabled. Adapt nothing. - adapter.set_cpu_adaptation(false); - adapter.OnCpuLoadUpdated(1, 1, 0.55f, 0.98f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - adapter.set_cpu_adaptation(true); - - // System load is high, but time has not elaspsed. Adapt nothing. - adapter.set_cpu_load_min_samples(2); - adapter.OnCpuLoadUpdated(1, 1, 0.55f, 0.98f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Process load is medium, but system load is high. Downgrade. - UpdateCpuLoad(&adapter, 1, 1, 0.55f, 0.98f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Test reason for adapting is CPU. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU, - adapter.adapt_reason()); - - // Server format request 320x200. Same as CPU. Do nothing. - format.width = 320; - format.height = 200; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Test reason for adapting is CPU and VIEW. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU + - CoordinatedVideoAdapter::ADAPTREASON_VIEW, - adapter.adapt_reason()); - - // Process load and system load are normal. Adapt nothing. - UpdateCpuLoad(&adapter, 1, 1, 0.5f, 0.8f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Process load and system load are low, but view is still low. Adapt nothing. - UpdateCpuLoad(&adapter, 1, 1, 0.2f, 0.3f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Test reason for adapting is VIEW. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_VIEW, - adapter.adapt_reason()); - - // Server format request 640x400. Cpu is still low. Upgrade. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Test reason for adapting is CPU. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU, - adapter.adapt_reason()); - - // Encoder resolution request: downgrade. - adapter.OnEncoderResolutionRequest(480, 300, - CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Test reason for adapting is BANDWIDTH. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH, - adapter.adapt_reason()); - - // Process load and system load are low. Constrained by GD. Adapt nothing - adapter.OnCpuLoadUpdated(1, 1, 0.2f, 0.3f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Encoder resolution request: upgrade. - adapter.OnEncoderResolutionRequest(320, 200, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Encoder resolution request: upgrade. Constrained by CPU. - adapter.OnEncoderResolutionRequest(480, 300, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Server format request 640x400. Constrained by CPU. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); -} - -TEST(CoordinatedVideoAdapterTest, TestCoordinatedWithCpuRequest) { - CoordinatedVideoAdapter adapter; - adapter.set_cpu_adaptation(true); - EXPECT_FALSE(adapter.cpu_smoothing()); - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - - // Server format request 640x400. - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // CPU resolution request: downgrade. Adapt down. - adapter.OnCpuResolutionRequest(CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // CPU resolution request: keep. Do nothing. - adapter.OnCpuResolutionRequest(CoordinatedVideoAdapter::KEEP); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // CPU resolution request: downgrade, but cpu adaptation disabled. - // Adapt nothing. - adapter.set_cpu_adaptation(false); - adapter.OnCpuResolutionRequest(CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // CPU resolution request: downgrade. Adapt down. - adapter.set_cpu_adaptation(true); - adapter.OnCpuResolutionRequest(CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Test reason for adapting is CPU. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU, - adapter.adapt_reason()); - - // CPU resolution request: downgrade, but already at minimum. Do nothing. - adapter.OnCpuResolutionRequest(CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Server format request 320x200. Same as CPU. Do nothing. - format.width = 320; - format.height = 200; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Test reason for adapting is CPU and VIEW. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU + - CoordinatedVideoAdapter::ADAPTREASON_VIEW, - adapter.adapt_reason()); - - // CPU resolution request: upgrade, but view request still low. Do nothing. - adapter.OnCpuResolutionRequest(CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Test reason for adapting is VIEW. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_VIEW, - adapter.adapt_reason()); - - // Server format request 640x400. Cpu is still low. Upgrade. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Test reason for adapting is CPU. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU, - adapter.adapt_reason()); - - // Encoder resolution request: downgrade. - adapter.OnEncoderResolutionRequest(480, 300, - CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Test reason for adapting is BANDWIDTH. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH, - adapter.adapt_reason()); - - // Process load and system load are low. Constrained by GD. Adapt nothing - adapter.OnCpuLoadUpdated(1, 1, 0.2f, 0.3f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Encoder resolution request: upgrade. - adapter.OnEncoderResolutionRequest(320, 200, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Encoder resolution request: upgrade. Constrained by CPU. - adapter.OnEncoderResolutionRequest(480, 300, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Server format request 640x400. Constrained by CPU. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); -} - -TEST(CoordinatedVideoAdapterTest, TestViewRequestPlusCameraSwitch) { - CoordinatedVideoAdapter adapter; - adapter.set_view_switch(true); - +TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { // Start at HD. - VideoFormat format(1280, 720, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - EXPECT_EQ(format, adapter.input_format()); - EXPECT_TRUE(adapter.output_format().IsSize0x0()); + VideoFormat format(1280, 720, VideoFormat::FpsToInterval(30), 0); + adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + VideoFormat out_format = + adapter_.AdaptFrameResolution(format.width, format.height); + EXPECT_EQ(format, adapter_.input_format()); + EXPECT_EQ(out_format, adapter_.input_format()); - // View request for VGA. + // Format request for VGA. format.width = 640; format.height = 360; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(360, adapter.output_format().height); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_VIEW, adapter.adapt_reason()); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(640, out_format.width); + EXPECT_EQ(360, out_format.height); // Now, the camera reopens at VGA. // Both the frame and the output format should be 640x360. - const VideoFormat out_format = adapter.AdaptFrameResolution(640, 360); + out_format = adapter_.AdaptFrameResolution(640, 360); EXPECT_EQ(640, out_format.width); EXPECT_EQ(360, out_format.height); - // At this point, the view is no longer adapted, since the input has resized - // small enough to fit the last view request. - EXPECT_EQ(0, adapter.adapt_reason()); // And another view request comes in for 640x360, which should have no // real impact. - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(360, adapter.output_format().height); - EXPECT_EQ(0, adapter.adapt_reason()); + adapter_.OnOutputFormatRequest(format); + out_format = adapter_.AdaptFrameResolution(640, 360); + EXPECT_EQ(640, out_format.width); + EXPECT_EQ(360, out_format.height); } -TEST(CoordinatedVideoAdapterTest, TestVGAWidth) { - CoordinatedVideoAdapter adapter; - adapter.set_view_switch(true); +TEST_F(VideoAdapterTest, TestVGAWidth) { + // Reqeuested Output format is 640x360. + VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); + adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + adapter_.OnOutputFormatRequest(format); - // Start at 640x480, for cameras that don't support 640x360. - VideoFormat format(640, 480, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - EXPECT_EQ(format, adapter.input_format()); - EXPECT_TRUE(adapter.output_format().IsSize0x0()); - - // Output format is 640x360, though. - format.width = 640; - format.height = 360; - adapter.SetOutputFormat(format); - - // And also a view request comes for 640x360. - adapter.OnOutputFormatRequest(format); + VideoFormat out_format = adapter_.AdaptFrameResolution(640, 480); // At this point, we have to adapt down to something lower. - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(360, adapter.output_format().height); + EXPECT_EQ(480, out_format.width); + EXPECT_EQ(360, out_format.height); // But if frames come in at 640x360, we shouldn't adapt them down. - // Fake a 640x360 frame. - VideoFormat out_format = adapter.AdaptFrameResolution(640, 360); + out_format = adapter_.AdaptFrameResolution(640, 360); EXPECT_EQ(640, out_format.width); EXPECT_EQ(360, out_format.height); - // Similarly, no-op adapt requests for other reasons shouldn't change - // adaptation state (before a previous bug, the previous EXPECTs would - // fail and the following would succeed, as the no-op CPU request would - // fix the adaptation state). - adapter.set_cpu_adaptation(true); - UpdateCpuLoad(&adapter, 1, 1, 0.7f, 0.7f); - out_format = adapter.AdaptFrameResolution(640, 360); + out_format = adapter_.AdaptFrameResolution(640, 480); + EXPECT_EQ(480, out_format.width); + EXPECT_EQ(360, out_format.height); +} +TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { + VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(1280, out_format.width); + EXPECT_EQ(720, out_format.height); + + // Adapt down one step. + adapter_.OnResolutionRequest(rtc::Optional(1280 * 720 - 1), + rtc::Optional()); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(960, out_format.width); + EXPECT_EQ(540, out_format.height); + + // Adapt down one step more. + adapter_.OnResolutionRequest(rtc::Optional(960 * 540 - 1), + rtc::Optional()); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(640, out_format.width); + EXPECT_EQ(360, out_format.height); + + // Adapt down one step more. + adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), + rtc::Optional()); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(480, out_format.width); + EXPECT_EQ(270, out_format.height); + + // Adapt up one step. + adapter_.OnResolutionRequest(rtc::Optional(), + rtc::Optional(480 * 270)); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(640, out_format.width); + EXPECT_EQ(360, out_format.height); + + // Adapt up one step more. + adapter_.OnResolutionRequest(rtc::Optional(), + rtc::Optional(640 * 360)); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(960, out_format.width); + EXPECT_EQ(540, out_format.height); + + // Adapt up one step more. + adapter_.OnResolutionRequest(rtc::Optional(), + rtc::Optional(960 * 720)); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(1280, out_format.width); + EXPECT_EQ(720, out_format.height); +} + +TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) { + VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(1280, out_format.width); + EXPECT_EQ(720, out_format.height); + + adapter_.OnResolutionRequest(rtc::Optional(0), rtc::Optional()); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(0, out_format.width); + EXPECT_EQ(0, out_format.height); +} + +TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { + adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), + rtc::Optional()); + VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(480, out_format.width); + EXPECT_EQ(270, out_format.height); + + adapter_.OnResolutionRequest(rtc::Optional(), + rtc::Optional(960 * 720)); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(1280, out_format.width); + EXPECT_EQ(720, out_format.height); +} + +TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { + adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), + rtc::Optional()); + VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(480, out_format.width); + EXPECT_EQ(270, out_format.height); + + VideoFormat new_format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); + adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + adapter_.OnOutputFormatRequest(new_format); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(480, out_format.width); + EXPECT_EQ(270, out_format.height); + + adapter_.OnResolutionRequest(rtc::Optional(), + rtc::Optional(960 * 720)); + out_format = adapter_.AdaptFrameResolution(1280, 720); EXPECT_EQ(640, out_format.width); EXPECT_EQ(360, out_format.height); } -// When adapting resolution for CPU or GD, the quantity of pixels that the -// request is based on is reduced to half or double, and then an actual -// resolution is snapped to, rounding to the closest actual resolution. -// This works well for some tolerance to 3/4, odd widths and aspect ratios -// that dont exactly match, but is not best behavior for ViewRequests which -// need to be be strictly respected to avoid going over the resolution budget -// given to the codec - 854x480 total pixels. -// ViewRequest must find a lower resolution. -TEST(CoordinatedVideoAdapterTest, TestCoordinatedViewRequestDown) { - CoordinatedVideoAdapter adapter; - adapter.set_cpu_adaptation(false); - - VideoFormat format(960, 540, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - adapter.set_scale_third(true); - EXPECT_EQ(format, adapter.input_format()); - EXPECT_TRUE(adapter.output_format().IsSize0x0()); - - // Server format request 640x400. Expect HVGA. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(360, adapter.output_format().height); - - // Test reason for adapting is VIEW. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_VIEW, adapter.adapt_reason()); -} - -// Test that we downgrade video for cpu up to two times. -TEST(CoordinatedVideoAdapterTest, TestCpuDowngradeTimes) { - CoordinatedVideoAdapter adapter; - adapter.set_cpu_adaptation(true); - EXPECT_FALSE(adapter.cpu_smoothing()); - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - - // Server format request 640x400. - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Process load and system load are low. Do not change the cpu desired format - // and do not adapt. - adapter.OnCpuLoadUpdated(1, 1, 0.2f, 0.3f); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // System load is high. Downgrade. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // System load is high. Downgrade again. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // System load is still high. Do not downgrade any more. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Process load and system load are low. Upgrade. - UpdateCpuLoad(&adapter, 1, 1, 0.2f, 0.3f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // System load is high. Downgrade. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // System load is still high. Do not downgrade any more. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); -} - -// Test that we respect CPU adapter threshold values. -TEST(CoordinatedVideoAdapterTest, TestAdapterCpuThreshold) { - CoordinatedVideoAdapter adapter; - adapter.set_cpu_adaptation(true); - EXPECT_FALSE(adapter.cpu_smoothing()); - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - - // Server format request 640x400. - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Process load and system load are low. Do not change the cpu desired format - // and do not adapt. - adapter.OnCpuLoadUpdated(1, 1, 0.2f, 0.3f); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // System load is high. Downgrade. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Test reason for adapting is CPU. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU, adapter.adapt_reason()); - - // System load is high. Normally downgrade but threshold is high. Do nothing. - adapter.set_high_system_threshold(0.98f); // Set threshold high. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // System load is medium. Normally do nothing, threshold is low. Adapt down. - adapter.set_high_system_threshold(0.75f); // Set threshold low. - UpdateCpuLoad(&adapter, 1, 1, 0.8f, 0.8f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); -} - - -// Test that for an upgrade cpu request, we actually upgrade the desired format; -// for a downgrade request, we downgrade from the output format. -TEST(CoordinatedVideoAdapterTest, TestRealCpuUpgrade) { - CoordinatedVideoAdapter adapter; - adapter.set_cpu_adaptation(true); - adapter.set_cpu_smoothing(true); - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - - // Server format request 640x400. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Process load and system load are low. Do not change the cpu desired format - // and do not adapt. - UpdateCpuLoad(&adapter, 1, 1, 0.2f, 0.3f); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Server format request 320x200. - format.width = 320; - format.height = 200; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Process load and system load are low. Do not change the cpu desired format - // and do not adapt. - UpdateCpuLoad(&adapter, 1, 1, 0.2f, 0.3f); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Server format request 640x400. Set to 640x400 immediately. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Server format request 320x200. - format.width = 320; - format.height = 200; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Process load is high, but system is not. Do not change the cpu desired - // format and do not adapt. - for (size_t i = 0; i < 10; ++i) { - UpdateCpuLoad(&adapter, 1, 1, 0.75f, 0.8f); - } - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); -} - -// Test that for an upgrade encoder request, we actually upgrade the desired -// format; for a downgrade request, we downgrade from the output format. -TEST(CoordinatedVideoAdapterTest, TestRealEncoderUpgrade) { - CoordinatedVideoAdapter adapter; - adapter.set_cpu_adaptation(true); - adapter.set_cpu_smoothing(true); - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - - // Server format request 640x400. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Encoder resolution request. Do not change the encoder desired format and - // do not adapt. - adapter.OnEncoderResolutionRequest(640, 400, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(400, adapter.output_format().height); - - // Server format request 320x200. - format.width = 320; - format.height = 200; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Encoder resolution request. Do not change the encoder desired format and - // do not adapt. - adapter.OnEncoderResolutionRequest(320, 200, - CoordinatedVideoAdapter::UPGRADE); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Server format request 640x400. Set to 640x400 immediately. - format.width = 640; - format.height = 400; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(300, adapter.output_format().height); - - // Test reason for adapting is BANDWIDTH. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH, - adapter.adapt_reason()); - - // Server format request 320x200. - format.width = 320; - format.height = 200; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(200, adapter.output_format().height); - - // Encoder resolution request. Downgrade from 320x200. - adapter.OnEncoderResolutionRequest(320, 200, - CoordinatedVideoAdapter::DOWNGRADE); - EXPECT_EQ(240, adapter.output_format().width); - EXPECT_EQ(150, adapter.output_format().height); -} - -TEST(CoordinatedVideoAdapterTest, TestNormalizeOutputFormat) { - CoordinatedVideoAdapter adapter; - // The input format is 640x360 and the output is limited to 16:9. - VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - - format.width = 320; - format.height = 180; - format.interval = VideoFormat::FpsToInterval(15); - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(180, adapter.output_format().height); - EXPECT_EQ(VideoFormat::FpsToInterval(15), adapter.output_format().interval); - - format.width = 320; - format.height = 200; - format.interval = VideoFormat::FpsToInterval(40); - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(180, adapter.output_format().height); - EXPECT_EQ(VideoFormat::FpsToInterval(30), adapter.output_format().interval); - - // Test reason for adapting is VIEW. Should work even with normalization. - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_VIEW, - adapter.adapt_reason()); - - format.width = 320; - format.height = 240; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(180, adapter.output_format().height); - - // The input format is 640x480 and the output will be 4:3. - format.width = 640; - format.height = 480; - adapter.SetInputFormat(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(240, adapter.output_format().height); - - format.width = 320; - format.height = 240; - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(240, adapter.output_format().height); - - // The input format is initialized after the output. At that time, the output - // height is adjusted. - format.width = 0; - format.height = 0; - adapter.SetInputFormat(format); - - format.width = 320; - format.height = 240; - format.interval = VideoFormat::FpsToInterval(30); - adapter.OnOutputFormatRequest(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(240, adapter.output_format().height); - EXPECT_EQ(VideoFormat::FpsToInterval(30), adapter.output_format().interval); - - format.width = 640; - format.height = 480; - format.interval = VideoFormat::FpsToInterval(15); - adapter.SetInputFormat(format); - EXPECT_EQ(320, adapter.output_format().width); - EXPECT_EQ(240, adapter.output_format().height); - EXPECT_EQ(VideoFormat::FpsToInterval(15), adapter.output_format().interval); -} - -// Test that we downgrade video for cpu up to two times. -TEST_F(VideoAdapterTest, CpuDowngradeAndSignal) { - CoordinatedVideoAdapter adapter; - CpuAdapterListener cpu_listener; - adapter.SignalCpuAdaptationUnable.connect( - &cpu_listener, &CpuAdapterListener::OnCpuAdaptationSignalled); - - adapter.set_cpu_adaptation(true); - EXPECT_FALSE(adapter.cpu_smoothing()); - VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - adapter.OnOutputFormatRequest(format); - - // System load is high. Downgrade. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - - // System load is high. Downgrade again. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - - // System load is still high. Do not downgrade any more. Ensure we have not - // signalled until after the cpu warning though. - EXPECT_TRUE(!cpu_listener.received_cpu_signal()); - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - EXPECT_TRUE_WAIT(cpu_listener.received_cpu_signal(), kWaitTimeout); -} - -// Test that we downgrade video for cpu up to two times. -TEST_F(VideoAdapterTest, CpuDowngradeAndDontSignal) { - CoordinatedVideoAdapter adapter; - CpuAdapterListener cpu_listener; - adapter.SignalCpuAdaptationUnable.connect( - &cpu_listener, &CpuAdapterListener::OnCpuAdaptationSignalled); - - adapter.set_cpu_adaptation(true); - adapter.set_cpu_smoothing(true); - VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - adapter.OnOutputFormatRequest(format); - - // System load is high. Downgrade. - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - - // System load is high, process is not, Do not downgrade again. - UpdateCpuLoad(&adapter, 1, 1, 0.25f, 0.95f); - - // System load is high, process is not, Do not downgrade again and do not - // signal. - adapter.set_cpu_adaptation(false); - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - rtc::Thread::Current()->ProcessMessages(kShortWaitTimeout); - EXPECT_TRUE(!cpu_listener.received_cpu_signal()); - adapter.set_cpu_adaptation(true); -} - -// Test that we require enough time before we downgrade. -TEST_F(VideoAdapterTest, CpuMinTimeRequirement) { - CoordinatedVideoAdapter adapter; - CpuAdapterListener cpu_listener; - adapter.SignalCpuAdaptationUnable.connect( - &cpu_listener, &CpuAdapterListener::OnCpuAdaptationSignalled); - - adapter.set_cpu_adaptation(true); - adapter.set_cpu_smoothing(true); - VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - adapter.OnOutputFormatRequest(format); - - EXPECT_EQ(3, adapter.cpu_load_min_samples()); - adapter.set_cpu_load_min_samples(5); - - for (size_t i = 0; i < 4; ++i) { - adapter.OnCpuLoadUpdated(1, 1, 1.0f, 1.0f); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(360, adapter.output_format().height); - } - // The computed cpu load should now be around 93.5%, with the coefficient of - // 0.4 and a seed value of 0.5. That should be high enough to adapt, but it - // isn't enough samples, so we shouldn't have adapted on any of the previous - // samples. - - // One more sample is enough, though, once enough time has passed. - adapter.OnCpuLoadUpdated(1, 1, 1.0f, 1.0f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(270, adapter.output_format().height); - - // Now the cpu is lower, but we still need enough samples to upgrade. - for (size_t i = 0; i < 4; ++i) { - adapter.OnCpuLoadUpdated(1, 1, 0.1f, 0.1f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(270, adapter.output_format().height); - } - - // One more sample is enough, once time has elapsed. - adapter.OnCpuLoadUpdated(1, 1, 1.0f, 1.0f); - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(360, adapter.output_format().height); -} - -TEST_F(VideoAdapterTest, CpuIgnoresSpikes) { - CoordinatedVideoAdapter adapter; - CpuAdapterListener cpu_listener; - adapter.SignalCpuAdaptationUnable.connect( - &cpu_listener, &CpuAdapterListener::OnCpuAdaptationSignalled); - - adapter.set_cpu_adaptation(true); - adapter.set_cpu_smoothing(true); - VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter.SetInputFormat(format); - adapter.OnOutputFormatRequest(format); - - // System load is high. Downgrade. - for (size_t i = 0; i < 5; ++i) { - UpdateCpuLoad(&adapter, 1, 1, 0.95f, 0.95f); - } - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(270, adapter.output_format().height); - - // Now we're in a state where we could upgrade or downgrade, so get to a - // steady state of about 75% cpu usage. - for (size_t i = 0; i < 5; ++i) { - UpdateCpuLoad(&adapter, 1, 1, 0.75f, 0.75f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(270, adapter.output_format().height); - } - - // Now, the cpu spikes for two samples, but then goes back to - // normal. This shouldn't cause adaptation. - UpdateCpuLoad(&adapter, 1, 1, 0.90f, 0.90f); - UpdateCpuLoad(&adapter, 1, 1, 0.90f, 0.90f); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(270, adapter.output_format().height); - // Back to the steady state for awhile. - for (size_t i = 0; i < 5; ++i) { - UpdateCpuLoad(&adapter, 1, 1, 0.75, 0.75); - EXPECT_EQ(480, adapter.output_format().width); - EXPECT_EQ(270, adapter.output_format().height); - } - - // Now, system cpu usage is starting to drop down. But it takes a bit before - // it gets all the way there. - for (size_t i = 0; i < 10; ++i) { - UpdateCpuLoad(&adapter, 1, 1, 0.5f, 0.5f); - } - EXPECT_EQ(640, adapter.output_format().width); - EXPECT_EQ(360, adapter.output_format().height); +TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { + VideoFormat out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(1280, out_format.width); + EXPECT_EQ(720, out_format.height); + + adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), + rtc::Optional()); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(480, out_format.width); + EXPECT_EQ(270, out_format.height); + + adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional()); + out_format = adapter_.AdaptFrameResolution(1280, 720); + EXPECT_EQ(1280, out_format.width); + EXPECT_EQ(720, out_format.height); } } // namespace cricket -#endif // HAVE_WEBRTC_VIDEO diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc index 57dc79f7b1..94f6514353 100644 --- a/webrtc/media/base/videocapturer.cc +++ b/webrtc/media/base/videocapturer.cc @@ -213,8 +213,8 @@ void VideoCapturer::OnSinkWantsChanged(const rtc::VideoSinkWants& wants) { } if (video_adapter()) { - video_adapter()->OnCpuResolutionRequest(wants.max_pixel_count, - wants.max_pixel_count_step_up); + video_adapter()->OnResolutionRequest(wants.max_pixel_count, + wants.max_pixel_count_step_up); } } diff --git a/webrtc/media/base/videocapturer.h b/webrtc/media/base/videocapturer.h index 5ec743aaee..270315582a 100644 --- a/webrtc/media/base/videocapturer.h +++ b/webrtc/media/base/videocapturer.h @@ -230,7 +230,7 @@ class VideoCapturer : public sigslot::has_slots<>, // SignalFrameCaptured or OnFrameCaptured. void OnFrame(VideoCapturer* capturer, const VideoFrame* frame); - CoordinatedVideoAdapter* video_adapter() { return &video_adapter_; } + VideoAdapter* video_adapter() { return &video_adapter_; } void SetCaptureState(CaptureState state); @@ -248,8 +248,7 @@ class VideoCapturer : public sigslot::has_slots<>, if (capture_format_) { ASSERT(capture_format_->interval > 0 && "Capture format expected to have positive interval."); - // Video adapter really only cares about capture format interval. - video_adapter_.SetInputFormat(*capture_format_); + video_adapter_.SetExpectedInputFrameInterval(capture_format_->interval); } } @@ -293,7 +292,7 @@ class VideoCapturer : public sigslot::has_slots<>, rtc::VideoBroadcaster broadcaster_; bool enable_video_adapter_; - CoordinatedVideoAdapter video_adapter_; + VideoAdapter video_adapter_; rtc::CriticalSection frame_stats_crit_; // The captured frame size before potential adapation. diff --git a/webrtc/media/base/videocapturer_unittest.cc b/webrtc/media/base/videocapturer_unittest.cc index 40c770608f..bd145e33ea 100644 --- a/webrtc/media/base/videocapturer_unittest.cc +++ b/webrtc/media/base/videocapturer_unittest.cc @@ -256,9 +256,11 @@ TEST_F(VideoCapturerTest, SinkWantsMaxPixelAndMaxPixelCountStepUp) { EXPECT_EQ(1280, renderer_.width()); EXPECT_EQ(720, renderer_.height()); - // Request a lower resolution. + // Request a lower resolution. The output resolution will have a resolution + // with less than or equal to |wants.max_pixel_count| depending on how the + // capturer can scale the input frame size. rtc::VideoSinkWants wants; - wants.max_pixel_count = rtc::Optional(1280 * 720 / 2); + wants.max_pixel_count = rtc::Optional(1280 * 720 * 3 / 5); capturer_->AddOrUpdateSink(&renderer_, wants); EXPECT_TRUE(capturer_->CaptureFrame()); EXPECT_EQ(2, renderer_.num_rendered_frames()); @@ -267,7 +269,7 @@ TEST_F(VideoCapturerTest, SinkWantsMaxPixelAndMaxPixelCountStepUp) { // Request a lower resolution. wants.max_pixel_count = - rtc::Optional(renderer_.width() * renderer_.height() / 2); + rtc::Optional(renderer_.width() * renderer_.height() * 3 / 5); capturer_->AddOrUpdateSink(&renderer_, wants); EXPECT_TRUE(capturer_->CaptureFrame()); EXPECT_EQ(3, renderer_.num_rendered_frames()); @@ -306,6 +308,17 @@ TEST_F(VideoCapturerTest, SinkWantsMaxPixelAndMaxPixelCountStepUp) { EXPECT_EQ(3, renderer2.num_rendered_frames()); EXPECT_EQ(960, renderer2.width()); EXPECT_EQ(540, renderer2.height()); + + // But resetting the wants should reset the resolution to what the camera is + // opened with. + capturer_->AddOrUpdateSink(&renderer_, rtc::VideoSinkWants()); + EXPECT_TRUE(capturer_->CaptureFrame()); + EXPECT_EQ(7, renderer_.num_rendered_frames()); + EXPECT_EQ(1280, renderer_.width()); + EXPECT_EQ(720, renderer_.height()); + EXPECT_EQ(4, renderer2.num_rendered_frames()); + EXPECT_EQ(1280, renderer2.width()); + EXPECT_EQ(720, renderer2.height()); } TEST_F(VideoCapturerTest, ScreencastScaledSuperLarge) { diff --git a/webrtc/media/engine/webrtcvideoengine2.cc b/webrtc/media/engine/webrtcvideoengine2.cc index a65f799bd6..969c6e5b57 100644 --- a/webrtc/media/engine/webrtcvideoengine2.cc +++ b/webrtc/media/engine/webrtcvideoengine2.cc @@ -371,6 +371,9 @@ static const int kDefaultQpMax = 56; static const int kDefaultRtcpReceiverReportSsrc = 1; +// Down grade resolution at most 2 times for CPU reasons. +static const int kMaxCpuDowngrades = 2; + std::vector DefaultVideoCodecList() { std::vector codecs; codecs.push_back(MakeVideoCodecWithDefaultFeedbackParams(kDefaultVp8PlType, @@ -1835,7 +1838,7 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::SetSendParameters( << "RecreateWebRtcStream (send) because of SetSendParameters"; RecreateWebRtcStream(); } - } // release |lock_| + } // release |lock_| // |capturer_->AddOrUpdateSink| may not be called while holding |lock_| since // that might cause a lock order inversion. @@ -2026,8 +2029,14 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate(Load load) { rtc::Optional max_pixel_count; rtc::Optional max_pixel_count_step_up; if (load == kOveruse) { - max_pixel_count = rtc::Optional( - (last_dimensions_.height * last_dimensions_.width) / 2); + if (cpu_restricted_counter_ >= kMaxCpuDowngrades) { + return; + } + // The input video frame size will have a resolution with less than or + // equal to |max_pixel_count| depending on how the capturer can scale the + // input frame size. + max_pixel_count = rtc::Optional( + (last_dimensions_.height * last_dimensions_.width * 3) / 5); // Increase |number_of_cpu_adapt_changes_| if // sink_wants_.max_pixel_count will be changed since // last time |capturer_->AddOrUpdateSink| was called. That is, this will @@ -2039,6 +2048,9 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate(Load load) { } } else { RTC_DCHECK(load == kUnderuse); + // The input video frame size will have a resolution with "one step up" + // pixels than |max_pixel_count_step_up| where "one step up" depends on + // how the capturer can scale the input frame size. max_pixel_count_step_up = rtc::Optional(last_dimensions_.height * last_dimensions_.width); // Increase |number_of_cpu_adapt_changes_| if @@ -2088,16 +2100,15 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() { stats = stream_->GetStats(); } info.adapt_changes = number_of_cpu_adapt_changes_; - info.adapt_reason = cpu_restricted_counter_ <= 0 - ? CoordinatedVideoAdapter::ADAPTREASON_NONE - : CoordinatedVideoAdapter::ADAPTREASON_CPU; + info.adapt_reason = + cpu_restricted_counter_ <= 0 ? ADAPTREASON_NONE : ADAPTREASON_CPU; // Get bandwidth limitation info from stream_->GetStats(). // Input resolution (output from video_adapter) can be further scaled down or // higher video layer(s) can be dropped due to bitrate constraints. // Note, adapt_changes only include changes from the video_adapter. if (stats.bw_limited_resolution) - info.adapt_reason |= CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH; + info.adapt_reason |= ADAPTREASON_BANDWIDTH; info.encoder_implementation_name = stats.encoder_implementation_name; info.ssrc_groups = ssrc_groups_; diff --git a/webrtc/media/engine/webrtcvideoengine2.h b/webrtc/media/engine/webrtcvideoengine2.h index 8c05c60f68..ba2dbd8a02 100644 --- a/webrtc/media/engine/webrtcvideoengine2.h +++ b/webrtc/media/engine/webrtcvideoengine2.h @@ -175,6 +175,15 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, public webrtc::Transport { // Implemented for VideoMediaChannelTest. bool sending() const { return sending_; } + // AdaptReason is used for expressing why a WebRtcVideoSendStream request + // a lower input frame size than the currently configured camera input frame + // size. There can be more than one reason OR:ed together. + enum AdaptReason { + ADAPTREASON_NONE = 0, + ADAPTREASON_CPU = 1, + ADAPTREASON_BANDWIDTH = 2, + }; + private: class WebRtcVideoReceiveStream; struct VideoCodecSettings { diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc index 2f5ad6dd2a..e1f540660f 100644 --- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc @@ -2036,29 +2036,99 @@ TEST_F(WebRtcVideoChannel2Test, AdaptsOnOveruseAndChangeResolution) { EXPECT_EQ(1280 * 2 / 4, send_stream->GetLastWidth()); EXPECT_EQ(720 * 2 / 4, send_stream->GetLastHeight()); + // Trigger overuse again. This should not decrease the resolution since we + // should only adapt the resolution down max two steps. + overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse); + EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420)); + EXPECT_EQ(4, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1280 * 2 / 4, send_stream->GetLastWidth()); + EXPECT_EQ(720 * 2 / 4, send_stream->GetLastHeight()); + // Change input resolution. EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420)); - EXPECT_EQ(4, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(5, send_stream->GetNumberOfSwappedFrames()); EXPECT_EQ(1284 / 2, send_stream->GetLastWidth()); EXPECT_EQ(724 / 2, send_stream->GetLastHeight()); // Trigger underuse which should go back up in resolution. overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse); EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420)); - EXPECT_EQ(5, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(6, send_stream->GetNumberOfSwappedFrames()); EXPECT_EQ(1284 * 3 / 4, send_stream->GetLastWidth()); EXPECT_EQ(724 * 3 / 4, send_stream->GetLastHeight()); // Trigger underuse which should go back up in resolution. overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse); EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420)); - EXPECT_EQ(6, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(7, send_stream->GetNumberOfSwappedFrames()); EXPECT_EQ(1284, send_stream->GetLastWidth()); EXPECT_EQ(724, send_stream->GetLastHeight()); EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, NULL)); } +TEST_F(WebRtcVideoChannel2Test, PreviousAdaptationDoesNotApplyToScreenshare) { + cricket::VideoCodec codec = kVp8Codec720p; + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(codec); + + MediaConfig media_config = MediaConfig(); + channel_.reset( + engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions())); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + AddSendStream(); + + cricket::FakeVideoCapturer capturer; + ASSERT_TRUE(channel_->SetCapturer(last_ssrc_, &capturer)); + ASSERT_EQ(cricket::CS_RUNNING, + capturer.Start(capturer.GetSupportedFormats()->front())); + ASSERT_TRUE(channel_->SetSend(true)); + cricket::VideoOptions camera_options; + channel_->SetVideoSend(last_ssrc_, true /* enable */, &camera_options); + + ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size()); + FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front(); + webrtc::LoadObserver* overuse_callback = + send_stream->GetConfig().overuse_callback; + ASSERT_TRUE(overuse_callback != NULL); + + EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420)); + EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1280, send_stream->GetLastWidth()); + EXPECT_EQ(720, send_stream->GetLastHeight()); + + // Trigger overuse. + overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse); + EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420)); + EXPECT_EQ(2, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1280 * 3 / 4, send_stream->GetLastWidth()); + EXPECT_EQ(720 * 3 / 4, send_stream->GetLastHeight()); + + // Switch to screen share. Expect no CPU adaptation. + cricket::FakeVideoCapturer screen_share(true); + ASSERT_EQ(cricket::CS_RUNNING, + screen_share.Start(screen_share.GetSupportedFormats()->front())); + ASSERT_TRUE(channel_->SetCapturer(last_ssrc_, &screen_share)); + cricket::VideoOptions screenshare_options; + screenshare_options.is_screencast = rtc::Optional(true); + channel_->SetVideoSend(last_ssrc_, true /* enable */, &screenshare_options); + EXPECT_TRUE(screen_share.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420)); + EXPECT_EQ(3, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1284, send_stream->GetLastWidth()); + EXPECT_EQ(724, send_stream->GetLastHeight()); + + // Switch back to the normal capturer. Expect the frame to be CPU adapted. + ASSERT_TRUE(channel_->SetCapturer(last_ssrc_, &capturer)); + channel_->SetVideoSend(last_ssrc_, true /* enable */, &camera_options); + EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420)); + EXPECT_EQ(4, send_stream->GetNumberOfSwappedFrames()); + EXPECT_EQ(1280 * 3 / 4, send_stream->GetLastWidth()); + EXPECT_EQ(720 * 3 / 4, send_stream->GetLastHeight()); + + EXPECT_TRUE(channel_->SetCapturer(last_ssrc_, NULL)); +} + void WebRtcVideoChannel2Test::TestCpuAdaptation(bool enable_overuse, bool is_screenshare) { cricket::VideoCodec codec = kVp8Codec720p; @@ -2783,8 +2853,7 @@ TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationStats) { EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); EXPECT_EQ(1, info.senders[0].adapt_changes); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU, - info.senders[0].adapt_reason); + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_CPU, info.senders[0].adapt_reason); // Trigger upgrade and verify that we adapt back up to VGA. overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse); @@ -2793,7 +2862,7 @@ TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationStats) { EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); EXPECT_EQ(2, info.senders[0].adapt_changes); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_NONE, + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_NONE, info.senders[0].adapt_reason); // No capturer (no adapter). Adapt changes from old adapter should be kept. @@ -2802,7 +2871,7 @@ TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationStats) { EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); EXPECT_EQ(2, info.senders[0].adapt_changes); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_NONE, + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_NONE, info.senders[0].adapt_reason); // Set new capturer, capture format HD. @@ -2819,8 +2888,7 @@ TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationStats) { EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); EXPECT_EQ(3, info.senders[0].adapt_changes); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU, - info.senders[0].adapt_reason); + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_CPU, info.senders[0].adapt_reason); EXPECT_TRUE(channel_->SetCapturer(kSsrcs3[0], NULL)); } @@ -2855,8 +2923,7 @@ TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationAndBandwidthStats) { cricket::VideoMediaInfo info; EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU, - info.senders[0].adapt_reason); + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_CPU, info.senders[0].adapt_reason); // Set bandwidth limitation stats for the stream -> adapt CPU + BW. webrtc::VideoSendStream::Stats stats; @@ -2865,8 +2932,8 @@ TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationAndBandwidthStats) { info.Clear(); EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_CPU + - CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH, + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_CPU | + WebRtcVideoChannel2::ADAPTREASON_BANDWIDTH, info.senders[0].adapt_reason); // Trigger upgrade -> adapt BW. @@ -2875,7 +2942,7 @@ TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationAndBandwidthStats) { info.Clear(); EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH, + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_BANDWIDTH, info.senders[0].adapt_reason); // Reset bandwidth limitation state -> adapt NONE. @@ -2884,7 +2951,7 @@ TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationAndBandwidthStats) { info.Clear(); EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_NONE, + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_NONE, info.senders[0].adapt_reason); EXPECT_TRUE(channel_->SetCapturer(kSsrcs3[0], NULL)); @@ -2900,7 +2967,7 @@ TEST_F(WebRtcVideoChannel2Test, cricket::VideoMediaInfo info; EXPECT_TRUE(channel_->GetStats(&info)); ASSERT_EQ(1U, info.senders.size()); - EXPECT_EQ(CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH, + EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_BANDWIDTH, info.senders[0].adapt_reason); }