From e22d81ce4d12c183ad7ce296f89cf10ac68858b0 Mon Sep 17 00:00:00 2001 From: "marpan@webrtc.org" Date: Tue, 20 Mar 2012 18:21:53 +0000 Subject: [PATCH] Updates to resolution adpatation: -moved calculation of selected frame size & frame rate to qm_select class. -various updates to qm_select class (switch to 1/2 from 2 stages of 3/4, include native frame rate for going up temporal, favor spatial action for temporal layers,..). -updates to unittest. Review URL: https://webrtc-codereview.appspot.com/450008 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1914 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/source/media_optimization.cc | 44 +- .../main/source/media_optimization.h | 2 - .../video_coding/main/source/qm_select.cc | 167 +++++-- .../video_coding/main/source/qm_select.h | 42 +- .../video_coding/main/source/qm_select_data.h | 13 +- .../main/source/qm_select_unittest.cc | 455 +++++++++++------- 6 files changed, 474 insertions(+), 249 deletions(-) diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index 552aaab79d..a85c05b836 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -24,8 +24,6 @@ _maxBitRate(0), _sendCodecType(kVideoCodecUnknown), _codecWidth(0), _codecHeight(0), -_initCodecWidth(0), -_initCodecHeight(0), _userFrameRate(0), _packetLossEnc(0), _fractionLost(0), @@ -286,8 +284,6 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, _userFrameRate = static_cast(frameRate); _codecWidth = width; _codecHeight = height; - _initCodecWidth = width; - _initCodecHeight = height; _numLayers = (numLayers <= 1) ? 1 : numLayers; // Can also be zero. WebRtc_Word32 ret = VCM_OK; ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate, @@ -577,41 +573,39 @@ VCMMediaOptimization::checkStatusForQMchange() bool VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) { // Check for no change - if (!qm->change_resolution) { + if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) { return false; } // Check for change in frame rate. - if (qm->temporal_fact != 1.0f) { - _incomingFrameRate = _incomingFrameRate / qm->temporal_fact + 0.5f; + if (qm->change_resolution_temporal) { + _incomingFrameRate = qm->frame_rate; + // Reset frame rate estimate. memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); } // Check for change in frame size. - if (qm->spatial_height_fact != 1.0 || qm->spatial_width_fact != 1.0) { - _codecWidth = static_cast(_codecWidth / - qm->spatial_width_fact); - _codecHeight = static_cast(_codecHeight / - qm->spatial_height_fact); - // New frame sizes should not exceed original size from SetEncodingData(). - assert(_codecWidth <= _initCodecWidth); - assert(_codecHeight <= _initCodecHeight); - // Check that new frame sizes are multiples of two. - assert(_codecWidth % 2 == 0); - assert(_codecHeight % 2 == 0); + if (qm->change_resolution_spatial) { + _codecWidth = qm->codec_width; + _codecHeight = qm->codec_height; } WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, _id, - "Quality Mode Update: W = %d, H = %d, FR = %f", - _codecWidth, _codecHeight, _incomingFrameRate); + "Resolution change from QM select: W = %d, H = %d, FR = %f", + qm->codec_width, qm->codec_height, qm->frame_rate); - // Update VPM with new target frame rate and size - _videoQMSettingsCallback->SetVideoQMSettings(_incomingFrameRate, + // Update VPM with new target frame rate and frame size. + // Note: use |qm->frame_rate| instead of |_incomingFrameRate| for updating + // target frame rate in VPM frame dropper. The quantity |_incomingFrameRate| + // will vary/fluctuate, and since we don't want to change the state of the + // VPM frame dropper, unless a temporal action was selected, we use the + // quantity |qm->frame_rate| for updating. + _videoQMSettingsCallback->SetVideoQMSettings(qm->frame_rate, _codecWidth, _codecHeight); - - _content->UpdateFrameRate(_incomingFrameRate); - _qmResolution->UpdateCodecFrameSize(_codecWidth, _codecHeight); + _content->UpdateFrameRate(qm->frame_rate); + _qmResolution->UpdateCodecParameters(qm->frame_rate, _codecWidth, + _codecHeight); return true; } diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h index 14e5d1a2d7..7d87a6d041 100644 --- a/src/modules/video_coding/main/source/media_optimization.h +++ b/src/modules/video_coding/main/source/media_optimization.h @@ -168,8 +168,6 @@ private: VideoCodecType _sendCodecType; WebRtc_UWord16 _codecWidth; WebRtc_UWord16 _codecHeight; - WebRtc_UWord16 _initCodecWidth; - WebRtc_UWord16 _initCodecHeight; float _userFrameRate; VCMFrameDropper* _frameDropper; diff --git a/src/modules/video_coding/main/source/qm_select.cc b/src/modules/video_coding/main/source/qm_select.cc index ef273c0726..67b3bcab44 100644 --- a/src/modules/video_coding/main/source/qm_select.cc +++ b/src/modules/video_coding/main/source/qm_select.cc @@ -26,9 +26,11 @@ VCMQmMethod::VCMQmMethod() : content_metrics_(NULL), width_(0), height_(0), + user_frame_rate_(0.0f), native_width_(0), native_height_(0), - framerate_level_(kDefault), + native_frame_rate_(0.0f), + framerate_level_(kFrameRateHigh), init_(false) { ResetQM(); } @@ -134,13 +136,15 @@ ImageType VCMQmMethod::FindClosestImageType(uint16_t width, uint16_t height) { return static_cast(isel); } -LevelClass VCMQmMethod::FrameRateLevel(float avg_framerate) { +FrameRateLevelClass VCMQmMethod::FrameRateLevel(float avg_framerate) { if (avg_framerate < kLowFrameRate) { - return kLow; - } else if (avg_framerate > kHighFrameRate) { - return kHigh; + return kFrameRateLow; + } else if (avg_framerate < kMiddleFrameRate) { + return kFrameRateMiddle1; + } else if (avg_framerate < kHighFrameRate) { + return kFrameRateMiddle2; } else { - return kDefault; + return kFrameRateHigh; } } @@ -179,7 +183,6 @@ void VCMQmResolution::ResetDownSamplingState() { void VCMQmResolution::Reset() { target_bitrate_ = 0.0f; - user_framerate_ = 0.0f; incoming_framerate_ = 0.0f; buffer_level_ = 0.0f; per_frame_bandwidth_ =0.0f; @@ -212,23 +215,26 @@ int VCMQmResolution::Initialize(float bitrate, } Reset(); target_bitrate_ = bitrate; - user_framerate_ = user_framerate; incoming_framerate_ = user_framerate; - UpdateCodecFrameSize(width, height); + UpdateCodecParameters(user_framerate, width, height); native_width_ = width; native_height_ = height; + native_frame_rate_ = user_framerate; num_layers_ = num_layers; // Initial buffer level. buffer_level_ = kOptBufferLevel * target_bitrate_; // Per-frame bandwidth. - per_frame_bandwidth_ = target_bitrate_ / user_framerate_; + per_frame_bandwidth_ = target_bitrate_ / user_framerate; init_ = true; return VCM_OK; } -void VCMQmResolution::UpdateCodecFrameSize(uint16_t width, uint16_t height) { +void VCMQmResolution::UpdateCodecParameters(float frame_rate, uint16_t width, + uint16_t height) { width_ = width; height_ = height; + // |user_frame_rate| is the target frame rate for VPM frame dropper. + user_frame_rate_ = frame_rate; image_type_ = GetImageType(width, height); } @@ -256,11 +262,9 @@ void VCMQmResolution::UpdateRates(float target_bitrate, float encoder_sent_rate, float incoming_framerate, uint8_t packet_loss) { - // Sum the target bitrate and incoming frame rate: - // these values are the encoder rates (from previous update ~1sec), - // i.e, before the update for next ~1sec. + // Sum the target bitrate: this is the encoder rate from previous update + // (~1sec), i.e, before the update for next ~1sec. sum_target_rate_ += target_bitrate_; - sum_incoming_framerate_ += incoming_framerate_; update_rate_cnt_++; // Sum the received (from RTCP reports) packet loss rates. @@ -281,6 +285,7 @@ void VCMQmResolution::UpdateRates(float target_bitrate, // these values are ones the encoder will use for the current/next ~1sec target_bitrate_ = target_bitrate; incoming_framerate_ = incoming_framerate; + sum_incoming_framerate_ += incoming_framerate_; // Update the per_frame_bandwidth: // this is the per_frame_bw for the current/next ~1sec @@ -296,8 +301,8 @@ void VCMQmResolution::UpdateRates(float target_bitrate, // In the current version the following constraints are imposed: // 1) We only allow for one action, either down or up, at a given time. -// 2) The possible down-sampling actions are: spatial 1/2x1/2, 3/4x3/4; -// temporal 1/2 and 2/3. +// 2) The possible down-sampling actions are: spatial by 1/2x1/2, 3/4x3/4; +// temporal/frame rate reduction by 1/2 and 2/3. // 3) The action for going back up is the reverse of last (spatial or temporal) // down-sampling action. The list of down-sampling actions from the // Initialize() state are kept in |down_action_history_|. @@ -313,10 +318,6 @@ int VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { return VCM_OK; } - // Default settings: no action. - SetDefaultAction(); - *qm = qm_; - // Compute content class for selection. content_class_ = ComputeContentClass(); @@ -326,6 +327,10 @@ int VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { // Get the encoder state. ComputeEncoderState(); + // Default settings: no action. + SetDefaultAction(); + *qm = qm_; + // Check for going back up in resolution, if we have had some down-sampling // relative to native state in Initialize(). if (down_action_history_[0].spatial != kNoChangeSpatial || @@ -348,10 +353,14 @@ int VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { } void VCMQmResolution::SetDefaultAction() { + qm_->codec_width = width_; + qm_->codec_height = height_; + qm_->frame_rate = user_frame_rate_; + qm_->change_resolution_spatial = false; + qm_->change_resolution_temporal = false; qm_->spatial_width_fact = 1.0f; qm_->spatial_height_fact = 1.0f; qm_->temporal_fact = 1.0f; - qm_->change_resolution = false; action_.spatial = kNoChangeSpatial; action_.temporal = kNoChangeTemporal; } @@ -385,7 +394,9 @@ void VCMQmResolution::ComputeRatesForSelection() { (1.0 - kWeightRate) * target_bitrate_; avg_incoming_framerate_ = kWeightRate * avg_incoming_framerate_ + (1.0 - kWeightRate) * incoming_framerate_; - framerate_level_ = FrameRateLevel(avg_incoming_framerate_); + // Use base layer frame rate for temporal layers: this will favor spatial. + framerate_level_ = FrameRateLevel(avg_incoming_framerate_ / + pow(2, num_layers_ - 1)); } void VCMQmResolution::ComputeEncoderState() { @@ -461,6 +472,7 @@ bool VCMQmResolution::ConditionForGoingUp(float fac_width, float scale_fac) { float estimated_transition_rate_up = GetTransitionRate(fac_width, fac_height, fac_temp, scale_fac); + // Go back up if: // 1) target rate is above threshold and current encoder state is stable, or // 2) encoder state is easy (encoder is significantly under-shooting target). @@ -527,11 +539,15 @@ bool VCMQmResolution::GoingDownResolution() { } } - // TODO(marpan): If num_layers_ > 1, adjust/favor spatial over temporal ? + // Only allow for one action (spatial or temporal) at a given time. + assert(action_.temporal == kNoChangeTemporal || + action_.spatial == kNoChangeSpatial); // Adjust cases not captured in tables, mainly based on frame rate. AdjustAction(); + ConvertSpatialFractionalToWhole(); + CheckForEvenFrameSize(); // Update down-sampling state. @@ -552,8 +568,14 @@ float VCMQmResolution::GetTransitionRate(float fac_width, static_cast(fac_width * width_), static_cast(fac_height * height_)); - LevelClass framerate_level = + FrameRateLevelClass framerate_level = FrameRateLevel(fac_temp * avg_incoming_framerate_); + // If we are checking for going up temporally, and this is the last + // temporal action, then use native frame rate. + if (down_action_history_[1].temporal == kNoChangeTemporal && + fac_temp > 1.0f) { + framerate_level = FrameRateLevel(native_frame_rate_); + } // The maximum allowed rate below which down-sampling is allowed: // Nominal values based on image format (frame size and frame rate). @@ -570,7 +592,6 @@ float VCMQmResolution::GetTransitionRate(float fac_width, } void VCMQmResolution::UpdateDownsamplingState(UpDownAction up_down) { - qm_->change_resolution = true; if (up_down == kUpResolution) { qm_->spatial_width_fact = 1.0f / kFactorWidthSpatial[action_.spatial]; qm_->spatial_height_fact = 1.0f / kFactorHeightSpatial[action_.spatial]; @@ -587,9 +608,9 @@ void VCMQmResolution::UpdateDownsamplingState(UpDownAction up_down) { // has been selected. assert(false); } + UpdateCodecResolution(); state_dec_factor_spatial_ = state_dec_factor_spatial_ * - qm_->spatial_width_fact * - qm_->spatial_height_fact; + qm_->spatial_width_fact * qm_->spatial_height_fact; state_dec_factor_temporal_ = state_dec_factor_temporal_ * qm_->temporal_fact; assert(state_dec_factor_spatial_ >= 1.0f); assert(state_dec_factor_spatial_ <= kMaxSpatialDown); @@ -597,6 +618,33 @@ void VCMQmResolution::UpdateDownsamplingState(UpDownAction up_down) { assert(state_dec_factor_temporal_ <= kMaxTempDown); } +void VCMQmResolution::UpdateCodecResolution() { + if (action_.spatial != kNoChangeSpatial) { + qm_->change_resolution_spatial = true; + qm_->codec_width = static_cast(width_ / qm_->spatial_width_fact); + qm_->codec_height = static_cast + (height_ / qm_->spatial_height_fact); + // Size can never exceed native sizes. + assert(qm_->codec_width <= native_width_); + assert(qm_->codec_height <= native_height_); + // Size should be multiple of 2. + assert(qm_->codec_width % 2 == 0); + assert(qm_->codec_height % 2 == 0); + } + if (action_.temporal != kNoChangeTemporal) { + qm_->change_resolution_temporal = true; + // Update the frame rate based on the average incoming frame rate. + qm_->frame_rate = avg_incoming_framerate_ / qm_->temporal_fact + 0.5f; + if (down_action_history_[0].temporal == 0) { + // When we undo the last temporal-down action, make sure we go back up + // to the native frame rate. Since the incoming frame rate may + // fluctuate over time, |avg_incoming_framerate_| scaled back up may + // be smaller than |native_frame rate_|. + qm_->frame_rate = native_frame_rate_; + } + } +} + uint8_t VCMQmResolution::RateClass(float transition_rate) { return avg_target_rate_ < (kFacLowRate * transition_rate) ? 0: (avg_target_rate_ >= transition_rate ? 2 : 1); @@ -607,17 +655,67 @@ void VCMQmResolution::AdjustAction() { // is not high, then safer to take frame rate reduction if the // average incoming frame rate is high. if (spatial_.level == kDefault && motion_.level != kHigh && - framerate_level_ == kHigh) { + framerate_level_ == kFrameRateHigh) { action_.spatial = kNoChangeSpatial; action_.temporal = kOneHalfTemporal; } - // If both motion and spatial level are low, and temporal down-action + // If both motion and spatial level are low, and temporal down action // was selected, switch to spatial 3/4x3/4 if the frame rate is low. if (motion_.level == kLow && spatial_.level == kLow && - framerate_level_ == kLow && action_.temporal != kNoChangeTemporal) { + framerate_level_ == kFrameRateLow && + action_.temporal != kNoChangeTemporal) { action_.spatial = kOneHalfSpatialUniform; action_.temporal = kNoChangeTemporal; } + + // If too much spatial action, and temporal action has not yet been chosen, + // then change to temporal action if the average frame rate is not low. + if (action_.spatial == kOneQuarterSpatialUniform && + down_action_history_[0].spatial == kOneQuarterSpatialUniform && + down_action_history_[0].temporal == kNoChangeTemporal && + framerate_level_ != kFrameRateLow) { + action_.spatial = kNoChangeSpatial; + action_.temporal = kOneHalfTemporal; + } + + // Never use temporal action if number of temporal layers is above 2. + if (num_layers_ > 2) { + if (action_.temporal != kNoChangeTemporal) { + action_.spatial = kOneHalfSpatialUniform; + } + action_.temporal = kNoChangeTemporal; + } +} + +void VCMQmResolution::ConvertSpatialFractionalToWhole() { + // If 3/4 spatial is selected, check if there has been another 3/4, + // and if so, combine them into 1/2. 1/2 scaling is more efficient than 9/16. + // Note we define 3/4x3/4 spatial as kOneHalfSpatialUniform. + if (action_.spatial == kOneHalfSpatialUniform) { + bool found = false; + int isel = kDownActionHistorySize; + for (int i = 0; i < kDownActionHistorySize; ++i) { + if (down_action_history_[i].spatial == kOneHalfSpatialUniform) { + isel = i; + found = true; + break; + } + } + if (found) { + // Update state for removing 3/4 spatial. + state_dec_factor_spatial_ = state_dec_factor_spatial_ / + (kFactorWidthSpatial[kOneHalfSpatialUniform] * + kFactorHeightSpatial[kOneHalfSpatialUniform]); + width_ = width_ * kFactorWidthSpatial[kOneHalfSpatialUniform]; + height_ = height_ * kFactorHeightSpatial[kOneHalfSpatialUniform]; + // Remove 3/4 from the history. + for (int i = isel; i < kDownActionHistorySize - 1; ++i) { + down_action_history_[i].spatial = down_action_history_[i + 1].spatial; + } + // Update current selection action to be 1/2x1/2 (=1/4) spatial. + action_.spatial = kOneQuarterSpatialUniform; + } + } } void VCMQmResolution::CheckForEvenFrameSize() { @@ -669,11 +767,11 @@ void VCMQmResolution::ConstrainAmountOfDownSampling() { // No spatial sampling if current frame size is too small (QCIF), // or if the amount of spatial down-sampling will be too much. float new_dec_factor_spatial = state_dec_factor_spatial_ * - qm_->spatial_width_fact * - qm_->spatial_height_fact; + qm_->spatial_width_fact * qm_->spatial_height_fact; if ((width_ * height_) <= kMinImageSize || new_dec_factor_spatial > kMaxSpatialDown) { action_.spatial = kNoChangeSpatial; + qm_->change_resolution_spatial = false; qm_->spatial_width_fact = 1.0f; qm_->spatial_height_fact = 1.0f; } @@ -683,6 +781,7 @@ void VCMQmResolution::ConstrainAmountOfDownSampling() { if (avg_incoming_framerate_ <= kMinFrameRate || new_dec_factor_temp >= kMaxTempDown) { action_.temporal = kNoChangeTemporal; + qm_->change_resolution_temporal = false; qm_->temporal_fact = 1.0f; } } @@ -698,7 +797,7 @@ void VCMQmResolution::PickSpatialOrTemporal() { } } -// TODO(marpan): Update this when we allow for 1/2 spatial down-sampling. +// TODO(marpan): Update when we allow for directional spatial down-sampling. void VCMQmResolution::SelectSpatialDirectionMode(float transition_rate) { // Default is 4/3x4/3 // For bit rates well below transitional rate, we select 2x2. diff --git a/src/modules/video_coding/main/source/qm_select.h b/src/modules/video_coding/main/source/qm_select.h index b76e32afba..f5b9d210d3 100644 --- a/src/modules/video_coding/main/source/qm_select.h +++ b/src/modules/video_coding/main/source/qm_select.h @@ -23,15 +23,23 @@ struct VideoContentMetrics; struct VCMResolutionScale { VCMResolutionScale() - : spatial_width_fact(1.0f), + : codec_width(640), + codec_height(480), + frame_rate(30.0f), + spatial_width_fact(1.0f), spatial_height_fact(1.0f), temporal_fact(1.0f), - change_resolution(false) { + change_resolution_spatial(false), + change_resolution_temporal(false) { } + uint16_t codec_width; + uint16_t codec_height; + float frame_rate; float spatial_width_fact; float spatial_height_fact; float temporal_fact; - bool change_resolution; + bool change_resolution_spatial; + bool change_resolution_temporal; }; enum ImageType { @@ -50,7 +58,14 @@ enum ImageType { const uint32_t kSizeOfImageType[kNumImageTypes] = { 25344, 57024, 76800, 101376, 172800, 307200, 518400, 921600, 2073600 }; -enum LevelClass { +enum FrameRateLevelClass { + kFrameRateLow, + kFrameRateMiddle1, + kFrameRateMiddle2, + kFrameRateHigh +}; + +enum ContentLevelClass { kLow, kHigh, kDefault @@ -66,7 +81,7 @@ struct VCMContFeature { level = kDefault; } float value; - LevelClass level; + ContentLevelClass level; }; enum UpDownAction { @@ -146,7 +161,7 @@ class VCMQmMethod { ImageType FindClosestImageType(uint16_t width, uint16_t height); // Get the frame rate level. - LevelClass FrameRateLevel(float frame_rate); + FrameRateLevelClass FrameRateLevel(float frame_rate); protected: // Content Data. @@ -155,12 +170,14 @@ class VCMQmMethod { // Encoder frame sizes and native frame sizes. uint16_t width_; uint16_t height_; + float user_frame_rate_; uint16_t native_width_; uint16_t native_height_; + float native_frame_rate_; float aspect_ratio_; // Image type and frame rate leve, for the current encoder resolution. ImageType image_type_; - LevelClass framerate_level_; + FrameRateLevelClass framerate_level_; // Content class data. VCMContFeature motion_; VCMContFeature spatial_; @@ -195,7 +212,7 @@ class VCMQmResolution : public VCMQmMethod { int num_layers); // Update the encoder frame size. - void UpdateCodecFrameSize(uint16_t width, uint16_t height); + void UpdateCodecParameters(float frame_rate, uint16_t width, uint16_t height); // Update with actual bit rate (size of the latest encoded frame) // and frame type, after every encoded frame. @@ -214,6 +231,7 @@ class VCMQmResolution : public VCMQmMethod { // Output: the spatial and/or temporal scale change. int SelectResolution(VCMResolutionScale** qm); + private: // Set the default resolution action. void SetDefaultAction(); @@ -248,12 +266,18 @@ class VCMQmResolution : public VCMQmMethod { // Update the down-sampling state. void UpdateDownsamplingState(UpDownAction up_down); + // Update the codec frame size and frame rate. + void UpdateCodecResolution(); + // Return a state based on average target rate relative transition rate. uint8_t RateClass(float transition_rate); // Adjust the action selected from the table. void AdjustAction(); + // Covert 2 stages of 3/4 (=9/16) spatial decimation to 1/2. + void ConvertSpatialFractionalToWhole(); + // Check if the new frame sizes are still divisible by 2. void CheckForEvenFrameSize(); @@ -273,13 +297,11 @@ class VCMQmResolution : public VCMQmMethod { // Select the directional (1x2 or 2x1) spatial down-sampling action. void SelectSpatialDirectionMode(float transition_rate); - private: enum { kDownActionHistorySize = 10}; VCMResolutionScale* qm_; // Encoder rate control parameters. float target_bitrate_; - float user_framerate_; float incoming_framerate_; float per_frame_bandwidth_; float buffer_level_; diff --git a/src/modules/video_coding/main/source/qm_select_data.h b/src/modules/video_coding/main/source/qm_select_data.h index ec1e9888cc..1c9858d505 100644 --- a/src/modules/video_coding/main/source/qm_select_data.h +++ b/src/modules/video_coding/main/source/qm_select_data.h @@ -53,7 +53,6 @@ const float kPacketLossThr = 0.1f; // Factor for reducing transitonal bitrate under packet loss. const float kPacketLossRateFac = 1.0f; - // Maximum possible transitional rate for down-sampling: // (units in kbps), for 30fps. const uint16_t kMaxRateQm[9] = { @@ -69,10 +68,11 @@ const uint16_t kMaxRateQm[9] = { }; // Frame rate scale for maximum transition rate. -const float kFrameRateFac[3] = { - 0.7f, // L - 1.0f, // H - 0.8f // D +const float kFrameRateFac[4] = { + 0.5f, // Low + 0.7f, // Middle level 1 + 0.85f, // Middle level 2 + 1.0f, // High }; // Scale for transitional rate: based on content class @@ -180,7 +180,7 @@ const uint8_t kTemporalAction[27] = { // Control the total amount of down-sampling allowed. const float kMaxSpatialDown = 8.0f; const float kMaxTempDown = 4.0f; -const float kMaxDownSample = 16.0f; +const float kMaxDownSample = 12.0f; // Minimum image size for a spatial down-sampling. const int kMinImageSize= 176 * 144; @@ -199,6 +199,7 @@ const int kMinFrameRate = 8; // Thresholds for frame rate: const int kLowFrameRate = 10; +const int kMiddleFrameRate = 15; const int kHighFrameRate = 25; // Thresholds for motion: motion level is from NFD diff --git a/src/modules/video_coding/main/source/qm_select_unittest.cc b/src/modules/video_coding/main/source/qm_select_unittest.cc index 5b670fb657..f6990a6403 100644 --- a/src/modules/video_coding/main/source/qm_select_unittest.cc +++ b/src/modules/video_coding/main/source/qm_select_unittest.cc @@ -62,7 +62,10 @@ class QmSelectTest : public ::testing::Test { bool IsSelectedActionCorrect(VCMResolutionScale* qm_scale, float fac_width, float fac_height, - float fac_temp); + float fac_temp, + uint16_t new_width, + uint16_t new_height, + float new_frame_rate); void TearDown() { delete qm_resolution_; @@ -84,7 +87,8 @@ TEST_F(QmSelectTest, HandleInputs) { qm_resolution_->UpdateContent(content_metrics); // Content metrics are NULL: Expect success and no down-sampling action. EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0, 1.0, 1.0)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0, 1.0, 1.0, 640, 480, + 30.0f)); } // No down-sampling action at high rates. @@ -95,7 +99,7 @@ TEST_F(QmSelectTest, NoActionHighRate) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -111,7 +115,8 @@ TEST_F(QmSelectTest, NoActionHighRate) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(0, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 640, 480, + 30.0f)); } // Rate is well below transition, down-sampling action is taken, @@ -123,7 +128,7 @@ TEST_F(QmSelectTest, DownActionLowRate) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -140,35 +145,40 @@ TEST_F(QmSelectTest, DownActionLowRate) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, low spatial: 2/3 temporal is expected. UpdateQmContentData(kTemporalLow, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(0, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f, 640, 480, + 20.5f)); qm_resolution_->ResetDownSamplingState(); // Medium motion, low spatial: 2x2 spatial expected. UpdateQmContentData(kTemporalMedium, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); qm_resolution_->ResetDownSamplingState(); - // High motion, high spatial: 1/2 temporal expected. + // High motion, high spatial: 2/3 temporal expected. UpdateQmContentData(kTemporalHigh, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(4, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f, 640, 480, + 20.5f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial: 1/2 temporal expected. UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 640, 480, + 15.5f)); qm_resolution_->ResetDownSamplingState(); // Medium motion, high spatial: 1/2 temporal expected. @@ -176,7 +186,8 @@ TEST_F(QmSelectTest, DownActionLowRate) { kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 640, 480, + 15.5f)); qm_resolution_->ResetDownSamplingState(); // High motion, medium spatial: 2x2 spatial expected. @@ -184,7 +195,9 @@ TEST_F(QmSelectTest, DownActionLowRate) { kSpatialMedium); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(5, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + // Target frame rate for frame dropper should be the same as previous == 15. + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, medium spatial: high frame rate, so 1/2 temporal expected. @@ -192,7 +205,8 @@ TEST_F(QmSelectTest, DownActionLowRate) { kSpatialMedium); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(2, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 640, 480, + 15.5f)); qm_resolution_->ResetDownSamplingState(); // Medium motion, medium spatial: high frame rate, so 1/2 temporal expected. @@ -200,7 +214,8 @@ TEST_F(QmSelectTest, DownActionLowRate) { kSpatialMedium); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(8, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 640, 480, + 15.5f)); } // Rate mis-match is high, and we have over-shooting. @@ -212,7 +227,7 @@ TEST_F(QmSelectTest, DownActionHighRateMMOvershoot) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -230,14 +245,15 @@ TEST_F(QmSelectTest, DownActionHighRateMMOvershoot) { EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStressedEncoding, qm_resolution_->GetEncoderState()); EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, - 1.0f)); + 1.0f, 480, 360, 30.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f, 640, 480, + 20.5f)); } // Rate mis-match is high, target rate is below max for down-sampling, @@ -249,7 +265,7 @@ TEST_F(QmSelectTest, NoActionHighRateMMUndershoot) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -266,14 +282,16 @@ TEST_F(QmSelectTest, NoActionHighRateMMUndershoot) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kEasyEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 640, 480, + 30.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 640, 480, + 30.0f)); } // Buffer is underflowing, and target rate is below max for down-sampling, @@ -285,7 +303,7 @@ TEST_F(QmSelectTest, DownActionBufferUnderflow) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update with encoded size over a number of frames. @@ -308,14 +326,15 @@ TEST_F(QmSelectTest, DownActionBufferUnderflow) { EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStressedEncoding, qm_resolution_->GetEncoderState()); EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, - 1.0f)); + 1.0f, 480, 360, 30.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f, 640, 480, + 20.5f)); } // Target rate is below max for down-sampling, but buffer level is stable, @@ -327,7 +346,7 @@ TEST_F(QmSelectTest, NoActionBufferStable) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update with encoded size over a number of frames. @@ -349,14 +368,16 @@ TEST_F(QmSelectTest, NoActionBufferStable) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 640, 480, + 30.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 640, 480, + 30.0f)); } // Very low rate, but no spatial down-sampling below some size (QCIF). @@ -367,7 +388,7 @@ TEST_F(QmSelectTest, LimitDownSpatialAction) { // Update with encoder frame size. uint16_t codec_width = 176; uint16_t codec_height = 144; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(0, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -384,7 +405,8 @@ TEST_F(QmSelectTest, LimitDownSpatialAction) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 176, 144, + 30.0f)); } // Very low rate, but no frame reduction below some frame_rate (8fps). @@ -395,7 +417,7 @@ TEST_F(QmSelectTest, LimitDownTemporalAction) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(8.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -413,7 +435,8 @@ TEST_F(QmSelectTest, LimitDownTemporalAction) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(2, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 640, 480, + 8.0f)); } // Two stages: spatial down-sample and then back up spatially, @@ -425,7 +448,7 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatial) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -442,11 +465,12 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatial) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); // Reset and go up in rate: expected to go back up. qm_resolution_->ResetRates(); - qm_resolution_->UpdateCodecFrameSize(320, 240); + qm_resolution_->UpdateCodecParameters(30.0f, 320, 240); EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); // Update rates for a sequence of intervals. int target_rate2[] = {400, 400, 400, 400, 400}; @@ -457,7 +481,8 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatial) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f, 640, 480, + 30.0f)); } // Two stages: spatial down-sample and then back up spatially, since encoder @@ -469,7 +494,7 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatialUndershoot) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -486,11 +511,12 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatialUndershoot) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); // Reset rates and simulate under-shooting scenario.: expect to go back up. qm_resolution_->ResetRates(); - qm_resolution_->UpdateCodecFrameSize(320, 240); + qm_resolution_->UpdateCodecParameters(30.0f, 320, 240); EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); // Update rates for a sequence of intervals. int target_rate2[] = {200, 200, 200, 200, 200}; @@ -501,7 +527,8 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatialUndershoot) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kEasyEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f, 640, 480, + 30.0f)); } // Two stages: spatial down-sample and then no action to go up, @@ -513,7 +540,7 @@ TEST_F(QmSelectTest, 2StageDownSpatialNoActionUp) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -530,11 +557,12 @@ TEST_F(QmSelectTest, 2StageDownSpatialNoActionUp) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); // Reset and simulate large rate mis-match: expect no action to go back up. qm_resolution_->ResetRates(); - qm_resolution_->UpdateCodecFrameSize(320, 240); + qm_resolution_->UpdateCodecParameters(30.0f, 320, 240); EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); // Update rates for a sequence of intervals. int target_rate2[] = {400, 400, 400, 400, 400}; @@ -545,8 +573,10 @@ TEST_F(QmSelectTest, 2StageDownSpatialNoActionUp) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kStressedEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 320, 240, + 30.0f)); } + // Two stages: temporally down-sample and then back up temporally, // as rate as increased. TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporal) { @@ -556,91 +586,7 @@ TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporal) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); - - // Update rates for a sequence of intervals. - int target_rate[] = {100, 100, 100}; - int encoder_sent_rate[] = {100, 100, 100}; - int incoming_frame_rate[] = {30, 30, 30}; - uint8_t fraction_lost[] = {10, 10, 10}; - UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, - fraction_lost, 3); - - // Update content: motion level, and 3 spatial prediction errors. - // Low motion, high spatial. - UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); - EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); - EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); - - // Reset rates and go up in rate: expect to go back up. - qm_resolution_->ResetRates(); - // Update rates for a sequence of intervals. - int target_rate2[] = {400, 400, 400, 400, 400}; - int encoder_sent_rate2[] = {400, 400, 400, 400, 400}; - int incoming_frame_rate2[] = {15, 15, 15, 15, 15}; - uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; - UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, - fraction_lost2, 5); - EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); - EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f)); -} - -// Two stages: temporal down-sample and then back up temporally, since encoder -// is under-shooting target even though rate has not increased much. -TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporalUndershoot) { - // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480, 1); - - // Update with encoder frame size. - uint16_t codec_width = 640; - uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); - - // Update rates for a sequence of intervals. - int target_rate[] = {100, 100, 100}; - int encoder_sent_rate[] = {100, 100, 100}; - int incoming_frame_rate[] = {30, 30, 30}; - uint8_t fraction_lost[] = {10, 10, 10}; - UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, - fraction_lost, 3); - - // Update content: motion level, and 3 spatial prediction errors. - // Low motion, high spatial. - UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); - EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); - EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); - - // Reset rates and simulate under-shooting scenario.: expect to go back up. - qm_resolution_->ResetRates(); - // Update rates for a sequence of intervals. - int target_rate2[] = {200, 200, 200, 200, 200}; - int encoder_sent_rate2[] = {50, 50, 50, 50, 50}; - int incoming_frame_rate2[] = {15, 15, 15, 15, 15}; - uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; - UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, - fraction_lost2, 5); - EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); - EXPECT_EQ(kEasyEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f)); -} - -// Two stages: temporal down-sample and then no action to go up, -// as encoding rate mis-match is too high. -TEST_F(QmSelectTest, 2StageDownTemporalNoActionUp) { - // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480, 1); - - // Update with encoder frame size. - uint16_t codec_width = 640; - uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -657,9 +603,98 @@ TEST_F(QmSelectTest, 2StageDownTemporalNoActionUp) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 640, 480, + 15.5f)); + + // Reset rates and go up in rate: expect to go back up. + qm_resolution_->ResetRates(); + // Update rates for a sequence of intervals. + int target_rate2[] = {400, 400, 400, 400, 400}; + int encoder_sent_rate2[] = {400, 400, 400, 400, 400}; + int incoming_frame_rate2[] = {15, 15, 15, 15, 15}; + uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, + fraction_lost2, 5); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f, 640, 480, + 30.0f)); +} + +// Two stages: temporal down-sample and then back up temporally, since encoder +// is under-shooting target even though rate has not increased much. +TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporalUndershoot) { + // Initialize with bitrate, frame rate, and native system width/height. + InitQmNativeData(100, 30, 640, 480, 1); + + // Update with encoder frame size. + uint16_t codec_width = 640; + uint16_t codec_height = 480; + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); + + // Update rates for a sequence of intervals. + int target_rate[] = {100, 100, 100}; + int encoder_sent_rate[] = {100, 100, 100}; + int incoming_frame_rate[] = {30, 30, 30}; + uint8_t fraction_lost[] = {10, 10, 10}; + UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, + fraction_lost, 3); + + // Update content: motion level, and 3 spatial prediction errors. + // Low motion, high spatial. + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 640, 480, + 15.5f)); + + // Reset rates and simulate under-shooting scenario.: expect to go back up. + qm_resolution_->ResetRates(); + // Update rates for a sequence of intervals. + int target_rate2[] = {200, 200, 200, 200, 200}; + int encoder_sent_rate2[] = {50, 50, 50, 50, 50}; + int incoming_frame_rate2[] = {15, 15, 15, 15, 15}; + uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, + fraction_lost2, 5); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(kEasyEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f, 640, 480, + 30.0f)); +} + +// Two stages: temporal down-sample and then no action to go up, +// as encoding rate mis-match is too high. +TEST_F(QmSelectTest, 2StageDownTemporalNoActionUp) { + // Initialize with bitrate, frame rate, and native system width/height. + InitQmNativeData(100, 30, 640, 480, 1); + + // Update with encoder frame size. + uint16_t codec_width = 640; + uint16_t codec_height = 480; + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); + + // Update rates for a sequence of intervals. + int target_rate[] = {100, 100, 100}; + int encoder_sent_rate[] = {100, 100, 100}; + int incoming_frame_rate[] = {30, 30, 30}; + uint8_t fraction_lost[] = {10, 10, 10}; + UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, + fraction_lost, 3); + + // Update content: motion level, and 3 spatial prediction errors. + // Low motion, high spatial. + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2, 640, 480, 15.5f)); // Reset and simulate large rate mis-match: expect no action to go back up. + qm_resolution_->UpdateCodecParameters(15.0f, codec_width, codec_height); qm_resolution_->ResetRates(); // Update rates for a sequence of intervals. int target_rate2[] = {600, 600, 600, 600, 600}; @@ -670,7 +705,8 @@ TEST_F(QmSelectTest, 2StageDownTemporalNoActionUp) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kStressedEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 640, 480, + 15.0f)); } // 3 stages: spatial down-sample, followed by temporal down-sample, // and then go up to full state, as encoding rate has increased. @@ -681,7 +717,7 @@ TEST_F(QmSelectTest, 3StageDownSpatialTemporlaUpSpatialTemporal) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -698,10 +734,11 @@ TEST_F(QmSelectTest, 3StageDownSpatialTemporlaUpSpatialTemporal) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); // Change content data: expect temporal down-sample. - qm_resolution_->UpdateCodecFrameSize(320, 240); + qm_resolution_->UpdateCodecParameters(30.0f, 320, 240); EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); // Update content: motion level, and 3 spatial prediction errors. @@ -710,7 +747,8 @@ TEST_F(QmSelectTest, 3StageDownSpatialTemporlaUpSpatialTemporal) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 320, 240, + 15.5f)); // Reset rates and go high up in rate: expect to go back up both spatial // and temporally. @@ -726,7 +764,8 @@ TEST_F(QmSelectTest, 3StageDownSpatialTemporlaUpSpatialTemporal) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 0.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 0.5f, 640, 480, + 30.0f)); } // No down-sampling below some total amount. @@ -737,7 +776,7 @@ TEST_F(QmSelectTest, NoActionTooMuchDownSampling) { // Update with encoder frame size. uint16_t codec_width = 1280; uint16_t codec_height = 720; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(7, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. @@ -754,11 +793,12 @@ TEST_F(QmSelectTest, NoActionTooMuchDownSampling) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 640, 360, + 30.0f)); // Reset and lower rates to get another spatial action (3/4x3/4) qm_resolution_->ResetRates(); - qm_resolution_->UpdateCodecFrameSize(640, 360); + qm_resolution_->UpdateCodecParameters(30.0f, 640, 360); EXPECT_EQ(4, qm_resolution_->GetImageType(640, 360)); // Update rates for a sequence of intervals. int target_rate2[] = {80, 80, 80, 80, 80}; @@ -776,13 +816,13 @@ TEST_F(QmSelectTest, NoActionTooMuchDownSampling) { EXPECT_EQ(5, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, - 1.0f)); + 1.0f, 480, 270, 30.0f)); // Reset and go to very low rate: no action should be taken, // we went down too much already. qm_resolution_->ResetRates(); - qm_resolution_->UpdateCodecFrameSize(320, 180); - EXPECT_EQ(1, qm_resolution_->GetImageType(320, 180)); + qm_resolution_->UpdateCodecParameters(30.0f, 480, 270); + EXPECT_EQ(3, qm_resolution_->GetImageType(480, 270)); // Update rates for a sequence of intervals. int target_rate3[] = {10, 10, 10, 10, 10}; int encoder_sent_rate3[] = {10, 10, 10, 10, 10}; @@ -793,7 +833,8 @@ TEST_F(QmSelectTest, NoActionTooMuchDownSampling) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(5, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f, 480, 270, + 30.0f)); } // Multiple down-sampling stages and then undo all of them. @@ -807,7 +848,7 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory1) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Go down spatial 3/4x3/4. @@ -826,9 +867,9 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory1) { EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, - 1.0f)); + 1.0f, 480, 360, 30.0f)); // Go down 1/2 temporal. - qm_resolution_->UpdateCodecFrameSize(480, 360); + qm_resolution_->UpdateCodecParameters(30.0f, 480, 360); EXPECT_EQ(4, qm_resolution_->GetImageType(480, 360)); qm_resolution_->ResetRates(); int target_rate2[] = {100, 100, 100, 100, 100}; @@ -844,9 +885,11 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory1) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 480, 360, + 15.5f)); // Go down 1/2x1/2 spatial. + qm_resolution_->UpdateCodecParameters(15.0f, 480, 360); qm_resolution_->ResetRates(); int target_rate3[] = {50, 50, 50, 50, 50}; int encoder_sent_rate3[] = {50, 50, 50, 50, 50}; @@ -861,14 +904,15 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory1) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 240, 180, + 15.0f)); // Reset rates and go high up in rate: expect to go up: // should go up first: 1/2x1x2 and 1/2 temporally, // and second: 3/4x3/4 spatial. // Go up 1/2x1/2 spatially and 1/2 temporally - qm_resolution_->UpdateCodecFrameSize(240, 180); + qm_resolution_->UpdateCodecParameters(15.0f, 240, 180); EXPECT_EQ(1, qm_resolution_->GetImageType(240, 180)); qm_resolution_->ResetRates(); // Update rates for a sequence of intervals. @@ -882,10 +926,11 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory1) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 0.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 0.5f, 480, 360, + 30.0f)); // Go up 3/4x3/4 spatially. - qm_resolution_->UpdateCodecFrameSize(480, 360); + qm_resolution_->UpdateCodecParameters(30.0f, 480, 360); EXPECT_EQ(4, qm_resolution_->GetImageType(480, 360)); qm_resolution_->ResetRates(); // Update rates for a sequence of intervals. @@ -900,7 +945,7 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory1) { EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 3.0f / 4.0f, 3.0f / 4.0f, - 1.0f)); + 1.0f, 640, 480, 30.0f)); } // Multiple down-sampling and up-sample stages, with partial undoing. @@ -914,7 +959,7 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory2) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Go down 1/2x1/2 spatial. @@ -932,10 +977,11 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory2) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); // Go down 2/3 temporal. - qm_resolution_->UpdateCodecFrameSize(320, 240); + qm_resolution_->UpdateCodecParameters(30.0f, 320, 240); EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); qm_resolution_->ResetRates(); int target_rate2[] = {80, 80, 80, 80, 80}; @@ -952,9 +998,11 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory2) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f, 320, 240, + 20.5f)); // Go up 1/2x1/2 spatially. + qm_resolution_->UpdateCodecParameters(20.0f, 320, 240); qm_resolution_->ResetRates(); // Update rates for a sequence of intervals. int target_rate3[] = {300, 300, 300, 300, 300}; @@ -967,10 +1015,11 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory2) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f, 640, 480, + 20.0f)); // Go down 1/2 temporal. - qm_resolution_->UpdateCodecFrameSize(640, 480); + qm_resolution_->UpdateCodecParameters(20.0f, 640, 480); EXPECT_EQ(5, qm_resolution_->GetImageType(640, 480)); qm_resolution_->ResetRates(); int target_rate4[] = {100, 100, 100, 100, 100}; @@ -986,7 +1035,8 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory2) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 640, 480, + 10.5f)); // Go up 1/2 temporal. qm_resolution_->ResetRates(); @@ -1001,7 +1051,8 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory2) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f, 640, 480, + 20.5f)); } // Multiple down-sampling and up-sample stages, with partial undoing. @@ -1015,7 +1066,7 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory3) { // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; - qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Go down 3/4x3/4 spatial. @@ -1034,10 +1085,10 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory3) { EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, - 1.0f)); + 1.0f, 480, 360, 30.0f)); // Go down 1/2 temporal. - qm_resolution_->UpdateCodecFrameSize(480, 360); + qm_resolution_->UpdateCodecParameters(30.0f, 480, 360); EXPECT_EQ(4, qm_resolution_->GetImageType(480, 360)); qm_resolution_->ResetRates(); int target_rate2[] = {100, 100, 100, 100, 100}; @@ -1053,7 +1104,8 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory3) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f, 480, 360, + 15.5f)); // Go up 1/2 temporal. qm_resolution_->ResetRates(); @@ -1068,15 +1120,16 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory3) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f, 480, 360, + 30.0f)); // Go down 2/3 temporal. - qm_resolution_->UpdateCodecFrameSize(640, 480); + qm_resolution_->UpdateCodecParameters(30.0f, 640, 480); EXPECT_EQ(5, qm_resolution_->GetImageType(640, 480)); qm_resolution_->ResetRates(); - int target_rate4[] = {150, 150, 150, 150, 150}; - int encoder_sent_rate4[] = {150, 150, 150, 150, 150}; - int incoming_frame_rate4[] = {20, 20, 20, 20, 20}; + int target_rate4[] = {200, 200, 200, 200, 200}; + int encoder_sent_rate4[] = {200, 200, 200, 200, 200}; + int incoming_frame_rate4[] = {30, 30, 30, 30, 30}; uint8_t fraction_lost4[] = {30, 30, 30, 30, 30}; UpdateQmRateData(target_rate4, encoder_sent_rate4, incoming_frame_rate4, fraction_lost4, 5); @@ -1088,7 +1141,8 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory3) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f, 640, 480, + 20.5f)); // Go up 2/3 temporal. qm_resolution_->ResetRates(); @@ -1103,7 +1157,58 @@ TEST_F(QmSelectTest, MultipleStagesCheckActionHistory3) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f / 3.0f)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f / 3.0f, 640, + 480, 30.0f)); +} + +// Two stages of 3/4x3/4 converted to one stage of 1/2x1/2. +TEST_F(QmSelectTest, ConvertThreeQuartersToOneHalf) { + // Initialize with bitrate, frame rate, and native system width/height. + InitQmNativeData(200, 30, 640, 480, 1); + + // Update with encoder frame size. + uint16_t codec_width = 640; + uint16_t codec_height = 480; + qm_resolution_->UpdateCodecParameters(30.0f, codec_width, codec_height); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); + + // Go down 3/4x3/4 spatial. + // Update rates for a sequence of intervals. + int target_rate[] = {200, 200, 200}; + int encoder_sent_rate[] = {200, 200, 200}; + int incoming_frame_rate[] = {30, 30, 30}; + uint8_t fraction_lost[] = {10, 10, 10}; + UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, + fraction_lost, 3); + + // Update content: motion level, and 3 spatial prediction errors. + // Medium motion, low spatial. + UpdateQmContentData(kTemporalMedium, kSpatialLow, kSpatialLow, kSpatialLow); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, + 1.0f, 480, 360, 30.0f)); + + // Set rates to go down another 3/4 spatial. Should be converted ton 1/2. + qm_resolution_->UpdateCodecParameters(30.0f, 480, 360); + EXPECT_EQ(4, qm_resolution_->GetImageType(480, 360)); + qm_resolution_->ResetRates(); + int target_rate2[] = {150, 150, 150, 150, 150}; + int encoder_sent_rate2[] = {150, 150, 150, 150, 150}; + int incoming_frame_rate2[] = {30, 30, 30, 30, 30}; + uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, + fraction_lost2, 5); + + // Update content: motion level, and 3 spatial prediction errors. + // Medium motion, low spatial. + UpdateQmContentData(kTemporalMedium, kSpatialLow, kSpatialLow, kSpatialLow); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f, 320, 240, + 30.0f)); } void QmSelectTest::InitQmNativeData(float initial_bit_rate, @@ -1160,10 +1265,16 @@ void QmSelectTest::UpdateQmRateData(int* target_rate, bool QmSelectTest::IsSelectedActionCorrect(VCMResolutionScale* qm_scale, float fac_width, float fac_height, - float fac_temp) { + float fac_temp, + uint16_t new_width, + uint16_t new_height, + float new_frame_rate) { if (qm_scale->spatial_width_fact == fac_width && qm_scale->spatial_height_fact == fac_height && - qm_scale->temporal_fact == fac_temp) { + qm_scale->temporal_fact == fac_temp && + qm_scale->codec_width == new_width && + qm_scale->codec_height == new_height && + qm_scale->frame_rate == new_frame_rate) { return true; } else { return false;