From accf607b3e0c98cf0c16314bbf258fa1ed581aa8 Mon Sep 17 00:00:00 2001 From: "marpan@webrtc.org" Date: Wed, 7 Mar 2012 17:16:10 +0000 Subject: [PATCH] Updates for resolution adaptation. 1) added support for two additional modes: -3/4 spatial down-sampling -2/3 frame rate reduction 2) updated unittest and added a few more tests 3) some code refactoring Review URL: https://webrtc-codereview.appspot.com/429005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1854 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/source/media_optimization.cc | 88 +- .../video_coding/main/source/qm_select.cc | 814 ++++++++++-------- .../video_coding/main/source/qm_select.h | 241 ++++-- .../video_coding/main/source/qm_select_data.h | 117 ++- .../main/source/qm_select_unittest.cc | 618 ++++++++++--- 5 files changed, 1229 insertions(+), 649 deletions(-) diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index cae491229f..552aaab79d 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -180,7 +180,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, // Update encoding rates following protection settings _frameDropper->SetRates(static_cast(_targetBitRate), 0); - if (_enableQm && _numLayers == 1) + if (_enableQm) { // Update QM with rates _qmResolution->UpdateRates((float)_targetBitRate, sent_video_rate, @@ -291,7 +291,7 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, _numLayers = (numLayers <= 1) ? 1 : numLayers; // Can also be zero. WebRtc_Word32 ret = VCM_OK; ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate, - _codecWidth, _codecHeight); + _codecWidth, _codecHeight, _numLayers); return ret; } @@ -575,68 +575,44 @@ VCMMediaOptimization::checkStatusForQMchange() } -bool -VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) -{ - // Check for no change - if (qm->spatialHeightFact == 1 && - qm->spatialWidthFact == 1 && - qm->temporalFact == 1) { - return false; - } +bool VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) { + // Check for no change + if (!qm->change_resolution) { + return false; + } - // Temporal - WebRtc_UWord32 frameRate = static_cast - (_incomingFrameRate + 0.5f); + // Check for change in frame rate. + if (qm->temporal_fact != 1.0f) { + _incomingFrameRate = _incomingFrameRate / qm->temporal_fact + 0.5f; + memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); + } - // Check if go back up in temporal resolution - if (qm->temporalFact == 0) { - // Currently only allow for 1/2 frame rate reduction per action. - // TODO (marpan): allow for 2/3 reduction. - frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; - } - // go down in temporal resolution - else { - frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); - } - // Reset _incomingFrameRate if temporal action was selected. - if (qm->temporalFact != 1) { - memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); - _incomingFrameRate = frameRate; - } - - // Spatial - WebRtc_UWord32 height = _codecHeight; - WebRtc_UWord32 width = _codecWidth; - // Check if go back up in spatial resolution, and update frame sizes. - // Currently only allow for 2x2 spatial down-sampling. - // TODO (marpan): allow for 1x2, 2x1, and 4/3x4/3 (or 3/2x3/2). - if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) { - width = _codecWidth * 2; - height = _codecHeight * 2; - } else { - width = _codecWidth / qm->spatialWidthFact; - height = _codecHeight / qm->spatialHeightFact; - } - _codecWidth = width; - _codecHeight = height; - - // New frame sizes should never exceed the original sizes - // from SetEncodingData(). + // Check for change in frame size. + if (qm->spatial_height_fact != 1.0 || qm->spatial_width_fact != 1.0) { + _codecWidth = static_cast(_codecWidth / + qm->spatial_width_fact); + _codecHeight = static_cast(_codecHeight / + qm->spatial_height_fact); + // New frame sizes should not exceed original size from SetEncodingData(). assert(_codecWidth <= _initCodecWidth); assert(_codecHeight <= _initCodecHeight); + // Check that new frame sizes are multiples of two. + assert(_codecWidth % 2 == 0); + assert(_codecHeight % 2 == 0); + } - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, _id, + WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, _id, "Quality Mode Update: W = %d, H = %d, FR = %f", - width, height, frameRate); + _codecWidth, _codecHeight, _incomingFrameRate); - // Update VPM with new target frame rate and size - _videoQMSettingsCallback->SetVideoQMSettings(frameRate, width, height); + // Update VPM with new target frame rate and size + _videoQMSettingsCallback->SetVideoQMSettings(_incomingFrameRate, + _codecWidth, + _codecHeight); - _content->UpdateFrameRate(frameRate); - _qmResolution->UpdateCodecFrameSize(width, height); - - return true; + _content->UpdateFrameRate(_incomingFrameRate); + _qmResolution->UpdateCodecFrameSize(_codecWidth, _codecHeight); + return true; } void diff --git a/src/modules/video_coding/main/source/qm_select.cc b/src/modules/video_coding/main/source/qm_select.cc index c4bd707812..ef273c0726 100644 --- a/src/modules/video_coding/main/source/qm_select.cc +++ b/src/modules/video_coding/main/source/qm_select.cc @@ -23,13 +23,13 @@ namespace webrtc { // QM-METHOD class VCMQmMethod::VCMQmMethod() - : _contentMetrics(NULL), - _width(0), - _height(0), - _nativeWidth(0), - _nativeHeight(0), - _frameRateLevel(kDefault), - _init(false) { + : content_metrics_(NULL), + width_(0), + height_(0), + native_width_(0), + native_height_(0), + framerate_level_(kDefault), + init_(false) { ResetQM(); } @@ -37,86 +37,107 @@ VCMQmMethod::~VCMQmMethod() { } void VCMQmMethod::ResetQM() { - _aspectRatio = 1.0f; - _imageType = 2; - _motion.Reset(); - _spatial.Reset(); - _contentClass = 0; + aspect_ratio_ = 1.0f; + image_type_ = kVGA; + motion_.Reset(); + spatial_.Reset(); + content_class_ = 0; } uint8_t VCMQmMethod::ComputeContentClass() { ComputeMotionNFD(); ComputeSpatial(); - return _contentClass = 3 * _motion.level + _spatial.level; + return content_class_ = 3 * motion_.level + spatial_.level; } void VCMQmMethod::UpdateContent(const VideoContentMetrics* contentMetrics) { - _contentMetrics = contentMetrics; + content_metrics_ = contentMetrics; } void VCMQmMethod::ComputeMotionNFD() { - if (_contentMetrics) { - _motion.value = _contentMetrics->motion_magnitude; + if (content_metrics_) { + motion_.value = content_metrics_->motion_magnitude; } // Determine motion level. - if (_motion.value < kLowMotionNfd) { - _motion.level = kLow; - } else if (_motion.value > kHighMotionNfd) { - _motion.level = kHigh; + if (motion_.value < kLowMotionNfd) { + motion_.level = kLow; + } else if (motion_.value > kHighMotionNfd) { + motion_.level = kHigh; } else { - _motion.level = kDefault; + motion_.level = kDefault; } } void VCMQmMethod::ComputeSpatial() { - float spatialErr = 0.0; - float spatialErrH = 0.0; - float spatialErrV = 0.0; - if (_contentMetrics) { - spatialErr = _contentMetrics->spatial_pred_err; - spatialErrH = _contentMetrics->spatial_pred_err_h; - spatialErrV = _contentMetrics->spatial_pred_err_v; + float spatial_err = 0.0; + float spatial_err_h = 0.0; + float spatial_err_v = 0.0; + if (content_metrics_) { + spatial_err = content_metrics_->spatial_pred_err; + spatial_err_h = content_metrics_->spatial_pred_err_h; + spatial_err_v = content_metrics_->spatial_pred_err_v; } // Spatial measure: take average of 3 prediction errors. - _spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f; + spatial_.value = (spatial_err + spatial_err_h + spatial_err_v) / 3.0f; - // Reduce thresholds for large scenes/higher pixel correlation (~>=WHD). - float scale2 = _imageType > 3 ? kScaleTexture : 1.0; + // Reduce thresholds for large scenes/higher pixel correlation. + float scale2 = image_type_ > kVGA ? kScaleTexture : 1.0; - if (_spatial.value > scale2 * kHighTexture) { - _spatial.level = kHigh; - } else if (_spatial.value < scale2 * kLowTexture) { - _spatial.level = kLow; + if (spatial_.value > scale2 * kHighTexture) { + spatial_.level = kHigh; + } else if (spatial_.value < scale2 * kLowTexture) { + spatial_.level = kLow; } else { - _spatial.level = kDefault; + spatial_.level = kDefault; } } -uint8_t VCMQmMethod::GetImageType(uint16_t width, - uint16_t height) { - // Get the closest image type for encoder frame size. - uint32_t imageSize = width * height; - if (imageSize < kFrameSizeTh[0]) { - return 0; // QCIF - } else if (imageSize < kFrameSizeTh[1]) { - return 1; // CIF - } else if (imageSize < kFrameSizeTh[2]) { - return 2; // VGA - } else if (imageSize < kFrameSizeTh[3]) { - return 3; // 4CIF - } else if (imageSize < kFrameSizeTh[4]) { - return 4; // 720,4:3 - } else if (imageSize < kFrameSizeTh[5]) { - return 5; // WHD +ImageType VCMQmMethod::GetImageType(uint16_t width, + uint16_t height) { + // Get the image type for the encoder frame size. + uint32_t image_size = width * height; + if (image_size == kSizeOfImageType[kQCIF]) { + return kQCIF; + } else if (image_size == kSizeOfImageType[kHCIF]) { + return kHCIF; + } else if (image_size == kSizeOfImageType[kQVGA]) { + return kQVGA; + } else if (image_size == kSizeOfImageType[kCIF]) { + return kCIF; + } else if (image_size == kSizeOfImageType[kHVGA]) { + return kHVGA; + } else if (image_size == kSizeOfImageType[kVGA]) { + return kVGA; + } else if (image_size == kSizeOfImageType[kQFULLHD]) { + return kQFULLHD; + } else if (image_size == kSizeOfImageType[kWHD]) { + return kWHD; + } else if (image_size == kSizeOfImageType[kFULLHD]) { + return kFULLHD; } else { - return 6; // HD + // No exact match, find closet one. + return FindClosestImageType(width, height); } } -LevelClass VCMQmMethod::FrameRateLevel(float avgFrameRate) { - if (avgFrameRate < kLowFrameRate) { +ImageType VCMQmMethod::FindClosestImageType(uint16_t width, uint16_t height) { + float size = static_cast(width * height); + float min = size; + int isel = 0; + for (int i = 0; i < kNumImageTypes; ++i) { + float dist = fabs(size - kSizeOfImageType[i]); + if (dist < min) { + min = dist; + isel = i; + } + } + return static_cast(isel); +} + +LevelClass VCMQmMethod::FrameRateLevel(float avg_framerate) { + if (avg_framerate < kLowFrameRate) { return kLow; - } else if (avgFrameRate > kHighFrameRate) { + } else if (avg_framerate > kHighFrameRate) { return kHigh; } else { return kDefault; @@ -126,139 +147,146 @@ LevelClass VCMQmMethod::FrameRateLevel(float avgFrameRate) { // RESOLUTION CLASS VCMQmResolution::VCMQmResolution() - : _qm(new VCMResolutionScale()) { + : qm_(new VCMResolutionScale()) { Reset(); } VCMQmResolution::~VCMQmResolution() { - delete _qm; + delete qm_; } void VCMQmResolution::ResetRates() { - _sumTargetRate = 0.0f; - _sumIncomingFrameRate = 0.0f; - _sumRateMM = 0.0f; - _sumRateMMSgn = 0; - _sumPacketLoss = 0.0f; - _frameCnt = 0; - _frameCntDelta = 0; - _lowBufferCnt = 0; - _updateRateCnt = 0; + sum_target_rate_ = 0.0f; + sum_incoming_framerate_ = 0.0f; + sum_rate_MM_ = 0.0f; + sum_rate_MM_sgn_ = 0.0f; + sum_packet_loss_ = 0.0f; + buffer_level_ = kOptBufferLevel * target_bitrate_; + frame_cnt_ = 0; + frame_cnt_delta_ = 0; + low_buffer_cnt_ = 0; + update_rate_cnt_ = 0; } void VCMQmResolution::ResetDownSamplingState() { - _stateDecFactorSpatial = 1; - _stateDecFactorTemp = 1; + state_dec_factor_spatial_ = 1.0; + state_dec_factor_temporal_ = 1.0; + for (int i = 0; i < kDownActionHistorySize; i++) { + down_action_history_[i].spatial = kNoChangeSpatial; + down_action_history_[i].temporal = kNoChangeTemporal; + } } void VCMQmResolution::Reset() { - _targetBitRate = 0.0f; - _userFrameRate = 0.0f; - _incomingFrameRate = 0.0f; - _perFrameBandwidth =0.0f; - _bufferLevel = 0.0f; - _avgTargetRate = 0.0f; - _avgIncomingFrameRate = 0.0f; - _avgRatioBufferLow = 0.0f; - _avgRateMisMatch = 0.0f; - _avgRateMisMatchSgn = 0.0f; - _avgPacketLoss = 0.0f; - _encoderState = kStableEncoding; + target_bitrate_ = 0.0f; + user_framerate_ = 0.0f; + incoming_framerate_ = 0.0f; + buffer_level_ = 0.0f; + per_frame_bandwidth_ =0.0f; + avg_target_rate_ = 0.0f; + avg_incoming_framerate_ = 0.0f; + avg_ratio_buffer_low_ = 0.0f; + avg_rate_mismatch_ = 0.0f; + avg_rate_mismatch_sgn_ = 0.0f; + avg_packet_loss_ = 0.0f; + encoder_state_ = kStableEncoding; + num_layers_ = 1; ResetRates(); ResetDownSamplingState(); ResetQM(); } EncoderState VCMQmResolution::GetEncoderState() { - return _encoderState; + return encoder_state_; } // Initialize state after re-initializing the encoder, // i.e., after SetEncodingData() in mediaOpt. -int VCMQmResolution::Initialize(float bitRate, - float userFrameRate, +int VCMQmResolution::Initialize(float bitrate, + float user_framerate, uint16_t width, - uint16_t height) { - if (userFrameRate == 0.0f || width == 0 || height == 0) { + uint16_t height, + int num_layers) { + if (user_framerate == 0.0f || width == 0 || height == 0) { return VCM_PARAMETER_ERROR; } Reset(); - _targetBitRate = bitRate; - _userFrameRate = userFrameRate; - _incomingFrameRate = userFrameRate; + target_bitrate_ = bitrate; + user_framerate_ = user_framerate; + incoming_framerate_ = user_framerate; UpdateCodecFrameSize(width, height); - _nativeWidth = width; - _nativeHeight = height; + native_width_ = width; + native_height_ = height; + num_layers_ = num_layers; // Initial buffer level. - _bufferLevel = kInitBufferLevel * _targetBitRate; + buffer_level_ = kOptBufferLevel * target_bitrate_; // Per-frame bandwidth. - _perFrameBandwidth = _targetBitRate / _userFrameRate; - _init = true; + per_frame_bandwidth_ = target_bitrate_ / user_framerate_; + init_ = true; return VCM_OK; } void VCMQmResolution::UpdateCodecFrameSize(uint16_t width, uint16_t height) { - _width = width; - _height = height; - // Set the imageType for the encoder width/height. - _imageType = GetImageType(width, height); + width_ = width; + height_ = height; + image_type_ = GetImageType(width, height); } // Update rate data after every encoded frame. -void VCMQmResolution::UpdateEncodedSize(int encodedSize, - FrameType encodedFrameType) { - _frameCnt++; +void VCMQmResolution::UpdateEncodedSize(int encoded_size, + FrameType encoded_frame_type) { + frame_cnt_++; // Convert to Kbps. - float encodedSizeKbits = static_cast((encodedSize * 8.0) / 1000.0); + float encoded_size_kbits = static_cast((encoded_size * 8.0) / 1000.0); // Update the buffer level: // Note this is not the actual encoder buffer level. - // |_bufferLevel| is reset to 0 every time SelectResolution is called, and - // does not account for frame dropping by encoder or VCM. - _bufferLevel += _perFrameBandwidth - encodedSizeKbits; + // |buffer_level_| is reset to average value every time SelectResolution is + // called, and does not account for frame dropping by encoder or VCM. + buffer_level_ += per_frame_bandwidth_ - encoded_size_kbits; // Counter for occurrences of low buffer level: // low/negative values means encoder is likely dropping frames. - if (_bufferLevel <= kPercBufferThr * kOptBufferLevel * _targetBitRate) { - _lowBufferCnt++; + if (buffer_level_ <= kPercBufferThr * kOptBufferLevel * target_bitrate_) { + low_buffer_cnt_++; } } // Update various quantities after SetTargetRates in MediaOpt. -void VCMQmResolution::UpdateRates(float targetBitRate, - float encoderSentRate, - float incomingFrameRate, - uint8_t packetLoss) { +void VCMQmResolution::UpdateRates(float target_bitrate, + float encoder_sent_rate, + float incoming_framerate, + uint8_t packet_loss) { // Sum the target bitrate and incoming frame rate: // these values are the encoder rates (from previous update ~1sec), // i.e, before the update for next ~1sec. - _sumTargetRate += _targetBitRate; - _sumIncomingFrameRate += _incomingFrameRate; - _updateRateCnt++; + sum_target_rate_ += target_bitrate_; + sum_incoming_framerate_ += incoming_framerate_; + update_rate_cnt_++; // Sum the received (from RTCP reports) packet loss rates. - _sumPacketLoss += static_cast(packetLoss / 255.0); + sum_packet_loss_ += static_cast(packet_loss / 255.0); // Sum the sequence rate mismatch: // Mismatch here is based on the difference between the target rate // used (in previous ~1sec) and the average actual encoding rate measured // at previous ~1sec. - float diff = _targetBitRate - encoderSentRate; - if (_targetBitRate > 0.0) - _sumRateMM += fabs(diff) / _targetBitRate; + float diff = target_bitrate_ - encoder_sent_rate; + if (target_bitrate_ > 0.0) + sum_rate_MM_ += fabs(diff) / target_bitrate_; int sgnDiff = diff > 0 ? 1 : (diff < 0 ? -1 : 0); // To check for consistent under(+)/over_shooting(-) of target rate. - _sumRateMMSgn += sgnDiff; + sum_rate_MM_sgn_ += sgnDiff; // Update with the current new target and frame rate: // these values are ones the encoder will use for the current/next ~1sec - _targetBitRate = targetBitRate; - _incomingFrameRate = incomingFrameRate; + target_bitrate_ = target_bitrate; + incoming_framerate_ = incoming_framerate; // Update the per_frame_bandwidth: // this is the per_frame_bw for the current/next ~1sec - _perFrameBandwidth = 0.0f; - if (_incomingFrameRate > 0.0f) { - _perFrameBandwidth = _targetBitRate / _incomingFrameRate; + per_frame_bandwidth_ = 0.0f; + if (incoming_framerate_ > 0.0f) { + per_frame_bandwidth_ = target_bitrate_ / incoming_framerate_; } } @@ -267,31 +295,30 @@ void VCMQmResolution::UpdateRates(float targetBitRate, // (if a previous down-sampling action was taken). // In the current version the following constraints are imposed: -// 1) we only allow for one action (either down or back up) at a given time. -// 2) the possible down-sampling actions are: 2x2 spatial and 1/2 temporal. -// 3) the total amount of down-sampling (spatial and/or temporal) from the -// initial (native) resolution is limited by various factors. - -// TODO(marpan): extend to allow options for: 4/3x4/3, 1x2, 2x1 spatial, -// and 2/3 temporal (i.e., skip every third frame). +// 1) We only allow for one action, either down or up, at a given time. +// 2) The possible down-sampling actions are: spatial 1/2x1/2, 3/4x3/4; +// temporal 1/2 and 2/3. +// 3) The action for going back up is the reverse of last (spatial or temporal) +// down-sampling action. The list of down-sampling actions from the +// Initialize() state are kept in |down_action_history_|. +// 4) The total amount of down-sampling (spatial and/or temporal) from the +// Initialize() state (native resolution) is limited by various factors. int VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { - if (!_init) { + if (!init_) { return VCM_UNINITIALIZED; } - if (_contentMetrics == NULL) { + if (content_metrics_ == NULL) { Reset(); - *qm = _qm; + *qm = qm_; return VCM_OK; } // Default settings: no action. - _qm->spatialWidthFact = 1; - _qm->spatialHeightFact = 1; - _qm->temporalFact = 1; - *qm = _qm; + SetDefaultAction(); + *qm = qm_; // Compute content class for selection. - _contentClass = ComputeContentClass(); + content_class_ = ComputeContentClass(); // Compute various rate quantities for selection. ComputeRatesForSelection(); @@ -300,133 +327,146 @@ int VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { ComputeEncoderState(); // Check for going back up in resolution, if we have had some down-sampling - // relative to native state in Initialize (i.e., after SetEncodingData() - // in mediaOpt.). - if (_stateDecFactorSpatial > 1 || _stateDecFactorTemp > 1) { + // relative to native state in Initialize(). + if (down_action_history_[0].spatial != kNoChangeSpatial || + down_action_history_[0].temporal != kNoChangeTemporal) { if (GoingUpResolution()) { - *qm = _qm; + *qm = qm_; return VCM_OK; } } // Check for going down in resolution, only if current total amount of // down-sampling state is below threshold. - if (_stateDecFactorTemp * _stateDecFactorSpatial < kMaxDownSample) { + if (state_dec_factor_temporal_ * state_dec_factor_spatial_ < kMaxDownSample) { if (GoingDownResolution()) { - *qm = _qm; + *qm = qm_; return VCM_OK; } } return VCM_OK; } +void VCMQmResolution::SetDefaultAction() { + qm_->spatial_width_fact = 1.0f; + qm_->spatial_height_fact = 1.0f; + qm_->temporal_fact = 1.0f; + qm_->change_resolution = false; + action_.spatial = kNoChangeSpatial; + action_.temporal = kNoChangeTemporal; +} + void VCMQmResolution::ComputeRatesForSelection() { - _avgTargetRate = 0.0f; - _avgIncomingFrameRate = 0.0f; - _avgRatioBufferLow = 0.0f; - _avgRateMisMatch = 0.0f; - _avgRateMisMatchSgn = 0.0f; - _avgPacketLoss = 0.0f; - if (_frameCnt > 0) { - _avgRatioBufferLow = static_cast(_lowBufferCnt) / - static_cast(_frameCnt); + avg_target_rate_ = 0.0f; + avg_incoming_framerate_ = 0.0f; + avg_ratio_buffer_low_ = 0.0f; + avg_rate_mismatch_ = 0.0f; + avg_rate_mismatch_sgn_ = 0.0f; + avg_packet_loss_ = 0.0f; + if (frame_cnt_ > 0) { + avg_ratio_buffer_low_ = static_cast(low_buffer_cnt_) / + static_cast(frame_cnt_); } - if (_updateRateCnt > 0) { - _avgRateMisMatch = static_cast(_sumRateMM) / - static_cast(_updateRateCnt); - _avgRateMisMatchSgn = static_cast(_sumRateMMSgn) / - static_cast(_updateRateCnt); - _avgTargetRate = static_cast(_sumTargetRate) / - static_cast(_updateRateCnt); - _avgIncomingFrameRate = static_cast(_sumIncomingFrameRate) / - static_cast(_updateRateCnt); - _avgPacketLoss = static_cast(_sumPacketLoss) / - static_cast(_updateRateCnt); + if (update_rate_cnt_ > 0) { + avg_rate_mismatch_ = static_cast(sum_rate_MM_) / + static_cast(update_rate_cnt_); + avg_rate_mismatch_sgn_ = static_cast(sum_rate_MM_sgn_) / + static_cast(update_rate_cnt_); + avg_target_rate_ = static_cast(sum_target_rate_) / + static_cast(update_rate_cnt_); + avg_incoming_framerate_ = static_cast(sum_incoming_framerate_) / + static_cast(update_rate_cnt_); + avg_packet_loss_ = static_cast(sum_packet_loss_) / + static_cast(update_rate_cnt_); } // For selection we may want to weight some quantities more heavily // with the current (i.e., next ~1sec) rate values. - float weight = 0.7f; - _avgTargetRate = weight * _avgTargetRate + (1.0 - weight) * _targetBitRate; - _avgIncomingFrameRate = weight * _avgIncomingFrameRate + - (1.0 - weight) * _incomingFrameRate; - _frameRateLevel = FrameRateLevel(_avgIncomingFrameRate); + avg_target_rate_ = kWeightRate * avg_target_rate_ + + (1.0 - kWeightRate) * target_bitrate_; + avg_incoming_framerate_ = kWeightRate * avg_incoming_framerate_ + + (1.0 - kWeightRate) * incoming_framerate_; + framerate_level_ = FrameRateLevel(avg_incoming_framerate_); } void VCMQmResolution::ComputeEncoderState() { // Default. - _encoderState = kStableEncoding; + encoder_state_ = kStableEncoding; // Assign stressed state if: // 1) occurrences of low buffer levels is high, or // 2) rate mis-match is high, and consistent over-shooting by encoder. - if ((_avgRatioBufferLow > kMaxBufferLow) || - ((_avgRateMisMatch > kMaxRateMisMatch) && - (_avgRateMisMatchSgn < -kRateOverShoot))) { - _encoderState = kStressedEncoding; + if ((avg_ratio_buffer_low_ > kMaxBufferLow) || + ((avg_rate_mismatch_ > kMaxRateMisMatch) && + (avg_rate_mismatch_sgn_ < -kRateOverShoot))) { + encoder_state_ = kStressedEncoding; } // Assign easy state if: // 1) rate mis-match is high, and // 2) consistent under-shooting by encoder. - if ((_avgRateMisMatch > kMaxRateMisMatch) && - (_avgRateMisMatchSgn > kRateUnderShoot)) { - _encoderState = kEasyEncoding; + if ((avg_rate_mismatch_ > kMaxRateMisMatch) && + (avg_rate_mismatch_sgn_ > kRateUnderShoot)) { + encoder_state_ = kEasyEncoding; } } bool VCMQmResolution::GoingUpResolution() { + // For going up, we check for undoing the previous down-sampling action. + float fac_width = kFactorWidthSpatial[down_action_history_[0].spatial]; + float fac_height = kFactorHeightSpatial[down_action_history_[0].spatial]; + float fac_temp = kFactorTemporal[down_action_history_[0].temporal]; + // Check if we should go up both spatially and temporally. - if (_stateDecFactorSpatial > 1 && _stateDecFactorTemp > 1) { - if (ConditionForGoingUp(2, 2, 2, kTransRateScaleUpSpatialTemp)) { - _qm->spatialHeightFact = 0; - _qm->spatialWidthFact = 0; - _qm->temporalFact = 0; - UpdateDownsamplingState(kUpResolution); - return true; - } - } else { - // Check if we should go up either spatially or temporally. - bool selectedUpS = false; - bool selectedUpT = false; - if (_stateDecFactorSpatial > 1) { - selectedUpS = ConditionForGoingUp(2, 2, 1, kTransRateScaleUpSpatial); - } - if (_stateDecFactorTemp > 1) { - selectedUpT = ConditionForGoingUp(1, 1, 2, kTransRateScaleUpTemp); - } - if (selectedUpS && !selectedUpT) { - _qm->spatialHeightFact = 0; - _qm->spatialWidthFact = 0; - UpdateDownsamplingState(kUpResolution); - return true; - } else if (!selectedUpS && selectedUpT) { - _qm->temporalFact = 0; - UpdateDownsamplingState(kUpResolution); - return true; - } else if (selectedUpS && selectedUpT) { - // TODO(marpan): which one to pick? - // pickSpatialOrTemporal() - // For now take spatial over temporal. - _qm->spatialHeightFact = 0; - _qm->spatialWidthFact = 0; + if (down_action_history_[0].spatial != kNoChangeSpatial && + down_action_history_[0].temporal != kNoChangeTemporal) { + if (ConditionForGoingUp(fac_width, fac_height, fac_temp, + kTransRateScaleUpSpatialTemp)) { + action_.spatial = down_action_history_[0].spatial; + action_.temporal = down_action_history_[0].temporal; UpdateDownsamplingState(kUpResolution); return true; } } + // Check if we should go up either spatially or temporally. + bool selected_up_spatial = false; + bool selected_up_temporal = false; + if (down_action_history_[0].spatial != kNoChangeSpatial) { + selected_up_spatial = ConditionForGoingUp(fac_width, fac_height, 1.0f, + kTransRateScaleUpSpatial); + } + if (down_action_history_[0].temporal != kNoChangeTemporal) { + selected_up_temporal = ConditionForGoingUp(1.0f, 1.0f, fac_temp, + kTransRateScaleUpTemp); + } + if (selected_up_spatial && !selected_up_temporal) { + action_.spatial = down_action_history_[0].spatial; + action_.temporal = kNoChangeTemporal; + UpdateDownsamplingState(kUpResolution); + return true; + } else if (!selected_up_spatial && selected_up_temporal) { + action_.spatial = kNoChangeSpatial; + action_.temporal = down_action_history_[0].temporal; + UpdateDownsamplingState(kUpResolution); + return true; + } else if (selected_up_spatial && selected_up_temporal) { + PickSpatialOrTemporal(); + UpdateDownsamplingState(kUpResolution); + return true; + } return false; } -bool VCMQmResolution::ConditionForGoingUp(uint8_t facWidth, - uint8_t facHeight, - uint8_t facTemp, - float scaleFac) { - float estimatedTransitionRateUp = GetTransitionRate(facWidth, facHeight, - facTemp, scaleFac); +bool VCMQmResolution::ConditionForGoingUp(float fac_width, + float fac_height, + float fac_temp, + float scale_fac) { + float estimated_transition_rate_up = GetTransitionRate(fac_width, fac_height, + fac_temp, scale_fac); // Go back up if: // 1) target rate is above threshold and current encoder state is stable, or // 2) encoder state is easy (encoder is significantly under-shooting target). - if (((_avgTargetRate > estimatedTransitionRateUp) && - (_encoderState == kStableEncoding)) || - (_encoderState == kEasyEncoding)) { + if (((avg_target_rate_ > estimated_transition_rate_up) && + (encoder_state_ == kStableEncoding)) || + (encoder_state_ == kEasyEncoding)) { return true; } else { return false; @@ -434,79 +474,69 @@ bool VCMQmResolution::ConditionForGoingUp(uint8_t facWidth, } bool VCMQmResolution::GoingDownResolution() { - float estimatedTransitionRateDown = GetTransitionRate(1, 1, 1, 1.0); - float maxRate = kFrameRateFac[_frameRateLevel] * kMaxRateQm[_imageType]; - - // TODO(marpan): Bias down-sampling based on packet loss conditions. + float estimated_transition_rate_down = + GetTransitionRate(1.0f, 1.0f, 1.0f, 1.0f); + float max_rate = kFrameRateFac[framerate_level_] * kMaxRateQm[image_type_]; // Resolution reduction if: // (1) target rate is below transition rate, or // (2) encoder is in stressed state and target rate below a max threshold. - if ((_avgTargetRate < estimatedTransitionRateDown ) || - (_encoderState == kStressedEncoding && _avgTargetRate < maxRate)) { - // Get the down-sampling action. - uint8_t spatialFact = kSpatialAction[_contentClass]; - uint8_t tempFact = kTemporalAction[_contentClass]; + if ((avg_target_rate_ < estimated_transition_rate_down ) || + (encoder_state_ == kStressedEncoding && avg_target_rate_ < max_rate)) { + // Get the down-sampling action: based on content class, and how low + // average target rate is relative to transition rate. + uint8_t spatial_fact = + kSpatialAction[content_class_ + + 9 * RateClass(estimated_transition_rate_down)]; + uint8_t temp_fact = + kTemporalAction[content_class_ + + 9 * RateClass(estimated_transition_rate_down)]; - switch (spatialFact) { + switch (spatial_fact) { case 4: { - _qm->spatialWidthFact = 2; - _qm->spatialHeightFact = 2; + action_.spatial = kOneQuarterSpatialUniform; break; } case 2: { - assert(false); // Currently not used. - // Select 1x2,2x1, or 4/3x4/3. - // SelectSpatialDirectionMode((float) estimatedTransitionRateDown); + action_.spatial = kOneHalfSpatialUniform; break; } case 1: { - _qm->spatialWidthFact = 1; - _qm->spatialHeightFact = 1; + action_.spatial = kNoChangeSpatial; break; } default: { assert(false); } } - switch (tempFact) { + switch (temp_fact) { + case 3: { + action_.temporal = kTwoThirdsTemporal; + break; + } case 2: { - _qm->temporalFact = 2; + action_.temporal = kOneHalfTemporal; break; } case 1: { - _qm->temporalFact = 1; + action_.temporal = kNoChangeTemporal; break; } default: { assert(false); } } - // Adjust some cases based on frame rate. - // TODO(marpan): will be modified when we add 1/2 spatial and 2/3 temporal. + + // TODO(marpan): If num_layers_ > 1, adjust/favor spatial over temporal ? + + // Adjust cases not captured in tables, mainly based on frame rate. AdjustAction(); - // Sanity checks on down-sampling selection: - // override the settings for too small image size and/or frame rate. - // Also check the limit on current down-sampling states. - - // No spatial sampling if current frame size is too small (QCIF), - // or if amount of spatial down-sampling is already too much. - if ((_width * _height) <= kMinImageSize || - _stateDecFactorSpatial >= kMaxSpatialDown) { - _qm->spatialWidthFact = 1; - _qm->spatialHeightFact = 1; - } - // No frame rate reduction if average frame rate is below some point, - // or if the amount of temporal down-sampling is already too much. - if (_avgIncomingFrameRate <= kMinFrameRate || - _stateDecFactorTemp >= kMaxTempDown) { - _qm->temporalFact = 1; - } + CheckForEvenFrameSize(); // Update down-sampling state. - if (_qm->spatialWidthFact != 1 || _qm->spatialHeightFact != 1 || - _qm->temporalFact != 1) { + if (action_.spatial != kNoChangeSpatial || + action_.temporal != kNoChangeTemporal) { UpdateDownsamplingState(kDownResolution); return true; } @@ -514,97 +544,197 @@ bool VCMQmResolution::GoingDownResolution() { return false; } -float VCMQmResolution::GetTransitionRate(uint8_t facWidth, - uint8_t facHeight, - uint8_t facTemp, - float scaleFac) { - uint8_t imageType = GetImageType(facWidth * _width, - facHeight * _height); - LevelClass frameRateLevel = FrameRateLevel(facTemp * _avgIncomingFrameRate); +float VCMQmResolution::GetTransitionRate(float fac_width, + float fac_height, + float fac_temp, + float scale_fac) { + ImageType image_type = GetImageType( + static_cast(fac_width * width_), + static_cast(fac_height * height_)); + + LevelClass framerate_level = + FrameRateLevel(fac_temp * avg_incoming_framerate_); // The maximum allowed rate below which down-sampling is allowed: // Nominal values based on image format (frame size and frame rate). - float maxRate = kFrameRateFac[frameRateLevel] * kMaxRateQm[imageType]; + float max_rate = kFrameRateFac[framerate_level] * kMaxRateQm[image_type]; - uint8_t imageClass = imageType > 3 ? 1: 0; - uint8_t tableIndex = imageClass * 9 + _contentClass; + uint8_t image_class = image_type > kVGA ? 1: 0; + uint8_t table_index = image_class * 9 + content_class_; // Scale factor for down-sampling transition threshold: // factor based on the content class and the image size. - float scaleTransRate = kScaleTransRateQm[tableIndex]; + float scaleTransRate = kScaleTransRateQm[table_index]; // Threshold bitrate for resolution action. - return static_cast (scaleFac * facTemp * _incomingFrameRate * - scaleTransRate * maxRate / 30); + return static_cast (scale_fac * scaleTransRate * max_rate); } -void VCMQmResolution::UpdateDownsamplingState(ResolutionAction action) { - // Assumes for now only actions are 1/2 frame rate of 2x2 spatial. - if (action == kUpResolution) { - if (_qm->spatialHeightFact == 0 && _qm->spatialWidthFact == 0) { - _stateDecFactorSpatial = _stateDecFactorSpatial / 4; - assert(_stateDecFactorSpatial >= 1); - } - if (_qm->temporalFact == 0) { - _stateDecFactorTemp = _stateDecFactorTemp / 2; - assert(_stateDecFactorTemp >= 1); - } - } else if (action == kDownResolution) { - _stateDecFactorSpatial = _stateDecFactorSpatial * _qm->spatialWidthFact - * _qm->spatialHeightFact; - _stateDecFactorTemp = _stateDecFactorTemp * _qm->temporalFact; - assert(_stateDecFactorSpatial >= 1); - assert(_stateDecFactorTemp >= 1); +void VCMQmResolution::UpdateDownsamplingState(UpDownAction up_down) { + qm_->change_resolution = true; + if (up_down == kUpResolution) { + qm_->spatial_width_fact = 1.0f / kFactorWidthSpatial[action_.spatial]; + qm_->spatial_height_fact = 1.0f / kFactorHeightSpatial[action_.spatial]; + qm_->temporal_fact = 1.0f / kFactorTemporal[action_.temporal]; + RemoveLastDownAction(); + } else if (up_down == kDownResolution) { + qm_->spatial_width_fact = kFactorWidthSpatial[action_.spatial]; + qm_->spatial_height_fact = kFactorHeightSpatial[action_.spatial]; + qm_->temporal_fact = kFactorTemporal[action_.temporal]; + ConstrainAmountOfDownSampling(); + InsertLatestDownAction(); } else { + // This function should only be called if either the Up or Down action + // has been selected. assert(false); } + state_dec_factor_spatial_ = state_dec_factor_spatial_ * + qm_->spatial_width_fact * + qm_->spatial_height_fact; + state_dec_factor_temporal_ = state_dec_factor_temporal_ * qm_->temporal_fact; + assert(state_dec_factor_spatial_ >= 1.0f); + assert(state_dec_factor_spatial_ <= kMaxSpatialDown); + assert(state_dec_factor_temporal_ >= 1.0f); + assert(state_dec_factor_temporal_ <= kMaxTempDown); +} + +uint8_t VCMQmResolution::RateClass(float transition_rate) { + return avg_target_rate_ < (kFacLowRate * transition_rate) ? 0: + (avg_target_rate_ >= transition_rate ? 2 : 1); } void VCMQmResolution::AdjustAction() { - if (_spatial.level == kDefault && _motion.level != kHigh && - _frameRateLevel == kHigh) { - _qm->temporalFact = 2; - _qm->spatialWidthFact = 1; - _qm->spatialHeightFact = 1; + // If the spatial level is default state (neither low or high) and motion + // is not high, then safer to take frame rate reduction if the + // average incoming frame rate is high. + if (spatial_.level == kDefault && motion_.level != kHigh && + framerate_level_ == kHigh) { + action_.spatial = kNoChangeSpatial; + action_.temporal = kOneHalfTemporal; + } + // If both motion and spatial level are low, and temporal down-action + // was selected, switch to spatial 3/4x3/4 if the frame rate is low. + if (motion_.level == kLow && spatial_.level == kLow && + framerate_level_ == kLow && action_.temporal != kNoChangeTemporal) { + action_.spatial = kOneHalfSpatialUniform; + action_.temporal = kNoChangeTemporal; + } +} + +void VCMQmResolution::CheckForEvenFrameSize() { + // If 3/4 is selected, check that the new frame size is still multiple of 2, + // otherwise take 1/2. + if (action_.spatial == kOneHalfSpatialUniform) { + if ((width_ * 3 / 4)%2 != 0 || (height_ * 3 / 4)%2 != 0) { + action_.spatial = kOneQuarterSpatialUniform; + } + } +} + +void VCMQmResolution::InsertLatestDownAction() { + if (action_.spatial != kNoChangeSpatial) { + for (int i = kDownActionHistorySize - 1; i > 0; --i) { + down_action_history_[i].spatial = down_action_history_[i - 1].spatial; + } + down_action_history_[0].spatial = action_.spatial; + } + if (action_.temporal != kNoChangeTemporal) { + for (int i = kDownActionHistorySize - 1; i > 0; --i) { + down_action_history_[i].temporal = down_action_history_[i - 1].temporal; + } + down_action_history_[0].temporal = action_.temporal; + } +} + +void VCMQmResolution::RemoveLastDownAction() { + if (action_.spatial != kNoChangeSpatial) { + for (int i = 0; i< kDownActionHistorySize - 1; ++i) { + down_action_history_[i].spatial = down_action_history_[i + 1].spatial; + } + down_action_history_[kDownActionHistorySize - 1].spatial = kNoChangeSpatial; + } + if (action_.temporal != kNoChangeTemporal) { + for (int i = 0; i< kDownActionHistorySize - 1; ++i) { + down_action_history_[i].temporal = down_action_history_[i + 1].temporal; + } + down_action_history_[kDownActionHistorySize - 1].temporal = + kNoChangeTemporal; + } +} + +void VCMQmResolution::ConstrainAmountOfDownSampling() { + // Sanity checks on down-sampling selection: + // override the settings for too small image size and/or frame rate. + // Also check the limit on current down-sampling states. + + // No spatial sampling if current frame size is too small (QCIF), + // or if the amount of spatial down-sampling will be too much. + float new_dec_factor_spatial = state_dec_factor_spatial_ * + qm_->spatial_width_fact * + qm_->spatial_height_fact; + if ((width_ * height_) <= kMinImageSize || + new_dec_factor_spatial > kMaxSpatialDown) { + action_.spatial = kNoChangeSpatial; + qm_->spatial_width_fact = 1.0f; + qm_->spatial_height_fact = 1.0f; + } + // No frame rate reduction if average frame rate is below some point, + // or if the amount of temporal down-sampling will be too much. + float new_dec_factor_temp = state_dec_factor_temporal_ * qm_->temporal_fact; + if (avg_incoming_framerate_ <= kMinFrameRate || + new_dec_factor_temp >= kMaxTempDown) { + action_.temporal = kNoChangeTemporal; + qm_->temporal_fact = 1.0f; + } +} + +void VCMQmResolution::PickSpatialOrTemporal() { + // Pick the one that has had the most down-sampling thus far. + if (state_dec_factor_spatial_ > state_dec_factor_temporal_) { + action_.spatial = down_action_history_[0].spatial; + action_.temporal = kNoChangeTemporal; + } else { + action_.spatial = kNoChangeSpatial; + action_.temporal = down_action_history_[0].temporal; } } // TODO(marpan): Update this when we allow for 1/2 spatial down-sampling. -void VCMQmResolution::SelectSpatialDirectionMode(float transRate) { - // Default is 1x2 (H) +void VCMQmResolution::SelectSpatialDirectionMode(float transition_rate) { + // Default is 4/3x4/3 // For bit rates well below transitional rate, we select 2x2. - if (_targetBitRate < transRate * kRateRedSpatial2X2) { - _qm->spatialWidthFact = 2; - _qm->spatialHeightFact = 2; + if (avg_target_rate_ < transition_rate * kRateRedSpatial2X2) { + qm_->spatial_width_fact = 2.0f; + qm_->spatial_height_fact = 2.0f; } // Otherwise check prediction errors and aspect ratio. - float spatialErr = 0.0; - float spatialErrH = 0.0; - float spatialErrV = 0.0; - if (_contentMetrics) { - spatialErr = _contentMetrics->spatial_pred_err; - spatialErrH = _contentMetrics->spatial_pred_err_h; - spatialErrV = _contentMetrics->spatial_pred_err_v; + float spatial_err = 0.0f; + float spatial_err_h = 0.0f; + float spatial_err_v = 0.0f; + if (content_metrics_) { + spatial_err = content_metrics_->spatial_pred_err; + spatial_err_h = content_metrics_->spatial_pred_err_h; + spatial_err_v = content_metrics_->spatial_pred_err_v; } // Favor 1x2 if aspect_ratio is 16:9. - if (_aspectRatio >= 16.0f / 9.0f) { + if (aspect_ratio_ >= 16.0f / 9.0f) { // Check if 1x2 has lowest prediction error. - if (spatialErrH < spatialErr && spatialErrH < spatialErrV) { - _qm->spatialWidthFact = 2; - _qm->spatialHeightFact = 1; + if (spatial_err_h < spatial_err && spatial_err_h < spatial_err_v) { + qm_->spatial_width_fact = 2.0f; + qm_->spatial_height_fact = 1.0f; } } - // Check for 2x2 selection: favor 2x2 over 1x2 and 2x1. - if (spatialErr < spatialErrH * (1.0f + kSpatialErr2x2VsHoriz) && - spatialErr < spatialErrV * (1.0f + kSpatialErr2X2VsVert)) { - _qm->spatialWidthFact = 2; - _qm->spatialHeightFact = 2; + // Check for 4/3x4/3 selection: favor 2x2 over 1x2 and 2x1. + if (spatial_err < spatial_err_h * (1.0f + kSpatialErr2x2VsHoriz) && + spatial_err < spatial_err_v * (1.0f + kSpatialErr2X2VsVert)) { + qm_->spatial_width_fact = 4.0f / 3.0f; + qm_->spatial_height_fact = 4.0f / 3.0f; } // Check for 2x1 selection. - if (spatialErrV < spatialErrH * (1.0f - kSpatialErrVertVsHoriz) && - spatialErrV < spatialErr * (1.0f - kSpatialErr2X2VsVert)) { - _qm->spatialWidthFact = 1; - _qm->spatialHeightFact = 2; + if (spatial_err_v < spatial_err_h * (1.0f - kSpatialErrVertVsHoriz) && + spatial_err_v < spatial_err * (1.0f - kSpatialErr2X2VsVert)) { + qm_->spatial_width_fact = 1.0f; + qm_->spatial_height_fact = 2.0f; } } @@ -618,25 +748,25 @@ VCMQmRobustness::~VCMQmRobustness() { } void VCMQmRobustness::Reset() { - _prevTotalRate = 0.0f; - _prevRttTime = 0; - _prevPacketLoss = 0; - _prevCodeRateDelta = 0; + prev_total_rate_ = 0.0f; + prev_rtt_time_ = 0; + prev_packet_loss_ = 0; + prev_code_rate_delta_ = 0; ResetQM(); } // Adjust the FEC rate based on the content and the network state // (packet loss rate, total rate/bandwidth, round trip time). // Note that packetLoss here is the filtered loss value. -float VCMQmRobustness::AdjustFecFactor(uint8_t codeRateDelta, - float totalRate, - float frameRate, - uint32_t rttTime, - uint8_t packetLoss) { +float VCMQmRobustness::AdjustFecFactor(uint8_t code_rate_delta, + float total_rate, + float framerate, + uint32_t rtt_time, + uint8_t packet_loss) { // Default: no adjustment - float adjustFec = 1.0f; - if (_contentMetrics == NULL) { - return adjustFec; + float adjust_fec = 1.0f; + if (content_metrics_ == NULL) { + return adjust_fec; } // Compute class state of the content. ComputeMotionNFD(); @@ -646,18 +776,18 @@ float VCMQmRobustness::AdjustFecFactor(uint8_t codeRateDelta, // Keep track of previous values of network state: // adjustment may be also based on pattern of changes in network state. - _prevTotalRate = totalRate; - _prevRttTime = rttTime; - _prevPacketLoss = packetLoss; - _prevCodeRateDelta = codeRateDelta; - return adjustFec; + prev_total_rate_ = total_rate; + prev_rtt_time_ = rtt_time; + prev_packet_loss_ = packet_loss; + prev_code_rate_delta_ = code_rate_delta; + return adjust_fec; } // Set the UEP (unequal-protection across packets) on/off for the FEC. -bool VCMQmRobustness::SetUepProtection(uint8_t codeRateDelta, - float totalRate, - uint8_t packetLoss, - bool frameType) { +bool VCMQmRobustness::SetUepProtection(uint8_t code_rate_delta, + float total_rate, + uint8_t packet_loss, + bool frame_type) { // Default. return false; } diff --git a/src/modules/video_coding/main/source/qm_select.h b/src/modules/video_coding/main/source/qm_select.h index 1859530adb..b76e32afba 100644 --- a/src/modules/video_coding/main/source/qm_select.h +++ b/src/modules/video_coding/main/source/qm_select.h @@ -23,15 +23,33 @@ struct VideoContentMetrics; struct VCMResolutionScale { VCMResolutionScale() - : spatialWidthFact(1), - spatialHeightFact(1), - temporalFact(1) { + : spatial_width_fact(1.0f), + spatial_height_fact(1.0f), + temporal_fact(1.0f), + change_resolution(false) { } - uint8_t spatialWidthFact; - uint8_t spatialHeightFact; - uint8_t temporalFact; + float spatial_width_fact; + float spatial_height_fact; + float temporal_fact; + bool change_resolution; }; +enum ImageType { + kQCIF = 0, // 176x144 + kHCIF, // 264x216 = half(~3/4x3/4) CIF. + kQVGA, // 320x240 = quarter VGA. + kCIF, // 352x288 + kHVGA, // 480x360 = half(~3/4x3/4) VGA. + kVGA, // 640x480 + kQFULLHD, // 960x540 = quarter FULLHD, and half(~3/4x3/4) WHD. + kWHD, // 1280x720 + kFULLHD, // 1920x1080 + kNumImageTypes +}; + +const uint32_t kSizeOfImageType[kNumImageTypes] = +{ 25344, 57024, 76800, 101376, 172800, 307200, 518400, 921600, 2073600 }; + enum LevelClass { kLow, kHigh, @@ -51,12 +69,44 @@ struct VCMContFeature { LevelClass level; }; -enum ResolutionAction { - kDownResolution, +enum UpDownAction { kUpResolution, - kNoChangeResolution + kDownResolution }; +enum SpatialAction { + kNoChangeSpatial, + kOneHalfSpatialUniform, // 3/4 x 3/4: 9/6 ~1/2 pixel reduction. + kOneQuarterSpatialUniform, // 1/2 x 1/2: 1/4 pixel reduction. + kNumModesSpatial +}; + +enum TemporalAction { + kNoChangeTemporal, + kTwoThirdsTemporal, // 2/3 frame rate reduction + kOneHalfTemporal, // 1/2 frame rate reduction + kNumModesTemporal +}; + +struct ResolutionAction { + ResolutionAction() + : spatial(kNoChangeSpatial), + temporal(kNoChangeTemporal) { + } + SpatialAction spatial; + TemporalAction temporal; +}; + +// Down-sampling factors for spatial (width and height), and temporal. +const float kFactorWidthSpatial[kNumModesSpatial] = + { 1.0f, 4.0f / 3.0f, 2.0f }; + +const float kFactorHeightSpatial[kNumModesSpatial] = + { 1.0f, 4.0f / 3.0f, 2.0f }; + +const float kFactorTemporal[kNumModesTemporal] = + { 1.0f, 1.5f, 2.0f }; + enum EncoderState { kStableEncoding, // Low rate mis-match, stable buffer levels. kStressedEncoding, // Significant over-shooting of target rate, @@ -79,7 +129,7 @@ class VCMQmMethod { uint8_t ComputeContentClass(); // Update with the content metrics. - void UpdateContent(const VideoContentMetrics* contentMetrics); + void UpdateContent(const VideoContentMetrics* content_metrics); // Compute spatial texture magnitude and level. // Spatial texture is a spatial prediction error measure. @@ -90,29 +140,32 @@ class VCMQmMethod { void ComputeMotionNFD(); // Get the imageType (CIF, VGA, HD, etc) for the system width/height. - uint8_t GetImageType(uint16_t width, uint16_t height); + ImageType GetImageType(uint16_t width, uint16_t height); + + // Return the closest image type. + ImageType FindClosestImageType(uint16_t width, uint16_t height); // Get the frame rate level. LevelClass FrameRateLevel(float frame_rate); protected: // Content Data. - const VideoContentMetrics* _contentMetrics; + const VideoContentMetrics* content_metrics_; // Encoder frame sizes and native frame sizes. - uint16_t _width; - uint16_t _height; - uint16_t _nativeWidth; - uint16_t _nativeHeight; - float _aspectRatio; + uint16_t width_; + uint16_t height_; + uint16_t native_width_; + uint16_t native_height_; + float aspect_ratio_; // Image type and frame rate leve, for the current encoder resolution. - uint8_t _imageType; - LevelClass _frameRateLevel; + ImageType image_type_; + LevelClass framerate_level_; // Content class data. - VCMContFeature _motion; - VCMContFeature _spatial; - uint8_t _contentClass; - bool _init; + VCMContFeature motion_; + VCMContFeature spatial_; + uint8_t content_class_; + bool init_; }; // Resolution settings class @@ -135,27 +188,35 @@ class VCMQmResolution : public VCMQmMethod { EncoderState GetEncoderState(); // Initialize after SetEncodingData in media_opt. - int Initialize(float bitRate, float userFrameRate, - uint16_t width, uint16_t height); + int Initialize(float bitrate, + float user_framerate, + uint16_t width, + uint16_t height, + int num_layers); // Update the encoder frame size. void UpdateCodecFrameSize(uint16_t width, uint16_t height); // Update with actual bit rate (size of the latest encoded frame) // and frame type, after every encoded frame. - void UpdateEncodedSize(int encodedSize, - FrameType encodedFrameType); + void UpdateEncodedSize(int encoded_size, + FrameType encoded_frame_type); // Update with new target bitrate, actual encoder sent rate, frame_rate, // loss rate: every ~1 sec from SetTargetRates in media_opt. - void UpdateRates(float targetBitRate, float encoderSentRate, - float incomingFrameRate, uint8_t packetLoss); + void UpdateRates(float target_bitrate, + float encoder_sent_rate, + float incoming_framerate, + uint8_t packet_loss); // Extract ST (spatio-temporal) resolution action. // Inputs: qm: Reference to the quality modes pointer. // Output: the spatial and/or temporal scale change. int SelectResolution(VCMResolutionScale** qm); + // Set the default resolution action. + void SetDefaultAction(); + // Compute rates for the selection of down-sampling action. void ComputeRatesForSelection(); @@ -171,57 +232,89 @@ class VCMQmResolution : public VCMQmMethod { // Check the condition for going up in resolution by the scale factors: // |facWidth|, |facHeight|, |facTemp|. // |scaleFac| is a scale factor for the transition rate. - bool ConditionForGoingUp(uint8_t facWidth, uint8_t facHeight, - uint8_t facTemp, - float scaleFac); + bool ConditionForGoingUp(float fac_width, + float fac_height, + float fac_temp, + float scale_fac); // Get the bitrate threshold for the resolution action. // The case |facWidth|=|facHeight|=|facTemp|==1 is for down-sampling action. // |scaleFac| is a scale factor for the transition rate. - float GetTransitionRate(uint8_t facWidth, uint8_t facHeight, - uint8_t facTemp, float scaleFac); + float GetTransitionRate(float fac_width, + float fac_height, + float fac_temp, + float scale_fac); - // Update the downsampling state. - void UpdateDownsamplingState(ResolutionAction action); + // Update the down-sampling state. + void UpdateDownsamplingState(UpDownAction up_down); + // Return a state based on average target rate relative transition rate. + uint8_t RateClass(float transition_rate); + + // Adjust the action selected from the table. void AdjustAction(); + // Check if the new frame sizes are still divisible by 2. + void CheckForEvenFrameSize(); + + // Insert latest down-sampling action into the history list. + void InsertLatestDownAction(); + + // Remove the last (first element) down-sampling action from the list. + void RemoveLastDownAction(); + + // Check constraints on the amount of down-sampling allowed. + void ConstrainAmountOfDownSampling(); + + // For going up in resolution: pick spatial or temporal action, + // if both actions were separately selected. + void PickSpatialOrTemporal(); + // Select the directional (1x2 or 2x1) spatial down-sampling action. - void SelectSpatialDirectionMode(float transRate); + void SelectSpatialDirectionMode(float transition_rate); private: - VCMResolutionScale* _qm; + enum { kDownActionHistorySize = 10}; + + VCMResolutionScale* qm_; // Encoder rate control parameters. - float _targetBitRate; - float _userFrameRate; - float _incomingFrameRate; - float _perFrameBandwidth; - float _bufferLevel; + float target_bitrate_; + float user_framerate_; + float incoming_framerate_; + float per_frame_bandwidth_; + float buffer_level_; // Data accumulated every ~1sec from MediaOpt. - float _sumTargetRate; - float _sumIncomingFrameRate; - float _sumRateMM; - float _sumRateMMSgn; - float _sumPacketLoss; + float sum_target_rate_; + float sum_incoming_framerate_; + float sum_rate_MM_; + float sum_rate_MM_sgn_; + float sum_packet_loss_; // Counters. - uint32_t _frameCnt; - uint32_t _frameCntDelta; - uint32_t _updateRateCnt; - uint32_t _lowBufferCnt; + uint32_t frame_cnt_; + uint32_t frame_cnt_delta_; + uint32_t update_rate_cnt_; + uint32_t low_buffer_cnt_; // Resolution state parameters. - uint8_t _stateDecFactorSpatial; - uint8_t _stateDecFactorTemp; + float state_dec_factor_spatial_; + float state_dec_factor_temporal_; // Quantities used for selection. - float _avgTargetRate; - float _avgIncomingFrameRate; - float _avgRatioBufferLow; - float _avgRateMisMatch; - float _avgRateMisMatchSgn; - float _avgPacketLoss; - EncoderState _encoderState; + float avg_target_rate_; + float avg_incoming_framerate_; + float avg_ratio_buffer_low_; + float avg_rate_mismatch_; + float avg_rate_mismatch_sgn_; + float avg_packet_loss_; + EncoderState encoder_state_; + ResolutionAction action_; + // Short history of the down-sampling actions from the Initialize() state. + // This is needed for going up in resolution. Since the total amount of + // down-sampling actions are constrained, the length of the list need not be + // large: i.e., (4/3) ^{kDownActionHistorySize} <= kMaxDownSample. + ResolutionAction down_action_history_[kDownActionHistorySize]; + int num_layers_; }; // Robustness settings class. @@ -235,24 +328,24 @@ class VCMQmRobustness : public VCMQmMethod { // Adjust FEC rate based on content: every ~1 sec from SetTargetRates. // Returns an adjustment factor. - float AdjustFecFactor(uint8_t codeRateDelta, - float totalRate, - float frameRate, - uint32_t rttTime, - uint8_t packetLoss); + float AdjustFecFactor(uint8_t code_rate_delta, + float total_rate, + float framerate, + uint32_t rtt_time, + uint8_t packet_loss); // Set the UEP protection on/off. - bool SetUepProtection(uint8_t codeRateDelta, - float totalRate, - uint8_t packetLoss, - bool frameType); + bool SetUepProtection(uint8_t code_rate_delta, + float total_rate, + uint8_t packet_loss, + bool frame_type); private: // Previous state of network parameters. - float _prevTotalRate; - uint32_t _prevRttTime; - uint8_t _prevPacketLoss; - uint8_t _prevCodeRateDelta; + float prev_total_rate_; + uint32_t prev_rtt_time_; + uint8_t prev_packet_loss_; + uint8_t prev_code_rate_delta_; }; } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_ diff --git a/src/modules/video_coding/main/source/qm_select_data.h b/src/modules/video_coding/main/source/qm_select_data.h index d4af642291..ec1e9888cc 100644 --- a/src/modules/video_coding/main/source/qm_select_data.h +++ b/src/modules/video_coding/main/source/qm_select_data.h @@ -23,11 +23,8 @@ namespace webrtc { // PARAMETERS FOR RESOLUTION ADAPTATION // -// Initial level of buffer in secs: should corresponds to wrapper settings. -const float kInitBufferLevel = 0.5f; - // Optimal level of buffer in secs: should corresponds to wrapper settings. -const float kOptBufferLevel = 0.6f; +const float kOptBufferLevel = 0.5f; // Threshold of (max) buffer size below which we consider too low (underflow). const float kPercBufferThr = 0.10f; @@ -42,6 +39,9 @@ const float kMaxRateMisMatch = 0.5f; const float kRateOverShoot = 0.75f; const float kRateUnderShoot = 0.75f; +// Factor to favor weighting the average rates with the current/last data. +const float kWeightRate = 0.70f; + // Factor for transitional rate for going back up in resolution. const float kTransRateScaleUpSpatial = 1.25f; const float kTransRateScaleUpTemp = 1.25f; @@ -53,16 +53,19 @@ const float kPacketLossThr = 0.1f; // Factor for reducing transitonal bitrate under packet loss. const float kPacketLossRateFac = 1.0f; + // Maximum possible transitional rate for down-sampling: // (units in kbps), for 30fps. -const uint16_t kMaxRateQm[7] = { - 100, // QCIF +const uint16_t kMaxRateQm[9] = { + 50, // QCIF + 100, // kHCIF + 175, // kQVGA 250, // CIF + 350, // HVGA 500, // VGA - 800, // 4CIF - 1000, // 720 HD 4:3, - 1500, // 720 HD 16:9 - 2000 // 1080HD + 1000, // QFULLHD + 1500, // WHD + 2000 // FULLHD }; // Frame rate scale for maximum transition rate. @@ -75,32 +78,37 @@ const float kFrameRateFac[3] = { // Scale for transitional rate: based on content class // motion=L/H/D,spatial==L/H/D: for low, high, middle levels const float kScaleTransRateQm[18] = { - // 4CIF and lower + // VGA and lower 0.50f, // L, L 0.50f, // L, H 0.50f, // L, D 0.50f, // H ,L - 0.25f, // H, H - 0.25f, // H, D + 0.35f, // H, H + 0.35f, // H, D 0.50f, // D, L 0.50f, // D, D - 0.25f, // D, H + 0.35f, // D, H - // over 4CIF: WHD, HD + // over VGA 0.50f, // L, L 0.50f, // L, H 0.50f, // L, D 0.50f, // H ,L - 0.25f, // H, H - 0.25f, // H, D + 0.35f, // H, H + 0.35f, // H, D 0.50f, // D, L 0.50f, // D, D - 0.25f, // D, H + 0.35f, // D, H }; +// Threshold on the target rate relative to transitional rate. +const float kFacLowRate = 0.75f; + // Action for down-sampling: -// motion=L/H/D,spatial==L/H/D: for low, high, middle levels -const uint8_t kSpatialAction[9] = { +// motion=L/H/D,spatial==L/H/D, for low, high, middle levels; +// rate = 0/1/2, for target rate state relative to transition rate. +const uint8_t kSpatialAction[27] = { +// rateClass = 0: 1, // L, L 1, // L, H 1, // L, D @@ -109,25 +117,70 @@ const uint8_t kSpatialAction[9] = { 4, // H, D 4, // D, L 1, // D, H - 1, // D, D + 2, // D, D + +// rateClass = 1: + 1, // L, L + 1, // L, H + 1, // L, D + 4, // H ,L + 1, // H, H + 2, // H, D + 2, // D, L + 1, // D, H + 2, // D, D + +// rateClass = 2: + 1, // L, L + 1, // L, H + 1, // L, D + 2, // H ,L + 1, // H, H + 2, // H, D + 2, // D, L + 1, // D, H + 2, // D, D }; -const uint8_t kTemporalAction[9] = { - 1, // L, L +const uint8_t kTemporalAction[27] = { +// rateClass = 0: + 3, // L, L 2, // L, H 2, // L, D 1, // H ,L - 2, // H, H + 3, // H, H 1, // H, D 1, // D, L 2, // D, H 1, // D, D + +// rateClass = 1: + 3, // L, L + 2, // L, H + 3, // L, D + 1, // H ,L + 3, // H, H + 1, // H, D + 1, // D, L + 3, // D, H + 1, // D, D + +// rateClass = 2: + 1, // L, L + 3, // L, H + 3, // L, D + 1, // H ,L + 3, // H, H + 1, // H, D + 1, // D, L + 3, // D, H + 1, // D, D }; // Control the total amount of down-sampling allowed. -const int kMaxSpatialDown = 16; -const int kMaxTempDown = 4; -const int kMaxDownSample = 16; +const float kMaxSpatialDown = 8.0f; +const float kMaxTempDown = 4.0f; +const float kMaxDownSample = 16.0f; // Minimum image size for a spatial down-sampling. const int kMinImageSize= 176 * 144; @@ -136,16 +189,6 @@ const int kMinImageSize= 176 * 144; // no frame rate reduction if incomingFrameRate <= MIN_FRAME_RATE const int kMinFrameRate = 8; -// Boundaries for the closest standard frame size -const uint32_t kFrameSizeTh[6] = { - 63360, // between 176*144 and 352*288 - 204288, // between 352*288 and 640*480 - 356352, // between 640*480 and 704*576 - 548352, // between 704*576 and 960*720 - 806400, // between 960*720 and 1280*720 - 1497600, // between 1280*720 and 1920*1080 -}; - // // PARAMETERS FOR FEC ADJUSTMENT: TODO (marpan) // diff --git a/src/modules/video_coding/main/source/qm_select_unittest.cc b/src/modules/video_coding/main/source/qm_select_unittest.cc index e0ab7bf035..5b670fb657 100644 --- a/src/modules/video_coding/main/source/qm_select_unittest.cc +++ b/src/modules/video_coding/main/source/qm_select_unittest.cc @@ -20,6 +20,15 @@ namespace webrtc { +// Representative values of content metrics for: low/high/medium(default) state, +// based on parameters settings in qm_select_data.h. +const float kSpatialLow = 0.01f; +const float kSpatialMedium = 0.03f; +const float kSpatialHigh = 0.1f; +const float kTemporalLow = 0.01f; +const float kTemporalMedium = 0.06f; +const float kTemporalHigh = 0.1f; + class QmSelectTest : public ::testing::Test { protected: QmSelectTest() @@ -31,8 +40,11 @@ class QmSelectTest : public ::testing::Test { VideoContentMetrics* content_metrics_; VCMResolutionScale* qm_scale_; - void InitQmNativeData(float initial_bit_rate, int user_frame_rate, - int native_width, int native_height); + void InitQmNativeData(float initial_bit_rate, + int user_frame_rate, + int native_width, + int native_height, + int num_layers); void UpdateQmEncodedFrame(int* encoded_size, int num_updates); @@ -48,9 +60,9 @@ class QmSelectTest : public ::testing::Test { float spatial_metric_vert); bool IsSelectedActionCorrect(VCMResolutionScale* qm_scale, - uint8_t fac_width, - uint8_t fac_height, - uint8_t fac_temp); + float fac_width, + float fac_height, + float fac_temp); void TearDown() { delete qm_resolution_; @@ -60,31 +72,31 @@ class QmSelectTest : public ::testing::Test { TEST_F(QmSelectTest, HandleInputs) { // Expect parameter error. Initialize with invalid inputs. - EXPECT_EQ(-4, qm_resolution_->Initialize(1000, 0, 640, 480)); - EXPECT_EQ(-4, qm_resolution_->Initialize(1000, 30, 640, 0)); - EXPECT_EQ(-4, qm_resolution_->Initialize(1000, 30, 0, 480)); + EXPECT_EQ(-4, qm_resolution_->Initialize(1000, 0, 640, 480, 1)); + EXPECT_EQ(-4, qm_resolution_->Initialize(1000, 30, 640, 0, 1)); + EXPECT_EQ(-4, qm_resolution_->Initialize(1000, 30, 0, 480, 1)); // Expect uninitialized error.: No valid initialization before selection. EXPECT_EQ(-7, qm_resolution_->SelectResolution(&qm_scale_)); VideoContentMetrics* content_metrics = NULL; - EXPECT_EQ(0, qm_resolution_->Initialize(1000, 30, 640, 480)); + EXPECT_EQ(0, qm_resolution_->Initialize(1000, 30, 640, 480, 1)); qm_resolution_->UpdateContent(content_metrics); // Content metrics are NULL: Expect success and no down-sampling action. EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0, 1.0, 1.0)); } // No down-sampling action at high rates. TEST_F(QmSelectTest, NoActionHighRate) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(800, 30, 640, 480); + InitQmNativeData(800, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {800, 800, 800}; @@ -95,24 +107,24 @@ TEST_F(QmSelectTest, NoActionHighRate) { fraction_lost, 3); // Update content: motion level, and 3 spatial prediction errors. - UpdateQmContentData(0.01f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalLow, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(0, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); } // Rate is well below transition, down-sampling action is taken, // depending on the content state. TEST_F(QmSelectTest, DownActionLowRate) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480); + InitQmNativeData(100, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {100, 100, 100}; @@ -124,80 +136,84 @@ TEST_F(QmSelectTest, DownActionLowRate) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial: 2x2 spatial expected. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); qm_resolution_->ResetDownSamplingState(); - // Low motion, low spatial: no action expected: content is too low. - UpdateQmContentData(0.01f, 0.01f, 0.01f, 0.01f); + // Low motion, low spatial: 2/3 temporal is expected. + UpdateQmContentData(kTemporalLow, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(0, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); qm_resolution_->ResetDownSamplingState(); // Medium motion, low spatial: 2x2 spatial expected. - UpdateQmContentData(0.06f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalMedium, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); qm_resolution_->ResetDownSamplingState(); // High motion, high spatial: 1/2 temporal expected. - UpdateQmContentData(0.1f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalHigh, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(4, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial: 1/2 temporal expected. - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); qm_resolution_->ResetDownSamplingState(); // Medium motion, high spatial: 1/2 temporal expected. - UpdateQmContentData(0.06f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalMedium, kSpatialHigh, kSpatialHigh, + kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); qm_resolution_->ResetDownSamplingState(); // High motion, medium spatial: 2x2 spatial expected. - UpdateQmContentData(0.1f, 0.03f, 0.03f, 0.03f); + UpdateQmContentData(kTemporalHigh, kSpatialMedium, kSpatialMedium, + kSpatialMedium); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(5, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, medium spatial: high frame rate, so 1/2 temporal expected. - UpdateQmContentData(0.01f, 0.03f, 0.03f, 0.03f); + UpdateQmContentData(kTemporalLow, kSpatialMedium, kSpatialMedium, + kSpatialMedium); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(2, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); qm_resolution_->ResetDownSamplingState(); // Medium motion, medium spatial: high frame rate, so 1/2 temporal expected. - UpdateQmContentData(0.06f, 0.03f, 0.03f, 0.03f); + UpdateQmContentData(kTemporalMedium, kSpatialMedium, kSpatialMedium, + kSpatialMedium); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(8, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); } // Rate mis-match is high, and we have over-shooting. // since target rate is below max for down-sampling, down-sampling is selected. TEST_F(QmSelectTest, DownActionHighRateMMOvershoot) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(450, 30, 640, 480); + InitQmNativeData(450, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {450, 450, 450}; @@ -209,31 +225,32 @@ TEST_F(QmSelectTest, DownActionHighRateMMOvershoot) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStressedEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, + 1.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); } // Rate mis-match is high, target rate is below max for down-sampling, // but since we have consistent under-shooting, no down-sampling action. TEST_F(QmSelectTest, NoActionHighRateMMUndershoot) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(450, 30, 640, 480); + InitQmNativeData(450, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {450, 450, 450}; @@ -245,31 +262,31 @@ TEST_F(QmSelectTest, NoActionHighRateMMUndershoot) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kEasyEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); } // Buffer is underflowing, and target rate is below max for down-sampling, // so action is taken. TEST_F(QmSelectTest, DownActionBufferUnderflow) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(450, 30, 640, 480); + InitQmNativeData(450, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update with encoded size over a number of frames. // per-frame bandwidth = 15 = 450/30: simulate (decoder) buffer underflow: @@ -286,31 +303,32 @@ TEST_F(QmSelectTest, DownActionBufferUnderflow) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStressedEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, + 1.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); } // Target rate is below max for down-sampling, but buffer level is stable, // so no action is taken. TEST_F(QmSelectTest, NoActionBufferStable) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(450, 30, 640, 480); + InitQmNativeData(450, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update with encoded size over a number of frames. // per-frame bandwidth = 15 = 450/30: simulate stable (decoder) buffer levels. @@ -327,24 +345,24 @@ TEST_F(QmSelectTest, NoActionBufferStable) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); qm_resolution_->ResetDownSamplingState(); // Low motion, high spatial - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); } // Very low rate, but no spatial down-sampling below some size (QCIF). TEST_F(QmSelectTest, LimitDownSpatialAction) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(10, 30, 176, 144); + InitQmNativeData(10, 30, 176, 144, 1); // Update with encoder frame size. uint16_t codec_width = 176; @@ -362,23 +380,23 @@ TEST_F(QmSelectTest, LimitDownSpatialAction) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); } // Very low rate, but no frame reduction below some frame_rate (8fps). TEST_F(QmSelectTest, LimitDownTemporalAction) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(10, 8, 640, 480); + InitQmNativeData(10, 8, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {10, 10, 10}; @@ -390,24 +408,25 @@ TEST_F(QmSelectTest, LimitDownTemporalAction) { // Update content: motion level, and 3 spatial prediction errors. // Low motion, medium spatial. - UpdateQmContentData(0.01f, 0.03f, 0.03f, 0.03f); + UpdateQmContentData(kTemporalLow, kSpatialMedium, kSpatialMedium, + kSpatialMedium); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(2, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); } // Two stages: spatial down-sample and then back up spatially, // as rate as increased. TEST_F(QmSelectTest, 2StageDownSpatialUpSpatial) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480); + InitQmNativeData(100, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {100, 100, 100}; @@ -419,16 +438,16 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatial) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); // Reset and go up in rate: expected to go back up. qm_resolution_->ResetRates(); qm_resolution_->UpdateCodecFrameSize(320, 240); - EXPECT_EQ(1, qm_resolution_->GetImageType(320, 240)); + EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); // Update rates for a sequence of intervals. int target_rate2[] = {400, 400, 400, 400, 400}; int encoder_sent_rate2[] = {400, 400, 400, 400, 400}; @@ -438,20 +457,20 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatial) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0, 0, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f)); } // Two stages: spatial down-sample and then back up spatially, since encoder // is under-shooting target even though rate has not increased much. TEST_F(QmSelectTest, 2StageDownSpatialUpSpatialUndershoot) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480); + InitQmNativeData(100, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {100, 100, 100}; @@ -463,16 +482,16 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatialUndershoot) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); // Reset rates and simulate under-shooting scenario.: expect to go back up. qm_resolution_->ResetRates(); qm_resolution_->UpdateCodecFrameSize(320, 240); - EXPECT_EQ(1, qm_resolution_->GetImageType(320, 240)); + EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); // Update rates for a sequence of intervals. int target_rate2[] = {200, 200, 200, 200, 200}; int encoder_sent_rate2[] = {50, 50, 50, 50, 50}; @@ -482,20 +501,20 @@ TEST_F(QmSelectTest, 2StageDownSpatialUpSpatialUndershoot) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kEasyEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0, 0, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f)); } // Two stages: spatial down-sample and then no action to go up, // as encoding rate mis-match is too high. TEST_F(QmSelectTest, 2StageDownSpatialNoActionUp) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480); + InitQmNativeData(100, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {100, 100, 100}; @@ -507,16 +526,16 @@ TEST_F(QmSelectTest, 2StageDownSpatialNoActionUp) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); // Reset and simulate large rate mis-match: expect no action to go back up. qm_resolution_->ResetRates(); qm_resolution_->UpdateCodecFrameSize(320, 240); - EXPECT_EQ(1, qm_resolution_->GetImageType(320, 240)); + EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); // Update rates for a sequence of intervals. int target_rate2[] = {400, 400, 400, 400, 400}; int encoder_sent_rate2[] = {1000, 1000, 1000, 1000, 1000}; @@ -526,19 +545,19 @@ TEST_F(QmSelectTest, 2StageDownSpatialNoActionUp) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kStressedEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); } // Two stages: temporally down-sample and then back up temporally, // as rate as increased. TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporal) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480); + InitQmNativeData(100, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {100, 100, 100}; @@ -550,11 +569,11 @@ TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporal) { // Update content: motion level, and 3 spatial prediction errors. // Low motion, high spatial. - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); // Reset rates and go up in rate: expect to go back up. qm_resolution_->ResetRates(); @@ -567,20 +586,20 @@ TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporal) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 0)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f)); } // Two stages: temporal down-sample and then back up temporally, since encoder // is under-shooting target even though rate has not increased much. TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporalUndershoot) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480); + InitQmNativeData(100, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {100, 100, 100}; @@ -592,11 +611,11 @@ TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporalUndershoot) { // Update content: motion level, and 3 spatial prediction errors. // Low motion, high spatial. - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); // Reset rates and simulate under-shooting scenario.: expect to go back up. qm_resolution_->ResetRates(); @@ -609,20 +628,20 @@ TEST_F(QmSelectTest, 2StatgeDownTemporalUpTemporalUndershoot) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kEasyEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 0)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f)); } // Two stages: temporal down-sample and then no action to go up, // as encoding rate mis-match is too high. TEST_F(QmSelectTest, 2StageDownTemporalNoActionUp) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480); + InitQmNativeData(100, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. int target_rate[] = {100, 100, 100}; @@ -634,7 +653,7 @@ TEST_F(QmSelectTest, 2StageDownTemporalNoActionUp) { // Update content: motion level, and 3 spatial prediction errors. // Low motion, high spatial. - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); @@ -651,23 +670,23 @@ TEST_F(QmSelectTest, 2StageDownTemporalNoActionUp) { fraction_lost2, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(kStressedEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); } // 3 stages: spatial down-sample, followed by temporal down-sample, // and then go up to full state, as encoding rate has increased. TEST_F(QmSelectTest, 3StageDownSpatialTemporlaUpSpatialTemporal) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(100, 30, 640, 480); + InitQmNativeData(100, 30, 640, 480, 1); // Update with encoder frame size. uint16_t codec_width = 640; uint16_t codec_height = 480; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(2, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. - int target_rate[] = {100, 100, 100}; - int encoder_sent_rate[] = {100, 100, 100}; + int target_rate[] = {80, 80, 80}; + int encoder_sent_rate[] = {80, 80, 80}; int incoming_frame_rate[] = {30, 30, 30}; uint8_t fraction_lost[] = {10, 10, 10}; UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, @@ -675,24 +694,23 @@ TEST_F(QmSelectTest, 3StageDownSpatialTemporlaUpSpatialTemporal) { // Update content: motion level, and 3 spatial prediction errors. // High motion, low spatial. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); - // Reset rate and change content data: expect temporal down-sample. - qm_resolution_->ResetRates(); + // Change content data: expect temporal down-sample. qm_resolution_->UpdateCodecFrameSize(320, 240); - EXPECT_EQ(1, qm_resolution_->GetImageType(320, 240)); + EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); // Update content: motion level, and 3 spatial prediction errors. // Low motion, high spatial. - UpdateQmContentData(0.01f, 0.1f, 0.1f, 0.1f); + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 2)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); // Reset rates and go high up in rate: expect to go back up both spatial // and temporally. @@ -708,57 +726,63 @@ TEST_F(QmSelectTest, 3StageDownSpatialTemporlaUpSpatialTemporal) { EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0, 0, 0)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 0.5f)); } -// No down-sampling below some totol amount (factor of 16) +// No down-sampling below some total amount. TEST_F(QmSelectTest, NoActionTooMuchDownSampling) { // Initialize with bitrate, frame rate, and native system width/height. - InitQmNativeData(400, 30, 1280, 720); + InitQmNativeData(400, 30, 1280, 720, 1); // Update with encoder frame size. uint16_t codec_width = 1280; uint16_t codec_height = 720; qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); - EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); + EXPECT_EQ(7, qm_resolution_->GetImageType(codec_width, codec_height)); // Update rates for a sequence of intervals. - int target_rate[] = {400, 400, 400}; - int encoder_sent_rate[] = {400, 400, 400}; + int target_rate[] = {200, 200, 200}; + int encoder_sent_rate[] = {200, 200, 200}; int incoming_frame_rate[] = {30, 30, 30}; uint8_t fraction_lost[] = {10, 10, 10}; UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, fraction_lost, 3); // Update content: motion level, and 3 spatial prediction errors. - // High motion, low spatial: 2x2 spatial expected. - UpdateQmContentData(0.1f, 0.01f, 0.01f, 0.01f); + // High motion, low spatial. + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); - // Reset and lower rates to get another spatial action. + // Reset and lower rates to get another spatial action (3/4x3/4) qm_resolution_->ResetRates(); qm_resolution_->UpdateCodecFrameSize(640, 360); - EXPECT_EQ(2, qm_resolution_->GetImageType(640, 360)); + EXPECT_EQ(4, qm_resolution_->GetImageType(640, 360)); // Update rates for a sequence of intervals. - int target_rate2[] = {100, 100, 100, 100, 100}; - int encoder_sent_rate2[] = {100, 100, 100, 100, 100}; + int target_rate2[] = {80, 80, 80, 80, 80}; + int encoder_sent_rate2[] = {80, 80, 80, 80, 80}; int incoming_frame_rate2[] = {30, 30, 30, 30, 30}; uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, fraction_lost2, 5); - EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); - EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); - EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2, 2, 1)); - // Reset and go to low rate: no action should be taken, + // Update content: motion level, and 3 spatial prediction errors. + // High motion, medium spatial. + UpdateQmContentData(kTemporalHigh, kSpatialMedium, kSpatialMedium, + kSpatialMedium); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(5, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, + 1.0f)); + + // Reset and go to very low rate: no action should be taken, // we went down too much already. qm_resolution_->ResetRates(); qm_resolution_->UpdateCodecFrameSize(320, 180); - EXPECT_EQ(0, qm_resolution_->GetImageType(320, 180)); + EXPECT_EQ(1, qm_resolution_->GetImageType(320, 180)); // Update rates for a sequence of intervals. int target_rate3[] = {10, 10, 10, 10, 10}; int encoder_sent_rate3[] = {10, 10, 10, 10, 10}; @@ -767,17 +791,331 @@ TEST_F(QmSelectTest, NoActionTooMuchDownSampling) { UpdateQmRateData(target_rate3, encoder_sent_rate3, incoming_frame_rate3, fraction_lost3, 5); EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(5, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.0f)); +} + +// Multiple down-sampling stages and then undo all of them. +// Spatial down-sample 3/4x3/4, followed by temporal down-sample 2/3, +// followed by spatial 1/2x1/2. Then go up to full state, +// as encoding rate has increased. +TEST_F(QmSelectTest, MultipleStagesCheckActionHistory1) { + // Initialize with bitrate, frame rate, and native system width/height. + InitQmNativeData(200, 30, 640, 480, 1); + + // Update with encoder frame size. + uint16_t codec_width = 640; + uint16_t codec_height = 480; + qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); + + // Go down spatial 3/4x3/4. + // Update rates for a sequence of intervals. + int target_rate[] = {200, 200, 200}; + int encoder_sent_rate[] = {200, 200, 200}; + int incoming_frame_rate[] = {30, 30, 30}; + uint8_t fraction_lost[] = {10, 10, 10}; + UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, + fraction_lost, 3); + + // Update content: motion level, and 3 spatial prediction errors. + // Medium motion, low spatial. + UpdateQmContentData(kTemporalMedium, kSpatialLow, kSpatialLow, kSpatialLow); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, + 1.0f)); + // Go down 1/2 temporal. + qm_resolution_->UpdateCodecFrameSize(480, 360); + EXPECT_EQ(4, qm_resolution_->GetImageType(480, 360)); + qm_resolution_->ResetRates(); + int target_rate2[] = {100, 100, 100, 100, 100}; + int encoder_sent_rate2[] = {100, 100, 100, 100, 100}; + int incoming_frame_rate2[] = {30, 30, 30, 30, 30}; + uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, + fraction_lost2, 5); + + // Update content: motion level, and 3 spatial prediction errors. + // Low motion, high spatial. + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + + // Go down 1/2x1/2 spatial. + qm_resolution_->ResetRates(); + int target_rate3[] = {50, 50, 50, 50, 50}; + int encoder_sent_rate3[] = {50, 50, 50, 50, 50}; + int incoming_frame_rate3[] = {15, 15, 15, 15, 15}; + uint8_t fraction_lost3[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate3, encoder_sent_rate3, incoming_frame_rate3, + fraction_lost3, 5); + + // Update content: motion level, and 3 spatial prediction errors. + // High motion, low spatial. + UpdateQmContentData(kTemporalHigh, kSpatialLow, kSpatialLow, kSpatialLow); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); - EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1, 1, 1)); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + + // Reset rates and go high up in rate: expect to go up: + // should go up first: 1/2x1x2 and 1/2 temporally, + // and second: 3/4x3/4 spatial. + + // Go up 1/2x1/2 spatially and 1/2 temporally + qm_resolution_->UpdateCodecFrameSize(240, 180); + EXPECT_EQ(1, qm_resolution_->GetImageType(240, 180)); + qm_resolution_->ResetRates(); + // Update rates for a sequence of intervals. + int target_rate4[] = {1000, 1000, 1000, 1000, 1000}; + int encoder_sent_rate4[] = {1000, 1000, 1000, 1000, 1000}; + int incoming_frame_rate4[] = {15, 15, 15, 15, 15}; + uint8_t fraction_lost4[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate4, encoder_sent_rate4, incoming_frame_rate4, + fraction_lost4, 5); + + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 0.5f)); + + // Go up 3/4x3/4 spatially. + qm_resolution_->UpdateCodecFrameSize(480, 360); + EXPECT_EQ(4, qm_resolution_->GetImageType(480, 360)); + qm_resolution_->ResetRates(); + // Update rates for a sequence of intervals. + int target_rate5[] = {1000, 1000, 1000, 1000, 1000}; + int encoder_sent_rate5[] = {1000, 1000, 1000, 1000, 1000}; + int incoming_frame_rate5[] = {30, 30, 30, 30, 30}; + uint8_t fraction_lost5[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate5, encoder_sent_rate5, incoming_frame_rate5, + fraction_lost5, 5); + + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(3, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 3.0f / 4.0f, 3.0f / 4.0f, + 1.0f)); +} + +// Multiple down-sampling and up-sample stages, with partial undoing. +// Spatial down-sample 1/2x1/2, followed by temporal down-sample 2/3, +// undo the spatial 1/2x1/2, then another temporal 1/2, and undo +// the 1/2 temporal. +TEST_F(QmSelectTest, MultipleStagesCheckActionHistory2) { + // Initialize with bitrate, frame rate, and native system width/height. + InitQmNativeData(100, 30, 640, 480, 1); + + // Update with encoder frame size. + uint16_t codec_width = 640; + uint16_t codec_height = 480; + qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); + + // Go down 1/2x1/2 spatial. + // Update rates for a sequence of intervals. + int target_rate[] = {100, 100, 100}; + int encoder_sent_rate[] = {100, 100, 100}; + int incoming_frame_rate[] = {30, 30, 30}; + uint8_t fraction_lost[] = {10, 10, 10}; + UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, + fraction_lost, 3); + + // Update content: motion level, and 3 spatial prediction errors. + // Medium motion, low spatial. + UpdateQmContentData(kTemporalMedium, kSpatialLow, kSpatialLow, kSpatialLow); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 2.0f, 2.0f, 1.0f)); + + // Go down 2/3 temporal. + qm_resolution_->UpdateCodecFrameSize(320, 240); + EXPECT_EQ(2, qm_resolution_->GetImageType(320, 240)); + qm_resolution_->ResetRates(); + int target_rate2[] = {80, 80, 80, 80, 80}; + int encoder_sent_rate2[] = {80, 80, 80, 80, 80}; + int incoming_frame_rate2[] = {30, 30, 30, 30, 30}; + uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, + fraction_lost2, 5); + + // Update content: motion level, and 3 spatial prediction errors. + // Medium motion, high spatial. + UpdateQmContentData(kTemporalMedium, kSpatialHigh, kSpatialHigh, + kSpatialHigh); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); + + // Go up 1/2x1/2 spatially. + qm_resolution_->ResetRates(); + // Update rates for a sequence of intervals. + int target_rate3[] = {300, 300, 300, 300, 300}; + int encoder_sent_rate3[] = {300, 300, 300, 300, 300}; + int incoming_frame_rate3[] = {20, 20, 20, 20, 20}; + uint8_t fraction_lost3[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate3, encoder_sent_rate3, incoming_frame_rate3, + fraction_lost3, 5); + + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 0.5f, 0.5f, 1.0f)); + + // Go down 1/2 temporal. + qm_resolution_->UpdateCodecFrameSize(640, 480); + EXPECT_EQ(5, qm_resolution_->GetImageType(640, 480)); + qm_resolution_->ResetRates(); + int target_rate4[] = {100, 100, 100, 100, 100}; + int encoder_sent_rate4[] = {100, 100, 100, 100, 100}; + int incoming_frame_rate4[] = {20, 20, 20, 20, 20}; + uint8_t fraction_lost4[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate4, encoder_sent_rate4, incoming_frame_rate4, + fraction_lost4, 5); + + // Update content: motion level, and 3 spatial prediction errors. + // Low motion, high spatial. + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + + // Go up 1/2 temporal. + qm_resolution_->ResetRates(); + // Update rates for a sequence of intervals. + int target_rate5[] = {1000, 1000, 1000, 1000, 1000}; + int encoder_sent_rate5[] = {1000, 1000, 1000, 1000, 1000}; + int incoming_frame_rate5[] = {10, 10, 10, 10, 10}; + uint8_t fraction_lost5[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate5, encoder_sent_rate5, incoming_frame_rate5, + fraction_lost5, 5); + + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f)); +} + +// Multiple down-sampling and up-sample stages, with partial undoing. +// Spatial down-sample 3/4x3/4, followed by temporal down-sample 1/2, +// undo the temporal 1/2, then another temporal 2/3 down, and undo +// the 2/3 temporal. +TEST_F(QmSelectTest, MultipleStagesCheckActionHistory3) { + // Initialize with bitrate, frame rate, and native system width/height. + InitQmNativeData(200, 30, 640, 480, 1); + + // Update with encoder frame size. + uint16_t codec_width = 640; + uint16_t codec_height = 480; + qm_resolution_->UpdateCodecFrameSize(codec_width, codec_height); + EXPECT_EQ(5, qm_resolution_->GetImageType(codec_width, codec_height)); + + // Go down 3/4x3/4 spatial. + // Update rates for a sequence of intervals. + int target_rate[] = {200, 200, 200}; + int encoder_sent_rate[] = {200, 200, 200}; + int incoming_frame_rate[] = {30, 30, 30}; + uint8_t fraction_lost[] = {10, 10, 10}; + UpdateQmRateData(target_rate, encoder_sent_rate, incoming_frame_rate, + fraction_lost, 3); + + // Update content: motion level, and 3 spatial prediction errors. + // Medium motion, low spatial. + UpdateQmContentData(kTemporalMedium, kSpatialLow, kSpatialLow, kSpatialLow); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(6, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 4.0f / 3.0f, 4.0f / 3.0f, + 1.0f)); + + // Go down 1/2 temporal. + qm_resolution_->UpdateCodecFrameSize(480, 360); + EXPECT_EQ(4, qm_resolution_->GetImageType(480, 360)); + qm_resolution_->ResetRates(); + int target_rate2[] = {100, 100, 100, 100, 100}; + int encoder_sent_rate2[] = {100, 100, 100, 100, 100}; + int incoming_frame_rate2[] = {30, 30, 30, 30, 30}; + uint8_t fraction_lost2[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate2, encoder_sent_rate2, incoming_frame_rate2, + fraction_lost2, 5); + + // Update content: motion level, and 3 spatial prediction errors. + // Low motion, high spatial. + UpdateQmContentData(kTemporalLow, kSpatialHigh, kSpatialHigh, kSpatialHigh); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f)); + + // Go up 1/2 temporal. + qm_resolution_->ResetRates(); + // Update rates for a sequence of intervals. + int target_rate3[] = {300, 300, 300, 300, 300}; + int encoder_sent_rate3[] = {300, 300, 300, 300, 300}; + int incoming_frame_rate3[] = {15, 15, 15, 15, 15}; + uint8_t fraction_lost3[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate3, encoder_sent_rate3, incoming_frame_rate3, + fraction_lost3, 5); + + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(1, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 0.5f)); + + // Go down 2/3 temporal. + qm_resolution_->UpdateCodecFrameSize(640, 480); + EXPECT_EQ(5, qm_resolution_->GetImageType(640, 480)); + qm_resolution_->ResetRates(); + int target_rate4[] = {150, 150, 150, 150, 150}; + int encoder_sent_rate4[] = {150, 150, 150, 150, 150}; + int incoming_frame_rate4[] = {20, 20, 20, 20, 20}; + uint8_t fraction_lost4[] = {30, 30, 30, 30, 30}; + UpdateQmRateData(target_rate4, encoder_sent_rate4, incoming_frame_rate4, + fraction_lost4, 5); + + // Update content: motion level, and 3 spatial prediction errors. + // Medium motion, high spatial. + UpdateQmContentData(kTemporalMedium, kSpatialHigh, kSpatialHigh, + kSpatialHigh); + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 1.5f)); + + // Go up 2/3 temporal. + qm_resolution_->ResetRates(); + // Update rates for a sequence of intervals. + int target_rate5[] = {500, 500, 500, 500, 500}; + int encoder_sent_rate5[] = {500, 500, 500, 500, 500}; + int incoming_frame_rate5[] = {20, 20, 20, 20, 20}; + uint8_t fraction_lost5[] = {10, 10, 10, 10, 10}; + UpdateQmRateData(target_rate5, encoder_sent_rate5, incoming_frame_rate5, + fraction_lost5, 5); + + EXPECT_EQ(0, qm_resolution_->SelectResolution(&qm_scale_)); + EXPECT_EQ(7, qm_resolution_->ComputeContentClass()); + EXPECT_EQ(kStableEncoding, qm_resolution_->GetEncoderState()); + EXPECT_TRUE(IsSelectedActionCorrect(qm_scale_, 1.0f, 1.0f, 2.0f / 3.0f)); } void QmSelectTest::InitQmNativeData(float initial_bit_rate, int user_frame_rate, int native_width, - int native_height) { - EXPECT_EQ(0, qm_resolution_->Initialize(initial_bit_rate, user_frame_rate, - native_width, native_height)); + int native_height, + int num_layers) { + EXPECT_EQ(0, qm_resolution_->Initialize(initial_bit_rate, + user_frame_rate, + native_width, + native_height, + num_layers)); } void QmSelectTest::UpdateQmContentData(float motion_metric, @@ -793,7 +1131,7 @@ void QmSelectTest::UpdateQmContentData(float motion_metric, void QmSelectTest::UpdateQmEncodedFrame(int* encoded_size, int num_updates) { FrameType frame_type = kVideoFrameDelta; - for (int i = 0; i < num_updates; i++) { + for (int i = 0; i < num_updates; ++i) { // Convert to bytes. int32_t encoded_size_update = 1000 * encoded_size[i] / 8; qm_resolution_->UpdateEncodedSize(encoded_size_update, frame_type); @@ -805,7 +1143,7 @@ void QmSelectTest::UpdateQmRateData(int* target_rate, int* incoming_frame_rate, uint8_t* fraction_lost, int num_updates) { - for (int i = 0; i < num_updates; i++) { + for (int i = 0; i < num_updates; ++i) { float target_rate_update = target_rate[i]; float encoder_sent_rate_update = encoder_sent_rate[i]; float incoming_frame_rate_update = incoming_frame_rate[i]; @@ -820,12 +1158,12 @@ void QmSelectTest::UpdateQmRateData(int* target_rate, // Check is the selected action from the QmResolution class is the same // as the expected scales from |fac_width|, |fac_height|, |fac_temp|. bool QmSelectTest::IsSelectedActionCorrect(VCMResolutionScale* qm_scale, - uint8_t fac_width, - uint8_t fac_height, - uint8_t fac_temp) { - if (qm_scale->spatialWidthFact == fac_width && - qm_scale->spatialHeightFact == fac_height && - qm_scale->temporalFact == fac_temp) { + float fac_width, + float fac_height, + float fac_temp) { + if (qm_scale->spatial_width_fact == fac_width && + qm_scale->spatial_height_fact == fac_height && + qm_scale->temporal_fact == fac_temp) { return true; } else { return false;