diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc index f621f8bf57..c6ab9fbfbf 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc @@ -240,7 +240,7 @@ VCMEncodedFrameCallback::EncodedBytes() void VCMEncodedFrameCallback::SetMediaOpt( - media_optimization::VCMMediaOptimization *mediaOpt) + media_optimization::MediaOptimization *mediaOpt) { _mediaOpt = mediaOpt; } diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.h b/webrtc/modules/video_coding/main/source/generic_encoder.h index a5b87e795b..0c2d287a9d 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.h +++ b/webrtc/modules/video_coding/main/source/generic_encoder.h @@ -19,7 +19,7 @@ namespace webrtc { namespace media_optimization { -class VCMMediaOptimization; +class MediaOptimization; } // namespace media_optimization /*************************************/ @@ -49,7 +49,7 @@ public: /** * Set media Optimization */ - void SetMediaOpt (media_optimization::VCMMediaOptimization* mediaOpt); + void SetMediaOpt (media_optimization::MediaOptimization* mediaOpt); void SetPayloadType(uint8_t payloadType) { _payloadType = payloadType; }; void SetCodecType(VideoCodecType codecType) {_codecType = codecType;}; @@ -64,7 +64,7 @@ private: RTPVideoHeader** rtp); VCMPacketizationCallback* _sendCallback; - media_optimization::VCMMediaOptimization* _mediaOpt; + media_optimization::MediaOptimization* _mediaOpt; uint32_t _encodedBytes; uint8_t _payloadType; VideoCodecType _codecType; diff --git a/webrtc/modules/video_coding/main/source/media_optimization.cc b/webrtc/modules/video_coding/main/source/media_optimization.cc index 7c6966d605..b2715e7399 100644 --- a/webrtc/modules/video_coding/main/source/media_optimization.cc +++ b/webrtc/modules/video_coding/main/source/media_optimization.cc @@ -18,661 +18,574 @@ namespace webrtc { namespace media_optimization { -VCMMediaOptimization::VCMMediaOptimization(int32_t id, - Clock* clock): -_id(id), -_clock(clock), -_maxBitRate(0), -_sendCodecType(kVideoCodecUnknown), -_codecWidth(0), -_codecHeight(0), -_userFrameRate(0), -_fractionLost(0), -_sendStatisticsZeroEncode(0), -_maxPayloadSize(1460), -_targetBitRate(0), -_incomingFrameRate(0), -_enableQm(false), -_videoProtectionCallback(NULL), -_videoQMSettingsCallback(NULL), -_encodedFrameSamples(), -_avgSentBitRateBps(0), -_avgSentFramerate(0), -_keyFrameCnt(0), -_deltaFrameCnt(0), -_lastQMUpdateTime(0), -_lastChangeTime(0), -_numLayers(0) -{ - memset(_sendStatistics, 0, sizeof(_sendStatistics)); - memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); +MediaOptimization::MediaOptimization(int32_t id, Clock* clock) + : id_(id), + clock_(clock), + max_bit_rate_(0), + send_codec_type_(kVideoCodecUnknown), + codec_width_(0), + codec_height_(0), + user_frame_rate_(0), + fraction_lost_(0), + send_statistics_zero_encode_(0), + max_payload_size_(1460), + target_bit_rate_(0), + incoming_frame_rate_(0), + enable_qm_(false), + video_protection_callback_(NULL), + video_qmsettings_callback_(NULL), + encoded_frame_samples_(), + avg_sent_bit_rate_bps_(0), + avg_sent_framerate_(0), + key_frame_cnt_(0), + delta_frame_cnt_(0), + last_qm_update_time_(0), + last_change_time_(0), + num_layers_(0) { + memset(send_statistics_, 0, sizeof(send_statistics_)); + memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); - _frameDropper = new FrameDropper; - _lossProtLogic = new VCMLossProtectionLogic(_clock->TimeInMilliseconds()); - _content = new VCMContentMetricsProcessing(); - _qmResolution = new VCMQmResolution(); + frame_dropper_ = new FrameDropper; + loss_prot_logic_ = new VCMLossProtectionLogic(clock_->TimeInMilliseconds()); + content_ = new VCMContentMetricsProcessing(); + qm_resolution_ = new VCMQmResolution(); } -VCMMediaOptimization::~VCMMediaOptimization(void) -{ - _lossProtLogic->Release(); - delete _lossProtLogic; - delete _frameDropper; - delete _content; - delete _qmResolution; +MediaOptimization::~MediaOptimization(void) { + loss_prot_logic_->Release(); + delete loss_prot_logic_; + delete frame_dropper_; + delete content_; + delete qm_resolution_; } -int32_t -VCMMediaOptimization::Reset() -{ - memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); - _incomingFrameRate = 0.0; - _frameDropper->Reset(); - _lossProtLogic->Reset(_clock->TimeInMilliseconds()); - _frameDropper->SetRates(0, 0); - _content->Reset(); - _qmResolution->Reset(); - _lossProtLogic->UpdateFrameRate(_incomingFrameRate); - _lossProtLogic->Reset(_clock->TimeInMilliseconds()); - _sendStatisticsZeroEncode = 0; - _targetBitRate = 0; - _codecWidth = 0; - _codecHeight = 0; - _userFrameRate = 0; - _keyFrameCnt = 0; - _deltaFrameCnt = 0; - _lastQMUpdateTime = 0; - _lastChangeTime = 0; - _encodedFrameSamples.clear(); - _avgSentBitRateBps = 0; - _numLayers = 1; - return VCM_OK; +int32_t MediaOptimization::Reset() { + memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); + incoming_frame_rate_ = 0.0; + frame_dropper_->Reset(); + loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); + frame_dropper_->SetRates(0, 0); + content_->Reset(); + qm_resolution_->Reset(); + loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_); + loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); + send_statistics_zero_encode_ = 0; + target_bit_rate_ = 0; + codec_width_ = 0; + codec_height_ = 0; + user_frame_rate_ = 0; + key_frame_cnt_ = 0; + delta_frame_cnt_ = 0; + last_qm_update_time_ = 0; + last_change_time_ = 0; + encoded_frame_samples_.clear(); + avg_sent_bit_rate_bps_ = 0; + num_layers_ = 1; + return VCM_OK; } -uint32_t -VCMMediaOptimization::SetTargetRates(uint32_t target_bitrate, - uint8_t &fractionLost, - uint32_t roundTripTimeMs) -{ - // TODO(holmer): Consider putting this threshold only on the video bitrate, - // and not on protection. - if (_maxBitRate > 0 && - target_bitrate > static_cast(_maxBitRate)) { - target_bitrate = _maxBitRate; - } - VCMProtectionMethod *selectedMethod = _lossProtLogic->SelectedMethod(); - float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; - _lossProtLogic->UpdateBitRate(target_bitrate_kbps); - _lossProtLogic->UpdateRtt(roundTripTimeMs); - _lossProtLogic->UpdateResidualPacketLoss(static_cast(fractionLost)); +uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate, + uint8_t fraction_lost, + uint32_t round_trip_time_ms) { + // TODO(holmer): Consider putting this threshold only on the video bitrate, + // and not on protection. + if (max_bit_rate_ > 0 && + target_bitrate > static_cast(max_bit_rate_)) { + target_bitrate = max_bit_rate_; + } + VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod(); + float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; + loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); + loss_prot_logic_->UpdateRtt(round_trip_time_ms); + loss_prot_logic_->UpdateResidualPacketLoss(static_cast(fraction_lost)); - // Get frame rate for encoder: this is the actual/sent frame rate - float actualFrameRate = SentFrameRate(); + // Get frame rate for encoder: this is the actual/sent frame rate. + float actual_frame_rate = SentFrameRate(); - // sanity - if (actualFrameRate < 1.0) - { - actualFrameRate = 1.0; - } + // Sanity check. + if (actual_frame_rate < 1.0) { + actual_frame_rate = 1.0; + } - // Update frame rate for the loss protection logic class: frame rate should - // be the actual/sent rate - _lossProtLogic->UpdateFrameRate(actualFrameRate); + // Update frame rate for the loss protection logic class: frame rate should + // be the actual/sent rate. + loss_prot_logic_->UpdateFrameRate(actual_frame_rate); - _fractionLost = fractionLost; + fraction_lost_ = fraction_lost; - // Returns the filtered packet loss, used for the protection setting. - // The filtered loss may be the received loss (no filter), or some - // filtered value (average or max window filter). - // Use max window filter for now. - FilterPacketLossMode filter_mode = kMaxFilter; - uint8_t packetLossEnc = _lossProtLogic->FilteredLoss( - _clock->TimeInMilliseconds(), filter_mode, fractionLost); + // Returns the filtered packet loss, used for the protection setting. + // The filtered loss may be the received loss (no filter), or some + // filtered value (average or max window filter). + // Use max window filter for now. + FilterPacketLossMode filter_mode = kMaxFilter; + uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss( + clock_->TimeInMilliseconds(), filter_mode, fraction_lost); - // For now use the filtered loss for computing the robustness settings - _lossProtLogic->UpdateFilteredLossPr(packetLossEnc); + // For now use the filtered loss for computing the robustness settings. + loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc); - // Rate cost of the protection methods - uint32_t protection_overhead_bps = 0; + // Rate cost of the protection methods. + uint32_t protection_overhead_bps = 0; - // Update protection settings, when applicable - float sent_video_rate_kbps = 0.0f; - if (selectedMethod) - { - // Update protection method with content metrics - selectedMethod->UpdateContentMetrics(_content->ShortTermAvgData()); + // Update protection settings, when applicable. + float sent_video_rate_kbps = 0.0f; + if (selected_method) { + // Update protection method with content metrics. + selected_method->UpdateContentMetrics(content_->ShortTermAvgData()); - // Update method will compute the robustness settings for the given - // protection method and the overhead cost - // the protection method is set by the user via SetVideoProtection. - _lossProtLogic->UpdateMethod(); + // Update method will compute the robustness settings for the given + // protection method and the overhead cost + // the protection method is set by the user via SetVideoProtection. + loss_prot_logic_->UpdateMethod(); - // Update protection callback with protection settings. - uint32_t sent_video_rate_bps = 0; - uint32_t sent_nack_rate_bps = 0; - uint32_t sent_fec_rate_bps = 0; - // Get the bit cost of protection method, based on the amount of - // overhead data actually transmitted (including headers) the last - // second. - UpdateProtectionCallback(selectedMethod, - &sent_video_rate_bps, - &sent_nack_rate_bps, - &sent_fec_rate_bps); - uint32_t sent_total_rate_bps = sent_video_rate_bps + - sent_nack_rate_bps + sent_fec_rate_bps; - // Estimate the overhead costs of the next second as staying the same - // wrt the source bitrate. - if (sent_total_rate_bps > 0) { - protection_overhead_bps = static_cast(target_bitrate * + // Update protection callback with protection settings. + uint32_t sent_video_rate_bps = 0; + uint32_t sent_nack_rate_bps = 0; + uint32_t sent_fec_rate_bps = 0; + // Get the bit cost of protection method, based on the amount of + // overhead data actually transmitted (including headers) the last + // second. + UpdateProtectionCallback(selected_method, + &sent_video_rate_bps, + &sent_nack_rate_bps, + &sent_fec_rate_bps); + uint32_t sent_total_rate_bps = + sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; + // Estimate the overhead costs of the next second as staying the same + // wrt the source bitrate. + if (sent_total_rate_bps > 0) { + protection_overhead_bps = static_cast( + target_bitrate * static_cast(sent_nack_rate_bps + sent_fec_rate_bps) / - sent_total_rate_bps + 0.5); - } - // Cap the overhead estimate to 50%. - if (protection_overhead_bps > target_bitrate / 2) - protection_overhead_bps = target_bitrate / 2; - - // Get the effective packet loss for encoder ER - // when applicable, should be passed to encoder via fractionLost - packetLossEnc = selectedMethod->RequiredPacketLossER(); - sent_video_rate_kbps = - static_cast(sent_video_rate_bps) / 1000.0f; + sent_total_rate_bps + + 0.5); } + // Cap the overhead estimate to 50%. + if (protection_overhead_bps > target_bitrate / 2) + protection_overhead_bps = target_bitrate / 2; - // Source coding rate: total rate - protection overhead - _targetBitRate = target_bitrate - protection_overhead_bps; + // Get the effective packet loss for encoder ER when applicable. Should be + // passed to encoder via fraction_lost. + packet_loss_enc = selected_method->RequiredPacketLossER(); + sent_video_rate_kbps = static_cast(sent_video_rate_bps) / 1000.0f; + } - // Update encoding rates following protection settings - float target_video_bitrate_kbps = - static_cast(_targetBitRate) / 1000.0f; - _frameDropper->SetRates(target_video_bitrate_kbps, _incomingFrameRate); + // Source coding rate: total rate - protection overhead. + target_bit_rate_ = target_bitrate - protection_overhead_bps; - if (_enableQm) - { - // Update QM with rates - _qmResolution->UpdateRates(target_video_bitrate_kbps, - sent_video_rate_kbps, _incomingFrameRate, - _fractionLost); - // Check for QM selection - bool selectQM = CheckStatusForQMchange(); - if (selectQM) - { - SelectQuality(); - } - // Reset the short-term averaged content data. - _content->ResetShortTermAvgData(); + // Update encoding rates following protection settings. + float target_video_bitrate_kbps = + static_cast(target_bit_rate_) / 1000.0f; + frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); + + if (enable_qm_) { + // Update QM with rates. + qm_resolution_->UpdateRates(target_video_bitrate_kbps, + sent_video_rate_kbps, + incoming_frame_rate_, + fraction_lost_); + // Check for QM selection. + bool select_qm = CheckStatusForQMchange(); + if (select_qm) { + SelectQuality(); } + // Reset the short-term averaged content data. + content_->ResetShortTermAvgData(); + } - return _targetBitRate; + return target_bit_rate_; } -int VCMMediaOptimization::UpdateProtectionCallback( - VCMProtectionMethod *selected_method, +int32_t MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, + int32_t max_bit_rate, + uint32_t frame_rate, + uint32_t target_bitrate, + uint16_t width, + uint16_t height, + int num_layers) { + // Everything codec specific should be reset here since this means the codec + // has changed. If native dimension values have changed, then either user + // initiated change, or QM initiated change. Will be able to determine only + // after the processing of the first frame. + last_change_time_ = clock_->TimeInMilliseconds(); + content_->Reset(); + content_->UpdateFrameRate(frame_rate); + + max_bit_rate_ = max_bit_rate; + send_codec_type_ = send_codec_type; + target_bit_rate_ = target_bitrate; + float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; + loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); + loss_prot_logic_->UpdateFrameRate(static_cast(frame_rate)); + loss_prot_logic_->UpdateFrameSize(width, height); + loss_prot_logic_->UpdateNumLayers(num_layers); + frame_dropper_->Reset(); + frame_dropper_->SetRates(target_bitrate_kbps, static_cast(frame_rate)); + user_frame_rate_ = static_cast(frame_rate); + codec_width_ = width; + codec_height_ = height; + num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. + int32_t ret = VCM_OK; + ret = qm_resolution_->Initialize(target_bitrate_kbps, + user_frame_rate_, + codec_width_, + codec_height_, + num_layers_); + return ret; +} + +void MediaOptimization::EnableProtectionMethod(bool enable, + VCMProtectionMethodEnum method) { + bool updated = false; + if (enable) { + updated = loss_prot_logic_->SetMethod(method); + } else { + loss_prot_logic_->RemoveMethod(method); + } + if (updated) { + loss_prot_logic_->UpdateMethod(); + } +} + +bool MediaOptimization::IsProtectionMethodEnabled( + VCMProtectionMethodEnum method) { + return (loss_prot_logic_->SelectedType() == method); +} + +uint32_t MediaOptimization::InputFrameRate() { + ProcessIncomingFrameRate(clock_->TimeInMilliseconds()); + return uint32_t(incoming_frame_rate_ + 0.5f); +} + +uint32_t MediaOptimization::SentFrameRate() { + PurgeOldFrameSamples(clock_->TimeInMilliseconds()); + UpdateSentFramerate(); + return avg_sent_framerate_; +} + +uint32_t MediaOptimization::SentBitRate() { + const int64_t now_ms = clock_->TimeInMilliseconds(); + PurgeOldFrameSamples(now_ms); + UpdateSentBitrate(now_ms); + return avg_sent_bit_rate_bps_; +} + +int32_t MediaOptimization::UpdateWithEncodedData(int encoded_length, + uint32_t timestamp, + FrameType encoded_frame_type) { + const int64_t now_ms = clock_->TimeInMilliseconds(); + PurgeOldFrameSamples(now_ms); + if (encoded_frame_samples_.size() > 0 && + encoded_frame_samples_.back().timestamp == timestamp) { + // Frames having the same timestamp are generated from the same input + // frame. We don't want to double count them, but only increment the + // size_bytes. + encoded_frame_samples_.back().size_bytes += encoded_length; + encoded_frame_samples_.back().time_complete_ms = now_ms; + } else { + encoded_frame_samples_.push_back( + EncodedFrameSample(encoded_length, timestamp, now_ms)); + } + UpdateSentBitrate(now_ms); + UpdateSentFramerate(); + if (encoded_length > 0) { + const bool delta_frame = (encoded_frame_type != kVideoFrameKey && + encoded_frame_type != kVideoFrameGolden); + + frame_dropper_->Fill(encoded_length, delta_frame); + if (max_payload_size_ > 0 && encoded_length > 0) { + const float min_packets_per_frame = + encoded_length / static_cast(max_payload_size_); + if (delta_frame) { + loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame, + clock_->TimeInMilliseconds()); + } else { + loss_prot_logic_->UpdatePacketsPerFrameKey( + min_packets_per_frame, clock_->TimeInMilliseconds()); + } + + if (enable_qm_) { + // Update quality select with encoded length. + qm_resolution_->UpdateEncodedSize(encoded_length, encoded_frame_type); + } + } + if (!delta_frame && encoded_length > 0) { + loss_prot_logic_->UpdateKeyFrameSize(static_cast(encoded_length)); + } + + // Updating counters. + if (delta_frame) { + delta_frame_cnt_++; + } else { + key_frame_cnt_++; + } + } + + return VCM_OK; +} + +int32_t MediaOptimization::RegisterProtectionCallback( + VCMProtectionCallback* protection_callback) { + video_protection_callback_ = protection_callback; + return VCM_OK; +} + +int32_t MediaOptimization::RegisterVideoQMCallback( + VCMQMSettingsCallback* video_qmsettings) { + video_qmsettings_callback_ = video_qmsettings; + // Callback setting controls QM. + if (video_qmsettings_callback_ != NULL) { + enable_qm_ = true; + } else { + enable_qm_ = false; + } + return VCM_OK; +} + +void MediaOptimization::EnableFrameDropper(bool enable) { + frame_dropper_->Enable(enable); +} + +bool MediaOptimization::DropFrame() { + // Leak appropriate number of bytes. + frame_dropper_->Leak((uint32_t)(InputFrameRate() + 0.5f)); + + return frame_dropper_->DropFrame(); +} + +int32_t MediaOptimization::SentFrameCount(VCMFrameCount* frame_count) const { + frame_count->numDeltaFrames = delta_frame_cnt_; + frame_count->numKeyFrames = key_frame_cnt_; + return VCM_OK; +} + +void MediaOptimization::UpdateIncomingFrameRate() { + int64_t now = clock_->TimeInMilliseconds(); + if (incoming_frame_times_[0] == 0) { + // No shifting if this is the first time. + } else { + // Shift all times one step. + for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) { + incoming_frame_times_[i + 1] = incoming_frame_times_[i]; + } + } + incoming_frame_times_[0] = now; + ProcessIncomingFrameRate(now); +} + +void MediaOptimization::UpdateContentData( + const VideoContentMetrics* content_metrics) { + // Updating content metrics. + if (content_metrics == NULL) { + // Disable QM if metrics are NULL. + enable_qm_ = false; + qm_resolution_->Reset(); + } else { + content_->UpdateContentData(content_metrics); + } +} + +int32_t MediaOptimization::SelectQuality() { + // Reset quantities for QM select. + qm_resolution_->ResetQM(); + + // Update QM will long-term averaged content metrics. + qm_resolution_->UpdateContent(content_->LongTermAvgData()); + + // Select quality mode. + VCMResolutionScale* qm = NULL; + int32_t ret = qm_resolution_->SelectResolution(&qm); + if (ret < 0) { + return ret; + } + + // Check for updates to spatial/temporal modes. + QMUpdate(qm); + + // Reset all the rate and related frame counters quantities. + qm_resolution_->ResetRates(); + + // Reset counters. + last_qm_update_time_ = clock_->TimeInMilliseconds(); + + // Reset content metrics. + content_->Reset(); + + return VCM_OK; +} + +// Private methods below this line. + +int MediaOptimization::UpdateProtectionCallback( + VCMProtectionMethod* selected_method, uint32_t* video_rate_bps, uint32_t* nack_overhead_rate_bps, - uint32_t* fec_overhead_rate_bps) -{ - if (!_videoProtectionCallback) - { - return VCM_OK; - } - FecProtectionParams delta_fec_params; - FecProtectionParams key_fec_params; - // Get the FEC code rate for Key frames (set to 0 when NA) - key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK(); + uint32_t* fec_overhead_rate_bps) { + if (!video_protection_callback_) { + return VCM_OK; + } + FecProtectionParams delta_fec_params; + FecProtectionParams key_fec_params; + // Get the FEC code rate for Key frames (set to 0 when NA). + key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK(); - // Get the FEC code rate for Delta frames (set to 0 when NA) - delta_fec_params.fec_rate = - selected_method->RequiredProtectionFactorD(); + // Get the FEC code rate for Delta frames (set to 0 when NA). + delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD(); - // Get the FEC-UEP protection status for Key frames: UEP on/off - key_fec_params.use_uep_protection = - selected_method->RequiredUepProtectionK(); + // Get the FEC-UEP protection status for Key frames: UEP on/off. + key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK(); - // Get the FEC-UEP protection status for Delta frames: UEP on/off - delta_fec_params.use_uep_protection = - selected_method->RequiredUepProtectionD(); + // Get the FEC-UEP protection status for Delta frames: UEP on/off. + delta_fec_params.use_uep_protection = + selected_method->RequiredUepProtectionD(); - // The RTP module currently requires the same |max_fec_frames| for both - // key and delta frames. - delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); - key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); + // The RTP module currently requires the same |max_fec_frames| for both + // key and delta frames. + delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); + key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); - // Set the FEC packet mask type. |kFecMaskBursty| is more effective for - // consecutive losses and little/no packet re-ordering. As we currently - // do not have feedback data on the degree of correlated losses and packet - // re-ordering, we keep default setting to |kFecMaskRandom| for now. - delta_fec_params.fec_mask_type = kFecMaskRandom; - key_fec_params.fec_mask_type = kFecMaskRandom; + // Set the FEC packet mask type. |kFecMaskBursty| is more effective for + // consecutive losses and little/no packet re-ordering. As we currently + // do not have feedback data on the degree of correlated losses and packet + // re-ordering, we keep default setting to |kFecMaskRandom| for now. + delta_fec_params.fec_mask_type = kFecMaskRandom; + key_fec_params.fec_mask_type = kFecMaskRandom; - // TODO(Marco): Pass FEC protection values per layer. - return _videoProtectionCallback->ProtectionRequest(&delta_fec_params, + // TODO(Marco): Pass FEC protection values per layer. + return video_protection_callback_->ProtectionRequest(&delta_fec_params, &key_fec_params, video_rate_bps, nack_overhead_rate_bps, fec_overhead_rate_bps); } -bool -VCMMediaOptimization::DropFrame() -{ - // leak appropriate number of bytes - _frameDropper->Leak((uint32_t)(InputFrameRate() + 0.5f)); - - return _frameDropper->DropFrame(); -} - -int32_t -VCMMediaOptimization::SentFrameCount(VCMFrameCount &frameCount) const -{ - frameCount.numDeltaFrames = _deltaFrameCnt; - frameCount.numKeyFrames = _keyFrameCnt; - return VCM_OK; -} - -int32_t -VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, - int32_t maxBitRate, - uint32_t frameRate, - uint32_t target_bitrate, - uint16_t width, - uint16_t height, - int numLayers) -{ - // Everything codec specific should be reset here since this means the codec - // has changed. If native dimension values have changed, then either user - // initiated change, or QM initiated change. Will be able to determine only - // after the processing of the first frame. - _lastChangeTime = _clock->TimeInMilliseconds(); - _content->Reset(); - _content->UpdateFrameRate(frameRate); - - _maxBitRate = maxBitRate; - _sendCodecType = sendCodecType; - _targetBitRate = target_bitrate; - float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; - _lossProtLogic->UpdateBitRate(target_bitrate_kbps); - _lossProtLogic->UpdateFrameRate(static_cast(frameRate)); - _lossProtLogic->UpdateFrameSize(width, height); - _lossProtLogic->UpdateNumLayers(numLayers); - _frameDropper->Reset(); - _frameDropper->SetRates(target_bitrate_kbps, static_cast(frameRate)); - _userFrameRate = static_cast(frameRate); - _codecWidth = width; - _codecHeight = height; - _numLayers = (numLayers <= 1) ? 1 : numLayers; // Can also be zero. - int32_t ret = VCM_OK; - ret = _qmResolution->Initialize(target_bitrate_kbps, _userFrameRate, - _codecWidth, _codecHeight, _numLayers); - return ret; -} - -int32_t -VCMMediaOptimization::RegisterProtectionCallback(VCMProtectionCallback* - protectionCallback) -{ - _videoProtectionCallback = protectionCallback; - return VCM_OK; - -} - -void -VCMMediaOptimization::EnableFrameDropper(bool enable) -{ - _frameDropper->Enable(enable); -} - -void -VCMMediaOptimization::EnableProtectionMethod(bool enable, - VCMProtectionMethodEnum method) -{ - bool updated = false; - if (enable) - { - updated = _lossProtLogic->SetMethod(method); - } - else - { - _lossProtLogic->RemoveMethod(method); - } - if (updated) - { - _lossProtLogic->UpdateMethod(); - } -} - -bool -VCMMediaOptimization::IsProtectionMethodEnabled(VCMProtectionMethodEnum method) -{ - return (_lossProtLogic->SelectedType() == method); -} - -void -VCMMediaOptimization::SetMtu(int32_t mtu) -{ - _maxPayloadSize = mtu; -} - -uint32_t -VCMMediaOptimization::SentFrameRate() -{ - PurgeOldFrameSamples(_clock->TimeInMilliseconds()); - UpdateSentFramerate(); - return _avgSentFramerate; -} - -uint32_t -VCMMediaOptimization::SentBitRate() -{ - const int64_t now_ms = _clock->TimeInMilliseconds(); - PurgeOldFrameSamples(now_ms); - UpdateSentBitrate(now_ms); - return _avgSentBitRateBps; -} - -int32_t -VCMMediaOptimization::MaxBitRate() -{ - return _maxBitRate; -} - -int32_t -VCMMediaOptimization::UpdateWithEncodedData(int encodedLength, - uint32_t timestamp, - FrameType encodedFrameType) -{ - const int64_t now_ms = _clock->TimeInMilliseconds(); - PurgeOldFrameSamples(now_ms); - if (_encodedFrameSamples.size() > 0 && - _encodedFrameSamples.back().timestamp == timestamp) { - // Frames having the same timestamp are generated from the same input - // frame. We don't want to double count them, but only increment the - // size_bytes. - _encodedFrameSamples.back().size_bytes += encodedLength; - _encodedFrameSamples.back().time_complete_ms = now_ms; - } else { - _encodedFrameSamples.push_back(VCMEncodedFrameSample( - encodedLength, timestamp, now_ms)); - } - UpdateSentBitrate(now_ms); - UpdateSentFramerate(); - if(encodedLength > 0) - { - const bool deltaFrame = (encodedFrameType != kVideoFrameKey && - encodedFrameType != kVideoFrameGolden); - - _frameDropper->Fill(encodedLength, deltaFrame); - if (_maxPayloadSize > 0 && encodedLength > 0) - { - const float minPacketsPerFrame = encodedLength / - static_cast(_maxPayloadSize); - if (deltaFrame) - { - _lossProtLogic->UpdatePacketsPerFrame( - minPacketsPerFrame, _clock->TimeInMilliseconds()); - } - else - { - _lossProtLogic->UpdatePacketsPerFrameKey( - minPacketsPerFrame, _clock->TimeInMilliseconds()); - } - - if (_enableQm) - { - // update quality select with encoded length - _qmResolution->UpdateEncodedSize(encodedLength, - encodedFrameType); - } - } - if (!deltaFrame && encodedLength > 0) - { - _lossProtLogic->UpdateKeyFrameSize(static_cast(encodedLength)); - } - - // updating counters - if (deltaFrame) - { - _deltaFrameCnt++; - } - else - { - _keyFrameCnt++; - } - - } - - return VCM_OK; - -} - -void VCMMediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { - while (!_encodedFrameSamples.empty()) { - if (now_ms - _encodedFrameSamples.front().time_complete_ms > +void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { + while (!encoded_frame_samples_.empty()) { + if (now_ms - encoded_frame_samples_.front().time_complete_ms > kBitrateAverageWinMs) { - _encodedFrameSamples.pop_front(); + encoded_frame_samples_.pop_front(); } else { break; } } } -void VCMMediaOptimization::UpdateSentBitrate(int64_t now_ms) { - if (_encodedFrameSamples.empty()) { - _avgSentBitRateBps = 0; +void MediaOptimization::UpdateSentBitrate(int64_t now_ms) { + if (encoded_frame_samples_.empty()) { + avg_sent_bit_rate_bps_ = 0; return; } int framesize_sum = 0; - for (FrameSampleList::iterator it = _encodedFrameSamples.begin(); - it != _encodedFrameSamples.end(); ++it) { + for (FrameSampleList::iterator it = encoded_frame_samples_.begin(); + it != encoded_frame_samples_.end(); + ++it) { framesize_sum += it->size_bytes; } float denom = static_cast( - now_ms - _encodedFrameSamples.front().time_complete_ms); + now_ms - encoded_frame_samples_.front().time_complete_ms); if (denom >= 1.0f) { - _avgSentBitRateBps = static_cast(framesize_sum * 8 * 1000 / - denom + 0.5f); + avg_sent_bit_rate_bps_ = + static_cast(framesize_sum * 8 * 1000 / denom + 0.5f); } else { - _avgSentBitRateBps = framesize_sum * 8; + avg_sent_bit_rate_bps_ = framesize_sum * 8; } } -void VCMMediaOptimization::UpdateSentFramerate() { - if (_encodedFrameSamples.size() <= 1) { - _avgSentFramerate = _encodedFrameSamples.size(); +void MediaOptimization::UpdateSentFramerate() { + if (encoded_frame_samples_.size() <= 1) { + avg_sent_framerate_ = encoded_frame_samples_.size(); return; } - int denom = _encodedFrameSamples.back().timestamp - - _encodedFrameSamples.front().timestamp; + int denom = encoded_frame_samples_.back().timestamp - + encoded_frame_samples_.front().timestamp; if (denom > 0) { - _avgSentFramerate = (90000 * (_encodedFrameSamples.size() - 1) + denom / 2) - / denom; + avg_sent_framerate_ = + (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom; } else { - _avgSentFramerate = _encodedFrameSamples.size(); + avg_sent_framerate_ = encoded_frame_samples_.size(); } } -int32_t -VCMMediaOptimization::RegisterVideoQMCallback(VCMQMSettingsCallback* - videoQMSettings) -{ - _videoQMSettingsCallback = videoQMSettings; - // Callback setting controls QM - if (_videoQMSettingsCallback != NULL) - { - _enableQm = true; - } - else - { - _enableQm = false; - } - return VCM_OK; -} - -void -VCMMediaOptimization::UpdateContentData(const VideoContentMetrics* - contentMetrics) -{ - // Updating content metrics - if (contentMetrics == NULL) - { - // Disable QM if metrics are NULL - _enableQm = false; - _qmResolution->Reset(); - } - else - { - _content->UpdateContentData(contentMetrics); - } -} - -int32_t -VCMMediaOptimization::SelectQuality() -{ - // Reset quantities for QM select - _qmResolution->ResetQM(); - - // Update QM will long-term averaged content metrics. - _qmResolution->UpdateContent(_content->LongTermAvgData()); - - // Select quality mode - VCMResolutionScale* qm = NULL; - int32_t ret = _qmResolution->SelectResolution(&qm); - if (ret < 0) - { - return ret; - } - - // Check for updates to spatial/temporal modes - QMUpdate(qm); - - // Reset all the rate and related frame counters quantities - _qmResolution->ResetRates(); - - // Reset counters - _lastQMUpdateTime = _clock->TimeInMilliseconds(); - - // Reset content metrics - _content->Reset(); - - return VCM_OK; -} - - -// Check timing constraints and look for significant change in: -// (1) scene content -// (2) target bit rate - -bool -VCMMediaOptimization::CheckStatusForQMchange() -{ - - bool status = true; - - // Check that we do not call QMSelect too often, and that we waited some time - // (to sample the metrics) from the event lastChangeTime - // lastChangeTime is the time where user changed the size/rate/frame rate - // (via SetEncodingData) - int64_t now = _clock->TimeInMilliseconds(); - if ((now - _lastQMUpdateTime) < kQmMinIntervalMs || - (now - _lastChangeTime) < kQmMinIntervalMs) - { - status = false; - } - - return status; - -} - -bool VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) { - // Check for no change +bool MediaOptimization::QMUpdate(VCMResolutionScale* qm) { + // Check for no change. if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) { return false; } // Check for change in frame rate. if (qm->change_resolution_temporal) { - _incomingFrameRate = qm->frame_rate; + incoming_frame_rate_ = qm->frame_rate; // Reset frame rate estimate. - memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); + memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); } // Check for change in frame size. if (qm->change_resolution_spatial) { - _codecWidth = qm->codec_width; - _codecHeight = qm->codec_height; + codec_width_ = qm->codec_width; + codec_height_ = qm->codec_height; } - WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, _id, + WEBRTC_TRACE(webrtc::kTraceDebug, + webrtc::kTraceVideoCoding, + id_, "Resolution change from QM select: W = %d, H = %d, FR = %f", - qm->codec_width, qm->codec_height, qm->frame_rate); + qm->codec_width, + qm->codec_height, + qm->frame_rate); // Update VPM with new target frame rate and frame size. - // Note: use |qm->frame_rate| instead of |_incomingFrameRate| for updating - // target frame rate in VPM frame dropper. The quantity |_incomingFrameRate| + // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating + // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate| // will vary/fluctuate, and since we don't want to change the state of the // VPM frame dropper, unless a temporal action was selected, we use the // quantity |qm->frame_rate| for updating. - _videoQMSettingsCallback->SetVideoQMSettings(qm->frame_rate, - _codecWidth, - _codecHeight); - _content->UpdateFrameRate(qm->frame_rate); - _qmResolution->UpdateCodecParameters(qm->frame_rate, _codecWidth, - _codecHeight); + video_qmsettings_callback_->SetVideoQMSettings( + qm->frame_rate, codec_width_, codec_height_); + content_->UpdateFrameRate(qm->frame_rate); + qm_resolution_->UpdateCodecParameters( + qm->frame_rate, codec_width_, codec_height_); return true; } -void -VCMMediaOptimization::UpdateIncomingFrameRate() -{ - int64_t now = _clock->TimeInMilliseconds(); - if (_incomingFrameTimes[0] == 0) - { - // first no shift - } else - { - // shift - for(int32_t i = (kFrameCountHistorySize - 2); i >= 0 ; i--) - { - _incomingFrameTimes[i+1] = _incomingFrameTimes[i]; - } - } - _incomingFrameTimes[0] = now; - ProcessIncomingFrameRate(now); +// Check timing constraints and look for significant change in: +// (1) scene content, +// (2) target bit rate. +bool MediaOptimization::CheckStatusForQMchange() { + bool status = true; + + // Check that we do not call QMSelect too often, and that we waited some time + // (to sample the metrics) from the event last_change_time + // last_change_time is the time where user changed the size/rate/frame rate + // (via SetEncodingData). + int64_t now = clock_->TimeInMilliseconds(); + if ((now - last_qm_update_time_) < kQmMinIntervalMs || + (now - last_change_time_) < kQmMinIntervalMs) { + status = false; + } + + return status; } -// allowing VCM to keep track of incoming frame rate -void -VCMMediaOptimization::ProcessIncomingFrameRate(int64_t now) -{ - int32_t num = 0; - int32_t nrOfFrames = 0; - for (num = 1; num < (kFrameCountHistorySize - 1); num++) - { - if (_incomingFrameTimes[num] <= 0 || - // don't use data older than 2 s - now - _incomingFrameTimes[num] > kFrameHistoryWinMs) - { - break; - } else - { - nrOfFrames++; - } +// Allowing VCM to keep track of incoming frame rate. +void MediaOptimization::ProcessIncomingFrameRate(int64_t now) { + int32_t num = 0; + int32_t nr_of_frames = 0; + for (num = 1; num < (kFrameCountHistorySize - 1); ++num) { + if (incoming_frame_times_[num] <= 0 || + // don't use data older than 2 s + now - incoming_frame_times_[num] > kFrameHistoryWinMs) { + break; + } else { + nr_of_frames++; } - if (num > 1) - { - const int64_t diff = now - _incomingFrameTimes[num-1]; - _incomingFrameRate = 1.0; - if(diff >0) - { - _incomingFrameRate = nrOfFrames * 1000.0f / static_cast(diff); - } + } + if (num > 1) { + const int64_t diff = now - incoming_frame_times_[num - 1]; + incoming_frame_rate_ = 1.0; + if (diff > 0) { + incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast(diff); } -} - -uint32_t -VCMMediaOptimization::InputFrameRate() -{ - ProcessIncomingFrameRate(_clock->TimeInMilliseconds()); - return uint32_t (_incomingFrameRate + 0.5f); + } } } // namespace media_optimization diff --git a/webrtc/modules/video_coding/main/source/media_optimization.h b/webrtc/modules/video_coding/main/source/media_optimization.h index be20b58e17..4b0fd96b8c 100644 --- a/webrtc/modules/video_coding/main/source/media_optimization.h +++ b/webrtc/modules/video_coding/main/source/media_optimization.h @@ -8,8 +8,10 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_ -#define WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_ +#ifndef WEBRTC_MODULES_VIDEO_CODING_MAIN_SOURCE_MEDIA_OPTIMIZATION_H_ +#define WEBRTC_MODULES_VIDEO_CODING_MAIN_SOURCE_MEDIA_OPTIMIZATION_H_ + +#include #include "webrtc/modules/interface/module_common_types.h" #include "webrtc/modules/video_coding/main/interface/video_coding.h" @@ -17,22 +19,26 @@ #include "webrtc/modules/video_coding/main/source/qm_select.h" #include "webrtc/system_wrappers/interface/trace.h" -#include - namespace webrtc { +// Forward declarations. class Clock; class FrameDropper; class VCMContentMetricsProcessing; namespace media_optimization { -enum { kBitrateMaxFrameSamples = 60 }; -enum { kBitrateAverageWinMs = 1000 }; +enum { + kBitrateMaxFrameSamples = 60 +}; +enum { + kBitrateAverageWinMs = 1000 +}; -struct VCMEncodedFrameSample { - VCMEncodedFrameSample(int size_bytes, uint32_t timestamp, - int64_t time_complete_ms) +struct EncodedFrameSample { + EncodedFrameSample(int size_bytes, + uint32_t timestamp, + int64_t time_complete_ms) : size_bytes(size_bytes), timestamp(timestamp), time_complete_ms(time_complete_ms) {} @@ -42,181 +48,141 @@ struct VCMEncodedFrameSample { int64_t time_complete_ms; }; -class VCMMediaOptimization -{ -public: - VCMMediaOptimization(int32_t id, Clock* clock); - ~VCMMediaOptimization(void); - /* - * Reset the Media Optimization module - */ - int32_t Reset(); - /** - * Set target Rates for the encoder given the channel parameters - * Inputs: target bitrate - the encoder target bitrate in bits/s. - * fractionLost - packet loss in % in the network - * roundTripTimeMs - round trip time in milliseconds - * minBitRate - the bit rate of the end-point with lowest rate - * maxBitRate - the bit rate of the end-point with highest rate - */ - uint32_t SetTargetRates(uint32_t target_bitrate, - uint8_t &fractionLost, - uint32_t roundTripTimeMs); +class MediaOptimization { + public: + MediaOptimization(int32_t id, Clock* clock); + ~MediaOptimization(void); - /** - * Inform media optimization of initial encoding state - */ - int32_t SetEncodingData(VideoCodecType sendCodecType, - int32_t maxBitRate, - uint32_t frameRate, - uint32_t bitRate, - uint16_t width, - uint16_t height, - int numTemporalLayers); - /** - * Enable protection method - */ - void EnableProtectionMethod(bool enable, VCMProtectionMethodEnum method); - /** - * Returns weather or not protection method is enabled - */ - bool IsProtectionMethodEnabled(VCMProtectionMethodEnum method); - /** - * Updates the max pay load size - */ - void SetMtu(int32_t mtu); - /* - * Get actual input frame rate - */ - uint32_t InputFrameRate(); + // Resets the Media Optimization module. + int32_t Reset(); - /* - * Get actual sent frame rate - */ - uint32_t SentFrameRate(); - /* - * Get actual sent bit rate - */ - uint32_t SentBitRate(); - /* - * Get maximum allowed bit rate - */ - int32_t MaxBitRate(); - /* - * Inform Media Optimization of encoding output: Length and frame type - */ - int32_t UpdateWithEncodedData(int encodedLength, - uint32_t timestamp, - FrameType encodedFrameType); - /* - * Register a protection callback to be used to inform the user about the - * protection methods used - */ - int32_t RegisterProtectionCallback(VCMProtectionCallback* - protectionCallback); - /* - * Register a quality settings callback to be used to inform VPM/user about - */ - int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* videoQMSettings); - void EnableFrameDropper(bool enable); + // Sets target rates for the encoder given the channel parameters. + // Inputs: target bitrate - the encoder target bitrate in bits/s. + // fraction_lost - packet loss rate in % in the network. + // round_trip_time_ms - round trip time in milliseconds. + // min_bit_rate - the bit rate of the end-point with lowest rate. + // max_bit_rate - the bit rate of the end-point with highest rate. + uint32_t SetTargetRates(uint32_t target_bitrate, + uint8_t fraction_lost, + uint32_t round_trip_time_ms); - bool DropFrame(); + // Informs media optimization of initial encoding state. + int32_t SetEncodingData(VideoCodecType send_codec_type, + int32_t max_bit_rate, + uint32_t frame_rate, + uint32_t bit_rate, + uint16_t width, + uint16_t height, + int num_temporal_layers); - /* - * Get number of key/delta frames encoded - */ - int32_t SentFrameCount(VCMFrameCount &frameCount) const; + // Enables protection method. + void EnableProtectionMethod(bool enable, VCMProtectionMethodEnum method); - /* - * update incoming frame rate value - */ - void UpdateIncomingFrameRate(); + // Returns weather or not protection method is enabled. + bool IsProtectionMethodEnabled(VCMProtectionMethodEnum method); - /** - * Update content metric Data - */ - void UpdateContentData(const VideoContentMetrics* contentMetrics); + // Returns the actual input frame rate. + uint32_t InputFrameRate(); - /** - * Compute new Quality Mode - */ - int32_t SelectQuality(); + // Returns the actual sent frame rate. + uint32_t SentFrameRate(); -private: - typedef std::list FrameSampleList; + // Returns the actual sent bit rate. + uint32_t SentBitRate(); - /* - * Update protection callback with protection settings - */ - int UpdateProtectionCallback(VCMProtectionMethod *selected_method, - uint32_t* total_video_rate_bps, - uint32_t* nack_overhead_rate_bps, - uint32_t* fec_overhead_rate_bps); + // Informs Media Optimization of encoding output: Length and frame type. + int32_t UpdateWithEncodedData(int encoded_length, + uint32_t timestamp, + FrameType encoded_frame_type); - void PurgeOldFrameSamples(int64_t now_ms); - void UpdateSentBitrate(int64_t nowMs); - void UpdateSentFramerate(); + // Registers a protection callback to be used to inform the user about the + // protection methods used. + int32_t RegisterProtectionCallback( + VCMProtectionCallback* protection_callback); - /* - * verify if QM settings differ from default, i.e. if an update is required - * Compute actual values, as will be sent to the encoder - */ - bool QMUpdate(VCMResolutionScale* qm); - /** - * check if we should make a QM change - * will return 1 if yes, 0 otherwise - */ - bool CheckStatusForQMchange(); + // Registers a quality settings callback to be used to inform VPM/user. + int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* video_qmsettings); - void ProcessIncomingFrameRate(int64_t now); + void EnableFrameDropper(bool enable); - enum { kFrameCountHistorySize = 90}; - enum { kFrameHistoryWinMs = 2000}; + bool DropFrame(); - int32_t _id; - Clock* _clock; - int32_t _maxBitRate; - VideoCodecType _sendCodecType; - uint16_t _codecWidth; - uint16_t _codecHeight; - float _userFrameRate; + // Returns the number of key/delta frames encoded. + int32_t SentFrameCount(VCMFrameCount* frame_count) const; - FrameDropper* _frameDropper; - VCMLossProtectionLogic* _lossProtLogic; - uint8_t _fractionLost; + // Updates incoming frame rate value. + void UpdateIncomingFrameRate(); + // Update content metric data. + void UpdateContentData(const VideoContentMetrics* content_metrics); - uint32_t _sendStatistics[4]; - uint32_t _sendStatisticsZeroEncode; - int32_t _maxPayloadSize; - uint32_t _targetBitRate; + // Computes new Quality Mode. + int32_t SelectQuality(); - float _incomingFrameRate; - int64_t _incomingFrameTimes[kFrameCountHistorySize]; + // Accessors and mutators. + int32_t max_bit_rate() const { return max_bit_rate_; } + void set_max_payload_size(int32_t mtu) { max_payload_size_ = mtu; } - bool _enableQm; + private: + typedef std::list FrameSampleList; + enum { + kFrameCountHistorySize = 90 + }; + enum { + kFrameHistoryWinMs = 2000 + }; - VCMProtectionCallback* _videoProtectionCallback; - VCMQMSettingsCallback* _videoQMSettingsCallback; + // Updates protection callback with protection settings. + int UpdateProtectionCallback(VCMProtectionMethod* selected_method, + uint32_t* total_video_rate_bps, + uint32_t* nack_overhead_rate_bps, + uint32_t* fec_overhead_rate_bps); - std::list _encodedFrameSamples; - uint32_t _avgSentBitRateBps; - uint32_t _avgSentFramerate; + void PurgeOldFrameSamples(int64_t now_ms); + void UpdateSentBitrate(int64_t now_ms); + void UpdateSentFramerate(); - uint32_t _keyFrameCnt; - uint32_t _deltaFrameCnt; + // Verifies if QM settings differ from default, i.e. if an update is required. + // Computes actual values, as will be sent to the encoder. + bool QMUpdate(VCMResolutionScale* qm); - VCMContentMetricsProcessing* _content; - VCMQmResolution* _qmResolution; + // Checks if we should make a QM change. Return true if yes, false otherwise. + bool CheckStatusForQMchange(); - int64_t _lastQMUpdateTime; - int64_t _lastChangeTime; // content/user triggered - int _numLayers; + void ProcessIncomingFrameRate(int64_t now); - -}; // end of VCMMediaOptimization class definition + int32_t id_; + Clock* clock_; + int32_t max_bit_rate_; + VideoCodecType send_codec_type_; + uint16_t codec_width_; + uint16_t codec_height_; + float user_frame_rate_; + FrameDropper* frame_dropper_; + VCMLossProtectionLogic* loss_prot_logic_; + uint8_t fraction_lost_; + uint32_t send_statistics_[4]; + uint32_t send_statistics_zero_encode_; + int32_t max_payload_size_; + uint32_t target_bit_rate_; + float incoming_frame_rate_; + int64_t incoming_frame_times_[kFrameCountHistorySize]; + bool enable_qm_; + VCMProtectionCallback* video_protection_callback_; + VCMQMSettingsCallback* video_qmsettings_callback_; + std::list encoded_frame_samples_; + uint32_t avg_sent_bit_rate_bps_; + uint32_t avg_sent_framerate_; + uint32_t key_frame_cnt_; + uint32_t delta_frame_cnt_; + VCMContentMetricsProcessing* content_; + VCMQmResolution* qm_resolution_; + int64_t last_qm_update_time_; + int64_t last_change_time_; // Content/user triggered. + int num_layers_; +}; // End of MediaOptimization class declaration. } // namespace media_optimization } // namespace webrtc -#endif // WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_ +#endif // WEBRTC_MODULES_VIDEO_CODING_MAIN_SOURCE_MEDIA_OPTIMIZATION_H_ diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.h b/webrtc/modules/video_coding/main/source/video_coding_impl.h index 5300589896..a7a6e4fc30 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.h +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.h @@ -107,7 +107,7 @@ class VideoSender { VCMGenericEncoder* _encoder; VCMEncodedFrameCallback _encodedFrameCallback; std::vector _nextFrameTypes; - media_optimization::VCMMediaOptimization _mediaOpt; + media_optimization::MediaOptimization _mediaOpt; VCMSendStatisticsCallback* _sendStatsCallback; FILE* _encoderInputFile; VCMCodecDataBase _codecDataBase; diff --git a/webrtc/modules/video_coding/main/source/video_sender.cc b/webrtc/modules/video_coding/main/source/video_sender.cc index c5d6d41dcf..26e31e0c67 100644 --- a/webrtc/modules/video_coding/main/source/video_sender.cc +++ b/webrtc/modules/video_coding/main/source/video_sender.cc @@ -124,7 +124,7 @@ int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, sendCodec->width, sendCodec->height, numLayers); - _mediaOpt.SetMtu(maxPayloadSize); + _mediaOpt.set_max_payload_size(maxPayloadSize); return VCM_OK; } @@ -369,7 +369,7 @@ int32_t VideoSender::EnableFrameDropper(bool enable) { int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) const { CriticalSectionScoped cs(_sendCritSect); - return _mediaOpt.SentFrameCount(*frameCount); + return _mediaOpt.SentFrameCount(frameCount); } int VideoSender::SetSenderNackMode(SenderNackMode mode) {