diff --git a/webrtc/modules/video_coding/main/source/media_optimization.cc b/webrtc/modules/video_coding/main/source/media_optimization.cc index 27fa681933..aeffac2551 100644 --- a/webrtc/modules/video_coding/main/source/media_optimization.cc +++ b/webrtc/modules/video_coding/main/source/media_optimization.cc @@ -17,6 +17,61 @@ namespace webrtc { namespace media_optimization { +namespace { +void UpdateProtectionCallback( + VCMProtectionMethod* selected_method, + uint32_t* video_rate_bps, + uint32_t* nack_overhead_rate_bps, + uint32_t* fec_overhead_rate_bps, + VCMProtectionCallback* video_protection_callback) { + FecProtectionParams delta_fec_params; + FecProtectionParams key_fec_params; + // Get the FEC code rate for Key frames (set to 0 when NA). + key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK(); + + // Get the FEC code rate for Delta frames (set to 0 when NA). + delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD(); + + // Get the FEC-UEP protection status for Key frames: UEP on/off. + key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK(); + + // Get the FEC-UEP protection status for Delta frames: UEP on/off. + delta_fec_params.use_uep_protection = + selected_method->RequiredUepProtectionD(); + + // The RTP module currently requires the same |max_fec_frames| for both + // key and delta frames. + delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); + key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); + + // Set the FEC packet mask type. |kFecMaskBursty| is more effective for + // consecutive losses and little/no packet re-ordering. As we currently + // do not have feedback data on the degree of correlated losses and packet + // re-ordering, we keep default setting to |kFecMaskRandom| for now. + delta_fec_params.fec_mask_type = kFecMaskRandom; + key_fec_params.fec_mask_type = kFecMaskRandom; + + // TODO(Marco): Pass FEC protection values per layer. + video_protection_callback->ProtectionRequest(&delta_fec_params, + &key_fec_params, + video_rate_bps, + nack_overhead_rate_bps, + fec_overhead_rate_bps); +} +} // namespace + +struct MediaOptimization::EncodedFrameSample { + EncodedFrameSample(int size_bytes, + uint32_t timestamp, + int64_t time_complete_ms) + : size_bytes(size_bytes), + timestamp(timestamp), + time_complete_ms(time_complete_ms) {} + + uint32_t size_bytes; + uint32_t timestamp; + int64_t time_complete_ms; +}; MediaOptimization::MediaOptimization(int32_t id, Clock* clock) : id_(id), @@ -35,8 +90,6 @@ MediaOptimization::MediaOptimization(int32_t id, Clock* clock) target_bit_rate_(0), incoming_frame_rate_(0), enable_qm_(false), - video_protection_callback_(NULL), - video_qmsettings_callback_(NULL), encoded_frame_samples_(), avg_sent_bit_rate_bps_(0), avg_sent_framerate_(0), @@ -59,7 +112,8 @@ MediaOptimization::~MediaOptimization(void) { loss_prot_logic_->Release(); } -int32_t MediaOptimization::Reset() { +void MediaOptimization::Reset() { + SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, max_payload_size_); memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); incoming_frame_rate_ = 0.0; frame_dropper_->Reset(); @@ -81,12 +135,52 @@ int32_t MediaOptimization::Reset() { encoded_frame_samples_.clear(); avg_sent_bit_rate_bps_ = 0; num_layers_ = 1; - return VCM_OK; } -uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate, - uint8_t fraction_lost, - uint32_t round_trip_time_ms) { +void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, + int32_t max_bit_rate, + uint32_t frame_rate, + uint32_t target_bitrate, + uint16_t width, + uint16_t height, + int num_layers, + int32_t mtu) { + // Everything codec specific should be reset here since this means the codec + // has changed. If native dimension values have changed, then either user + // initiated change, or QM initiated change. Will be able to determine only + // after the processing of the first frame. + last_change_time_ = clock_->TimeInMilliseconds(); + content_->Reset(); + content_->UpdateFrameRate(frame_rate); + + max_bit_rate_ = max_bit_rate; + send_codec_type_ = send_codec_type; + target_bit_rate_ = target_bitrate; + float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; + loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); + loss_prot_logic_->UpdateFrameRate(static_cast(frame_rate)); + loss_prot_logic_->UpdateFrameSize(width, height); + loss_prot_logic_->UpdateNumLayers(num_layers); + frame_dropper_->Reset(); + frame_dropper_->SetRates(target_bitrate_kbps, static_cast(frame_rate)); + user_frame_rate_ = static_cast(frame_rate); + codec_width_ = width; + codec_height_ = height; + num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. + max_payload_size_ = mtu; + qm_resolution_->Initialize(target_bitrate_kbps, + user_frame_rate_, + codec_width_, + codec_height_, + num_layers_); +} + +uint32_t MediaOptimization::SetTargetRates( + uint32_t target_bitrate, + uint8_t fraction_lost, + uint32_t round_trip_time_ms, + VCMProtectionCallback* protection_callback, + VCMQMSettingsCallback* qmsettings_callback) { // TODO(holmer): Consider putting this threshold only on the video bitrate, // and not on protection. if (max_bit_rate_ > 0 && @@ -145,10 +239,13 @@ uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate, // Get the bit cost of protection method, based on the amount of // overhead data actually transmitted (including headers) the last // second. - UpdateProtectionCallback(selected_method, - &sent_video_rate_bps, - &sent_nack_rate_bps, - &sent_fec_rate_bps); + if (protection_callback) { + UpdateProtectionCallback(selected_method, + &sent_video_rate_bps, + &sent_nack_rate_bps, + &sent_fec_rate_bps, + protection_callback); + } uint32_t sent_total_rate_bps = sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; // Estimate the overhead costs of the next second as staying the same @@ -178,7 +275,7 @@ uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate, static_cast(target_bit_rate_) / 1000.0f; frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); - if (enable_qm_) { + if (enable_qm_ && qmsettings_callback) { // Update QM with rates. qm_resolution_->UpdateRates(target_video_bitrate_kbps, sent_video_rate_kbps, @@ -187,7 +284,7 @@ uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate, // Check for QM selection. bool select_qm = CheckStatusForQMchange(); if (select_qm) { - SelectQuality(); + SelectQuality(qmsettings_callback); } // Reset the short-term averaged content data. content_->ResetShortTermAvgData(); @@ -198,44 +295,6 @@ uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate, return target_bit_rate_; } -int32_t MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, - int32_t max_bit_rate, - uint32_t frame_rate, - uint32_t target_bitrate, - uint16_t width, - uint16_t height, - int num_layers) { - // Everything codec specific should be reset here since this means the codec - // has changed. If native dimension values have changed, then either user - // initiated change, or QM initiated change. Will be able to determine only - // after the processing of the first frame. - last_change_time_ = clock_->TimeInMilliseconds(); - content_->Reset(); - content_->UpdateFrameRate(frame_rate); - - max_bit_rate_ = max_bit_rate; - send_codec_type_ = send_codec_type; - target_bit_rate_ = target_bitrate; - float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; - loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); - loss_prot_logic_->UpdateFrameRate(static_cast(frame_rate)); - loss_prot_logic_->UpdateFrameSize(width, height); - loss_prot_logic_->UpdateNumLayers(num_layers); - frame_dropper_->Reset(); - frame_dropper_->SetRates(target_bitrate_kbps, static_cast(frame_rate)); - user_frame_rate_ = static_cast(frame_rate); - codec_width_ = width; - codec_height_ = height; - num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. - int32_t ret = VCM_OK; - ret = qm_resolution_->Initialize(target_bitrate_kbps, - user_frame_rate_, - codec_width_, - codec_height_, - num_layers_); - return ret; -} - void MediaOptimization::EnableProtectionMethod(bool enable, VCMProtectionMethodEnum method) { bool updated = false; @@ -249,11 +308,6 @@ void MediaOptimization::EnableProtectionMethod(bool enable, } } -bool MediaOptimization::IsProtectionMethodEnabled( - VCMProtectionMethodEnum method) { - return (loss_prot_logic_->SelectedType() == method); -} - uint32_t MediaOptimization::InputFrameRate() { ProcessIncomingFrameRate(clock_->TimeInMilliseconds()); return uint32_t(incoming_frame_rate_ + 0.5f); @@ -272,6 +326,13 @@ uint32_t MediaOptimization::SentBitRate() { return avg_sent_bit_rate_bps_; } +VCMFrameCount MediaOptimization::SentFrameCount() { + VCMFrameCount count; + count.numDeltaFrames = delta_frame_cnt_; + count.numKeyFrames = key_frame_cnt_; + return count; +} + int32_t MediaOptimization::UpdateWithEncodedData(int encoded_length, uint32_t timestamp, FrameType encoded_frame_type) { @@ -325,29 +386,14 @@ int32_t MediaOptimization::UpdateWithEncodedData(int encoded_length, return VCM_OK; } -int32_t MediaOptimization::RegisterProtectionCallback( - VCMProtectionCallback* protection_callback) { - video_protection_callback_ = protection_callback; - return VCM_OK; -} - -int32_t MediaOptimization::RegisterVideoQMCallback( - VCMQMSettingsCallback* video_qmsettings) { - video_qmsettings_callback_ = video_qmsettings; - // Callback setting controls QM. - if (video_qmsettings_callback_ != NULL) { - enable_qm_ = true; - } else { - enable_qm_ = false; - } - return VCM_OK; -} +void MediaOptimization::EnableQM(bool enable) { enable_qm_ = enable; } void MediaOptimization::EnableFrameDropper(bool enable) { frame_dropper_->Enable(enable); } bool MediaOptimization::DropFrame() { + UpdateIncomingFrameRate(); // Leak appropriate number of bytes. frame_dropper_->Leak((uint32_t)(InputFrameRate() + 0.5f)); if (video_suspended_) { @@ -356,12 +402,6 @@ bool MediaOptimization::DropFrame() { return frame_dropper_->DropFrame(); } -int32_t MediaOptimization::SentFrameCount(VCMFrameCount* frame_count) const { - frame_count->numDeltaFrames = delta_frame_cnt_; - frame_count->numKeyFrames = key_frame_cnt_; - return VCM_OK; -} - void MediaOptimization::UpdateIncomingFrameRate() { int64_t now = clock_->TimeInMilliseconds(); if (incoming_frame_times_[0] == 0) { @@ -388,7 +428,8 @@ void MediaOptimization::UpdateContentData( } } -int32_t MediaOptimization::SelectQuality() { +int32_t MediaOptimization::SelectQuality( + VCMQMSettingsCallback* video_qmsettings_callback) { // Reset quantities for QM select. qm_resolution_->ResetQM(); @@ -403,7 +444,7 @@ int32_t MediaOptimization::SelectQuality() { } // Check for updates to spatial/temporal modes. - QMUpdate(qm); + QMUpdate(qm, video_qmsettings_callback); // Reset all the rate and related frame counters quantities. qm_resolution_->ResetRates(); @@ -426,50 +467,7 @@ void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps, video_suspended_ = false; } -// Private methods below this line. - -int MediaOptimization::UpdateProtectionCallback( - VCMProtectionMethod* selected_method, - uint32_t* video_rate_bps, - uint32_t* nack_overhead_rate_bps, - uint32_t* fec_overhead_rate_bps) { - if (!video_protection_callback_) { - return VCM_OK; - } - FecProtectionParams delta_fec_params; - FecProtectionParams key_fec_params; - // Get the FEC code rate for Key frames (set to 0 when NA). - key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK(); - - // Get the FEC code rate for Delta frames (set to 0 when NA). - delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD(); - - // Get the FEC-UEP protection status for Key frames: UEP on/off. - key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK(); - - // Get the FEC-UEP protection status for Delta frames: UEP on/off. - delta_fec_params.use_uep_protection = - selected_method->RequiredUepProtectionD(); - - // The RTP module currently requires the same |max_fec_frames| for both - // key and delta frames. - delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); - key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); - - // Set the FEC packet mask type. |kFecMaskBursty| is more effective for - // consecutive losses and little/no packet re-ordering. As we currently - // do not have feedback data on the degree of correlated losses and packet - // re-ordering, we keep default setting to |kFecMaskRandom| for now. - delta_fec_params.fec_mask_type = kFecMaskRandom; - key_fec_params.fec_mask_type = kFecMaskRandom; - - // TODO(Marco): Pass FEC protection values per layer. - return video_protection_callback_->ProtectionRequest(&delta_fec_params, - &key_fec_params, - video_rate_bps, - nack_overhead_rate_bps, - fec_overhead_rate_bps); -} +bool MediaOptimization::IsVideoSuspended() const { return video_suspended_; } void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { while (!encoded_frame_samples_.empty()) { @@ -518,7 +516,9 @@ void MediaOptimization::UpdateSentFramerate() { } } -bool MediaOptimization::QMUpdate(VCMResolutionScale* qm) { +bool MediaOptimization::QMUpdate( + VCMResolutionScale* qm, + VCMQMSettingsCallback* video_qmsettings_callback) { // Check for no change. if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) { return false; @@ -551,7 +551,7 @@ bool MediaOptimization::QMUpdate(VCMResolutionScale* qm) { // will vary/fluctuate, and since we don't want to change the state of the // VPM frame dropper, unless a temporal action was selected, we use the // quantity |qm->frame_rate| for updating. - video_qmsettings_callback_->SetVideoQMSettings( + video_qmsettings_callback->SetVideoQMSettings( qm->frame_rate, codec_width_, codec_height_); content_->UpdateFrameRate(qm->frame_rate); qm_resolution_->UpdateCodecParameters( diff --git a/webrtc/modules/video_coding/main/source/media_optimization.h b/webrtc/modules/video_coding/main/source/media_optimization.h index cde28d2328..3c3c8e4454 100644 --- a/webrtc/modules/video_coding/main/source/media_optimization.h +++ b/webrtc/modules/video_coding/main/source/media_optimization.h @@ -29,33 +29,25 @@ class VCMContentMetricsProcessing; namespace media_optimization { -enum { - kBitrateMaxFrameSamples = 60 -}; -enum { - kBitrateAverageWinMs = 1000 -}; - -struct EncodedFrameSample { - EncodedFrameSample(int size_bytes, - uint32_t timestamp, - int64_t time_complete_ms) - : size_bytes(size_bytes), - timestamp(timestamp), - time_complete_ms(time_complete_ms) {} - - uint32_t size_bytes; - uint32_t timestamp; - int64_t time_complete_ms; -}; - +// TODO(andresp): Make thread safe. class MediaOptimization { public: MediaOptimization(int32_t id, Clock* clock); - ~MediaOptimization(void); + ~MediaOptimization(); - // Resets the Media Optimization module. - int32_t Reset(); + // TODO(andresp): Can Reset and SetEncodingData be done at construction time + // only? + void Reset(); + + // Informs media optimization of initial encoding state. + void SetEncodingData(VideoCodecType send_codec_type, + int32_t max_bit_rate, + uint32_t frame_rate, + uint32_t bit_rate, + uint16_t width, + uint16_t height, + int num_temporal_layers, + int32_t mtu); // Sets target rates for the encoder given the channel parameters. // Inputs: target bitrate - the encoder target bitrate in bits/s. @@ -63,95 +55,64 @@ class MediaOptimization { // round_trip_time_ms - round trip time in milliseconds. // min_bit_rate - the bit rate of the end-point with lowest rate. // max_bit_rate - the bit rate of the end-point with highest rate. + // TODO(andresp): Find if the callbacks can be triggered only after releasing + // an internal critical section. uint32_t SetTargetRates(uint32_t target_bitrate, uint8_t fraction_lost, - uint32_t round_trip_time_ms); + uint32_t round_trip_time_ms, + VCMProtectionCallback* protection_callback, + VCMQMSettingsCallback* qmsettings_callback); - // Informs media optimization of initial encoding state. - int32_t SetEncodingData(VideoCodecType send_codec_type, - int32_t max_bit_rate, - uint32_t frame_rate, - uint32_t bit_rate, - uint16_t width, - uint16_t height, - int num_temporal_layers); - - // Enables protection method. void EnableProtectionMethod(bool enable, VCMProtectionMethodEnum method); + void EnableQM(bool enable); + void EnableFrameDropper(bool enable); - // Returns weather or not protection method is enabled. - bool IsProtectionMethodEnabled(VCMProtectionMethodEnum method); + // Lets the sender suspend video when the rate drops below + // |threshold_bps|, and turns back on when the rate goes back up above + // |threshold_bps| + |window_bps|. + void SuspendBelowMinBitrate(int threshold_bps, int window_bps); + bool IsVideoSuspended() const; - // Returns the actual input frame rate. - uint32_t InputFrameRate(); + bool DropFrame(); - // Returns the actual sent frame rate. - uint32_t SentFrameRate(); - - // Returns the actual sent bit rate. - uint32_t SentBitRate(); + void UpdateContentData(const VideoContentMetrics* content_metrics); // Informs Media Optimization of encoding output: Length and frame type. int32_t UpdateWithEncodedData(int encoded_length, uint32_t timestamp, FrameType encoded_frame_type); - // Registers a protection callback to be used to inform the user about the - // protection methods used. - int32_t RegisterProtectionCallback( - VCMProtectionCallback* protection_callback); - - // Registers a quality settings callback to be used to inform VPM/user. - int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* video_qmsettings); - - void EnableFrameDropper(bool enable); - - bool DropFrame(); - - // Returns the number of key/delta frames encoded. - int32_t SentFrameCount(VCMFrameCount* frame_count) const; - - // Updates incoming frame rate value. - void UpdateIncomingFrameRate(); - - // Update content metric data. - void UpdateContentData(const VideoContentMetrics* content_metrics); - - // Computes new Quality Mode. - int32_t SelectQuality(); - - // Lets the sender suspend video when the rate drops below - // |threshold_bps|, and turns back on when the rate goes back up above - // |threshold_bps| + |window_bps|. - void SuspendBelowMinBitrate(int threshold_bps, int window_bps); - - // Accessors and mutators. - int32_t max_bit_rate() const { return max_bit_rate_; } - void set_max_payload_size(int32_t mtu) { max_payload_size_ = mtu; } - bool video_suspended() const { return video_suspended_; } + uint32_t InputFrameRate(); + uint32_t SentFrameRate(); + uint32_t SentBitRate(); + VCMFrameCount SentFrameCount(); private: - typedef std::list FrameSampleList; enum { kFrameCountHistorySize = 90 }; enum { kFrameHistoryWinMs = 2000 }; + enum { + kBitrateAverageWinMs = 1000 + }; - // Updates protection callback with protection settings. - int UpdateProtectionCallback(VCMProtectionMethod* selected_method, - uint32_t* total_video_rate_bps, - uint32_t* nack_overhead_rate_bps, - uint32_t* fec_overhead_rate_bps); + struct EncodedFrameSample; + typedef std::list FrameSampleList; + void UpdateIncomingFrameRate(); void PurgeOldFrameSamples(int64_t now_ms); void UpdateSentBitrate(int64_t now_ms); void UpdateSentFramerate(); + // Computes new Quality Mode. + int32_t SelectQuality(VCMQMSettingsCallback* qmsettings_callback); + // Verifies if QM settings differ from default, i.e. if an update is required. // Computes actual values, as will be sent to the encoder. - bool QMUpdate(VCMResolutionScale* qm); + bool QMUpdate(VCMResolutionScale* qm, + VCMQMSettingsCallback* qmsettings_callback); // Checks if we should make a QM change. Return true if yes, false otherwise. bool CheckStatusForQMchange(); @@ -180,8 +141,6 @@ class MediaOptimization { float incoming_frame_rate_; int64_t incoming_frame_times_[kFrameCountHistorySize]; bool enable_qm_; - VCMProtectionCallback* video_protection_callback_; - VCMQMSettingsCallback* video_qmsettings_callback_; std::list encoded_frame_samples_; uint32_t avg_sent_bit_rate_bps_; uint32_t avg_sent_framerate_; @@ -196,8 +155,7 @@ class MediaOptimization { bool video_suspended_; int suspension_threshold_bps_; int suspension_window_bps_; -}; // End of MediaOptimization class declaration. - +}; } // namespace media_optimization } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/media_optimization_unittest.cc b/webrtc/modules/video_coding/main/source/media_optimization_unittest.cc index 1425dad837..d58ada651d 100644 --- a/webrtc/modules/video_coding/main/source/media_optimization_unittest.cc +++ b/webrtc/modules/video_coding/main/source/media_optimization_unittest.cc @@ -35,7 +35,6 @@ class TestMediaOptimization : public ::testing::Test { // This method mimics what happens in VideoSender::AddVideoFrame. void AddFrameAndAdvanceTime(int bitrate_bps, bool expect_frame_drop) { ASSERT_GE(bitrate_bps, 0); - media_opt_.UpdateIncomingFrameRate(); bool frame_dropped = media_opt_.DropFrame(); EXPECT_EQ(expect_frame_drop, frame_dropped); if (!frame_dropped) { @@ -63,12 +62,14 @@ TEST_F(TestMediaOptimization, VerifyMuting) { media_opt_.SuspendBelowMinBitrate(kThresholdBps, kWindowBps); // The video should not be suspended from the start. - EXPECT_FALSE(media_opt_.video_suspended()); + EXPECT_FALSE(media_opt_.IsVideoSuspended()); int target_bitrate_kbps = 100; media_opt_.SetTargetRates(target_bitrate_kbps * 1000, 0, // Lossrate. - 100); // RTT in ms. + 100, + NULL, + NULL); // RTT in ms. media_opt_.EnableFrameDropper(true); for (int time = 0; time < 2000; time += frame_time_ms_) { ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, false)); @@ -77,11 +78,13 @@ TEST_F(TestMediaOptimization, VerifyMuting) { // Set the target rate below the limit for muting. media_opt_.SetTargetRates(kThresholdBps - 1000, 0, // Lossrate. - 100); // RTT in ms. + 100, + NULL, + NULL); // RTT in ms. // Expect the muter to engage immediately and stay muted. // Test during 2 seconds. for (int time = 0; time < 2000; time += frame_time_ms_) { - EXPECT_TRUE(media_opt_.video_suspended()); + EXPECT_TRUE(media_opt_.IsVideoSuspended()); ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, true)); } @@ -89,22 +92,26 @@ TEST_F(TestMediaOptimization, VerifyMuting) { // limit + window. media_opt_.SetTargetRates(kThresholdBps + 1000, 0, // Lossrate. - 100); // RTT in ms. - // Expect the muter to stay muted. + 100, + NULL, + NULL); // RTT in ms. + // Expect the muter to stay muted. // Test during 2 seconds. for (int time = 0; time < 2000; time += frame_time_ms_) { - EXPECT_TRUE(media_opt_.video_suspended()); + EXPECT_TRUE(media_opt_.IsVideoSuspended()); ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, true)); } // Set the target above limit + window. media_opt_.SetTargetRates(kThresholdBps + kWindowBps + 1000, 0, // Lossrate. - 100); // RTT in ms. + 100, + NULL, + NULL); // RTT in ms. // Expect the muter to disengage immediately. // Test during 2 seconds. for (int time = 0; time < 2000; time += frame_time_ms_) { - EXPECT_FALSE(media_opt_.video_suspended()); + EXPECT_FALSE(media_opt_.IsVideoSuspended()); ASSERT_NO_FATAL_FAILURE( AddFrameAndAdvanceTime((kThresholdBps + kWindowBps) / 1000, false)); } diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.h b/webrtc/modules/video_coding/main/source/video_coding_impl.h index d9564c04bd..5ecb02aad3 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.h +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.h @@ -68,7 +68,8 @@ class VideoSender { uint8_t payloadType, bool internalSource); - int32_t CodecConfigParameters(uint8_t* buffer, int32_t size); + int32_t CodecConfigParameters(uint8_t* buffer, int32_t size) const; + int32_t SentFrameCount(VCMFrameCount* frameCount); int Bitrate(unsigned int* bitrate) const; int FrameRate(unsigned int* framerate) const; @@ -88,7 +89,6 @@ class VideoSender { int32_t IntraFrameRequest(int stream_index); int32_t EnableFrameDropper(bool enable); - int32_t SentFrameCount(VCMFrameCount* frameCount) const; int SetSenderNackMode(SenderNackMode mode); int SetSenderReferenceSelection(bool enable); @@ -122,6 +122,9 @@ class VideoSender { VCMCodecDataBase _codecDataBase; bool frame_dropper_enabled_; VCMProcessTimer _sendStatsTimer; + + VCMQMSettingsCallback* qm_settings_callback_; + VCMProtectionCallback* protection_callback_; }; class VideoReceiver { diff --git a/webrtc/modules/video_coding/main/source/video_sender.cc b/webrtc/modules/video_coding/main/source/video_sender.cc index 948218b83a..d19896666d 100644 --- a/webrtc/modules/video_coding/main/source/video_sender.cc +++ b/webrtc/modules/video_coding/main/source/video_sender.cc @@ -34,7 +34,9 @@ VideoSender::VideoSender(const int32_t id, Clock* clock) _encoderInputFile(NULL), _codecDataBase(id), frame_dropper_enabled_(true), - _sendStatsTimer(1000, clock_) {} + _sendStatsTimer(1000, clock_), + qm_settings_callback_(NULL), + protection_callback_(NULL) {} VideoSender::~VideoSender() { delete _sendCritSect; @@ -70,8 +72,6 @@ int32_t VideoSender::InitializeSender() { _codecDataBase.ResetSender(); _encoder = NULL; _encodedFrameCallback.SetTransportCallback(NULL); - // setting default bitRate and frameRate to 0 - _mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0); _mediaOpt.Reset(); // Resetting frame dropper return VCM_OK; } @@ -125,9 +125,8 @@ int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, sendCodec->startBitrate * 1000, sendCodec->width, sendCodec->height, - numLayers); - _mediaOpt.set_max_payload_size(maxPayloadSize); - + numLayers, + maxPayloadSize); return VCM_OK; } @@ -171,7 +170,8 @@ int32_t VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder, } // Get codec config parameters -int32_t VideoSender::CodecConfigParameters(uint8_t* buffer, int32_t size) { +int32_t VideoSender::CodecConfigParameters(uint8_t* buffer, + int32_t size) const { CriticalSectionScoped cs(_sendCritSect); if (_encoder != NULL) { return _encoder->CodecConfigParameters(buffer, size); @@ -179,6 +179,14 @@ int32_t VideoSender::CodecConfigParameters(uint8_t* buffer, int32_t size) { return VCM_UNINITIALIZED; } +// TODO(andresp): Make const once media_opt is thread-safe and this has a +// pointer to it. +int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) { + CriticalSectionScoped cs(_sendCritSect); + *frameCount = _mediaOpt.SentFrameCount(); + return VCM_OK; +} + // Get encode bitrate int VideoSender::Bitrate(unsigned int* bitrate) const { CriticalSectionScoped cs(_sendCritSect); @@ -208,8 +216,11 @@ int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate, int32_t ret = 0; { CriticalSectionScoped sendCs(_sendCritSect); - uint32_t targetRate = - _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt); + uint32_t targetRate = _mediaOpt.SetTargetRates(target_bitrate, + lossRate, + rtt, + protection_callback_, + qm_settings_callback_); if (_encoder != NULL) { ret = _encoder->SetChannelParameters(lossRate, rtt); if (ret < 0) { @@ -247,17 +258,19 @@ int32_t VideoSender::RegisterSendStatisticsCallback( // Register a video quality settings callback which will be called when frame // rate/dimensions need to be updated for video quality optimization int32_t VideoSender::RegisterVideoQMCallback( - VCMQMSettingsCallback* videoQMSettings) { + VCMQMSettingsCallback* qm_settings_callback) { CriticalSectionScoped cs(_sendCritSect); - return _mediaOpt.RegisterVideoQMCallback(videoQMSettings); + qm_settings_callback_ = qm_settings_callback; + _mediaOpt.EnableQM(qm_settings_callback_ != NULL); + return VCM_OK; } // Register a video protection callback which will be called to deliver the // requested FEC rate and NACK status (on/off). int32_t VideoSender::RegisterProtectionCallback( - VCMProtectionCallback* protection) { + VCMProtectionCallback* protection_callback) { CriticalSectionScoped cs(_sendCritSect); - _mediaOpt.RegisterProtectionCallback(protection); + protection_callback_ = protection_callback; return VCM_OK; } @@ -314,8 +327,6 @@ int32_t VideoSender::AddVideoFrame(const I420VideoFrame& videoFrame, if (_nextFrameTypes[0] == kFrameEmpty) { return VCM_OK; } - _mediaOpt.UpdateIncomingFrameRate(); - if (_mediaOpt.DropFrame()) { WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideoCoding, @@ -369,11 +380,6 @@ int32_t VideoSender::EnableFrameDropper(bool enable) { return VCM_OK; } -int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) const { - CriticalSectionScoped cs(_sendCritSect); - return _mediaOpt.SentFrameCount(frameCount); -} - int VideoSender::SetSenderNackMode(SenderNackMode mode) { CriticalSectionScoped cs(_sendCritSect); @@ -443,7 +449,7 @@ void VideoSender::SuspendBelowMinBitrate() { bool VideoSender::VideoSuspended() const { CriticalSectionScoped cs(_sendCritSect); - return _mediaOpt.video_suspended(); + return _mediaOpt.IsVideoSuspended(); } void VideoSender::RegisterPostEncodeImageCallback(