diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index b2d62f4125..d29932fc81 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -370,6 +370,7 @@ 'video_coding/nack_module_unittest.cc', 'video_coding/packet_buffer_unittest.cc', 'video_coding/percentile_filter_unittest.cc', + 'video_coding/protection_bitrate_calculator_unittest.cc', 'video_coding/receiver_unittest.cc', 'video_coding/session_info_unittest.cc', 'video_coding/sequence_number_util_unittest.cc', diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index 0ea5ea2455..5f865b3b19 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -54,6 +54,8 @@ source_set("video_coding") { "packet_buffer.h", "percentile_filter.cc", "percentile_filter.h", + "protection_bitrate_calculator.cc", + "protection_bitrate_calculator.h", "receiver.cc", "receiver.h", "rtp_frame_reference_finder.cc", diff --git a/webrtc/modules/video_coding/include/video_coding.h b/webrtc/modules/video_coding/include/video_coding.h index 0f8567963a..611206e325 100644 --- a/webrtc/modules/video_coding/include/video_coding.h +++ b/webrtc/modules/video_coding/include/video_coding.h @@ -188,14 +188,10 @@ class VideoCodingModule : public Module { // < 0, on error. virtual int32_t SetReceiveChannelParameters(int64_t rtt) = 0; + // Deprecated: This method currently does not have any effect. // Register a video protection callback which will be called to deliver // the requested FEC rate and NACK status (on/off). - // - // Input: - // - protection : The callback object to register. - // - // Return value : VCM_OK, on success. - // < 0, on error. + // TODO(perkj): Remove once no projects use it. virtual int32_t RegisterProtectionCallback( VCMProtectionCallback* protection) = 0; diff --git a/webrtc/modules/video_coding/media_optimization.cc b/webrtc/modules/video_coding/media_optimization.cc index d5fbadc122..7655649283 100644 --- a/webrtc/modules/video_coding/media_optimization.cc +++ b/webrtc/modules/video_coding/media_optimization.cc @@ -16,39 +16,6 @@ namespace webrtc { namespace media_optimization { -namespace { -void UpdateProtectionCallback( - VCMProtectionMethod* selected_method, - uint32_t* video_rate_bps, - uint32_t* nack_overhead_rate_bps, - uint32_t* fec_overhead_rate_bps, - VCMProtectionCallback* video_protection_callback) { - FecProtectionParams delta_fec_params; - FecProtectionParams key_fec_params; - // Get the FEC code rate for Key frames (set to 0 when NA). - key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK(); - - // Get the FEC code rate for Delta frames (set to 0 when NA). - delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD(); - - // The RTP module currently requires the same |max_fec_frames| for both - // key and delta frames. - delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); - key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); - - // Set the FEC packet mask type. |kFecMaskBursty| is more effective for - // consecutive losses and little/no packet re-ordering. As we currently - // do not have feedback data on the degree of correlated losses and packet - // re-ordering, we keep default setting to |kFecMaskRandom| for now. - delta_fec_params.fec_mask_type = kFecMaskRandom; - key_fec_params.fec_mask_type = kFecMaskRandom; - - // TODO(Marco): Pass FEC protection values per layer. - video_protection_callback->ProtectionRequest( - &delta_fec_params, &key_fec_params, video_rate_bps, - nack_overhead_rate_bps, fec_overhead_rate_bps); -} -} // namespace struct MediaOptimization::EncodedFrameSample { EncodedFrameSample(size_t size_bytes, @@ -67,13 +34,10 @@ MediaOptimization::MediaOptimization(Clock* clock) : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), clock_(clock), max_bit_rate_(0), - send_codec_type_(kVideoCodecUnknown), codec_width_(0), codec_height_(0), user_frame_rate_(0), frame_dropper_(new FrameDropper), - loss_prot_logic_( - new VCMLossProtectionLogic(clock_->TimeInMilliseconds())), fraction_lost_(0), send_statistics_zero_encode_(0), max_payload_size_(1460), @@ -82,8 +46,6 @@ MediaOptimization::MediaOptimization(Clock* clock) encoded_frame_samples_(), avg_sent_bit_rate_bps_(0), avg_sent_framerate_(0), - key_frame_cnt_(0), - delta_frame_cnt_(0), num_layers_(0), suspension_enabled_(false), video_suspended_(false), @@ -94,34 +56,26 @@ MediaOptimization::MediaOptimization(Clock* clock) } MediaOptimization::~MediaOptimization(void) { - loss_prot_logic_->Release(); } void MediaOptimization::Reset() { CriticalSectionScoped lock(crit_sect_.get()); - SetEncodingDataInternal(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, - max_payload_size_); + SetEncodingDataInternal(0, 0, 0, 0, 0, 0, max_payload_size_); memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); incoming_frame_rate_ = 0.0; frame_dropper_->Reset(); - loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); frame_dropper_->SetRates(0, 0); - loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_); - loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); send_statistics_zero_encode_ = 0; video_target_bitrate_ = 0; codec_width_ = 0; codec_height_ = 0; user_frame_rate_ = 0; - key_frame_cnt_ = 0; - delta_frame_cnt_ = 0; encoded_frame_samples_.clear(); avg_sent_bit_rate_bps_ = 0; num_layers_ = 1; } -void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, - int32_t max_bit_rate, +void MediaOptimization::SetEncodingData(int32_t max_bit_rate, uint32_t target_bitrate, uint16_t width, uint16_t height, @@ -129,12 +83,11 @@ void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, int num_layers, int32_t mtu) { CriticalSectionScoped lock(crit_sect_.get()); - SetEncodingDataInternal(send_codec_type, max_bit_rate, frame_rate, - target_bitrate, width, height, num_layers, mtu); + SetEncodingDataInternal(max_bit_rate, frame_rate, target_bitrate, width, + height, num_layers, mtu); } -void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type, - int32_t max_bit_rate, +void MediaOptimization::SetEncodingDataInternal(int32_t max_bit_rate, uint32_t frame_rate, uint32_t target_bitrate, uint16_t width, @@ -145,13 +98,8 @@ void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type, // has changed. max_bit_rate_ = max_bit_rate; - send_codec_type_ = send_codec_type; video_target_bitrate_ = target_bitrate; float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; - loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); - loss_prot_logic_->UpdateFrameRate(static_cast(frame_rate)); - loss_prot_logic_->UpdateFrameSize(width, height); - loss_prot_logic_->UpdateNumLayers(num_layers); frame_dropper_->Reset(); frame_dropper_->SetRates(target_bitrate_kbps, static_cast(frame_rate)); user_frame_rate_ = static_cast(frame_rate); @@ -161,16 +109,10 @@ void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type, max_payload_size_ = mtu; } -uint32_t MediaOptimization::SetTargetRates( - uint32_t target_bitrate, - uint8_t fraction_lost, - int64_t round_trip_time_ms, - VCMProtectionCallback* protection_callback) { +uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate, + uint8_t fraction_lost, + int64_t round_trip_time_ms) { CriticalSectionScoped lock(crit_sect_.get()); - VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod(); - float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; - loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); - loss_prot_logic_->UpdateRtt(round_trip_time_ms); // Get frame rate for encoder: this is the actual/sent frame rate. float actual_frame_rate = SentFrameRateInternal(); @@ -180,65 +122,9 @@ uint32_t MediaOptimization::SetTargetRates( actual_frame_rate = 1.0; } - // Update frame rate for the loss protection logic class: frame rate should - // be the actual/sent rate. - loss_prot_logic_->UpdateFrameRate(actual_frame_rate); - fraction_lost_ = fraction_lost; - // Returns the filtered packet loss, used for the protection setting. - // The filtered loss may be the received loss (no filter), or some - // filtered value (average or max window filter). - // Use max window filter for now. - FilterPacketLossMode filter_mode = kMaxFilter; - uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss( - clock_->TimeInMilliseconds(), filter_mode, fraction_lost); - - // For now use the filtered loss for computing the robustness settings. - loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc); - - // Rate cost of the protection methods. - float protection_overhead_rate = 0.0f; - - // Update protection settings, when applicable. - if (loss_prot_logic_->SelectedType() != kNone) { - // Update method will compute the robustness settings for the given - // protection method and the overhead cost - // the protection method is set by the user via SetVideoProtection. - loss_prot_logic_->UpdateMethod(); - - // Update protection callback with protection settings. - uint32_t sent_video_rate_bps = 0; - uint32_t sent_nack_rate_bps = 0; - uint32_t sent_fec_rate_bps = 0; - // Get the bit cost of protection method, based on the amount of - // overhead data actually transmitted (including headers) the last - // second. - if (protection_callback) { - UpdateProtectionCallback(selected_method, &sent_video_rate_bps, - &sent_nack_rate_bps, &sent_fec_rate_bps, - protection_callback); - } - uint32_t sent_total_rate_bps = - sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; - // Estimate the overhead costs of the next second as staying the same - // wrt the source bitrate. - if (sent_total_rate_bps > 0) { - protection_overhead_rate = - static_cast(sent_nack_rate_bps + sent_fec_rate_bps) / - sent_total_rate_bps; - } - // Cap the overhead estimate to 50%. - if (protection_overhead_rate > 0.5) - protection_overhead_rate = 0.5; - - // Get the effective packet loss for encoder ER when applicable. Should be - // passed to encoder via fraction_lost. - packet_loss_enc = selected_method->RequiredPacketLossER(); - } - - // Source coding rate: total rate - protection overhead. - video_target_bitrate_ = target_bitrate * (1.0 - protection_overhead_rate); + video_target_bitrate_ = target_bitrate; // Cap target video bitrate to codec maximum. if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) { @@ -255,11 +141,6 @@ uint32_t MediaOptimization::SetTargetRates( return video_target_bitrate_; } -void MediaOptimization::SetProtectionMethod(VCMProtectionMethodEnum method) { - CriticalSectionScoped lock(crit_sect_.get()); - loss_prot_logic_->SetMethod(method); -} - uint32_t MediaOptimization::InputFrameRate() { CriticalSectionScoped lock(crit_sect_.get()); return InputFrameRateInternal(); @@ -311,29 +192,7 @@ int32_t MediaOptimization::UpdateWithEncodedData( UpdateSentFramerate(); if (encoded_length > 0) { const bool delta_frame = encoded_image._frameType != kVideoFrameKey; - frame_dropper_->Fill(encoded_length, delta_frame); - if (max_payload_size_ > 0 && encoded_length > 0) { - const float min_packets_per_frame = - encoded_length / static_cast(max_payload_size_); - if (delta_frame) { - loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame, - clock_->TimeInMilliseconds()); - } else { - loss_prot_logic_->UpdatePacketsPerFrameKey( - min_packets_per_frame, clock_->TimeInMilliseconds()); - } - } - if (!delta_frame && encoded_length > 0) { - loss_prot_logic_->UpdateKeyFrameSize(static_cast(encoded_length)); - } - - // Updating counters. - if (delta_frame) { - delta_frame_cnt_++; - } else { - key_frame_cnt_++; - } } return VCM_OK; diff --git a/webrtc/modules/video_coding/media_optimization.h b/webrtc/modules/video_coding/media_optimization.h index 081b2a900a..a14bacdacf 100644 --- a/webrtc/modules/video_coding/media_optimization.h +++ b/webrtc/modules/video_coding/media_optimization.h @@ -38,8 +38,9 @@ class MediaOptimization { void Reset(); // Informs media optimization of initial encoding state. - void SetEncodingData(VideoCodecType send_codec_type, - int32_t max_bit_rate, + // TODO(perkj): Deprecate SetEncodingData once its not used for stats in + // VieEncoder. + void SetEncodingData(int32_t max_bit_rate, uint32_t bit_rate, uint16_t width, uint16_t height, @@ -53,14 +54,10 @@ class MediaOptimization { // round_trip_time_ms - round trip time in milliseconds. // min_bit_rate - the bit rate of the end-point with lowest rate. // max_bit_rate - the bit rate of the end-point with highest rate. - // TODO(andresp): Find if the callbacks can be triggered only after releasing - // an internal critical section. uint32_t SetTargetRates(uint32_t target_bitrate, uint8_t fraction_lost, - int64_t round_trip_time_ms, - VCMProtectionCallback* protection_callback); + int64_t round_trip_time_ms); - void SetProtectionMethod(VCMProtectionMethodEnum method); void EnableFrameDropper(bool enable); // Lets the sender suspend video when the rate drops below @@ -72,6 +69,8 @@ class MediaOptimization { bool DropFrame(); // Informs Media Optimization of encoded output. + // TODO(perkj): Deprecate SetEncodingData once its not used for stats in + // VieEncoder. int32_t UpdateWithEncodedData(const EncodedImage& encoded_image); // InputFrameRate 0 = no frame rate estimate available. @@ -101,8 +100,7 @@ class MediaOptimization { // changes the state of |video_suspended_| accordingly. void CheckSuspendConditions() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); - void SetEncodingDataInternal(VideoCodecType send_codec_type, - int32_t max_bit_rate, + void SetEncodingDataInternal(int32_t max_bit_rate, uint32_t frame_rate, uint32_t bit_rate, uint16_t width, @@ -120,13 +118,10 @@ class MediaOptimization { Clock* clock_ GUARDED_BY(crit_sect_); int32_t max_bit_rate_ GUARDED_BY(crit_sect_); - VideoCodecType send_codec_type_ GUARDED_BY(crit_sect_); uint16_t codec_width_ GUARDED_BY(crit_sect_); uint16_t codec_height_ GUARDED_BY(crit_sect_); float user_frame_rate_ GUARDED_BY(crit_sect_); std::unique_ptr frame_dropper_ GUARDED_BY(crit_sect_); - std::unique_ptr loss_prot_logic_ - GUARDED_BY(crit_sect_); uint8_t fraction_lost_ GUARDED_BY(crit_sect_); uint32_t send_statistics_[4] GUARDED_BY(crit_sect_); uint32_t send_statistics_zero_encode_ GUARDED_BY(crit_sect_); @@ -137,8 +132,6 @@ class MediaOptimization { std::list encoded_frame_samples_ GUARDED_BY(crit_sect_); uint32_t avg_sent_bit_rate_bps_ GUARDED_BY(crit_sect_); uint32_t avg_sent_framerate_ GUARDED_BY(crit_sect_); - uint32_t key_frame_cnt_ GUARDED_BY(crit_sect_); - uint32_t delta_frame_cnt_ GUARDED_BY(crit_sect_); int num_layers_ GUARDED_BY(crit_sect_); bool suspension_enabled_ GUARDED_BY(crit_sect_); bool video_suspended_ GUARDED_BY(crit_sect_); diff --git a/webrtc/modules/video_coding/media_optimization_unittest.cc b/webrtc/modules/video_coding/media_optimization_unittest.cc index e6a1bcccd9..2263099f23 100644 --- a/webrtc/modules/video_coding/media_optimization_unittest.cc +++ b/webrtc/modules/video_coding/media_optimization_unittest.cc @@ -64,9 +64,8 @@ TEST_F(TestMediaOptimization, VerifyMuting) { uint32_t target_bitrate_kbps = 100; media_opt_.SetTargetRates(target_bitrate_kbps * 1000, - 0, // Lossrate. - 100, // RTT in ms. - nullptr); + 0, // Lossrate. + 100); // RTT in ms. media_opt_.EnableFrameDropper(true); for (int time = 0; time < 2000; time += frame_time_ms_) { ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, false)); @@ -74,9 +73,8 @@ TEST_F(TestMediaOptimization, VerifyMuting) { // Set the target rate below the limit for muting. media_opt_.SetTargetRates(kThresholdBps - 1000, - 0, // Lossrate. - 100, // RTT in ms. - nullptr); + 0, // Lossrate. + 100); // RTT in ms. // Expect the muter to engage immediately and stay muted. // Test during 2 seconds. for (int time = 0; time < 2000; time += frame_time_ms_) { @@ -87,9 +85,8 @@ TEST_F(TestMediaOptimization, VerifyMuting) { // Set the target above the limit for muting, but not above the // limit + window. media_opt_.SetTargetRates(kThresholdBps + 1000, - 0, // Lossrate. - 100, // RTT in ms. - nullptr); + 0, // Lossrate. + 100); // RTT in ms. // Expect the muter to stay muted. // Test during 2 seconds. for (int time = 0; time < 2000; time += frame_time_ms_) { @@ -99,9 +96,8 @@ TEST_F(TestMediaOptimization, VerifyMuting) { // Set the target above limit + window. media_opt_.SetTargetRates(kThresholdBps + kWindowBps + 1000, - 0, // Lossrate. - 100, // RTT in ms. - nullptr); + 0, // Lossrate. + 100); // RTT in ms. // Expect the muter to disengage immediately. // Test during 2 seconds. for (int time = 0; time < 2000; time += frame_time_ms_) { @@ -111,44 +107,5 @@ TEST_F(TestMediaOptimization, VerifyMuting) { } } -TEST_F(TestMediaOptimization, ProtectsUsingFecBitrateAboveCodecMax) { - static const int kCodecBitrateBps = 100000; - static const int kMaxBitrateBps = 130000; - - class ProtectionCallback : public VCMProtectionCallback { - int ProtectionRequest(const FecProtectionParams* delta_params, - const FecProtectionParams* key_params, - uint32_t* sent_video_rate_bps, - uint32_t* sent_nack_rate_bps, - uint32_t* sent_fec_rate_bps) override { - *sent_video_rate_bps = kCodecBitrateBps; - *sent_nack_rate_bps = 0; - *sent_fec_rate_bps = fec_rate_bps_; - return 0; - } - - public: - uint32_t fec_rate_bps_; - } protection_callback; - - media_opt_.SetProtectionMethod(kFec); - media_opt_.SetEncodingData(kVideoCodecVP8, kCodecBitrateBps, kCodecBitrateBps, - 640, 480, 30, 1, 1000); - - // Using 10% of codec bitrate for FEC, should still be able to use all of it. - protection_callback.fec_rate_bps_ = kCodecBitrateBps / 10; - uint32_t target_bitrate = media_opt_.SetTargetRates( - kMaxBitrateBps, 0, 0, &protection_callback); - - EXPECT_EQ(kCodecBitrateBps, static_cast(target_bitrate)); - - // Using as much for codec bitrate as fec rate, new target rate should share - // both equally, but only be half of max (since that ceiling should be hit). - protection_callback.fec_rate_bps_ = kCodecBitrateBps; - target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 128, 100, - &protection_callback); - EXPECT_EQ(kMaxBitrateBps / 2, static_cast(target_bitrate)); -} - } // namespace media_optimization } // namespace webrtc diff --git a/webrtc/modules/video_coding/protection_bitrate_calculator.cc b/webrtc/modules/video_coding/protection_bitrate_calculator.cc new file mode 100644 index 0000000000..c92c3c445c --- /dev/null +++ b/webrtc/modules/video_coding/protection_bitrate_calculator.cc @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +namespace webrtc { + +using rtc::CritScope; + +struct ProtectionBitrateCalculator::EncodedFrameSample { + EncodedFrameSample(size_t size_bytes, + uint32_t timestamp, + int64_t time_complete_ms) + : size_bytes(size_bytes), + timestamp(timestamp), + time_complete_ms(time_complete_ms) {} + size_t size_bytes; + uint32_t timestamp; + int64_t time_complete_ms; +}; + +ProtectionBitrateCalculator::ProtectionBitrateCalculator( + Clock* clock, + VCMProtectionCallback* protection_callback) + : clock_(clock), + protection_callback_(protection_callback), + loss_prot_logic_(new media_optimization::VCMLossProtectionLogic( + clock_->TimeInMilliseconds())), + max_payload_size_(1460) {} + +ProtectionBitrateCalculator::~ProtectionBitrateCalculator(void) { + loss_prot_logic_->Release(); +} + +void ProtectionBitrateCalculator::SetEncodingData(uint32_t target_bitrate, + uint16_t width, + uint16_t height, + uint32_t frame_rate, + size_t num_temporal_layers, + size_t max_payload_size) { + CritScope lock(&crit_sect_); + // Everything codec specific should be reset here since this means the codec + // has changed. + float target_bitrate_kbps = static_cast(target_bitrate) / 1000.0f; + loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); + loss_prot_logic_->UpdateFrameRate(static_cast(frame_rate)); + loss_prot_logic_->UpdateFrameSize(width, height); + loss_prot_logic_->UpdateNumLayers(num_temporal_layers); + max_payload_size_ = max_payload_size; +} + +uint32_t ProtectionBitrateCalculator::SetTargetRates( + uint32_t estimated_bitrate_bps, + int actual_framerate_fps, + uint8_t fraction_lost, + int64_t round_trip_time_ms) { + float target_bitrate_kbps = + static_cast(estimated_bitrate_bps) / 1000.0f; + // Sanity check. + if (actual_framerate_fps < 1.0) { + actual_framerate_fps = 1.0; + } + + FecProtectionParams delta_fec_params; + FecProtectionParams key_fec_params; + { + CritScope lock(&crit_sect_); + + loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); + loss_prot_logic_->UpdateRtt(round_trip_time_ms); + + // Update frame rate for the loss protection logic class: frame rate should + // be the actual/sent rate. + loss_prot_logic_->UpdateFrameRate(actual_framerate_fps); + + // Returns the filtered packet loss, used for the protection setting. + // The filtered loss may be the received loss (no filter), or some + // filtered value (average or max window filter). + // Use max window filter for now. + media_optimization::FilterPacketLossMode filter_mode = + media_optimization::kMaxFilter; + uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss( + clock_->TimeInMilliseconds(), filter_mode, fraction_lost); + + // For now use the filtered loss for computing the robustness settings. + loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc); + + if (loss_prot_logic_->SelectedType() == media_optimization::kNone) { + return estimated_bitrate_bps; + } + + // Update method will compute the robustness settings for the given + // protection method and the overhead cost + // the protection method is set by the user via SetVideoProtection. + loss_prot_logic_->UpdateMethod(); + + // Get the bit cost of protection method, based on the amount of + // overhead data actually transmitted (including headers) the last + // second. + + // Get the FEC code rate for Key frames (set to 0 when NA). + key_fec_params.fec_rate = + loss_prot_logic_->SelectedMethod()->RequiredProtectionFactorK(); + + // Get the FEC code rate for Delta frames (set to 0 when NA). + delta_fec_params.fec_rate = + loss_prot_logic_->SelectedMethod()->RequiredProtectionFactorD(); + + // The RTP module currently requires the same |max_fec_frames| for both + // key and delta frames. + delta_fec_params.max_fec_frames = + loss_prot_logic_->SelectedMethod()->MaxFramesFec(); + key_fec_params.max_fec_frames = + loss_prot_logic_->SelectedMethod()->MaxFramesFec(); + } + + // Set the FEC packet mask type. |kFecMaskBursty| is more effective for + // consecutive losses and little/no packet re-ordering. As we currently + // do not have feedback data on the degree of correlated losses and packet + // re-ordering, we keep default setting to |kFecMaskRandom| for now. + delta_fec_params.fec_mask_type = kFecMaskRandom; + key_fec_params.fec_mask_type = kFecMaskRandom; + + // Update protection callback with protection settings. + uint32_t sent_video_rate_bps = 0; + uint32_t sent_nack_rate_bps = 0; + uint32_t sent_fec_rate_bps = 0; + // Rate cost of the protection methods. + float protection_overhead_rate = 0.0f; + + // TODO(Marco): Pass FEC protection values per layer. + protection_callback_->ProtectionRequest( + &delta_fec_params, &key_fec_params, &sent_video_rate_bps, + &sent_nack_rate_bps, &sent_fec_rate_bps); + + uint32_t sent_total_rate_bps = + sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; + // Estimate the overhead costs of the next second as staying the same + // wrt the source bitrate. + if (sent_total_rate_bps > 0) { + protection_overhead_rate = + static_cast(sent_nack_rate_bps + sent_fec_rate_bps) / + sent_total_rate_bps; + } + // Cap the overhead estimate to 50%. + if (protection_overhead_rate > 0.5) + protection_overhead_rate = 0.5; + + // Source coding rate: total rate - protection overhead. + return estimated_bitrate_bps * (1.0 - protection_overhead_rate); +} + +void ProtectionBitrateCalculator::SetProtectionMethod(bool enable_fec, + bool enable_nack) { + media_optimization::VCMProtectionMethodEnum method(media_optimization::kNone); + if (enable_fec && enable_nack) { + method = media_optimization::kNackFec; + } else if (enable_nack) { + method = media_optimization::kNack; + } else if (enable_fec) { + method = media_optimization::kFec; + } + CritScope lock(&crit_sect_); + loss_prot_logic_->SetMethod(method); +} + +void ProtectionBitrateCalculator::UpdateWithEncodedData( + const EncodedImage& encoded_image) { + const size_t encoded_length = encoded_image._length; + CritScope lock(&crit_sect_); + if (encoded_length > 0) { + const bool delta_frame = encoded_image._frameType != kVideoFrameKey; + + if (max_payload_size_ > 0 && encoded_length > 0) { + const float min_packets_per_frame = + encoded_length / static_cast(max_payload_size_); + if (delta_frame) { + loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame, + clock_->TimeInMilliseconds()); + } else { + loss_prot_logic_->UpdatePacketsPerFrameKey( + min_packets_per_frame, clock_->TimeInMilliseconds()); + } + } + if (!delta_frame && encoded_length > 0) { + loss_prot_logic_->UpdateKeyFrameSize(static_cast(encoded_length)); + } + } +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/protection_bitrate_calculator.h b/webrtc/modules/video_coding/protection_bitrate_calculator.h new file mode 100644 index 0000000000..cdbab102d4 --- /dev/null +++ b/webrtc/modules/video_coding/protection_bitrate_calculator.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_ +#define WEBRTC_MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_ + +#include +#include + +#include "webrtc/base/criticalsection.h" +#include "webrtc/modules/include/module_common_types.h" +#include "webrtc/modules/video_coding/include/video_coding.h" +#include "webrtc/modules/video_coding/media_opt_util.h" +#include "webrtc/system_wrappers/include/clock.h" + +namespace webrtc { + +// ProtectionBitrateCalculator calculates how much of the allocated network +// capacity that can be used by an encoder and how much that +// is needed for redundant packets such as FEC and NACK. It uses an +// implementation of |VCMProtectionCallback| to set new FEC parameters and get +// the bitrate currently used for FEC and NACK. +// Usage: +// Setup by calling SetProtectionMethod and SetEncodingData. +// For each encoded image, call UpdateWithEncodedData. +// Each time the bandwidth estimate change, call SetTargetRates. SetTargetRates +// will return the bitrate that can be used by an encoder. +// A lock is used to protect internal states, so methods can be called on an +// arbitrary thread. +class ProtectionBitrateCalculator { + public: + ProtectionBitrateCalculator(Clock* clock, + VCMProtectionCallback* protection_callback); + ~ProtectionBitrateCalculator(); + + void SetProtectionMethod(bool enable_fec, bool enable_nack); + + // Informs media optimization of initial encoding state. + void SetEncodingData(uint32_t estimated_bitrate_bps, + uint16_t width, + uint16_t height, + uint32_t frame_rate, + size_t num_temporal_layers, + size_t max_payload_size); + + // Returns target rate for the encoder given the channel parameters. + // Inputs: estimated_bitrate_bps - the estimated network bitrate in bits/s. + // actual_framerate - encoder frame rate. + // fraction_lost - packet loss rate in % in the network. + // round_trip_time_ms - round trip time in milliseconds. + uint32_t SetTargetRates(uint32_t estimated_bitrate_bps, + int actual_framerate, + uint8_t fraction_lost, + int64_t round_trip_time_ms); + + // Informs of encoded output. + void UpdateWithEncodedData(const EncodedImage& encoded_image); + + private: + struct EncodedFrameSample; + enum { kBitrateAverageWinMs = 1000 }; + + Clock* const clock_; + VCMProtectionCallback* const protection_callback_; + + rtc::CriticalSection crit_sect_; + std::unique_ptr loss_prot_logic_ + GUARDED_BY(crit_sect_); + size_t max_payload_size_ GUARDED_BY(crit_sect_); + + RTC_DISALLOW_COPY_AND_ASSIGN(ProtectionBitrateCalculator); +}; + +} // namespace webrtc +#endif // WEBRTC_MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_ diff --git a/webrtc/modules/video_coding/protection_bitrate_calculator_unittest.cc b/webrtc/modules/video_coding/protection_bitrate_calculator_unittest.cc new file mode 100644 index 0000000000..667dd6a359 --- /dev/null +++ b/webrtc/modules/video_coding/protection_bitrate_calculator_unittest.cc @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/modules/video_coding/protection_bitrate_calculator.h" +#include "webrtc/system_wrappers/include/clock.h" + +namespace webrtc { + +static const int kCodecBitrateBps = 100000; + +class ProtectionBitrateCalculatorTest : public ::testing::Test { + protected: + enum { + kSampleRate = 90000 // RTP timestamps per second. + }; + + class ProtectionCallback : public VCMProtectionCallback { + public: + int ProtectionRequest(const FecProtectionParams* delta_params, + const FecProtectionParams* key_params, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) override { + *sent_video_rate_bps = kCodecBitrateBps; + *sent_nack_rate_bps = nack_rate_bps_; + *sent_fec_rate_bps = fec_rate_bps_; + return 0; + } + + uint32_t fec_rate_bps_ = 0; + uint32_t nack_rate_bps_ = 0; + }; + + // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as + // a special case (e.g. frame rate in media optimization). + ProtectionBitrateCalculatorTest() + : clock_(1000), media_opt_(&clock_, &protection_callback_) {} + + SimulatedClock clock_; + ProtectionCallback protection_callback_; + ProtectionBitrateCalculator media_opt_; +}; + +TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingFecBitrate) { + static const uint32_t kMaxBitrateBps = 130000; + + media_opt_.SetProtectionMethod(true /*enable_fec*/, false /* enable_nack */); + media_opt_.SetEncodingData(kCodecBitrateBps, 640, 480, 30, 1, 1000); + + // Using 10% of codec bitrate for FEC. + protection_callback_.fec_rate_bps_ = kCodecBitrateBps / 10; + uint32_t target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 0, 0); + + EXPECT_GT(target_bitrate, 0u); + EXPECT_GT(kMaxBitrateBps, target_bitrate); + + // Using as much for codec bitrate as fec rate, new target rate should share + // both equally, but only be half of max (since that ceiling should be hit). + protection_callback_.fec_rate_bps_ = kCodecBitrateBps; + target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 128, 100); + EXPECT_EQ(kMaxBitrateBps / 2, target_bitrate); +} + +TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingNackBitrate) { + static const uint32_t kMaxBitrateBps = 130000; + + media_opt_.SetProtectionMethod(false /*enable_fec*/, true /* enable_nack */); + media_opt_.SetEncodingData(kCodecBitrateBps, 640, 480, 30, 1, 1000); + + uint32_t target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 0, 0); + + EXPECT_EQ(kMaxBitrateBps, target_bitrate); + + // Using as much for codec bitrate as nack rate, new target rate should share + // both equally, but only be half of max (since that ceiling should be hit). + protection_callback_.nack_rate_bps_ = kMaxBitrateBps; + target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 30, 128, 100); + EXPECT_EQ(kMaxBitrateBps / 2, target_bitrate); +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/video_coding.gypi b/webrtc/modules/video_coding/video_coding.gypi index 27454a4711..602f3686b7 100644 --- a/webrtc/modules/video_coding/video_coding.gypi +++ b/webrtc/modules/video_coding/video_coding.gypi @@ -50,6 +50,7 @@ 'packet.h', 'packet_buffer.h', 'percentile_filter.h', + 'protection_bitrate_calculator.h', 'receiver.h', 'rtt_filter.h', 'session_info.h', @@ -74,6 +75,7 @@ 'jitter_estimator.cc', 'media_opt_util.cc', 'media_optimization.cc', + 'protection_bitrate_calculator.cc', 'nack_module.cc', 'packet.cc', 'packet_buffer.cc', diff --git a/webrtc/modules/video_coding/video_coding_impl.cc b/webrtc/modules/video_coding/video_coding_impl.cc index 72bcc9a059..bb81ba9f9e 100644 --- a/webrtc/modules/video_coding/video_coding_impl.cc +++ b/webrtc/modules/video_coding/video_coding_impl.cc @@ -136,8 +136,6 @@ class VideoCodingModuleImpl : public VideoCodingModule { int32_t SetVideoProtection(VCMVideoProtection videoProtection, bool enable) override { // TODO(pbos): Remove enable from receive-side protection modes as well. - if (enable) - sender_.SetVideoProtection(videoProtection); return receiver_.SetVideoProtection(videoProtection, enable); } diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h index c9992b7f9c..2aabf05859 100644 --- a/webrtc/modules/video_coding/video_coding_impl.h +++ b/webrtc/modules/video_coding/video_coding_impl.h @@ -79,9 +79,9 @@ class VideoSender : public Module { int32_t SetChannelParameters(uint32_t target_bitrate, // bits/s. uint8_t lossRate, int64_t rtt); - + // Deprecated: + // TODO(perkj): Remove once no projects use it. int32_t RegisterProtectionCallback(VCMProtectionCallback* protection); - void SetVideoProtection(VCMVideoProtection videoProtection); int32_t AddVideoFrame(const VideoFrame& videoFrame, const CodecSpecificInfo* codecSpecificInfo); @@ -114,7 +114,6 @@ class VideoSender : public Module { VideoCodec current_codec_; rtc::ThreadChecker main_thread_; - VCMProtectionCallback* protection_callback_; rtc::CriticalSection params_crit_; EncoderParameters encoder_params_ GUARDED_BY(params_crit_); diff --git a/webrtc/modules/video_coding/video_sender.cc b/webrtc/modules/video_coding/video_sender.cc index f52b1c56e7..b9c5ea547a 100644 --- a/webrtc/modules/video_coding/video_sender.cc +++ b/webrtc/modules/video_coding/video_sender.cc @@ -37,7 +37,6 @@ VideoSender::VideoSender(Clock* clock, frame_dropper_enabled_(true), _sendStatsTimer(1000, clock_), current_codec_(), - protection_callback_(nullptr), encoder_params_({0, 0, 0, 0}), encoder_has_internal_source_(false), next_frame_types_(1, kVideoFrameDelta) { @@ -140,7 +139,7 @@ int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, << " start bitrate " << sendCodec->startBitrate << " max frame rate " << sendCodec->maxFramerate << " max payload size " << maxPayloadSize; - _mediaOpt.SetEncodingData(sendCodec->codecType, sendCodec->maxBitrate * 1000, + _mediaOpt.SetEncodingData(sendCodec->maxBitrate * 1000, sendCodec->startBitrate * 1000, sendCodec->width, sendCodec->height, sendCodec->maxFramerate, numLayers, maxPayloadSize); @@ -200,8 +199,8 @@ int VideoSender::FrameRate(unsigned int* framerate) const { int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate, uint8_t lossRate, int64_t rtt) { - uint32_t target_rate = _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt, - protection_callback_); + uint32_t target_rate = + _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt); uint32_t input_frame_rate = _mediaOpt.InputFrameRate(); @@ -243,35 +242,17 @@ void VideoSender::SetEncoderParameters(EncoderParameters params) { _encoder->SetEncoderParameters(params); } -// Register a video protection callback which will be called to deliver the -// requested FEC rate and NACK status (on/off). -// Note: this callback is assumed to only be registered once and before it is -// used in this class. +// Deprecated: +// TODO(perkj): Remove once no projects call this method. It currently do +// nothing. int32_t VideoSender::RegisterProtectionCallback( VCMProtectionCallback* protection_callback) { - RTC_DCHECK(protection_callback == nullptr || protection_callback_ == nullptr); - protection_callback_ = protection_callback; + // Deprecated: + // TODO(perkj): Remove once no projects call this method. It currently do + // nothing. return VCM_OK; } -// Enable or disable a video protection method. -void VideoSender::SetVideoProtection(VCMVideoProtection videoProtection) { - rtc::CritScope lock(&encoder_crit_); - switch (videoProtection) { - case kProtectionNone: - _mediaOpt.SetProtectionMethod(media_optimization::kNone); - break; - case kProtectionNack: - _mediaOpt.SetProtectionMethod(media_optimization::kNack); - break; - case kProtectionNackFEC: - _mediaOpt.SetProtectionMethod(media_optimization::kNackFec); - break; - case kProtectionFEC: - _mediaOpt.SetProtectionMethod(media_optimization::kFec); - break; - } -} // Add one raw video frame to the encoder, blocking. int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, const CodecSpecificInfo* codecSpecificInfo) { diff --git a/webrtc/video/send_statistics_proxy.cc b/webrtc/video/send_statistics_proxy.cc index d8c11a8c35..abaa438005 100644 --- a/webrtc/video/send_statistics_proxy.cc +++ b/webrtc/video/send_statistics_proxy.cc @@ -513,6 +513,11 @@ void SendStatisticsProxy::OnSendEncodedImage( static_cast(encoded_image._encodedHeight)); } +int SendStatisticsProxy::GetSendFrameRate() const { + rtc::CritScope lock(&crit_); + return stats_.encode_frame_rate; +} + void SendStatisticsProxy::OnIncomingFrame(int width, int height) { rtc::CritScope lock(&crit_); uma_container_->input_frame_rate_tracker_.AddSamples(1); diff --git a/webrtc/video/send_statistics_proxy.h b/webrtc/video/send_statistics_proxy.h index 74d5261ebd..60d50ec6f0 100644 --- a/webrtc/video/send_statistics_proxy.h +++ b/webrtc/video/send_statistics_proxy.h @@ -70,6 +70,8 @@ class SendStatisticsProxy : public CpuOveruseMetricsObserver, void OnEncodedFrameTimeMeasured(int encode_time_ms, const CpuOveruseMetrics& metrics) override; + int GetSendFrameRate() const; + protected: // From RtcpStatisticsCallback. void StatisticsUpdated(const RtcpStatistics& statistics, diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc index afd9909dc0..6af7ff6303 100644 --- a/webrtc/video/video_send_stream.cc +++ b/webrtc/video/video_send_stream.cc @@ -387,6 +387,7 @@ VideoSendStream::VideoSendStream( encoder_feedback_(Clock::GetRealTimeClock(), config.rtp.ssrcs, &vie_encoder_), + protection_bitrate_calculator_(Clock::GetRealTimeClock(), this), video_sender_(vie_encoder_.video_sender()), bandwidth_observer_(congestion_controller_->GetBitrateController() ->CreateRtcpBandwidthObserver()), @@ -422,8 +423,6 @@ VideoSendStream::VideoSendStream( congestion_controller_->packet_router()->AddRtpModule(rtp_rtcp); } - video_sender_->RegisterProtectionCallback(this); - for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) { const std::string& extension = config_.rtp.extensions[i].uri; int id = config_.rtp.extensions[i].id; @@ -571,6 +570,17 @@ void VideoSendStream::EncoderProcess() { stats_proxy_.OnInactiveSsrc(config_.rtp.ssrcs[i]); } + size_t number_of_temporal_layers = + encoder_settings->streams.back() + .temporal_layer_thresholds_bps.size() + + 1; + protection_bitrate_calculator_.SetEncodingData( + encoder_settings->video_codec.startBitrate * 1000, + encoder_settings->video_codec.width, + encoder_settings->video_codec.height, + encoder_settings->video_codec.maxFramerate, number_of_temporal_layers, + payload_router_.MaxPayloadLength()); + // We might've gotten new settings while configuring the encoder settings, // restart from the top to see if that's the case before trying to encode // a frame (which might correspond to the last frame size). @@ -627,6 +637,7 @@ int32_t VideoSendStream::Encoded(const EncodedImage& encoded_image, // |encoded_frame_proxy_| forwards frames to |config_.post_encode_callback|; encoded_frame_proxy_.Encoded(encoded_image, codec_specific_info, fragmentation); + protection_bitrate_calculator_.UpdateWithEncodedData(encoded_image); int32_t return_value = payload_router_.Encoded( encoded_image, codec_specific_info, fragmentation); @@ -706,8 +717,8 @@ void VideoSendStream::ConfigureProtection() { } } - vie_encoder_.SetProtectionMethod(enable_protection_nack, - enable_protection_fec); + protection_bitrate_calculator_.SetProtectionMethod(enable_protection_fec, + enable_protection_nack); } void VideoSendStream::ConfigureSsrcs() { @@ -785,7 +796,12 @@ void VideoSendStream::OnBitrateUpdated(uint32_t bitrate_bps, uint8_t fraction_loss, int64_t rtt) { payload_router_.SetTargetSendBitrate(bitrate_bps); - vie_encoder_.OnBitrateUpdated(bitrate_bps, fraction_loss, rtt); + // Get the encoder target rate. It is the estimated network rate - + // protection overhead. + uint32_t encoder_target_rate = protection_bitrate_calculator_.SetTargetRates( + bitrate_bps, stats_proxy_.GetSendFrameRate(), fraction_loss, rtt); + + vie_encoder_.OnBitrateUpdated(encoder_target_rate, fraction_loss, rtt); } int VideoSendStream::ProtectionRequest(const FecProtectionParams* delta_params, diff --git a/webrtc/video/video_send_stream.h b/webrtc/video/video_send_stream.h index 51d8a4600b..df3e3c4c63 100644 --- a/webrtc/video/video_send_stream.h +++ b/webrtc/video/video_send_stream.h @@ -19,6 +19,7 @@ #include "webrtc/base/criticalsection.h" #include "webrtc/call.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/modules/video_coding/protection_bitrate_calculator.h" #include "webrtc/video/encoded_frame_callback_adapter.h" #include "webrtc/video/encoder_state_feedback.h" #include "webrtc/video/payload_router.h" @@ -91,6 +92,7 @@ class VideoSendStream : public webrtc::VideoSendStream, uint8_t fraction_loss, int64_t rtt) override; + protected: // Implements webrtc::VCMProtectionCallback. int ProtectionRequest(const FecProtectionParams* delta_params, const FecProtectionParams* key_params, @@ -143,6 +145,8 @@ class VideoSendStream : public webrtc::VideoSendStream, OveruseFrameDetector overuse_detector_; ViEEncoder vie_encoder_; EncoderStateFeedback encoder_feedback_; + ProtectionBitrateCalculator protection_bitrate_calculator_; + vcm::VideoSender* const video_sender_; const std::unique_ptr bandwidth_observer_; diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index d10e8fc5cd..73b29f319e 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -264,18 +264,6 @@ void ViEEncoder::SendKeyFrame() { video_sender_.IntraFrameRequest(0); } -void ViEEncoder::SetProtectionMethod(bool nack, bool fec) { - // Set Video Protection for VCM. - VCMVideoProtection protection_mode; - if (fec) { - protection_mode = - nack ? webrtc::kProtectionNackFEC : kProtectionFEC; - } else { - protection_mode = nack ? kProtectionNack : kProtectionNone; - } - video_sender_.SetVideoProtection(protection_mode); -} - void ViEEncoder::OnSetRates(uint32_t bitrate_bps, int framerate) { if (stats_proxy_) stats_proxy_->OnSetRates(bitrate_bps, framerate); diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index 212962348a..4372bd11c9 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -84,13 +84,11 @@ class ViEEncoder : public VideoEncoderRateObserver, void EncodeVideoFrame(const VideoFrame& video_frame); void SendKeyFrame(); - uint32_t LastObservedBitrateBps() const; - // Loss protection. Must be called before SetEncoder() to have max packet size - // updated according to protection. - // TODO(pbos): Set protection method on construction. - void SetProtectionMethod(bool nack, bool fec); - // Implements VideoEncoderRateObserver. + // TODO(perkj): Refactor VideoEncoderRateObserver. This is only used for + // stats. The stats should be set in VideoSendStream instead. + // |bitrate_bps| is the target bitrate and |framerate| is the input frame + // rate so it has nothing to do with the actual encoder. void OnSetRates(uint32_t bitrate_bps, int framerate) override; // Implements EncodedImageCallback.