diff --git a/src/modules/remote_bitrate_estimator/include/bwe_defines.h b/src/modules/remote_bitrate_estimator/include/bwe_defines.h index 173a284329..395ea7ebd6 100644 --- a/src/modules/remote_bitrate_estimator/include/bwe_defines.h +++ b/src/modules/remote_bitrate_estimator/include/bwe_defines.h @@ -21,7 +21,7 @@ enum BandwidthUsage { kBwNormal, kBwOverusing, - kBwUnderUsing + kBwUnderusing }; enum RateControlState diff --git a/src/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h b/src/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h index d5678e439c..d30a9624b6 100644 --- a/src/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h +++ b/src/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h @@ -8,8 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -// RemoteBitrateEstimator -// This class estimates the incoming bitrate capacity. +// This class estimates the incoming available bandwidth. #ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_REMOTE_BITRATE_ESTIMATOR_H_ #define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_REMOTE_BITRATE_ESTIMATOR_H_ @@ -19,18 +18,21 @@ #include "modules/remote_bitrate_estimator/bitrate_estimator.h" #include "modules/remote_bitrate_estimator/overuse_detector.h" #include "modules/remote_bitrate_estimator/remote_rate_control.h" +#include "system_wrappers/interface/constructor_magic.h" #include "system_wrappers/interface/critical_section_wrapper.h" #include "system_wrappers/interface/scoped_ptr.h" #include "typedefs.h" +#include "video_engine/stream_synchronization.h" namespace webrtc { // RemoteBitrateObserver is used to signal changes in bitrate estimates for -// the incoming stream. +// the incoming streams. class RemoteBitrateObserver { public: // Called when a receive channel has a new bitrate estimate for the incoming // stream. + // TODO(holmer): Remove |ssrc| argument and remove SSRC map from VieRemb. virtual void OnReceiveBitrateChanged(unsigned int ssrc, unsigned int bitrate) = 0; @@ -39,54 +41,45 @@ class RemoteBitrateObserver { class RemoteBitrateEstimator { public: - RemoteBitrateEstimator(RemoteBitrateObserver* observer, - const OverUseDetectorOptions& options); - - // Called for each incoming packet. If this is a new SSRC, a new - // BitrateControl will be created. - void IncomingPacket(unsigned int ssrc, - int packet_size, - int64_t arrival_time, - uint32_t rtp_timestamp, - int64_t packet_send_time); - - // Triggers a new estimate calculation for the stream identified by |ssrc|. - void UpdateEstimate(unsigned int ssrc, int64_t time_now); - - // Set the current round-trip time experienced by the stream identified by - // |ssrc|. - void SetRtt(unsigned int ssrc); - - // Removes all data for |ssrc|. - void RemoveStream(unsigned int ssrc); - - // Returns true if a valid estimate exists for a stream identified by |ssrc| - // and sets |bitrate_bps| to the estimated bitrate in bits per second. - bool LatestEstimate(unsigned int ssrc, unsigned int* bitrate_bps) const; - - private: - struct BitrateControls { - explicit BitrateControls(const OverUseDetectorOptions& options) - : remote_rate(), - overuse_detector(options), - incoming_bitrate() { - } - BitrateControls(const BitrateControls& other) - : remote_rate(other.remote_rate), - overuse_detector(other.overuse_detector), - incoming_bitrate(other.incoming_bitrate) { - } - RemoteRateControl remote_rate; - OverUseDetector overuse_detector; - BitRateStats incoming_bitrate; + enum EstimationMode { + kMultiStreamEstimation, + kSingleStreamEstimation }; - typedef std::map SsrcBitrateControlsMap; + virtual ~RemoteBitrateEstimator() {} - const OverUseDetectorOptions& options_; - SsrcBitrateControlsMap bitrate_controls_; - RemoteBitrateObserver* observer_; - scoped_ptr crit_sect_; + static RemoteBitrateEstimator* Create(RemoteBitrateObserver* observer, + const OverUseDetectorOptions& options, + EstimationMode mode); + + // Stores an RTCP SR (NTP, RTP timestamp) tuple for a specific SSRC to be used + // in future RTP timestamp to NTP time conversions. As soon as any SSRC has + // two tuples the RemoteBitrateEstimator will switch to multi-stream mode. + virtual void IncomingRtcp(unsigned int ssrc, uint32_t ntp_secs, + uint32_t ntp_frac, uint32_t rtp_timestamp) = 0; + + // Called for each incoming packet. The first SSRC will immediately be used + // for overuse detection. Subsequent SSRCs will only be used when at least + // two RTCP SR reports with the same SSRC have been received. + virtual void IncomingPacket(unsigned int ssrc, + int packet_size, + int64_t arrival_time, + uint32_t rtp_timestamp) = 0; + + // Triggers a new estimate calculation. + virtual void UpdateEstimate(unsigned int ssrc, int64_t time_now) = 0; + + // Set the current round-trip time experienced by the streams going into this + // estimator. + virtual void SetRtt(unsigned int rtt) = 0; + + // Removes all data for |ssrc|. + virtual void RemoveStream(unsigned int ssrc) = 0; + + // Returns true if a valid estimate exists and sets |bitrate_bps| to the + // estimated bitrate in bits per second. + virtual bool LatestEstimate(unsigned int ssrc, + unsigned int* bitrate_bps) const = 0; }; } // namespace webrtc diff --git a/src/modules/remote_bitrate_estimator/overuse_detector.cc b/src/modules/remote_bitrate_estimator/overuse_detector.cc index 1c5bf9d831..084fa156da 100644 --- a/src/modules/remote_bitrate_estimator/overuse_detector.cc +++ b/src/modules/remote_bitrate_estimator/overuse_detector.cc @@ -23,11 +23,11 @@ extern MatlabEngine eng; // global variable defined elsewhere #endif -#define OVER_USING_TIME_THRESHOLD 100 -#define MIN_FRAME_PERIOD_HISTORY_LEN 60 +enum { kOverUsingTimeThreshold = 100 }; +enum { kMinFramePeriodHistoryLength = 60 }; namespace webrtc { -OverUseDetector::OverUseDetector(const OverUseDetectorOptions& options) +OveruseDetector::OveruseDetector(const OverUseDetectorOptions& options) : options_(options), current_frame_(), prev_frame_(), @@ -53,7 +53,7 @@ OverUseDetector::OverUseDetector(const OverUseDetectorOptions& options) sizeof(process_noise_)); } -OverUseDetector::~OverUseDetector() { +OveruseDetector::~OveruseDetector() { #ifdef WEBRTC_BWE_MATLAB if (plots_.plot1_) { eng.DeletePlot(plots_.plot1_); @@ -76,7 +76,8 @@ OverUseDetector::~OverUseDetector() { ts_delta_hist_.clear(); } -void OverUseDetector::Update(uint16_t packet_size, +void OveruseDetector::Update(uint16_t packet_size, + int64_t timestamp_ms, uint32_t timestamp, const int64_t now_ms) { #ifdef WEBRTC_BWE_MATLAB @@ -108,54 +109,51 @@ void OverUseDetector::Update(uint16_t packet_size, } #endif - - bool wrapped = false; - if (current_frame_.timestamp_ == -1) { - current_frame_.timestamp_ = timestamp; - } else if (OldTimestamp( - timestamp, - static_cast(current_frame_.timestamp_), - &wrapped)) { - // Don't update with old data - return; - } else if (timestamp != current_frame_.timestamp_) { + bool new_timestamp = (timestamp != current_frame_.timestamp); + if (timestamp_ms >= 0) { + if (prev_frame_.timestamp_ms == -1 && current_frame_.timestamp_ms == -1) { + SwitchTimeBase(); + } + new_timestamp = (timestamp_ms != current_frame_.timestamp_ms); + } + if (current_frame_.timestamp == -1) { + current_frame_.timestamp = timestamp; + current_frame_.timestamp_ms = timestamp_ms; + } else if (new_timestamp) { // First packet of a later frame, the previous frame sample is ready WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, - "Frame complete at %I64i", current_frame_.completeTimeMs_); - if (prev_frame_.completeTimeMs_ >= 0) { // This is our second frame + "Frame complete at %I64i", current_frame_.complete_time_ms); + if (prev_frame_.complete_time_ms >= 0) { // This is our second frame int64_t t_delta = 0; double ts_delta = 0; - // Check for wrap - OldTimestamp( - static_cast(prev_frame_.timestamp_), - static_cast(current_frame_.timestamp_), - &wrapped); - CompensatedTimeDelta(current_frame_, prev_frame_, t_delta, ts_delta, - wrapped); - UpdateKalman(t_delta, ts_delta, current_frame_.size_, - prev_frame_.size_); + if (!TimeDeltas(current_frame_, prev_frame_, + &t_delta, &ts_delta)) { + // Frame reordering, dropping this sample. + return; + } + UpdateKalman(t_delta, ts_delta, current_frame_.size, prev_frame_.size); } // The new timestamp is now the current frame, // and the old timestamp becomes the previous frame. prev_frame_ = current_frame_; - current_frame_.timestamp_ = timestamp; - current_frame_.size_ = 0; - current_frame_.completeTimeMs_ = -1; + current_frame_.timestamp = timestamp; + current_frame_.timestamp_ms = timestamp_ms; + current_frame_.size = 0; } // Accumulate the frame size - current_frame_.size_ += packet_size; - current_frame_.completeTimeMs_ = now_ms; + current_frame_.size += packet_size; + current_frame_.complete_time_ms = now_ms; } -BandwidthUsage OverUseDetector::State() const { +BandwidthUsage OveruseDetector::State() const { return hypothesis_; } -double OverUseDetector::NoiseVar() const { +double OveruseDetector::NoiseVar() const { return var_noise_; } -void OverUseDetector::SetRateControlRegion(RateControlRegion region) { +void OveruseDetector::SetRateControlRegion(RateControlRegion region) { switch (region) { case kRcMaxUnknown: { threshold_ = options_.initial_threshold; @@ -169,65 +167,82 @@ void OverUseDetector::SetRateControlRegion(RateControlRegion region) { } } -void OverUseDetector::CompensatedTimeDelta(const FrameSample& currentFrame, - const FrameSample& prevFrame, - int64_t& t_delta, - double& ts_delta, - bool wrapped) { +void OveruseDetector::SwitchTimeBase() { + current_frame_.size = 0; + current_frame_.complete_time_ms = -1; + current_frame_.timestamp = -1; + prev_frame_ = current_frame_; +} + +bool OveruseDetector::TimeDeltas(const FrameSample& current_frame, + const FrameSample& prev_frame, + int64_t* t_delta, + double* ts_delta) { + assert(t_delta); + assert(ts_delta); num_of_deltas_++; if (num_of_deltas_ > 1000) { num_of_deltas_ = 1000; } - // Add wrap-around compensation - int64_t wrapCompensation = 0; - if (wrapped) { - wrapCompensation = static_cast(1)<<32; + if (current_frame.timestamp_ms == -1) { + const uint32_t timestamp_diff = current_frame.timestamp - + prev_frame.timestamp; + if (timestamp_diff > 0x80000000) { + // Assume that a diff this big must be due to reordering. Don't update + // with reordered samples. + return false; + } + *ts_delta = timestamp_diff / 90.0; + } else { + *ts_delta = current_frame.timestamp_ms - prev_frame.timestamp_ms; + if (*ts_delta < 0) { + // Frame reordering. + return false; + } } - ts_delta = (currentFrame.timestamp_ - + wrapCompensation - - prevFrame.timestamp_) / 90.0; - t_delta = currentFrame.completeTimeMs_ - prevFrame.completeTimeMs_; - assert(ts_delta > 0); + *t_delta = current_frame.complete_time_ms - prev_frame.complete_time_ms; + assert(*ts_delta > 0); + return true; } -double OverUseDetector::CurrentDrift() { +double OveruseDetector::CurrentDrift() { return 1.0; } -void OverUseDetector::UpdateKalman(int64_t t_delta, +void OveruseDetector::UpdateKalman(int64_t t_delta, double ts_delta, uint32_t frame_size, uint32_t prev_frame_size) { - const double minFramePeriod = UpdateMinFramePeriod(ts_delta); + const double min_frame_period = UpdateMinFramePeriod(ts_delta); const double drift = CurrentDrift(); // Compensate for drift - const double tTsDelta = t_delta - ts_delta / drift; - double fsDelta = static_cast(frame_size) - prev_frame_size; + const double t_ts_delta = t_delta - ts_delta / drift; + double fs_delta = static_cast(frame_size) - prev_frame_size; // Update the Kalman filter - const double scaleFactor = minFramePeriod / (1000.0 / 30.0); - E_[0][0] += process_noise_[0] * scaleFactor; - E_[1][1] += process_noise_[1] * scaleFactor; + const double scale_factor = min_frame_period / (1000.0 / 30.0); + E_[0][0] += process_noise_[0] * scale_factor; + E_[1][1] += process_noise_[1] * scale_factor; if ((hypothesis_ == kBwOverusing && offset_ < prev_offset_) || - (hypothesis_ == kBwUnderUsing && offset_ > prev_offset_)) { - E_[1][1] += 10 * process_noise_[1] * scaleFactor; + (hypothesis_ == kBwUnderusing && offset_ > prev_offset_)) { + E_[1][1] += 10 * process_noise_[1] * scale_factor; } - const double h[2] = {fsDelta, 1.0}; + const double h[2] = {fs_delta, 1.0}; const double Eh[2] = {E_[0][0]*h[0] + E_[0][1]*h[1], E_[1][0]*h[0] + E_[1][1]*h[1]}; - const double residual = tTsDelta - slope_*h[0] - offset_; + const double residual = t_ts_delta - slope_*h[0] - offset_; const bool stable_state = (BWE_MIN(num_of_deltas_, 60) * fabsf(offset_) < threshold_); // We try to filter out very late frames. For instance periodic key // frames doesn't fit the Gaussian model well. if (fabsf(residual) < 3 * sqrt(var_noise_)) { - UpdateNoiseEstimate(residual, minFramePeriod, stable_state); + UpdateNoiseEstimate(residual, min_frame_period, stable_state); } else { - UpdateNoiseEstimate(3 * sqrt(var_noise_), minFramePeriod, stable_state); + UpdateNoiseEstimate(3 * sqrt(var_noise_), min_frame_period, stable_state); } const double denom = var_noise_ + h[0]*Eh[0] + h[1]*Eh[1]; @@ -268,7 +283,7 @@ void OverUseDetector::UpdateKalman(int64_t t_delta, #ifdef WEBRTC_BWE_MATLAB plots_.plot1_->Append("scatter", - static_cast(current_frame_.size_) - prev_frame_.size_, + static_cast(current_frame_.size) - prev_frame_.size, static_cast(t_delta - ts_delta)); plots_.plot1_->MakeTrend("scatter", "slope", slope_, offset_, "k-"); plots_.plot1_->MakeTrend("scatter", "thresholdPos", @@ -286,21 +301,21 @@ void OverUseDetector::UpdateKalman(int64_t t_delta, #endif } -double OverUseDetector::UpdateMinFramePeriod(double ts_delta) { - double minFramePeriod = ts_delta; - if (ts_delta_hist_.size() >= MIN_FRAME_PERIOD_HISTORY_LEN) { - std::list::iterator firstItem = ts_delta_hist_.begin(); - ts_delta_hist_.erase(firstItem); +double OveruseDetector::UpdateMinFramePeriod(double ts_delta) { + double min_frame_period = ts_delta; + if (ts_delta_hist_.size() >= kMinFramePeriodHistoryLength) { + std::list::iterator first_item = ts_delta_hist_.begin(); + ts_delta_hist_.erase(first_item); } std::list::iterator it = ts_delta_hist_.begin(); for (; it != ts_delta_hist_.end(); it++) { - minFramePeriod = BWE_MIN(*it, minFramePeriod); + min_frame_period = BWE_MIN(*it, min_frame_period); } ts_delta_hist_.push_back(ts_delta); - return minFramePeriod; + return min_frame_period; } -void OverUseDetector::UpdateNoiseEstimate(double residual, +void OveruseDetector::UpdateNoiseEstimate(double residual, double ts_delta, bool stable_state) { if (!stable_state) { @@ -325,7 +340,7 @@ void OverUseDetector::UpdateNoiseEstimate(double residual, } } -BandwidthUsage OverUseDetector::Detect(double ts_delta) { +BandwidthUsage OveruseDetector::Detect(double ts_delta) { if (num_of_deltas_ < 2) { return kBwNormal; } @@ -342,7 +357,7 @@ BandwidthUsage OverUseDetector::Detect(double ts_delta) { time_over_using_ += ts_delta; } over_use_counter_++; - if (time_over_using_ > OVER_USING_TIME_THRESHOLD + if (time_over_using_ > kOverUsingTimeThreshold && over_use_counter_ > 1) { if (offset_ >= prev_offset_) { #ifdef _DEBUG @@ -363,13 +378,13 @@ BandwidthUsage OverUseDetector::Detect(double ts_delta) { #endif } else { #ifdef _DEBUG - if (hypothesis_ != kBwUnderUsing) { + if (hypothesis_ != kBwUnderusing) { WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, "BWE: kBwUnderUsing"); } #endif time_over_using_ = -1; over_use_counter_ = 0; - hypothesis_ = kBwUnderUsing; + hypothesis_ = kBwUnderusing; } } else { #ifdef _DEBUG @@ -384,7 +399,7 @@ BandwidthUsage OverUseDetector::Detect(double ts_delta) { return hypothesis_; } -bool OverUseDetector::OldTimestamp(uint32_t new_timestamp, +bool OveruseDetector::OldTimestamp(uint32_t new_timestamp, uint32_t existing_timestamp, bool* wrapped) { bool tmpWrapped = diff --git a/src/modules/remote_bitrate_estimator/overuse_detector.h b/src/modules/remote_bitrate_estimator/overuse_detector.h index 3fd3933626..67a33508dd 100644 --- a/src/modules/remote_bitrate_estimator/overuse_detector.h +++ b/src/modules/remote_bitrate_estimator/overuse_detector.h @@ -23,24 +23,30 @@ namespace webrtc { enum RateControlRegion; -class OverUseDetector { +class OveruseDetector { public: - explicit OverUseDetector(const OverUseDetectorOptions& options); - ~OverUseDetector(); - void Update(const WebRtc_UWord16 packet_size, - const uint32_t timestamp, - const int64_t now_ms); + explicit OveruseDetector(const OverUseDetectorOptions& options); + ~OveruseDetector(); + void Update(uint16_t packet_size, + int64_t timestamp_ms, + uint32_t rtp_timestamp, + int64_t now_ms); BandwidthUsage State() const; double NoiseVar() const; void SetRateControlRegion(RateControlRegion region); private: struct FrameSample { - FrameSample() : size_(0), completeTimeMs_(-1), timestamp_(-1) {} + FrameSample() + : size(0), + complete_time_ms(-1), + timestamp(-1), + timestamp_ms(-1) {} - uint32_t size_; - int64_t completeTimeMs_; - int64_t timestamp_; + uint32_t size; + int64_t complete_time_ms; + int64_t timestamp; + int64_t timestamp_ms; }; struct DebugPlots { @@ -57,11 +63,14 @@ class OverUseDetector { uint32_t existing_timestamp, bool* wrapped); - void CompensatedTimeDelta(const FrameSample& current_frame, - const FrameSample& prev_frame, - int64_t& t_delta, - double& ts_delta, - bool wrapped); + // Prepares the overuse detector to start using timestamps in milliseconds + // instead of 90 kHz timestamps. + void SwitchTimeBase(); + + bool TimeDeltas(const FrameSample& current_frame, + const FrameSample& prev_frame, + int64_t* t_delta, + double* ts_delta); void UpdateKalman(int64_t t_delta, double ts_elta, uint32_t frame_size, diff --git a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi index 21ec7a9210..cb5107b632 100644 --- a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi +++ b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi @@ -35,9 +35,14 @@ 'bitrate_estimator.h', 'overuse_detector.cc', 'overuse_detector.h', - 'remote_bitrate_estimator.cc', + 'remote_bitrate_estimator_multi_stream.h', + 'remote_bitrate_estimator_multi_stream.cc', + 'remote_bitrate_estimator_single_stream.h', + 'remote_bitrate_estimator_single_stream.cc', 'remote_rate_control.cc', 'remote_rate_control.h', + '../../video_engine/stream_synchronization.cc', + '../../video_engine/stream_synchronization.h', ], # source }, ], # targets diff --git a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_multi_stream.cc b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_multi_stream.cc new file mode 100644 index 0000000000..fc0d2a8792 --- /dev/null +++ b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_multi_stream.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/remote_bitrate_estimator/remote_bitrate_estimator_multi_stream.h" + +#include "modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.h" +#include "system_wrappers/interface/tick_util.h" +#include "video_engine/stream_synchronization.h" + +namespace webrtc { + +RemoteBitrateEstimator* RemoteBitrateEstimator::Create( + RemoteBitrateObserver* observer, + const OverUseDetectorOptions& options, + EstimationMode mode) { + switch (mode) { + case kMultiStreamEstimation: + return new RemoteBitrateEstimatorMultiStream(observer, options); + case kSingleStreamEstimation: + return new RemoteBitrateEstimatorSingleStream(observer, options); + } + return NULL; +} + +RemoteBitrateEstimatorMultiStream::RemoteBitrateEstimatorMultiStream( + RemoteBitrateObserver* observer, + const OverUseDetectorOptions& options) + : remote_rate_(), + overuse_detector_(options), + incoming_bitrate_(), + observer_(observer), + streams_(), + crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), + initial_ssrc_(0), + multi_stream_(false) { + assert(observer_); +} + +void RemoteBitrateEstimatorMultiStream::IncomingRtcp(unsigned int ssrc, + uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t timestamp) { + CriticalSectionScoped cs(crit_sect_.get()); + if (ntp_secs == 0 && ntp_frac == 0) { + return; + } + // Insert a new RTCP list mapped to this SSRC if one doesn't already exist. + std::pair stream_insert_result = + streams_.insert(std::make_pair(ssrc, synchronization::RtcpList())); + StreamMap::iterator stream_it = stream_insert_result.first; + synchronization::RtcpList* rtcp_list = &stream_it->second; + synchronization::RtcpMeasurement measurement(ntp_secs, ntp_frac, timestamp); + // Make sure this RTCP is unique as we need two unique data points to + // calculate the RTP timestamp frequency. + for (synchronization::RtcpList::iterator it = rtcp_list->begin(); + it != rtcp_list->end(); ++it) { + if ((measurement.ntp_secs == (*it).ntp_secs && + measurement.ntp_frac == (*it).ntp_frac) || + measurement.rtp_timestamp == (*it).rtp_timestamp) { + return; + } + } + // If this stream will get two RTCPs when the new one is added we can switch + // to multi-stream mode. + if (!multi_stream_ && rtcp_list->size() >= 1) { + multi_stream_ = true; + } + if (rtcp_list->size() >= 2) { + rtcp_list->pop_back(); + } + rtcp_list->push_front(measurement); +} + +void RemoteBitrateEstimatorMultiStream::IncomingPacket(unsigned int ssrc, + int packet_size, + int64_t arrival_time, + uint32_t rtp_timestamp) { + CriticalSectionScoped cs(crit_sect_.get()); + incoming_bitrate_.Update(packet_size, arrival_time); + StreamMap::iterator stream_it = streams_.find(ssrc); + if (initial_ssrc_ == 0) { + initial_ssrc_ = ssrc; + } + if (!multi_stream_) { + if (ssrc != initial_ssrc_) { + // We can only handle the initial stream until we get into multi stream + // mode. + return; + } + } else if (stream_it == streams_.end() || stream_it->second.size() < 2) { + // We can't use this stream until we have received two RTCP SR reports. + return; + } + const BandwidthUsage prior_state = overuse_detector_.State(); + int64_t timestamp_in_ms = -1; + if (multi_stream_) { + synchronization::RtpToNtpMs(rtp_timestamp, stream_it->second, + ×tamp_in_ms); + } + overuse_detector_.Update(packet_size, timestamp_in_ms, rtp_timestamp, + arrival_time); + if (prior_state != kBwOverusing && + overuse_detector_.State() == kBwOverusing) { + // The first overuse should immediately trigger a new estimate. + UpdateEstimate(1, arrival_time); + } +} + +void RemoteBitrateEstimatorMultiStream::UpdateEstimate(unsigned int ssrc, + int64_t time_now) { + CriticalSectionScoped cs(crit_sect_.get()); + const RateControlInput input(overuse_detector_.State(), + incoming_bitrate_.BitRate(time_now), + overuse_detector_.NoiseVar()); + const RateControlRegion region = remote_rate_.Update(&input, time_now); + unsigned int target_bitrate = remote_rate_.UpdateBandwidthEstimate(time_now); + if (remote_rate_.ValidEstimate()) { + observer_->OnReceiveBitrateChanged(1, target_bitrate); + } + overuse_detector_.SetRateControlRegion(region); +} + +void RemoteBitrateEstimatorMultiStream::SetRtt(unsigned int rtt) { + CriticalSectionScoped cs(crit_sect_.get()); + remote_rate_.SetRtt(rtt); +} + +void RemoteBitrateEstimatorMultiStream::RemoveStream(unsigned int ssrc) { + CriticalSectionScoped cs(crit_sect_.get()); + streams_.erase(ssrc); +} + +bool RemoteBitrateEstimatorMultiStream::LatestEstimate( + unsigned int ssrc, + unsigned int* bitrate_bps) const { + CriticalSectionScoped cs(crit_sect_.get()); + assert(bitrate_bps != NULL); + if (!remote_rate_.ValidEstimate()) { + return false; + } + *bitrate_bps = remote_rate_.LatestEstimate(); + return true; +} + +} // namespace webrtc diff --git a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_multi_stream.h b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_multi_stream.h new file mode 100644 index 0000000000..567cc329a2 --- /dev/null +++ b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_multi_stream.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This class estimates the incoming available bandwidth. + +#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_REMOTE_BITRATE_ESTIMATOR_IMPL_H_ +#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_REMOTE_BITRATE_ESTIMATOR_IMPL_H_ + +#include + +#include "modules/remote_bitrate_estimator/bitrate_estimator.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "modules/remote_bitrate_estimator/overuse_detector.h" +#include "modules/remote_bitrate_estimator/remote_rate_control.h" +#include "system_wrappers/interface/constructor_magic.h" +#include "system_wrappers/interface/critical_section_wrapper.h" +#include "system_wrappers/interface/scoped_ptr.h" +#include "typedefs.h" +#include "video_engine/stream_synchronization.h" + +namespace webrtc { + +class RemoteBitrateEstimatorMultiStream : public RemoteBitrateEstimator { + public: + RemoteBitrateEstimatorMultiStream(RemoteBitrateObserver* observer, + const OverUseDetectorOptions& options); + + ~RemoteBitrateEstimatorMultiStream() {} + + // Stores an RTCP SR (NTP, RTP timestamp) tuple for a specific SSRC to be used + // in future RTP timestamp to NTP time conversions. As soon as any SSRC has + // two tuples the RemoteBitrateEstimator will switch to multi-stream mode. + void IncomingRtcp(unsigned int ssrc, uint32_t ntp_secs, uint32_t ntp_frac, + uint32_t rtp_timestamp); + + // Called for each incoming packet. The first SSRC will immediately be used + // for overuse detection. Subsequent SSRCs will only be used when at least + // two RTCP SR reports with the same SSRC have been received. + void IncomingPacket(unsigned int ssrc, + int packet_size, + int64_t arrival_time, + uint32_t rtp_timestamp); + + // Triggers a new estimate calculation. + void UpdateEstimate(unsigned int ssrc, int64_t time_now); + + // Set the current round-trip time experienced by the streams going into this + // estimator. + void SetRtt(unsigned int rtt); + + // Removes all data for |ssrc|. + void RemoveStream(unsigned int ssrc); + + // Returns true if a valid estimate exists and sets |bitrate_bps| to the + // estimated bitrate in bits per second. + bool LatestEstimate(unsigned int ssrc, unsigned int* bitrate_bps) const; + + private: + typedef std::map StreamMap; + + RemoteRateControl remote_rate_; + OveruseDetector overuse_detector_; + BitRateStats incoming_bitrate_; + RemoteBitrateObserver* observer_; + StreamMap streams_; + scoped_ptr crit_sect_; + unsigned int initial_ssrc_; + bool multi_stream_; + + DISALLOW_COPY_AND_ASSIGN(RemoteBitrateEstimatorMultiStream); +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_REMOTE_BITRATE_ESTIMATOR_IMPL_H_ diff --git a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator.cc b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.cc similarity index 73% rename from src/modules/remote_bitrate_estimator/remote_bitrate_estimator.cc rename to src/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.cc index 70713d56f8..45e301f590 100644 --- a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator.cc +++ b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.cc @@ -8,26 +8,25 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.h" #include "system_wrappers/interface/tick_util.h" namespace webrtc { -RemoteBitrateEstimator::RemoteBitrateEstimator( - RemoteBitrateObserver* observer, - const OverUseDetectorOptions& options) +RemoteBitrateEstimatorSingleStream::RemoteBitrateEstimatorSingleStream( + RemoteBitrateObserver* observer, const OverUseDetectorOptions& options) : options_(options), observer_(observer), crit_sect_(CriticalSectionWrapper::CreateCriticalSection()) { assert(observer_); } -void RemoteBitrateEstimator::IncomingPacket(unsigned int ssrc, - int packet_size, - int64_t arrival_time, - uint32_t rtp_timestamp, - int64_t packet_send_time) { +void RemoteBitrateEstimatorSingleStream::IncomingPacket( + unsigned int ssrc, + int packet_size, + int64_t arrival_time, + uint32_t rtp_timestamp) { CriticalSectionScoped cs(crit_sect_.get()); SsrcBitrateControlsMap::iterator it = bitrate_controls_.find(ssrc); if (it == bitrate_controls_.end()) { @@ -40,10 +39,10 @@ void RemoteBitrateEstimator::IncomingPacket(unsigned int ssrc, bitrate_controls_.insert(std::make_pair(ssrc, BitrateControls(options_))); it = bitrate_controls_.find(ssrc); } - OverUseDetector* overuse_detector = &it->second.overuse_detector; + OveruseDetector* overuse_detector = &it->second.overuse_detector; it->second.incoming_bitrate.Update(packet_size, arrival_time); const BandwidthUsage prior_state = overuse_detector->State(); - overuse_detector->Update(packet_size, rtp_timestamp, arrival_time); + overuse_detector->Update(packet_size, -1, rtp_timestamp, arrival_time); if (prior_state != overuse_detector->State() && overuse_detector->State() == kBwOverusing) { // The first overuse should immediately trigger a new estimate. @@ -51,14 +50,14 @@ void RemoteBitrateEstimator::IncomingPacket(unsigned int ssrc, } } -void RemoteBitrateEstimator::UpdateEstimate(unsigned int ssrc, - int64_t time_now) { +void RemoteBitrateEstimatorSingleStream::UpdateEstimate(unsigned int ssrc, + int64_t time_now) { CriticalSectionScoped cs(crit_sect_.get()); SsrcBitrateControlsMap::iterator it = bitrate_controls_.find(ssrc); if (it == bitrate_controls_.end()) { return; } - OverUseDetector* overuse_detector = &it->second.overuse_detector; + OveruseDetector* overuse_detector = &it->second.overuse_detector; RemoteRateControl* remote_rate = &it->second.remote_rate; const RateControlInput input(overuse_detector->State(), it->second.incoming_bitrate.BitRate(time_now), @@ -71,7 +70,7 @@ void RemoteBitrateEstimator::UpdateEstimate(unsigned int ssrc, overuse_detector->SetRateControlRegion(region); } -void RemoteBitrateEstimator::SetRtt(unsigned int rtt) { +void RemoteBitrateEstimatorSingleStream::SetRtt(unsigned int rtt) { CriticalSectionScoped cs(crit_sect_.get()); for (SsrcBitrateControlsMap::iterator it = bitrate_controls_.begin(); it != bitrate_controls_.end(); ++it) { @@ -79,14 +78,14 @@ void RemoteBitrateEstimator::SetRtt(unsigned int rtt) { } } -void RemoteBitrateEstimator::RemoveStream(unsigned int ssrc) { +void RemoteBitrateEstimatorSingleStream::RemoveStream(unsigned int ssrc) { CriticalSectionScoped cs(crit_sect_.get()); // Ignoring the return value which is the number of elements erased. bitrate_controls_.erase(ssrc); } -bool RemoteBitrateEstimator::LatestEstimate(unsigned int ssrc, - unsigned int* bitrate_bps) const { +bool RemoteBitrateEstimatorSingleStream::LatestEstimate( + unsigned int ssrc, unsigned int* bitrate_bps) const { CriticalSectionScoped cs(crit_sect_.get()); assert(bitrate_bps != NULL); SsrcBitrateControlsMap::const_iterator it = bitrate_controls_.find(ssrc); diff --git a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.h b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.h new file mode 100644 index 0000000000..241bb0d3cb --- /dev/null +++ b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This class estimates the incoming available bandwidth. + +#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_REMOTE_BITRATE_ESTIMATOR_SINGLE_STREAM_H_ +#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_REMOTE_BITRATE_ESTIMATOR_SINGLE_STREAM_H_ + +#include + +#include "modules/remote_bitrate_estimator/bitrate_estimator.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "modules/remote_bitrate_estimator/overuse_detector.h" +#include "modules/remote_bitrate_estimator/remote_rate_control.h" +#include "system_wrappers/interface/critical_section_wrapper.h" +#include "system_wrappers/interface/scoped_ptr.h" +#include "typedefs.h" + +namespace webrtc { + +class RemoteBitrateEstimatorSingleStream : public RemoteBitrateEstimator { + public: + RemoteBitrateEstimatorSingleStream(RemoteBitrateObserver* observer, + const OverUseDetectorOptions& options); + + void IncomingRtcp(unsigned int ssrc, uint32_t ntp_secs, uint32_t ntp_frac, + uint32_t rtp_timestamp) {} + + // Called for each incoming packet. If this is a new SSRC, a new + // BitrateControl will be created. + void IncomingPacket(unsigned int ssrc, + int packet_size, + int64_t arrival_time, + uint32_t rtp_timestamp); + + // Triggers a new estimate calculation for the stream identified by |ssrc|. + void UpdateEstimate(unsigned int ssrc, int64_t time_now); + + // Set the current round-trip time experienced by the stream identified by + // |ssrc|. + void SetRtt(unsigned int ssrc); + + // Removes all data for |ssrc|. + void RemoveStream(unsigned int ssrc); + + // Returns true if a valid estimate exists for a stream identified by |ssrc| + // and sets |bitrate_bps| to the estimated bitrate in bits per second. + bool LatestEstimate(unsigned int ssrc, unsigned int* bitrate_bps) const; + + private: + struct BitrateControls { + explicit BitrateControls(const OverUseDetectorOptions& options) + : remote_rate(), + overuse_detector(options), + incoming_bitrate() { + } + BitrateControls(const BitrateControls& other) + : remote_rate(other.remote_rate), + overuse_detector(other.overuse_detector), + incoming_bitrate(other.incoming_bitrate) { + } + RemoteRateControl remote_rate; + OveruseDetector overuse_detector; + BitRateStats incoming_bitrate; + }; + + typedef std::map SsrcBitrateControlsMap; + + const OverUseDetectorOptions& options_; + SsrcBitrateControlsMap bitrate_controls_; + RemoteBitrateObserver* observer_; + scoped_ptr crit_sect_; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_REMOTE_BITRATE_ESTIMATOR_SINGLE_STREAM_H_ diff --git a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest.cc b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest.cc index 9cd65f1525..70f066c180 100644 --- a/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest.cc +++ b/src/modules/remote_bitrate_estimator/remote_bitrate_estimator_unittest.cc @@ -8,13 +8,15 @@ * be found in the AUTHORS file in the root of the source tree. */ - // This file includes unit tests for RemoteBitrateEstimator. #include + +#include #include #include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "system_wrappers/interface/constructor_magic.h" #include "system_wrappers/interface/scoped_ptr.h" namespace webrtc { @@ -47,88 +49,249 @@ class TestBitrateObserver : public RemoteBitrateObserver { unsigned int latest_bitrate_; }; -class StreamGenerator { +class RtpStream { public: - struct Packet { + struct RtpPacket { int64_t send_time; int64_t arrival_time; uint32_t rtp_timestamp; unsigned int size; + unsigned int ssrc; }; - typedef std::list PacketList; + struct RtcpPacket { + uint32_t ntp_secs; + uint32_t ntp_frac; + uint32_t timestamp; + unsigned int ssrc; + }; - StreamGenerator(int fps, int bitrate_bps, int capacity, int64_t time_now) + typedef std::list PacketList; + + enum { kSendSideOffsetMs = 1000 }; + + RtpStream(int fps, int bitrate_bps, unsigned int ssrc, unsigned int frequency, + uint32_t timestamp_offset, int64_t rtcp_receive_time) : fps_(fps), bitrate_bps_(bitrate_bps), - capacity_(capacity), - time_now_(time_now), - prev_arrival_time_(time_now), - rtp_timestamp_offset_(0xFFFFF000) {} - - void SetCapacity(int capacity_bps) { - ASSERT_GT(capacity_bps, 0); - capacity_ = capacity_bps; + ssrc_(ssrc), + frequency_(frequency), + next_rtp_time_(0), + next_rtcp_time_(rtcp_receive_time), + rtp_timestamp_offset_(timestamp_offset), + kNtpFracPerMs(4.294967296E6) { + assert(fps_ > 0); } - void SetBitrate(int bitrate_bps) { + void set_rtp_timestamp_offset(uint32_t offset) { + rtp_timestamp_offset_ = offset; + } + + // Generates a new frame for this stream. If called too soon after the + // previous frame, no frame will be generated. The frame is split into + // packets. + int64_t GenerateFrame(double time_now, PacketList* packets) { + if (time_now < next_rtp_time_) { + return next_rtp_time_; + } + assert(packets != NULL); + int bits_per_frame = (bitrate_bps_ + fps_ / 2) / fps_; + int n_packets = std::max((bits_per_frame + 8 * kMtu) / (8 * kMtu), 1); + int packet_size = (bits_per_frame + 4 * n_packets) / (8 * n_packets); + assert(n_packets >= 0); + for (int i = 0; i < n_packets; ++i) { + RtpPacket* packet = new RtpPacket; + packet->send_time = time_now + kSendSideOffsetMs + 0.5f; + packet->size = packet_size; + packet->rtp_timestamp = rtp_timestamp_offset_ + static_cast( + (frequency_ / 1000.0) * packet->send_time + 0.5); + packet->ssrc = ssrc_; + packets->push_back(packet); + } + next_rtp_time_ = time_now + 1000.0 / static_cast(fps_); + return next_rtp_time_; + } + + // The send-side time when the next frame can be generated. + double next_rtp_time() const { + return next_rtp_time_; + } + + // Generates an RTCP packet. + RtcpPacket* Rtcp(double time_now) { + if (time_now < next_rtcp_time_) { + return NULL; + } + RtcpPacket* rtcp = new RtcpPacket; + int64_t send_time = RtpStream::kSendSideOffsetMs + time_now + 0.5; + rtcp->timestamp = rtp_timestamp_offset_ + static_cast( + (frequency_ / 1000.0) * send_time + 0.5); + rtcp->ntp_secs = send_time / 1000; + rtcp->ntp_frac = (send_time % 1000) * kNtpFracPerMs; + rtcp->ssrc = ssrc_; + next_rtcp_time_ = time_now + kRtcpIntervalMs; + return rtcp; + } + + void set_bitrate_bps(int bitrate_bps) { ASSERT_GE(bitrate_bps, 0); bitrate_bps_ = bitrate_bps; } - void SetRtpTimestampOffset(uint32_t offset) { - rtp_timestamp_offset_ = offset; + int bitrate_bps() const { + return bitrate_bps_; } - void GenerateFrame(PacketList* packets) { - ASSERT_FALSE(packets == NULL); - ASSERT_TRUE(packets->empty()); - ASSERT_GT(fps_, 0); - int bits_per_frame = bitrate_bps_ / fps_; - int n_packets = std::max(bits_per_frame / (8 * kMtu), 1); - int packet_size = bits_per_frame / (8 * n_packets); - ASSERT_GE(n_packets, 0); - for (int i = 0; i < n_packets; ++i) { - Packet* packet = new Packet; - packet->send_time = time_now_ + kSendSideOffsetMs; - ASSERT_GT(capacity_, 0); - packet->arrival_time = std::max( - prev_arrival_time_ + 8 * 1000 * packet_size / capacity_, - time_now_); - packet->size = packet_size; - packet->rtp_timestamp = rtp_timestamp_offset_ + 90 * packet->send_time; - prev_arrival_time_ = packet->arrival_time; - packets->push_back(packet); - } - time_now_ = time_now_ + 1000 / fps_; + unsigned int ssrc() const { + return ssrc_; } - int64_t TimeNow() const { - return time_now_; + static bool Compare(const std::pair& left, + const std::pair& right) { + return left.second->next_rtp_time_ < right.second->next_rtp_time_; } private: - enum { kSendSideOffsetMs = 1000 }; + enum { kRtcpIntervalMs = 1000 }; int fps_; int bitrate_bps_; - int capacity_; - int64_t time_now_; - int64_t prev_arrival_time_; + unsigned int ssrc_; + unsigned int frequency_; + double next_rtp_time_; + double next_rtcp_time_; uint32_t rtp_timestamp_offset_; + const double kNtpFracPerMs; + + DISALLOW_COPY_AND_ASSIGN(RtpStream); +}; + +class StreamGenerator { + public: + typedef std::list RtcpList; + + StreamGenerator(int capacity, double time_now) + : capacity_(capacity), + prev_arrival_time_(time_now) {} + + ~StreamGenerator() { + for (StreamMap::iterator it = streams_.begin(); it != streams_.end(); + ++it) { + delete it->second; + } + streams_.clear(); + } + + // Add a new stream. + void AddStream(RtpStream* stream) { + streams_[stream->ssrc()] = stream; + } + + // Set the link capacity. + void set_capacity_bps(int capacity_bps) { + ASSERT_GT(capacity_bps, 0); + capacity_ = capacity_bps; + } + + // Divides |bitrate_bps| among all streams. The allocated bitrate per stream + // is decided by the initial allocation ratios. + void set_bitrate_bps(int bitrate_bps) { + ASSERT_GE(streams_.size(), 0u); + double total_bitrate_before = 0; + for (StreamMap::iterator it = streams_.begin(); it != streams_.end(); + ++it) { + total_bitrate_before += it->second->bitrate_bps(); + } + int total_bitrate_after = 0; + for (StreamMap::iterator it = streams_.begin(); it != streams_.end(); + ++it) { + double ratio = it->second->bitrate_bps() / total_bitrate_before; + it->second->set_bitrate_bps(ratio * bitrate_bps + 0.5); + total_bitrate_after += it->second->bitrate_bps(); + } + EXPECT_NEAR(total_bitrate_after, bitrate_bps, 1); + } + + // Set the RTP timestamp offset for the stream identified by |ssrc|. + void set_rtp_timestamp_offset(unsigned int ssrc, uint32_t offset) { + streams_[ssrc]->set_rtp_timestamp_offset(offset); + } + + // TODO(holmer): Break out the channel simulation part from this class to make + // it possible to simulate different types of channels. + double GenerateFrame(RtpStream::PacketList* packets, double time_now) { + assert(packets != NULL); + assert(packets->empty()); + assert(capacity_ > 0); + StreamMap::iterator it = std::min_element(streams_.begin(), streams_.end(), + RtpStream::Compare); + (*it).second->GenerateFrame(time_now, packets); + for (RtpStream::PacketList::iterator packet_it = packets->begin(); + packet_it != packets->end(); ++packet_it) { + int required_network_time = + (8 * 1000 * (*packet_it)->size + capacity_ / 2) / capacity_; + prev_arrival_time_ = std::max(time_now + required_network_time, + prev_arrival_time_ + required_network_time); + (*packet_it)->arrival_time = prev_arrival_time_ + 0.5; + } + it = std::min_element(streams_.begin(), streams_.end(), RtpStream::Compare); + return (*it).second->next_rtp_time(); + } + + void Rtcps(RtcpList* rtcps, double time_now) const { + for (StreamMap::const_iterator it = streams_.begin(); it != streams_.end(); + ++it) { + RtpStream::RtcpPacket* rtcp = it->second->Rtcp(time_now); + if (rtcp) { + rtcps->push_front(rtcp); + } + } + } + + private: + typedef std::map StreamMap; + + // Capacity of the simulated channel in bits per second. + int capacity_; + // The time when the last packet arrived. + double prev_arrival_time_; + // All streams being transmitted on this simulated channel. + StreamMap streams_; + + DISALLOW_COPY_AND_ASSIGN(StreamGenerator); }; class RemoteBitrateEstimatorTest : public ::testing::Test { + public: + RemoteBitrateEstimatorTest() + : time_now_(0.0), + align_streams_(false) {} + explicit RemoteBitrateEstimatorTest(bool align_streams) + : time_now_(0.0), + align_streams_(align_streams) {} + protected: virtual void SetUp() { bitrate_observer_.reset(new TestBitrateObserver); - bitrate_estimator_.reset(new RemoteBitrateEstimator( - bitrate_observer_.get(), over_use_detector_options_)); - // Framerate: 30 fps; Start bitrate: 300 kbps; Link capacity: 1000 kbps, - // Start time: 0. - stream_generator_.reset(new StreamGenerator(30, 3e5, 1e6, 0)); + bitrate_estimator_.reset( + RemoteBitrateEstimator::Create( + bitrate_observer_.get(), + over_use_detector_options_, + RemoteBitrateEstimator::kMultiStreamEstimation)); + stream_generator_.reset(new StreamGenerator(1e6, // Capacity. + time_now_)); } + void AddDefaultStream() { + stream_generator_->AddStream(new RtpStream( + 30, // Frames per second. + 3e5, // Bitrate. + 1, // SSRC. + 90000, // RTP frequency. + 0xFFFFF000, // Timestamp offset. + 0)); // RTCP receive time. + } + // Generates a frame of packets belonging to a stream at a given bitrate and // with a given ssrc. The stream is pushed through a very simple simulated // network, and is then given to the receive-side bandwidth estimator. @@ -136,19 +299,31 @@ class RemoteBitrateEstimatorTest : public ::testing::Test { // The StreamGenerator::updated() should be used to check for any changes in // target bitrate after the call to this function. bool GenerateAndProcessFrame(unsigned int ssrc, unsigned int bitrate_bps) { - stream_generator_->SetBitrate(bitrate_bps); - StreamGenerator::PacketList packets; - stream_generator_->GenerateFrame(&packets); + stream_generator_->set_bitrate_bps(bitrate_bps); + RtpStream::PacketList packets; + time_now_ = stream_generator_->GenerateFrame(&packets, time_now_); int64_t last_arrival_time = -1; bool prev_was_decrease = false; bool overuse = false; while (!packets.empty()) { - StreamGenerator::Packet* packet = packets.front(); - bitrate_estimator_->IncomingPacket(ssrc, + RtpStream::RtpPacket* packet = packets.front(); + if (align_streams_) { + StreamGenerator::RtcpList rtcps; + stream_generator_->Rtcps(&rtcps, time_now_); + for (StreamGenerator::RtcpList::iterator it = rtcps.begin(); + it != rtcps.end(); ++it) { + bitrate_estimator_->IncomingRtcp((*it)->ssrc, + (*it)->ntp_secs, + (*it)->ntp_frac, + (*it)->timestamp); + delete *it; + } + } + bitrate_observer_->Reset(); + bitrate_estimator_->IncomingPacket(packet->ssrc, packet->size, packet->arrival_time, - packet->rtp_timestamp, - -1); + packet->rtp_timestamp); if (bitrate_observer_->updated()) { // Verify that new estimates only are triggered by an overuse and a // rate decrease. @@ -159,7 +334,6 @@ class RemoteBitrateEstimatorTest : public ::testing::Test { } else { prev_was_decrease = false; } - bitrate_observer_->Reset(); last_arrival_time = packet->arrival_time; delete packet; packets.pop_front(); @@ -196,39 +370,52 @@ class RemoteBitrateEstimatorTest : public ::testing::Test { return bitrate_bps; } + static const unsigned int kDefaultSsrc = 1; + + double time_now_; // Current time at the receiver. OverUseDetectorOptions over_use_detector_options_; scoped_ptr bitrate_estimator_; scoped_ptr bitrate_observer_; scoped_ptr stream_generator_; + const bool align_streams_; + + DISALLOW_COPY_AND_ASSIGN(RemoteBitrateEstimatorTest); +}; + +class RemoteBitrateEstimatorTestAlign : public RemoteBitrateEstimatorTest { + public: + RemoteBitrateEstimatorTestAlign() : RemoteBitrateEstimatorTest(true) {} + + private: + DISALLOW_COPY_AND_ASSIGN(RemoteBitrateEstimatorTestAlign); }; TEST_F(RemoteBitrateEstimatorTest, TestInitialBehavior) { unsigned int bitrate_bps = 0; - unsigned int ssrc = 0; int64_t time_now = 0; uint32_t timestamp = 0; - EXPECT_FALSE(bitrate_estimator_->LatestEstimate(ssrc, &bitrate_bps)); - bitrate_estimator_->UpdateEstimate(ssrc, time_now); - EXPECT_FALSE(bitrate_estimator_->LatestEstimate(ssrc, &bitrate_bps)); + EXPECT_FALSE(bitrate_estimator_->LatestEstimate(kDefaultSsrc, &bitrate_bps)); + bitrate_estimator_->UpdateEstimate(kDefaultSsrc, time_now); + EXPECT_FALSE(bitrate_estimator_->LatestEstimate(kDefaultSsrc, &bitrate_bps)); EXPECT_FALSE(bitrate_observer_->updated()); bitrate_observer_->Reset(); // Inserting a packet. Still no valid estimate. We need to wait 1 second. - bitrate_estimator_->IncomingPacket(ssrc, kMtu, time_now, - timestamp, -1); - bitrate_estimator_->UpdateEstimate(ssrc, time_now); - EXPECT_FALSE(bitrate_estimator_->LatestEstimate(ssrc, &bitrate_bps)); + bitrate_estimator_->IncomingPacket(kDefaultSsrc, kMtu, time_now, + timestamp); + bitrate_estimator_->UpdateEstimate(kDefaultSsrc, time_now); + EXPECT_FALSE(bitrate_estimator_->LatestEstimate(kDefaultSsrc, &bitrate_bps)); EXPECT_FALSE(bitrate_observer_->updated()); bitrate_observer_->Reset(); // Waiting more than one second gives us a valid estimate. // We need at least two packets for the incoming bitrate to be > 0 since the // window is 500 ms. time_now += 499; - bitrate_estimator_->IncomingPacket(ssrc, kMtu, time_now, - timestamp, -1); + bitrate_estimator_->IncomingPacket(kDefaultSsrc, kMtu, time_now, + timestamp); time_now += 2; - bitrate_estimator_->UpdateEstimate(ssrc, time_now); - EXPECT_TRUE(bitrate_estimator_->LatestEstimate(ssrc, &bitrate_bps)); - EXPECT_EQ(20607u, bitrate_bps); + bitrate_estimator_->UpdateEstimate(kDefaultSsrc, time_now); + EXPECT_TRUE(bitrate_estimator_->LatestEstimate(kDefaultSsrc, &bitrate_bps)); + EXPECT_EQ(20644u, bitrate_bps); EXPECT_TRUE(bitrate_observer_->updated()); bitrate_observer_->Reset(); EXPECT_EQ(bitrate_observer_->latest_bitrate(), bitrate_bps); @@ -236,14 +423,14 @@ TEST_F(RemoteBitrateEstimatorTest, TestInitialBehavior) { // Make sure we initially increase the bitrate as expected. TEST_F(RemoteBitrateEstimatorTest, TestRateIncreaseRtpTimestamps) { - const int kExpectedIterations = 277; + const int kExpectedIterations = 276; unsigned int bitrate_bps = 30000; - unsigned int ssrc = 0; int iterations = 0; + AddDefaultStream(); // Feed the estimator with a stream of packets and verify that it reaches // 500 kbps at the expected time. while (bitrate_bps < 5e5) { - bool overuse = GenerateAndProcessFrame(ssrc, bitrate_bps); + bool overuse = GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); if (overuse) { EXPECT_GT(bitrate_observer_->latest_bitrate(), bitrate_bps); bitrate_bps = bitrate_observer_->latest_bitrate(); @@ -255,73 +442,231 @@ TEST_F(RemoteBitrateEstimatorTest, TestRateIncreaseRtpTimestamps) { ++iterations; ASSERT_LE(iterations, kExpectedIterations); } - ASSERT_EQ(iterations, kExpectedIterations); + ASSERT_EQ(kExpectedIterations, iterations); } // Verify that the time it takes for the estimator to reduce the bitrate when // the capacity is tightened stays the same. TEST_F(RemoteBitrateEstimatorTest, TestCapacityDropRtpTimestamps) { - const unsigned int kSsrc = 0; const int kNumberOfFrames= 300; const int kStartBitrate = 900e3; const int kMinExpectedBitrate = 800e3; - const int kMaxExpectedBitrate = 1500e3; + const int kMaxExpectedBitrate = 1100e3; + AddDefaultStream(); // Run in steady state to make the estimator converge. - stream_generator_->SetCapacity(1000e3); - unsigned int bitrate_bps = SteadyStateRun(kSsrc, kNumberOfFrames, + stream_generator_->set_capacity_bps(1000e3); + unsigned int bitrate_bps = SteadyStateRun(kDefaultSsrc, kNumberOfFrames, kStartBitrate, kMinExpectedBitrate, kMaxExpectedBitrate); // Reduce the capacity and verify the decrease time. - stream_generator_->SetCapacity(500e3); - int64_t bitrate_drop_time = 0; - for (int i = 0; i < 1000; ++i) { - GenerateAndProcessFrame(kSsrc, bitrate_bps); + stream_generator_->set_capacity_bps(500e3); + int64_t bitrate_drop_time = -1; + for (int i = 0; i < 200; ++i) { + GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); // Check for either increase or decrease. if (bitrate_observer_->updated()) { - if (bitrate_observer_->latest_bitrate() <= 500e3) { - bitrate_drop_time = stream_generator_->TimeNow(); + if (bitrate_drop_time == -1 && + bitrate_observer_->latest_bitrate() <= 500e3) { + bitrate_drop_time = time_now_; } bitrate_bps = bitrate_observer_->latest_bitrate(); bitrate_observer_->Reset(); } } - EXPECT_EQ(42900, bitrate_drop_time); + EXPECT_EQ(10333, bitrate_drop_time); } // Verify that the time it takes for the estimator to reduce the bitrate when // the capacity is tightened stays the same. This test also verifies that we // handle wrap-arounds in this scenario. TEST_F(RemoteBitrateEstimatorTest, TestCapacityDropRtpTimestampsWrap) { - const unsigned int kSsrc = 0; const int kFramerate= 30; const int kStartBitrate = 900e3; const int kMinExpectedBitrate = 800e3; - const int kMaxExpectedBitrate = 1500e3; - const int kSteadyStateTime = 10; // Seconds. + const int kMaxExpectedBitrate = 1100e3; + const int kSteadyStateTime = 8; // Seconds. + AddDefaultStream(); // Trigger wrap right after the steady state run. - stream_generator_->SetRtpTimestampOffset( + stream_generator_->set_rtp_timestamp_offset(kDefaultSsrc, std::numeric_limits::max() - kSteadyStateTime * 90000); // Run in steady state to make the estimator converge. - unsigned int bitrate_bps = SteadyStateRun(kSsrc, + stream_generator_->set_capacity_bps(1000e3); + unsigned int bitrate_bps = SteadyStateRun(kDefaultSsrc, kSteadyStateTime * kFramerate, kStartBitrate, kMinExpectedBitrate, kMaxExpectedBitrate); + bitrate_observer_->Reset(); // Reduce the capacity and verify the decrease time. - stream_generator_->SetCapacity(500e3); - int64_t bitrate_drop_time = 0; - for (int i = 0; i < 1000; ++i) { - GenerateAndProcessFrame(kSsrc, bitrate_bps); + stream_generator_->set_capacity_bps(500e3); + int64_t bitrate_drop_time = -1; + for (int i = 0; i < 200; ++i) { + GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); // Check for either increase or decrease. if (bitrate_observer_->updated()) { - if (bitrate_observer_->latest_bitrate() <= 500e3) { - bitrate_drop_time = stream_generator_->TimeNow(); + if (bitrate_drop_time == -1 && + bitrate_observer_->latest_bitrate() <= 500e3) { + bitrate_drop_time = time_now_; } bitrate_bps = bitrate_observer_->latest_bitrate(); bitrate_observer_->Reset(); } } - EXPECT_EQ(42900, bitrate_drop_time); + EXPECT_EQ(8299, bitrate_drop_time); +} + +// Verify that the time it takes for the estimator to reduce the bitrate when +// the capacity is tightened stays the same. This test also verifies that we +// handle wrap-arounds in this scenario. This test also converts the timestamps +// to NTP time. +TEST_F(RemoteBitrateEstimatorTestAlign, TestCapacityDropRtpTimestampsWrap) { + const int kFramerate= 30; + const int kStartBitrate = 900e3; + const int kMinExpectedBitrate = 800e3; + const int kMaxExpectedBitrate = 1100e3; + const int kSteadyStateTime = 8; // Seconds. + AddDefaultStream(); + // Trigger wrap right after the steady state run. + stream_generator_->set_rtp_timestamp_offset(kDefaultSsrc, + std::numeric_limits::max() - kSteadyStateTime * 90000); + // Run in steady state to make the estimator converge. + stream_generator_->set_capacity_bps(1000e3); + unsigned int bitrate_bps = SteadyStateRun(kDefaultSsrc, + kSteadyStateTime * kFramerate, + kStartBitrate, + kMinExpectedBitrate, + kMaxExpectedBitrate); + bitrate_observer_->Reset(); + // Reduce the capacity and verify the decrease time. + stream_generator_->set_capacity_bps(500e3); + int64_t bitrate_drop_time = -1; + for (int i = 0; i < 200; ++i) { + GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); + // Check for either increase or decrease. + if (bitrate_observer_->updated()) { + if (bitrate_drop_time == -1 && + bitrate_observer_->latest_bitrate() <= 500e3) { + bitrate_drop_time = time_now_; + } + bitrate_bps = bitrate_observer_->latest_bitrate(); + bitrate_observer_->Reset(); + } + } + EXPECT_EQ(8299, bitrate_drop_time); +} + +// Verify that the time it takes for the estimator to reduce the bitrate when +// the capacity is tightened stays the same. This test also verifies that we +// handle wrap-arounds in this scenario. This is a multi-stream test. +TEST_F(RemoteBitrateEstimatorTestAlign, TwoStreamsCapacityDropWithWrap) { + const int kFramerate= 30; + const int kStartBitrate = 900e3; + const int kMinExpectedBitrate = 800e3; + const int kMaxExpectedBitrate = 1100e3; + const int kSteadyStateTime = 7; // Seconds. + stream_generator_->AddStream(new RtpStream( + 30, // Frames per second. + kStartBitrate/2, // Bitrate. + 1, // SSRC. + 90000, // RTP frequency. + 0xFFFFF000, // Timestamp offset. + 0)); // RTCP receive time. + + stream_generator_->AddStream(new RtpStream( + 15, // Frames per second. + kStartBitrate/2, // Bitrate. + 2, // SSRC. + 90000, // RTP frequency. + 0x00000FFF, // Timestamp offset. + 0)); // RTCP receive time. + // Trigger wrap right after the steady state run. + stream_generator_->set_rtp_timestamp_offset(kDefaultSsrc, + std::numeric_limits::max() - kSteadyStateTime * 90000); + // Run in steady state to make the estimator converge. + stream_generator_->set_capacity_bps(1000e3); + unsigned int bitrate_bps = SteadyStateRun(kDefaultSsrc, + kSteadyStateTime * kFramerate, + kStartBitrate, + kMinExpectedBitrate, + kMaxExpectedBitrate); + bitrate_observer_->Reset(); + // Reduce the capacity and verify the decrease time. + stream_generator_->set_capacity_bps(500e3); + int64_t bitrate_drop_time = -1; + for (int i = 0; i < 200; ++i) { + GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); + // Check for either increase or decrease. + if (bitrate_observer_->updated()) { + if (bitrate_drop_time == -1 && + bitrate_observer_->latest_bitrate() <= 500e3) { + bitrate_drop_time = time_now_; + } + bitrate_bps = bitrate_observer_->latest_bitrate(); + bitrate_observer_->Reset(); + } + } + EXPECT_EQ(4966, bitrate_drop_time); +} + +// Verify that the time it takes for the estimator to reduce the bitrate when +// the capacity is tightened stays the same. This test also verifies that we +// handle wrap-arounds in this scenario. This is a multi-stream test. +TEST_F(RemoteBitrateEstimatorTestAlign, ThreeStreams) { + const int kFramerate= 30; + const int kStartBitrate = 900e3; + const int kMinExpectedBitrate = 800e3; + const int kMaxExpectedBitrate = 1100e3; + const int kSteadyStateTime = 11; // Seconds. + stream_generator_->AddStream(new RtpStream( + 30, // Frames per second. + kStartBitrate/2, // Bitrate. + 1, // SSRC. + 90000, // RTP frequency. + 0xFFFFF000, // Timestamp offset. + 0)); // RTCP receive time. + + stream_generator_->AddStream(new RtpStream( + 30, // Frames per second. + kStartBitrate/3, // Bitrate. + 2, // SSRC. + 90000, // RTP frequency. + 0x00000FFF, // Timestamp offset. + 0)); // RTCP receive time. + + stream_generator_->AddStream(new RtpStream( + 30, // Frames per second. + kStartBitrate/6, // Bitrate. + 3, // SSRC. + 90000, // RTP frequency. + 0x00000FFF, // Timestamp offset. + 0)); // RTCP receive time. + // Trigger wrap right after the steady state run. + stream_generator_->set_rtp_timestamp_offset(kDefaultSsrc, + std::numeric_limits::max() - kSteadyStateTime * 90000); + // Run in steady state to make the estimator converge. + stream_generator_->set_capacity_bps(1000e3); + unsigned int bitrate_bps = SteadyStateRun(kDefaultSsrc, + kSteadyStateTime * kFramerate, + kStartBitrate, + kMinExpectedBitrate, + kMaxExpectedBitrate); + bitrate_observer_->Reset(); + // Reduce the capacity and verify the decrease time. + stream_generator_->set_capacity_bps(500e3); + int64_t bitrate_drop_time = -1; + for (int i = 0; i < 200; ++i) { + GenerateAndProcessFrame(kDefaultSsrc, bitrate_bps); + // Check for either increase or decrease. + if (bitrate_observer_->updated()) { + if (bitrate_drop_time == -1 && + bitrate_observer_->latest_bitrate() <= 500e3) { + bitrate_drop_time = time_now_; + } + bitrate_bps = bitrate_observer_->latest_bitrate(); + bitrate_observer_->Reset(); + } + } + EXPECT_EQ(3933, bitrate_drop_time); } } // namespace webrtc diff --git a/src/modules/remote_bitrate_estimator/remote_rate_control.cc b/src/modules/remote_bitrate_estimator/remote_rate_control.cc index ea15e21f8b..ae48cc239e 100644 --- a/src/modules/remote_bitrate_estimator/remote_rate_control.cc +++ b/src/modules/remote_bitrate_estimator/remote_rate_control.cc @@ -414,7 +414,7 @@ void RemoteRateControl::ChangeState(const RateControlInput& input, WebRtc_Word64 } break; } - case kBwUnderUsing: + case kBwUnderusing: { ChangeState(kRcHold); break; @@ -481,7 +481,7 @@ void RemoteRateControl::StateStr(BandwidthUsage state, char* str) case kBwOverusing: strncpy(str, "OVER USING", 11); break; - case kBwUnderUsing: + case kBwUnderusing: strncpy(str, "UNDER USING", 12); break; } diff --git a/src/modules/rtp_rtcp/interface/rtp_rtcp_defines.h b/src/modules/rtp_rtcp/interface/rtp_rtcp_defines.h index 5d7098c0d4..7399dc1100 100644 --- a/src/modules/rtp_rtcp/interface/rtp_rtcp_defines.h +++ b/src/modules/rtp_rtcp/interface/rtp_rtcp_defines.h @@ -151,8 +151,14 @@ public: virtual void OnRTCPPacketTimeout(const WebRtc_Word32 /*id*/) {}; + // |ntp_secs|, |ntp_frac| and |timestamp| are the NTP time and RTP timestamp + // parsed from the RTCP sender report from the sender with ssrc + // |senderSSRC|. virtual void OnSendReportReceived(const WebRtc_Word32 id, - const WebRtc_UWord32 senderSSRC) {}; + const WebRtc_UWord32 senderSSRC, + uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t timestamp) {}; virtual void OnReceiveReportReceived(const WebRtc_Word32 id, const WebRtc_UWord32 senderSSRC) {}; diff --git a/src/modules/rtp_rtcp/source/mock/mock_rtp_receiver_video.h b/src/modules/rtp_rtcp/source/mock/mock_rtp_receiver_video.h index 9498b74be3..37f08eba3f 100644 --- a/src/modules/rtp_rtcp/source/mock/mock_rtp_receiver_video.h +++ b/src/modules/rtp_rtcp/source/mock/mock_rtp_receiver_video.h @@ -17,7 +17,7 @@ namespace webrtc { class MockRTPReceiverVideo : public RTPReceiverVideo { public: - MockRTPReceiverVideo() : RTPReceiverVideo(0, NULL, NULL) {} + MockRTPReceiverVideo() : RTPReceiverVideo(0, NULL) {} MOCK_METHOD1(ChangeUniqueId, void(const WebRtc_Word32 id)); MOCK_METHOD3(ReceiveRecoveredPacketCallback, diff --git a/src/modules/rtp_rtcp/source/rtcp_format_remb_unittest.cc b/src/modules/rtp_rtcp/source/rtcp_format_remb_unittest.cc index 466fccd3dc..d229b6c66e 100644 --- a/src/modules/rtp_rtcp/source/rtcp_format_remb_unittest.cc +++ b/src/modules/rtp_rtcp/source/rtcp_format_remb_unittest.cc @@ -61,8 +61,10 @@ class RtcpFormatRembTest : public ::testing::Test { RtcpFormatRembTest() : over_use_detector_options_(), remote_bitrate_observer_(), - remote_bitrate_estimator_(&remote_bitrate_observer_, - over_use_detector_options_) {} + remote_bitrate_estimator_(RemoteBitrateEstimator::Create( + &remote_bitrate_observer_, + over_use_detector_options_, + RemoteBitrateEstimator::kMultiStreamEstimation)) {} virtual void SetUp(); virtual void TearDown(); @@ -73,7 +75,7 @@ class RtcpFormatRembTest : public ::testing::Test { RTCPReceiver* rtcp_receiver_; TestTransport* test_transport_; MockRemoteBitrateObserver remote_bitrate_observer_; - RemoteBitrateEstimator remote_bitrate_estimator_; + scoped_ptr remote_bitrate_estimator_; }; void RtcpFormatRembTest::SetUp() { @@ -82,7 +84,7 @@ void RtcpFormatRembTest::SetUp() { configuration.id = 0; configuration.audio = false; configuration.clock = system_clock_; - configuration.remote_bitrate_estimator = &remote_bitrate_estimator_; + configuration.remote_bitrate_estimator = remote_bitrate_estimator_.get(); dummy_rtp_rtcp_impl_ = new ModuleRtpRtcpImpl(configuration); rtcp_sender_ = new RTCPSender(0, false, system_clock_, dummy_rtp_rtcp_impl_); rtcp_receiver_ = new RTCPReceiver(0, system_clock_, dummy_rtp_rtcp_impl_); diff --git a/src/modules/rtp_rtcp/source/rtcp_receiver.cc b/src/modules/rtp_rtcp/source/rtcp_receiver.cc index e522d3f9e2..631cc1b0e9 100644 --- a/src/modules/rtp_rtcp/source/rtcp_receiver.cc +++ b/src/modules/rtp_rtcp/source/rtcp_receiver.cc @@ -377,6 +377,10 @@ RTCPReceiver::HandleSenderReceiverReport(RTCPUtility::RTCPParserV2& rtcpParser, // only signal that we have received a SR when we accept one rtcpPacketInformation.rtcpPacketTypeFlags |= kRtcpSr; + rtcpPacketInformation.ntp_secs = rtcpPacket.SR.NTPMostSignificant; + rtcpPacketInformation.ntp_frac = rtcpPacket.SR.NTPLeastSignificant; + rtcpPacketInformation.rtp_timestamp = rtcpPacket.SR.RTPTimestamp; + // We will only store the send report from one source, but // we will store all the receive block @@ -1261,7 +1265,10 @@ void RTCPReceiver::TriggerCallbacksFromRTCPPacket( if(_cbRtcpFeedback) { if(rtcpPacketInformation.rtcpPacketTypeFlags & kRtcpSr) { _cbRtcpFeedback->OnSendReportReceived(_id, - rtcpPacketInformation.remoteSSRC); + rtcpPacketInformation.remoteSSRC, + rtcpPacketInformation.ntp_secs, + rtcpPacketInformation.ntp_frac, + rtcpPacketInformation.rtp_timestamp); } else { _cbRtcpFeedback->OnReceiveReportReceived(_id, rtcpPacketInformation.remoteSSRC); diff --git a/src/modules/rtp_rtcp/source/rtcp_receiver_help.cc b/src/modules/rtp_rtcp/source/rtcp_receiver_help.cc index 81e33ac0c5..09c8ba4ffd 100644 --- a/src/modules/rtp_rtcp/source/rtcp_receiver_help.cc +++ b/src/modules/rtp_rtcp/source/rtcp_receiver_help.cc @@ -36,6 +36,9 @@ RTCPPacketInformation::RTCPPacketInformation() sliPictureId(0), rpsiPictureId(0), receiverEstimatedMaxBitrate(0), + ntp_secs(0), + ntp_frac(0), + rtp_timestamp(0), VoIPMetric(NULL) { } diff --git a/src/modules/rtp_rtcp/source/rtcp_receiver_help.h b/src/modules/rtp_rtcp/source/rtcp_receiver_help.h index a7214300be..8af7b66949 100644 --- a/src/modules/rtp_rtcp/source/rtcp_receiver_help.h +++ b/src/modules/rtp_rtcp/source/rtcp_receiver_help.h @@ -64,6 +64,10 @@ public: WebRtc_UWord64 rpsiPictureId; WebRtc_UWord32 receiverEstimatedMaxBitrate; + uint32_t ntp_secs; + uint32_t ntp_frac; + uint32_t rtp_timestamp; + RTCPVoIPMetric* VoIPMetric; }; diff --git a/src/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc b/src/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc index 2cb6d25f0e..2db06f96e8 100644 --- a/src/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc +++ b/src/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc @@ -185,8 +185,11 @@ class RtcpReceiverTest : public ::testing::Test { RtcpReceiverTest() : over_use_detector_options_(), remote_bitrate_observer_(), - remote_bitrate_estimator_(&remote_bitrate_observer_, - over_use_detector_options_) { + remote_bitrate_estimator_( + RemoteBitrateEstimator::Create( + &remote_bitrate_observer_, + over_use_detector_options_, + RemoteBitrateEstimator::kMultiStreamEstimation)) { // system_clock_ = ModuleRTPUtility::GetSystemClock(); system_clock_ = new FakeSystemClock(); test_transport_ = new TestTransport(); @@ -196,7 +199,7 @@ class RtcpReceiverTest : public ::testing::Test { configuration.audio = false; configuration.clock = system_clock_; configuration.outgoing_transport = test_transport_; - configuration.remote_bitrate_estimator = &remote_bitrate_estimator_; + configuration.remote_bitrate_estimator = remote_bitrate_estimator_.get(); rtp_rtcp_impl_ = new ModuleRtpRtcpImpl(configuration); rtcp_receiver_ = new RTCPReceiver(0, system_clock_, rtp_rtcp_impl_); test_transport_->SetRTCPReceiver(rtcp_receiver_); @@ -230,7 +233,7 @@ class RtcpReceiverTest : public ::testing::Test { TestTransport* test_transport_; RTCPHelp::RTCPPacketInformation rtcp_packet_info_; MockRemoteBitrateObserver remote_bitrate_observer_; - RemoteBitrateEstimator remote_bitrate_estimator_; + scoped_ptr remote_bitrate_estimator_; }; diff --git a/src/modules/rtp_rtcp/source/rtcp_sender_unittest.cc b/src/modules/rtp_rtcp/source/rtcp_sender_unittest.cc index 1d14f271cb..dffc9b3f2a 100644 --- a/src/modules/rtp_rtcp/source/rtcp_sender_unittest.cc +++ b/src/modules/rtp_rtcp/source/rtcp_sender_unittest.cc @@ -100,8 +100,11 @@ class RtcpSenderTest : public ::testing::Test { RtcpSenderTest() : over_use_detector_options_(), remote_bitrate_observer_(), - remote_bitrate_estimator_(&remote_bitrate_observer_, - over_use_detector_options_) { + remote_bitrate_estimator_( + RemoteBitrateEstimator::Create( + &remote_bitrate_observer_, + over_use_detector_options_, + RemoteBitrateEstimator::kMultiStreamEstimation)) { system_clock_ = ModuleRTPUtility::GetSystemClock(); test_transport_ = new TestTransport(); @@ -111,7 +114,7 @@ class RtcpSenderTest : public ::testing::Test { configuration.clock = system_clock_; configuration.incoming_data = test_transport_; configuration.outgoing_transport = test_transport_; - configuration.remote_bitrate_estimator = &remote_bitrate_estimator_; + configuration.remote_bitrate_estimator = remote_bitrate_estimator_.get(); rtp_rtcp_impl_ = new ModuleRtpRtcpImpl(configuration); rtcp_sender_ = new RTCPSender(0, false, system_clock_, rtp_rtcp_impl_); @@ -142,7 +145,7 @@ class RtcpSenderTest : public ::testing::Test { RTCPReceiver* rtcp_receiver_; TestTransport* test_transport_; MockRemoteBitrateObserver remote_bitrate_observer_; - RemoteBitrateEstimator remote_bitrate_estimator_; + scoped_ptr remote_bitrate_estimator_; enum {kMaxPacketLength = 1500}; uint8_t packet_[kMaxPacketLength]; diff --git a/src/modules/rtp_rtcp/source/rtp_receiver.cc b/src/modules/rtp_rtcp/source/rtp_receiver.cc index ca5cd54e9a..b4a7e52891 100644 --- a/src/modules/rtp_rtcp/source/rtp_receiver.cc +++ b/src/modules/rtp_rtcp/source/rtp_receiver.cc @@ -32,10 +32,9 @@ using ModuleRTPUtility::VideoPayload; RTPReceiver::RTPReceiver(const WebRtc_Word32 id, const bool audio, RtpRtcpClock* clock, - RemoteBitrateEstimator* remote_bitrate, ModuleRtpRtcpImpl* owner) : RTPReceiverAudio(id), - RTPReceiverVideo(id, remote_bitrate, owner), + RTPReceiverVideo(id, owner), Bitrate(clock), _id(id), _audio(audio), diff --git a/src/modules/rtp_rtcp/source/rtp_receiver.h b/src/modules/rtp_rtcp/source/rtp_receiver.h index b0e8e28817..3893727eed 100644 --- a/src/modules/rtp_rtcp/source/rtp_receiver.h +++ b/src/modules/rtp_rtcp/source/rtp_receiver.h @@ -35,7 +35,6 @@ public: RTPReceiver(const WebRtc_Word32 id, const bool audio, RtpRtcpClock* clock, - RemoteBitrateEstimator* remote_bitrate, ModuleRtpRtcpImpl* owner); virtual ~RTPReceiver(); diff --git a/src/modules/rtp_rtcp/source/rtp_receiver_video.cc b/src/modules/rtp_rtcp/source/rtp_receiver_video.cc index 0af375e2b5..08218c31aa 100644 --- a/src/modules/rtp_rtcp/source/rtp_receiver_video.cc +++ b/src/modules/rtp_rtcp/source/rtp_receiver_video.cc @@ -27,15 +27,12 @@ WebRtc_UWord32 BitRateBPS(WebRtc_UWord16 x ) } RTPReceiverVideo::RTPReceiverVideo(const WebRtc_Word32 id, - RemoteBitrateEstimator* remote_bitrate, ModuleRtpRtcpImpl* owner) : _id(id), _criticalSectionReceiverVideo( CriticalSectionWrapper::CreateCriticalSection()), _currentFecFrameDecoded(false), - _receiveFEC(NULL), - remote_bitrate_(remote_bitrate), - _packetOverHead(28) { + _receiveFEC(NULL) { } RTPReceiverVideo::~RTPReceiverVideo() { @@ -88,18 +85,6 @@ WebRtc_Word32 RTPReceiverVideo::ParseVideoCodecSpecific( _criticalSectionReceiverVideo->Enter(); - // Add headers, ideally we would like to include for instance - // Ethernet header here as well. - const WebRtc_UWord16 packetSize = payloadDataLength + _packetOverHead + - rtpHeader->header.headerLength + rtpHeader->header.paddingLength; - uint32_t compensated_timestamp = rtpHeader->header.timestamp + - rtpHeader->extension.transmissionTimeOffset; - remote_bitrate_->IncomingPacket(rtpHeader->header.ssrc, - packetSize, - nowMS, - compensated_timestamp, - -1); - if (isRED) { if(_receiveFEC == NULL) { _criticalSectionReceiverVideo->Leave(); @@ -127,7 +112,10 @@ WebRtc_Word32 RTPReceiverVideo::ParseVideoCodecSpecific( if(retVal != 0) { return retVal; } - retVal = CallbackOfReceivedPayloadData(NULL, 0, rtpHeader); + // Pass the length of FEC packets so that they can be accounted for in + // the bandwidth estimator. + retVal = CallbackOfReceivedPayloadData(NULL, payloadDataLength, + rtpHeader); } } else { // will leave the _criticalSectionReceiverVideo critsect @@ -344,8 +332,4 @@ WebRtc_Word32 RTPReceiverVideo::ReceiveGenericCodec( } return 0; } - -void RTPReceiverVideo::SetPacketOverHead(WebRtc_UWord16 packetOverHead) { - _packetOverHead = packetOverHead; -} } // namespace webrtc diff --git a/src/modules/rtp_rtcp/source/rtp_receiver_video.h b/src/modules/rtp_rtcp/source/rtp_receiver_video.h index e50ef60948..ddfa69e75a 100644 --- a/src/modules/rtp_rtcp/source/rtp_receiver_video.h +++ b/src/modules/rtp_rtcp/source/rtp_receiver_video.h @@ -16,9 +16,6 @@ #include "typedefs.h" -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" -#include "modules/remote_bitrate_estimator/overuse_detector.h" -#include "modules/remote_bitrate_estimator/remote_rate_control.h" #include "Bitrate.h" #include "scoped_ptr.h" @@ -30,7 +27,6 @@ class CriticalSectionWrapper; class RTPReceiverVideo { public: RTPReceiverVideo(const WebRtc_Word32 id, - RemoteBitrateEstimator* remote_bitrate, ModuleRtpRtcpImpl* owner); virtual ~RTPReceiverVideo(); @@ -104,10 +100,6 @@ class RTPReceiverVideo { // FEC bool _currentFecFrameDecoded; ReceiverFEC* _receiveFEC; - - // BWE - RemoteBitrateEstimator* remote_bitrate_; - WebRtc_UWord16 _packetOverHead; }; } // namespace webrtc #endif // WEBRTC_MODULES_RTP_RTCP_SOURCE_RTP_RECEIVER_VIDEO_H_ diff --git a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index fbd57f20d3..6002979237 100644 --- a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -54,7 +54,7 @@ RtpRtcp* RtpRtcp::CreateRtpRtcp(const RtpRtcp::Configuration& configuration) { ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) : _rtpSender(configuration.id, configuration.audio, configuration.clock), _rtpReceiver(configuration.id, configuration.audio, configuration.clock, - configuration.remote_bitrate_estimator, this), + this), _rtcpSender(configuration.id, configuration.audio, configuration.clock, this), _rtcpReceiver(configuration.id, configuration.clock, this), @@ -987,7 +987,6 @@ WebRtc_Word32 ModuleRtpRtcpImpl::SetTransportOverhead( // store new _packetOverHead = packetOverHead; - _rtpReceiver.SetPacketOverHead(_packetOverHead); WebRtc_UWord16 length = _rtpSender.MaxPayloadLength() - packetOverHeadDiff; return _rtpSender.SetMaxPayloadLength(length, _packetOverHead); } diff --git a/src/modules/rtp_rtcp/test/testAPI/test_api_rtcp.cc b/src/modules/rtp_rtcp/test/testAPI/test_api_rtcp.cc index 1e610da44e..ce77953cf1 100644 --- a/src/modules/rtp_rtcp/test/testAPI/test_api_rtcp.cc +++ b/src/modules/rtp_rtcp/test/testAPI/test_api_rtcp.cc @@ -51,7 +51,10 @@ class RtcpCallback : public RtcpFeedback, public RtcpIntraFrameObserver { EXPECT_STRCASEEQ("test", print_name); }; virtual void OnSendReportReceived(const WebRtc_Word32 id, - const WebRtc_UWord32 senderSSRC) { + const WebRtc_UWord32 senderSSRC, + uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t timestamp) { RTCPSenderInfo senderInfo; EXPECT_EQ(0, _rtpRtcpModule->RemoteRTCPStat(&senderInfo)); }; diff --git a/src/video_engine/include/vie_rtp_rtcp.h b/src/video_engine/include/vie_rtp_rtcp.h index f19171d683..aa0c466a20 100644 --- a/src/video_engine/include/vie_rtp_rtcp.h +++ b/src/video_engine/include/vie_rtp_rtcp.h @@ -48,6 +48,11 @@ enum StreamType { kViEStreamTypeRtx = 1 // Retransmission media stream }; +enum BandwidthEstimationMode { + kViEMultiStreamEstimation, + kViESingleStreamEstimation +}; + // This class declares an abstract interface for a user defined observer. It is // up to the VideoEngine user to implement a derived class which implements the // observer class. The observer is registered using RegisterRTPObserver() and @@ -209,6 +214,10 @@ class WEBRTC_DLLEXPORT ViERTP_RTCP { bool sender, bool receiver) = 0; + // Sets the bandwidth estimation mode. This can only be changed before + // adding a channel. + virtual int SetBandwidthEstimationMode(BandwidthEstimationMode mode) = 0; + // Enables RTP timestamp extension offset described in RFC 5450. This call // must be done before ViECodec::SetSendCodec is called. virtual int SetSendTimestampOffsetStatus(int video_channel, diff --git a/src/video_engine/test/auto_test/source/vie_autotest_loopback.cc b/src/video_engine/test/auto_test/source/vie_autotest_loopback.cc index ee31e59375..82241f8c2d 100644 --- a/src/video_engine/test/auto_test/source/vie_autotest_loopback.cc +++ b/src/video_engine/test/auto_test/source/vie_autotest_loopback.cc @@ -88,6 +88,41 @@ int VideoEngineSampleCode(void* window1, void* window2) return -1; } + webrtc::ViERTP_RTCP* ptrViERtpRtcp = + webrtc::ViERTP_RTCP::GetInterface(ptrViE); + if (ptrViERtpRtcp == NULL) + { + printf("ERROR in ViERTP_RTCP::GetInterface\n"); + return -1; + } + + printf("Bandwidth estimation modes:\n"); + printf("1. Multi-stream bandwidth estimation\n"); + printf("2. Single-stream bandwidth estimation\n"); + printf("Choose bandwidth estimation mode (default is 1): "); + std::string str; + std::getline(std::cin, str); + int bwe_mode_choice = atoi(str.c_str()); + webrtc::BandwidthEstimationMode bwe_mode; + switch (bwe_mode_choice) { + case 1: + bwe_mode = webrtc::kViEMultiStreamEstimation; + break; + case 2: + bwe_mode = webrtc::kViESingleStreamEstimation; + break; + default: + bwe_mode = webrtc::kViEMultiStreamEstimation; + break; + } + + error = ptrViERtpRtcp->SetBandwidthEstimationMode(bwe_mode); + if (error == -1) + { + printf("ERROR in ViERTP_RTCP::SetBandwidthEstimationMode\n"); + return -1; + } + int videoChannel = -1; error = ptrViEBase->CreateChannel(videoChannel); if (error == -1) @@ -181,13 +216,6 @@ int VideoEngineSampleCode(void* window1, void* window2) // // RTP/RTCP settings // - webrtc::ViERTP_RTCP* ptrViERtpRtcp = - webrtc::ViERTP_RTCP::GetInterface(ptrViE); - if (ptrViERtpRtcp == NULL) - { - printf("ERROR in ViERTP_RTCP::GetInterface\n"); - return -1; - } error = ptrViERtpRtcp->SetRTCPStatus(videoChannel, webrtc::kRtcpCompound_RFC4585); @@ -317,7 +345,6 @@ int VideoEngineSampleCode(void* window1, void* window2) } // Set spatial resolution option - std::string str; std::cout << std::endl; std::cout << "Enter frame size option (default is CIF):" << std::endl; std::cout << "1. QCIF (176X144) " << std::endl; diff --git a/src/video_engine/test/auto_test/source/vie_autotest_rtp_rtcp.cc b/src/video_engine/test/auto_test/source/vie_autotest_rtp_rtcp.cc index 0054830111..57eac00678 100644 --- a/src/video_engine/test/auto_test/source/vie_autotest_rtp_rtcp.cc +++ b/src/video_engine/test/auto_test/source/vie_autotest_rtp_rtcp.cc @@ -431,8 +431,22 @@ void ViEAutoTest::ViERtpRtcpAPITest() //*************************************************************** // Create VIE TbInterfaces ViE("ViERtpRtcpAPITest"); + + // Verify that we can set the bandwidth estimation mode, as that API only + // is valid to call before creating channels. + EXPECT_EQ(0, ViE.rtp_rtcp->SetBandwidthEstimationMode( + webrtc::kViESingleStreamEstimation)); + EXPECT_EQ(0, ViE.rtp_rtcp->SetBandwidthEstimationMode( + webrtc::kViEMultiStreamEstimation)); + // Create a video channel TbVideoChannel tbChannel(ViE, webrtc::kVideoCodecVP8); + + EXPECT_EQ(-1, ViE.rtp_rtcp->SetBandwidthEstimationMode( + webrtc::kViESingleStreamEstimation)); + EXPECT_EQ(-1, ViE.rtp_rtcp->SetBandwidthEstimationMode( + webrtc::kViEMultiStreamEstimation)); + // Create a capture device TbCaptureDevice tbCapture(ViE); tbCapture.ConnectTo(tbChannel.videoChannel); diff --git a/src/video_engine/test/auto_test/source/vie_autotest_simulcast.cc b/src/video_engine/test/auto_test/source/vie_autotest_simulcast.cc index 762c709152..c29558e5b1 100644 --- a/src/video_engine/test/auto_test/source/vie_autotest_simulcast.cc +++ b/src/video_engine/test/auto_test/source/vie_autotest_simulcast.cc @@ -126,6 +126,39 @@ int VideoEngineSimulcastTest(void* window1, void* window2) { } getchar(); + webrtc::ViERTP_RTCP* vie_rtp_rtcp = + webrtc::ViERTP_RTCP::GetInterface(video_engine); + if (vie_rtp_rtcp == NULL) { + printf("ERROR in ViERTP_RTCP::GetInterface\n"); + return -1; + } + + printf("Bandwidth estimation modes:\n"); + printf("1. Multi-stream bandwidth estimation\n"); + printf("2. Single-stream bandwidth estimation\n"); + printf("Choose bandwidth estimation mode (default is 1): "); + std::string str; + std::getline(std::cin, str); + int bwe_mode_choice = atoi(str.c_str()); + webrtc::BandwidthEstimationMode bwe_mode; + switch (bwe_mode_choice) { + case 1: + bwe_mode = webrtc::kViEMultiStreamEstimation; + break; + case 2: + bwe_mode = webrtc::kViESingleStreamEstimation; + break; + default: + bwe_mode = webrtc::kViEMultiStreamEstimation; + break; + } + + error = vie_rtp_rtcp->SetBandwidthEstimationMode(bwe_mode); + if (error == -1) { + printf("ERROR in ViERTP_RTCP::SetBandwidthEstimationMode\n"); + return -1; + } + int video_channel = -1; error = vie_base->CreateChannel(video_channel); if (error == -1) { @@ -215,13 +248,6 @@ int VideoEngineSimulcastTest(void* window1, void* window2) { } // RTP/RTCP settings. - webrtc::ViERTP_RTCP* vie_rtp_rtcp = - webrtc::ViERTP_RTCP::GetInterface(video_engine); - if (vie_rtp_rtcp == NULL) { - printf("ERROR in ViERTP_RTCP::GetInterface\n"); - return -1; - } - error = vie_rtp_rtcp->SetRTCPStatus(video_channel, webrtc::kRtcpCompound_RFC4585); if (error == -1) { @@ -354,7 +380,6 @@ int VideoEngineSimulcastTest(void* window1, void* window2) { } // Set start bit rate. - std::string str; std::cout << std::endl; std::cout << "Choose start rate (in kbps). Press enter for default: "; std::getline(std::cin, str); diff --git a/src/video_engine/vie_channel.cc b/src/video_engine/vie_channel.cc index 1a259e4730..2bd0ebba15 100644 --- a/src/video_engine/vie_channel.cc +++ b/src/video_engine/vie_channel.cc @@ -56,7 +56,7 @@ ViEChannel::ViEChannel(WebRtc_Word32 channel_id, ViEModuleId(engine_id, channel_id), num_socket_threads_)), #endif vcm_(*VideoCodingModule::Create(ViEModuleId(engine_id, channel_id))), - vie_receiver_(channel_id, &vcm_), + vie_receiver_(channel_id, &vcm_, remote_bitrate_estimator), vie_sender_(channel_id), vie_sync_(&vcm_, this), module_process_thread_(module_process_thread), @@ -2337,6 +2337,15 @@ void ViEChannel::OnApplicationDataReceived(const WebRtc_Word32 id, } } +void ViEChannel::OnSendReportReceived(const WebRtc_Word32 id, + const WebRtc_UWord32 senderSSRC, + uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t timestamp) { + vie_receiver_.OnSendReportReceived(id, senderSSRC, ntp_secs, ntp_frac, + timestamp); +} + WebRtc_Word32 ViEChannel::OnInitializeDecoder( const WebRtc_Word32 id, const WebRtc_Word8 payload_type, diff --git a/src/video_engine/vie_channel.h b/src/video_engine/vie_channel.h index bdf03fd566..062eb01143 100644 --- a/src/video_engine/vie_channel.h +++ b/src/video_engine/vie_channel.h @@ -182,6 +182,11 @@ class ViEChannel const WebRtc_UWord32 name, const WebRtc_UWord16 length, const WebRtc_UWord8* data); + virtual void OnSendReportReceived(const WebRtc_Word32 id, + const WebRtc_UWord32 senderSSRC, + uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t timestamp); // Implements RtpFeedback. virtual WebRtc_Word32 OnInitializeDecoder( const WebRtc_Word32 id, diff --git a/src/video_engine/vie_channel_group.cc b/src/video_engine/vie_channel_group.cc index 8d0e429968..6254c21e0e 100644 --- a/src/video_engine/vie_channel_group.cc +++ b/src/video_engine/vie_channel_group.cc @@ -20,11 +20,12 @@ namespace webrtc { ChannelGroup::ChannelGroup(ProcessThread* process_thread, - const OverUseDetectorOptions& options) + const OverUseDetectorOptions& options, + RemoteBitrateEstimator::EstimationMode mode) : remb_(new VieRemb(process_thread)), bitrate_controller_(BitrateController::CreateBitrateController()), - remote_bitrate_estimator_(new RemoteBitrateEstimator(remb_.get(), - options)) { + remote_bitrate_estimator_(RemoteBitrateEstimator::Create(remb_.get(), + options, mode)) { } ChannelGroup::~ChannelGroup() { diff --git a/src/video_engine/vie_channel_group.h b/src/video_engine/vie_channel_group.h index bcd58b2b27..14b55dbcb4 100644 --- a/src/video_engine/vie_channel_group.h +++ b/src/video_engine/vie_channel_group.h @@ -13,6 +13,7 @@ #include +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "system_wrappers/interface/scoped_ptr.h" namespace webrtc { @@ -20,8 +21,6 @@ namespace webrtc { class BitrateController; struct OverUseDetectorOptions; class ProcessThread; -class RemoteBitrateEstimator; -class RemoteBitrateObserver; class ViEChannel; class ViEEncoder; class VieRemb; @@ -31,7 +30,8 @@ class VieRemb; class ChannelGroup { public: ChannelGroup(ProcessThread* process_thread, - const OverUseDetectorOptions& options); + const OverUseDetectorOptions& options, + RemoteBitrateEstimator::EstimationMode mode); ~ChannelGroup(); void AddChannel(int channel_id); diff --git a/src/video_engine/vie_channel_manager.cc b/src/video_engine/vie_channel_manager.cc index 50c7fdd46f..bb4d09fd12 100644 --- a/src/video_engine/vie_channel_manager.cc +++ b/src/video_engine/vie_channel_manager.cc @@ -37,7 +37,8 @@ ViEChannelManager::ViEChannelManager( voice_sync_interface_(NULL), voice_engine_(NULL), module_process_thread_(NULL), - over_use_detector_options_(options) { + over_use_detector_options_(options), + bwe_mode_(RemoteBitrateEstimator::kMultiStreamEstimation) { WEBRTC_TRACE(kTraceMemory, kTraceVideo, ViEId(engine_id), "ViEChannelManager::ViEChannelManager(engine_id: %d)", engine_id); @@ -90,7 +91,8 @@ int ViEChannelManager::CreateChannel(int* channel_id) { // Create a new channel group and add this channel. ChannelGroup* group = new ChannelGroup(module_process_thread_, - over_use_detector_options_); + over_use_detector_options_, + bwe_mode_); BitrateController* bitrate_controller = group->GetBitrateController(); ViEEncoder* vie_encoder = new ViEEncoder(engine_id_, new_channel_id, number_of_cores_, @@ -326,6 +328,26 @@ bool ViEChannelManager::SetRembStatus(int channel_id, bool sender, encoder); } +bool ViEChannelManager::SetBandwidthEstimationMode( + BandwidthEstimationMode mode) { + CriticalSectionScoped cs(*channel_id_critsect_); + if (channel_groups_.size() > 0) { + return false; + } + switch (mode) { + case kViEMultiStreamEstimation: + bwe_mode_ = RemoteBitrateEstimator::kMultiStreamEstimation; + break; + case kViESingleStreamEstimation: + bwe_mode_ = RemoteBitrateEstimator::kSingleStreamEstimation; + break; + default: + assert(false); + return false; + } + return true; +} + bool ViEChannelManager::CreateChannelObject( int channel_id, ViEEncoder* vie_encoder, diff --git a/src/video_engine/vie_channel_manager.h b/src/video_engine/vie_channel_manager.h index 6294ab1325..758cdd31e1 100644 --- a/src/video_engine/vie_channel_manager.h +++ b/src/video_engine/vie_channel_manager.h @@ -17,6 +17,7 @@ #include "engine_configurations.h" // NOLINT #include "system_wrappers/interface/scoped_ptr.h" #include "typedefs.h" // NOLINT +#include "video_engine/include/vie_rtp_rtcp.h" #include "video_engine/vie_channel_group.h" #include "video_engine/vie_defines.h" #include "video_engine/vie_manager_base.h" @@ -74,6 +75,10 @@ class ViEChannelManager: private ViEManagerBase { // Adds a channel to include when sending REMB. bool SetRembStatus(int channel_id, bool sender, bool receiver); + // Sets the bandwidth estimation mode. This can only be changed before + // adding a channel. + bool SetBandwidthEstimationMode(BandwidthEstimationMode mode); + private: // Creates a channel object connected to |vie_encoder|. Assumed to be called // protected. @@ -125,6 +130,7 @@ class ViEChannelManager: private ViEManagerBase { VoiceEngine* voice_engine_; ProcessThread* module_process_thread_; const OverUseDetectorOptions& over_use_detector_options_; + RemoteBitrateEstimator::EstimationMode bwe_mode_; }; class ViEChannelManagerScoped: private ViEManagerScopedBase { diff --git a/src/video_engine/vie_receiver.cc b/src/video_engine/vie_receiver.cc index 955d3dc2ac..12d1847077 100644 --- a/src/video_engine/vie_receiver.cc +++ b/src/video_engine/vie_receiver.cc @@ -10,24 +10,31 @@ #include "video_engine/vie_receiver.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "modules/rtp_rtcp/interface/rtp_rtcp.h" #include "modules/utility/interface/rtp_dump.h" #include "modules/video_coding/main/interface/video_coding.h" #include "system_wrappers/interface/critical_section_wrapper.h" +#include "system_wrappers/interface/tick_util.h" #include "system_wrappers/interface/trace.h" namespace webrtc { +enum { kPacketOverheadBytes = 28 }; + ViEReceiver::ViEReceiver(const int32_t channel_id, - VideoCodingModule* module_vcm) + VideoCodingModule* module_vcm, + RemoteBitrateEstimator* remote_bitrate_estimator) : receive_cs_(CriticalSectionWrapper::CreateCriticalSection()), channel_id_(channel_id), rtp_rtcp_(NULL), vcm_(module_vcm), + remote_bitrate_estimator_(remote_bitrate_estimator), external_decryption_(NULL), decryption_buffer_(NULL), rtp_dump_(NULL), receiving_(false) { + assert(remote_bitrate_estimator); } ViEReceiver::~ViEReceiver() { @@ -118,6 +125,19 @@ WebRtc_Word32 ViEReceiver::OnReceivedPayloadData( return 0; } + // TODO(holmer): Make sure packets reconstructed using FEC are not passed to + // the bandwidth estimator. + // Add headers, ideally we would like to include for instance + // Ethernet header here as well. + const int packet_size = payload_size + kPacketOverheadBytes + + rtp_header->header.headerLength + rtp_header->header.paddingLength; + uint32_t compensated_timestamp = rtp_header->header.timestamp + + rtp_header->extension.transmissionTimeOffset; + remote_bitrate_estimator_->IncomingPacket(rtp_header->header.ssrc, + packet_size, + TickTime::MillisecondTimestamp(), + compensated_timestamp); + if (vcm_->IncomingPacket(payload_data, payload_size, *rtp_header) != 0) { // Check this... return -1; @@ -125,6 +145,15 @@ WebRtc_Word32 ViEReceiver::OnReceivedPayloadData( return 0; } +void ViEReceiver::OnSendReportReceived(const WebRtc_Word32 id, + const WebRtc_UWord32 senderSSRC, + uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t timestamp) { + remote_bitrate_estimator_->IncomingRtcp(senderSSRC, ntp_secs, ntp_frac, + timestamp); +} + int ViEReceiver::InsertRTPPacket(const WebRtc_Word8* rtp_packet, int rtp_packet_length) { // TODO(mflodman) Change decrypt to get rid of this cast. diff --git a/src/video_engine/vie_receiver.h b/src/video_engine/vie_receiver.h index 09fbc8f39d..40510ff4ef 100644 --- a/src/video_engine/vie_receiver.h +++ b/src/video_engine/vie_receiver.h @@ -24,13 +24,15 @@ namespace webrtc { class CriticalSectionWrapper; class Encryption; +class RemoteBitrateEstimator; class RtpDump; class RtpRtcp; class VideoCodingModule; class ViEReceiver : public UdpTransportData, public RtpData { public: - ViEReceiver(const int32_t channel_id, VideoCodingModule* module_vcm); + ViEReceiver(const int32_t channel_id, VideoCodingModule* module_vcm, + RemoteBitrateEstimator* remote_bitrate_estimator); ~ViEReceiver(); int RegisterExternalDecryption(Encryption* decryption); @@ -66,6 +68,12 @@ class ViEReceiver : public UdpTransportData, public RtpData { const WebRtc_UWord16 payload_size, const WebRtcRTPHeader* rtp_header); + void OnSendReportReceived(const WebRtc_Word32 id, + const WebRtc_UWord32 senderSSRC, + uint32_t ntp_secs, + uint32_t ntp_frac, + uint32_t timestamp); + private: int InsertRTPPacket(const WebRtc_Word8* rtp_packet, int rtp_packet_length); int InsertRTCPPacket(const WebRtc_Word8* rtcp_packet, int rtcp_packet_length); @@ -75,6 +83,7 @@ class ViEReceiver : public UdpTransportData, public RtpData { RtpRtcp* rtp_rtcp_; std::list rtp_rtcp_simulcast_; VideoCodingModule* vcm_; + RemoteBitrateEstimator* remote_bitrate_estimator_; Encryption* external_decryption_; WebRtc_UWord8* decryption_buffer_; diff --git a/src/video_engine/vie_remb.cc b/src/video_engine/vie_remb.cc index 4e18f250eb..496e6ad444 100644 --- a/src/video_engine/vie_remb.cc +++ b/src/video_engine/vie_remb.cc @@ -170,27 +170,32 @@ WebRtc_Word32 VieRemb::Process() { int num_bitrates = update_time_bitrates_.size(); - if (num_bitrates == 0) { + if (num_bitrates == 0 || receive_modules_.empty()) { list_crit_->Leave(); return 0; } // TODO(mflodman) Use std::vector and change RTP module API. - unsigned int* ssrcs = new unsigned int[num_bitrates]; + unsigned int* ssrcs = new unsigned int[receive_modules_.size()]; unsigned int total_bitrate = 0; - int idx = 0; for (it = update_time_bitrates_.begin(); it != update_time_bitrates_.end(); - ++it, ++idx) { + ++it) { total_bitrate += it->second.second; - ssrcs[idx] = it->first; + } + + int idx = 0; + RtpModules::iterator rtp_it; + for (rtp_it = receive_modules_.begin(); rtp_it != receive_modules_.end(); + ++rtp_it, ++idx) { + ssrcs[idx] = (*rtp_it)->RemoteSSRC(); } // Send a REMB packet. RtpRtcp* sender = NULL; if (!rtcp_sender_.empty()) { sender = rtcp_sender_.front(); - } else if (!receive_modules_.empty()) { + } else { sender = receive_modules_.front(); } last_send_bitrate_ = total_bitrate; diff --git a/src/video_engine/vie_rtp_rtcp_impl.cc b/src/video_engine/vie_rtp_rtcp_impl.cc index 8a444df113..c716e778d4 100644 --- a/src/video_engine/vie_rtp_rtcp_impl.cc +++ b/src/video_engine/vie_rtp_rtcp_impl.cc @@ -613,6 +613,15 @@ int ViERTP_RTCPImpl::SetRembStatus(int video_channel, bool sender, return 0; } +int ViERTP_RTCPImpl::SetBandwidthEstimationMode(BandwidthEstimationMode mode) { + WEBRTC_TRACE(kTraceApiCall, kTraceVideo, shared_data_->instance_id(), + "ViERTP_RTCPImpl::SetBandwidthEstimationMode(%d)", mode); + if (!shared_data_->channel_manager()->SetBandwidthEstimationMode(mode)) { + return -1; + } + return 0; +} + int ViERTP_RTCPImpl::SetSendTimestampOffsetStatus(int video_channel, bool enable, int id) { diff --git a/src/video_engine/vie_rtp_rtcp_impl.h b/src/video_engine/vie_rtp_rtcp_impl.h index 35db129325..577e0853ef 100644 --- a/src/video_engine/vie_rtp_rtcp_impl.h +++ b/src/video_engine/vie_rtp_rtcp_impl.h @@ -68,6 +68,7 @@ class ViERTP_RTCPImpl const ViEKeyFrameRequestMethod method); virtual int SetTMMBRStatus(const int video_channel, const bool enable); virtual int SetRembStatus(int video_channel, bool sender, bool receiver); + virtual int SetBandwidthEstimationMode(BandwidthEstimationMode mode); virtual int SetSendTimestampOffsetStatus(int video_channel, bool enable, int id);