From 821510637147d6e4ea358db8f0415dd756bb14af Mon Sep 17 00:00:00 2001 From: "solenberg@webrtc.org" Date: Mon, 21 Oct 2013 14:23:26 +0000 Subject: [PATCH] Framework for testing bandwidth estimation. BUG= R=mflodman@webrtc.org, stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/2317004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@5008 4adac7df-926f-26a2-2b94-8c16560cd09d --- webrtc/modules/modules.gyp | 4 + .../bwe_test_framework.cc | 30 + .../bwe_test_framework.h | 489 +++++++++++ .../bwe_test_framework_unittest.cc | 814 ++++++++++++++++++ .../include/remote_bitrate_estimator.h | 3 +- .../remote_bitrate_estimators_test.cc | 444 ++++++++++ 6 files changed, 1783 insertions(+), 1 deletion(-) create mode 100644 webrtc/modules/remote_bitrate_estimator/bwe_test_framework.cc create mode 100644 webrtc/modules/remote_bitrate_estimator/bwe_test_framework.h create mode 100644 webrtc/modules/remote_bitrate_estimator/bwe_test_framework_unittest.cc create mode 100644 webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index 5d80d31f0f..6c7912d5cf 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -174,9 +174,13 @@ 'pacing/paced_sender_unittest.cc', 'remote_bitrate_estimator/include/mock/mock_remote_bitrate_observer.h', 'remote_bitrate_estimator/bitrate_estimator_unittest.cc', + 'remote_bitrate_estimator/bwe_test_framework.cc', + 'remote_bitrate_estimator/bwe_test_framework.h', + 'remote_bitrate_estimator/bwe_test_framework_unittest.cc', 'remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc', 'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc', 'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h', + 'remote_bitrate_estimator/remote_bitrate_estimators_test.cc', 'remote_bitrate_estimator/rtp_to_ntp_unittest.cc', 'rtp_rtcp/source/mock/mock_rtp_payload_strategy.h', 'rtp_rtcp/source/fec_receiver_unittest.cc', diff --git a/webrtc/modules/remote_bitrate_estimator/bwe_test_framework.cc b/webrtc/modules/remote_bitrate_estimator/bwe_test_framework.cc new file mode 100644 index 0000000000..6a3b01da26 --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/bwe_test_framework.cc @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/remote_bitrate_estimator/bwe_test_framework.h" + +namespace webrtc { +namespace testing { +namespace bwe { + +bool IsTimeSorted(const Packets& packets) { + PacketsConstIt last_it = packets.begin(); + for (PacketsConstIt it = last_it; it != packets.end(); ++it) { + if (it != last_it && *it < *last_it) { + return false; + } + last_it = it; + } + return true; +} + +} // namespace bwe +} // namespace testing +} // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/bwe_test_framework.h b/webrtc/modules/remote_bitrate_estimator/bwe_test_framework.h new file mode 100644 index 0000000000..1029db925d --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/bwe_test_framework.h @@ -0,0 +1,489 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_BWE_TEST_FRAMEWORK_H_ +#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_BWE_TEST_FRAMEWORK_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/system_wrappers/interface/constructor_magic.h" + +namespace webrtc { +namespace testing { +namespace bwe { + +class Random { + public: + explicit Random(uint32_t seed) + : a_(0x531FDB97 ^ seed), + b_(0x6420ECA8 + seed) { + } + + // Return semi-random number in the interval [0.0, 1.0]. + float Rand() { + const float kScale = 1.0f / 0xffffffff; + float result = kScale * b_; + a_ ^= b_; + b_ += a_; + return result; + } + + // Normal Distribution. + int Gaussian(int mean, int standard_deviation) { + // Creating a Normal distribution variable from two independent uniform + // variables based on the Box-Muller transform, which is defined on the + // interval (0, 1], hence the mask+add below. + const double kPi = 3.14159265358979323846; + const double kScale = 1.0 / 0x80000000ul; + double u1 = kScale * ((a_ & 0x7ffffffful) + 1); + double u2 = kScale * ((b_ & 0x7ffffffful) + 1); + a_ ^= b_; + b_ += a_; + return static_cast(mean + standard_deviation * + std::sqrt(-2 * std::log(u1)) * std::cos(2 * kPi * u2)); + } + + private: + uint32_t a_; + uint32_t b_; + + DISALLOW_IMPLICIT_CONSTRUCTORS(Random); +}; + +template class Stats { + public: + Stats() + : data_(), + last_mean_count_(0), + last_variance_count_(0), + last_minmax_count_(0), + mean_(0), + variance_(0), + min_(0), + max_(0) { + } + + void Push(T data_point) { + data_.push_back(data_point); + } + + T GetMean() { + if (last_mean_count_ != data_.size()) { + last_mean_count_ = data_.size(); + mean_ = std::accumulate(data_.begin(), data_.end(), static_cast(0)); + assert(last_mean_count_ != 0); + mean_ /= static_cast(last_mean_count_); + } + return mean_; + } + T GetVariance() { + if (last_variance_count_ != data_.size()) { + last_variance_count_ = data_.size(); + T mean = GetMean(); + variance_ = 0; + for (typename std::vector::const_iterator it = data_.begin(); + it != data_.end(); ++it) { + T diff = (*it - mean); + variance_ += diff * diff; + } + assert(last_variance_count_ != 0); + variance_ /= static_cast(last_variance_count_); + } + return variance_; + } + T GetStdDev() { + return std::sqrt(static_cast(GetVariance())); + } + T GetMin() { + RefreshMinMax(); + return min_; + } + T GetMax() { + RefreshMinMax(); + return max_; + } + + void Log(const std::string& units) { + printf("%f %s\t+/-%f\t[%f,%f]", + GetMean(), units.c_str(), GetStdDev(), GetMin(), GetMax()); + } + + private: + void RefreshMinMax() { + if (last_minmax_count_ != data_.size()) { + last_minmax_count_ = data_.size(); + min_ = max_ = 0; + if (data_.empty()) { + return; + } + typename std::vector::const_iterator it = data_.begin(); + min_ = max_ = *it; + while (++it != data_.end()) { + min_ = std::min(min_, *it); + max_ = std::max(max_, *it); + } + } + } + + std::vector data_; + typename std::vector::size_type last_mean_count_; + typename std::vector::size_type last_variance_count_; + typename std::vector::size_type last_minmax_count_; + T mean_; + T variance_; + T min_; + T max_; +}; + +class BwePacket { + public: + BwePacket() + : send_time_us_(0), + payload_size_(0) { + memset(&header_, 0, sizeof(header_)); + } + + BwePacket(int64_t send_time_us, uint32_t payload_size, + const RTPHeader& header) + : send_time_us_(send_time_us), + payload_size_(payload_size), + header_(header) { + } + + BwePacket(int64_t send_time_us, uint32_t sequence_number) + : send_time_us_(send_time_us), + payload_size_(0) { + memset(&header_, 0, sizeof(header_)); + header_.sequenceNumber = sequence_number; + } + + bool operator<(const BwePacket& rhs) const { + return send_time_us_ < rhs.send_time_us_; + } + + void set_send_time_us(int64_t send_time_us) { + assert(send_time_us >= 0); + send_time_us_ = send_time_us; + } + int64_t send_time_us() const { return send_time_us_; } + uint32_t payload_size() const { return payload_size_; } + const RTPHeader& header() const { return header_; } + + private: + int64_t send_time_us_; // Time the packet left last processor touching it. + uint32_t payload_size_; // Size of the (non-existent, simulated) payload. + RTPHeader header_; // Actual contents. +}; + +typedef std::list Packets; +typedef std::list::iterator PacketsIt; +typedef std::list::const_iterator PacketsConstIt; + +bool IsTimeSorted(const Packets& packets); + +class PacketProcessorInterface { + public: + virtual ~PacketProcessorInterface() {} + + // Run simulation for |time_ms| micro seconds, consuming packets from, and + // producing packets into in_out. The outgoing packet list must be sorted on + // |send_time_us_|. The simulation time |time_ms| is optional to use. + virtual void RunFor(int64_t time_ms, Packets* in_out) = 0; +}; + +class VideoSender : public PacketProcessorInterface { + public: + VideoSender(float fps, uint32_t kbps, uint32_t ssrc, float first_frame_offset) + : kMaxPayloadSizeBytes(1000), + kTimestampBase(0xff80ff00ul), + frame_period_ms_(1000.0 / fps), + next_frame_ms_(frame_period_ms_ * first_frame_offset), + now_ms_(0.0), + bytes_per_second_(1000 * kbps / 8), + frame_size_bytes_(bytes_per_second_ / fps), + prototype_header_() { + assert(first_frame_offset >= 0.0f); + assert(first_frame_offset < 1.0f); + memset(&prototype_header_, 0, sizeof(prototype_header_)); + prototype_header_.ssrc = ssrc; + prototype_header_.sequenceNumber = 0xf000u; + } + virtual ~VideoSender() {} + + uint32_t max_payload_size_bytes() const { return kMaxPayloadSizeBytes; } + uint32_t bytes_per_second() const { return bytes_per_second_; } + + virtual void RunFor(int64_t time_ms, Packets* in_out) { + assert(in_out); + now_ms_ += time_ms; + Packets newPackets; + while (now_ms_ >= next_frame_ms_) { + prototype_header_.sequenceNumber++; + prototype_header_.timestamp = kTimestampBase + + static_cast(next_frame_ms_ * 90.0); + prototype_header_.extension.absoluteSendTime = (kTimestampBase + + ((static_cast(next_frame_ms_ * (1 << 18)) + 500) / 1000)) & + 0x00fffffful; + prototype_header_.extension.transmissionTimeOffset = 0; + + // Generate new packets for this frame, all with the same timestamp, + // but the payload size is capped, so if the whole frame doesn't fit in + // one packet, we will see a number of equally sized packets followed by + // one smaller at the tail. + int64_t send_time_us = next_frame_ms_ * 1000.0; + uint32_t payload_size = frame_size_bytes_; + while (payload_size > 0) { + uint32_t size = std::min(kMaxPayloadSizeBytes, payload_size); + newPackets.push_back(BwePacket(send_time_us, size, prototype_header_)); + payload_size -= size; + } + + next_frame_ms_ += frame_period_ms_; + } + in_out->merge(newPackets); + } + + private: + const uint32_t kMaxPayloadSizeBytes; + const uint32_t kTimestampBase; + double frame_period_ms_; + double next_frame_ms_; + double now_ms_; + uint32_t bytes_per_second_; + uint32_t frame_size_bytes_; + RTPHeader prototype_header_; + + DISALLOW_IMPLICIT_CONSTRUCTORS(VideoSender); +}; + +class RateCounterFilter : public PacketProcessorInterface { + public: + RateCounterFilter() + : kWindowSizeUs(1000000), + packets_per_second_(0), + bytes_per_second_(0), + last_accumulated_us_(0), + window_(), + pps_stats_(), + kbps_stats_() { + } + virtual ~RateCounterFilter() { + LogStats(); + } + + uint32_t packets_per_second() const { return packets_per_second_; } + uint32_t bits_per_second() const { return bytes_per_second_ * 8; } + + void LogStats() { + printf("RateCounterFilter "); + pps_stats_.Log("pps"); + printf("\n"); + printf("RateCounterFilter "); + kbps_stats_.Log("kbps"); + printf("\n"); + } + + virtual void RunFor(int64_t /*time_ms*/, Packets* in_out) { + assert(in_out); + for (PacketsConstIt it = in_out->begin(); it != in_out->end(); ++it) { + packets_per_second_++; + bytes_per_second_ += it->payload_size(); + last_accumulated_us_ = it->send_time_us(); + } + window_.insert(window_.end(), in_out->begin(), in_out->end()); + while (!window_.empty()) { + const BwePacket& packet = window_.front(); + if (packet.send_time_us() > (last_accumulated_us_ - kWindowSizeUs)) { + break; + } + assert(packets_per_second_ >= 1); + assert(bytes_per_second_ >= packet.payload_size()); + packets_per_second_--; + bytes_per_second_ -= packet.payload_size(); + window_.pop_front(); + } + pps_stats_.Push(packets_per_second_); + kbps_stats_.Push((bytes_per_second_ * 8) / 1000.0); + } + + private: + const int64_t kWindowSizeUs; + uint32_t packets_per_second_; + uint32_t bytes_per_second_; + int64_t last_accumulated_us_; + Packets window_; + Stats pps_stats_; + Stats kbps_stats_; + + DISALLOW_COPY_AND_ASSIGN(RateCounterFilter); +}; + +class LossFilter : public PacketProcessorInterface { + public: + LossFilter() : random_(0x12345678), loss_fraction_(0.0f) {} + virtual ~LossFilter() {} + + void SetLoss(float loss_percent) { + assert(loss_percent >= 0.0f); + assert(loss_percent <= 100.0f); + loss_fraction_ = loss_percent * 0.01f; + } + + virtual void RunFor(int64_t /*time_ms*/, Packets* in_out) { + assert(in_out); + for (PacketsIt it = in_out->begin(); it != in_out->end(); ) { + if (random_.Rand() < loss_fraction_) { + it = in_out->erase(it); + } else { + ++it; + } + } + } + + private: + Random random_; + float loss_fraction_; + + DISALLOW_COPY_AND_ASSIGN(LossFilter); +}; + +class DelayFilter : public PacketProcessorInterface { + public: + DelayFilter() : delay_us_(0), last_send_time_us_(0) {} + virtual ~DelayFilter() {} + + void SetDelay(int64_t delay_ms) { + assert(delay_ms >= 0); + delay_us_ = delay_ms * 1000; + } + + virtual void RunFor(int64_t /*time_ms*/, Packets* in_out) { + assert(in_out); + for (PacketsIt it = in_out->begin(); it != in_out->end(); ++it) { + int64_t new_send_time_us = it->send_time_us() + delay_us_; + last_send_time_us_ = std::max(last_send_time_us_, new_send_time_us); + it->set_send_time_us(last_send_time_us_); + } + } + + private: + int64_t delay_us_; + int64_t last_send_time_us_; + + DISALLOW_COPY_AND_ASSIGN(DelayFilter); +}; + +class JitterFilter : public PacketProcessorInterface { + public: + JitterFilter() + : random_(0x89674523), + stddev_jitter_us_(0), + last_send_time_us_(0) { + } + virtual ~JitterFilter() {} + + void SetJitter(int64_t stddev_jitter_ms) { + assert(stddev_jitter_ms >= 0); + stddev_jitter_us_ = stddev_jitter_ms * 1000; + } + + virtual void RunFor(int64_t /*time_ms*/, Packets* in_out) { + assert(in_out); + for (PacketsIt it = in_out->begin(); it != in_out->end(); ++it) { + int64_t new_send_time_us = it->send_time_us(); + new_send_time_us += random_.Gaussian(0, stddev_jitter_us_); + last_send_time_us_ = std::max(last_send_time_us_, new_send_time_us); + it->set_send_time_us(last_send_time_us_); + } + } + + private: + Random random_; + int64_t stddev_jitter_us_; + int64_t last_send_time_us_; + + DISALLOW_COPY_AND_ASSIGN(JitterFilter); +}; + +class ReorderFilter : public PacketProcessorInterface { + public: + ReorderFilter() : random_(0x27452389), reorder_fraction_(0.0f) {} + virtual ~ReorderFilter() {} + + void SetReorder(float reorder_percent) { + assert(reorder_percent >= 0.0f); + assert(reorder_percent <= 100.0f); + reorder_fraction_ = reorder_percent * 0.01f; + } + + virtual void RunFor(int64_t /*time_ms*/, Packets* in_out) { + assert(in_out); + if (in_out->size() >= 2) { + PacketsIt last_it = in_out->begin(); + PacketsIt it = last_it; + while (++it != in_out->end()) { + if (random_.Rand() < reorder_fraction_) { + int64_t t1 = last_it->send_time_us(); + int64_t t2 = it->send_time_us(); + std::swap(*last_it, *it); + last_it->set_send_time_us(t1); + it->set_send_time_us(t2); + } + last_it = it; + } + } + } + + private: + Random random_; + float reorder_fraction_; + + DISALLOW_COPY_AND_ASSIGN(ReorderFilter); +}; + +// Apply a bitrate choke with an infinite queue on the packet stream. +class ChokeFilter : public PacketProcessorInterface { + public: + ChokeFilter() : kbps_(1200), last_send_time_us_(0) {} + virtual ~ChokeFilter() {} + + void SetCapacity(uint32_t kbps) { + kbps_ = kbps; + } + + virtual void RunFor(int64_t /*time_ms*/, Packets* in_out) { + assert(in_out); + for (PacketsIt it = in_out->begin(); it != in_out->end(); ++it) { + int64_t earliest_send_time_us = last_send_time_us_ + + (it->payload_size() * 8 * 1000 + kbps_ / 2) / kbps_; + last_send_time_us_ = std::max(it->send_time_us(), earliest_send_time_us); + it->set_send_time_us(last_send_time_us_); + } + } + + private: + uint32_t kbps_; + int64_t last_send_time_us_; + + DISALLOW_COPY_AND_ASSIGN(ChokeFilter); +}; +} // namespace bwe +} // namespace testing +} // namespace webrtc + +#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_BWE_TEST_FRAMEWORK_H_ diff --git a/webrtc/modules/remote_bitrate_estimator/bwe_test_framework_unittest.cc b/webrtc/modules/remote_bitrate_estimator/bwe_test_framework_unittest.cc new file mode 100644 index 0000000000..0d1e7c1ee7 --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/bwe_test_framework_unittest.cc @@ -0,0 +1,814 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/remote_bitrate_estimator/bwe_test_framework.h" + +#include + +#include "gtest/gtest.h" +#include "webrtc/system_wrappers/interface/constructor_magic.h" + +using std::vector; + +namespace webrtc { +namespace testing { +namespace bwe { + +TEST(BweTestFramework_RandomTest, Gaussian) { + enum { + kN = 100000, + kBuckets = 100, + kMean = 49, + kStddev = 10 + }; + + Random random(0x12345678); + + int buckets[kBuckets] = {0}; + for (int i = 0; i < kN; ++i) { + int index = random.Gaussian(kMean, kStddev); + if (index >= 0 && index < kBuckets) { + buckets[index]++; + } + } + + const double kPi = 3.14159265358979323846; + const double kScale = kN / (kStddev * std::sqrt(2.0 * kPi)); + const double kDiv = -2.0 * kStddev * kStddev; + double self_corr = 0.0; + double bucket_corr = 0.0; + for (int n = 0; n < kBuckets; ++n) { + double normal_dist = kScale * std::exp((n - kMean) * (n - kMean) / kDiv); + self_corr += normal_dist * normal_dist; + bucket_corr += normal_dist * buckets[n]; + } + printf("Correlation: %f (random sample), %f (self), %f (quotient)\n", + bucket_corr, self_corr, bucket_corr / self_corr); + EXPECT_NEAR(1.0, bucket_corr / self_corr, 0.0004); +} + +static bool IsSequenceNumberSorted(const Packets& packets) { + PacketsConstIt last_it = packets.begin(); + for (PacketsConstIt it = last_it; it != packets.end(); ++it) { + if (IsNewerSequenceNumber(last_it->header().sequenceNumber, + it->header().sequenceNumber)) { + return false; + } + last_it = it; + } + return true; +} + +TEST(BweTestFramework_BwePacketTest, IsTimeSorted) { + Packets packets; + // Insert some packets in order... + EXPECT_TRUE(IsTimeSorted(packets)); + + packets.push_back(BwePacket(100, 0)); + EXPECT_TRUE(IsTimeSorted(packets)); + + packets.push_back(BwePacket(110, 0)); + EXPECT_TRUE(IsTimeSorted(packets)); + + // ...and one out-of-order... + packets.push_back(BwePacket(100, 0)); + EXPECT_FALSE(IsTimeSorted(packets)); + + // ...remove the out-of-order packet, insert another in-order packet. + packets.pop_back(); + packets.push_back(BwePacket(120, 0)); + EXPECT_TRUE(IsTimeSorted(packets)); +} + +TEST(BweTestFramework_BwePacketTest, IsSequenceNumberSorted) { + Packets packets; + // Insert some packets in order... + EXPECT_TRUE(IsSequenceNumberSorted(packets)); + + packets.push_back(BwePacket(0, 100)); + EXPECT_TRUE(IsSequenceNumberSorted(packets)); + + packets.push_back(BwePacket(0, 110)); + EXPECT_TRUE(IsSequenceNumberSorted(packets)); + + // ...and one out-of-order... + packets.push_back(BwePacket(0, 100)); + EXPECT_FALSE(IsSequenceNumberSorted(packets)); + + // ...remove the out-of-order packet, insert another in-order packet. + packets.pop_back(); + packets.push_back(BwePacket(0, 120)); + EXPECT_TRUE(IsSequenceNumberSorted(packets)); +} + +TEST(BweTestFramework_StatsTest, Mean) { + Stats stats; + EXPECT_EQ(0, stats.GetMean()); + + stats.Push(1); + stats.Push(3); + EXPECT_EQ(2, stats.GetMean()); + + // Integer division rounds (1+3-3)/3 to 0. + stats.Push(-3); + EXPECT_EQ(0, stats.GetMean()); +} + +TEST(BweTestFramework_StatsTest, Variance) { + Stats stats; + EXPECT_EQ(0, stats.GetVariance()); + + // Mean is 2 ; ((1-2)*(1-2)+(3-2)*(3-2))/2 = (1+1)/2 = 1 + stats.Push(1); + stats.Push(3); + EXPECT_EQ(1, stats.GetVariance()); + + // Integer division rounds 26/3 to 8 + // Mean is 0 ; (1*1+3*3+(-4)*(-4))/3 = (1+9+16)/3 = 8 + stats.Push(-4); + EXPECT_EQ(8, stats.GetVariance()); +} + +TEST(BweTestFramework_StatsTest, StdDev) { + Stats stats; + EXPECT_EQ(0, stats.GetStdDev()); + + // Variance is 1 ; sqrt(1) = 1 + stats.Push(1); + stats.Push(3); + EXPECT_EQ(1, stats.GetStdDev()); + + // Variance is 8 ; sqrt(8) = 2 with integers. + stats.Push(-4); + EXPECT_EQ(2, stats.GetStdDev()); +} + +TEST(BweTestFramework_StatsTest, MinMax) { + Stats stats; + EXPECT_EQ(0, stats.GetMin()); + EXPECT_EQ(0, stats.GetMax()); + + stats.Push(1); + EXPECT_EQ(1, stats.GetMin()); + EXPECT_EQ(1, stats.GetMax()); + + stats.Push(3); + EXPECT_EQ(1, stats.GetMin()); + EXPECT_EQ(3, stats.GetMax()); + + stats.Push(-4); + EXPECT_EQ(-4, stats.GetMin()); + EXPECT_EQ(3, stats.GetMax()); +} + +void TestVideoSender(VideoSender* sender, int64_t run_for_ms, + uint32_t expected_packets, + uint32_t expected_payload_size, + uint32_t expected_total_payload_size) { + assert(sender); + Packets packets; + sender->RunFor(run_for_ms, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + ASSERT_TRUE(IsSequenceNumberSorted(packets)); + EXPECT_EQ(expected_packets, packets.size()); + int64_t send_time_us = -1; + uint32_t total_payload_size = 0; + uint32_t absolute_send_time = 0; + uint32_t absolute_send_time_wraps = 0; + uint32_t rtp_timestamp = 0; + uint32_t rtp_timestamp_wraps = 0; + for (PacketsIt it = packets.begin(); it != packets.end(); ++it) { + EXPECT_LE(send_time_us, it->send_time_us()); + send_time_us = it->send_time_us(); + if (sender->max_payload_size_bytes() != it->payload_size()) { + EXPECT_EQ(expected_payload_size, it->payload_size()); + } + total_payload_size += it->payload_size(); + if (absolute_send_time > it->header().extension.absoluteSendTime) { + absolute_send_time_wraps++; + } + absolute_send_time = it->header().extension.absoluteSendTime; + if (rtp_timestamp > it->header().timestamp) { + rtp_timestamp_wraps++; + } + rtp_timestamp = it->header().timestamp; + } + EXPECT_EQ(expected_total_payload_size, total_payload_size); + EXPECT_GE(1u, absolute_send_time_wraps); + EXPECT_GE(1u, rtp_timestamp_wraps); +} + +TEST(BweTestFramework_VideoSenderTest, Fps1Kpbs80_1s) { + // 1 fps, 80 kbps + VideoSender sender(1.0f, 80, 0x1234, 0); + EXPECT_EQ(10000u, sender.bytes_per_second()); + // We're at 1 fps, so all packets should be generated on first call, giving 10 + // packets of each 1000 bytes, total 10000 bytes. + TestVideoSender(&sender, 1, 10, 1000, 10000); + // 999ms, should see no output here. + TestVideoSender(&sender, 998, 0, 0, 0); + // 1999ms, should get data for one more frame. + TestVideoSender(&sender, 1000, 10, 1000, 10000); + // 2000ms, one more frame. + TestVideoSender(&sender, 1, 10, 1000, 10000); + // 2999ms, should see nothing. + TestVideoSender(&sender, 999, 0, 0, 0); +} + +TEST(BweTestFramework_VideoSenderTest, Fps1Kpbs80_1s_Offset) { + // 1 fps, 80 kbps, offset 0.5 of a frame period, ==0.5s in this case. + VideoSender sender(1.0f, 80, 0x1234, 0.5f); + EXPECT_EQ(10000u, sender.bytes_per_second()); + // 499ms, no output. + TestVideoSender(&sender, 499, 0, 0, 0); + // 500ms, first frame (this is the offset we set), 10 packets of 1000 bytes. + TestVideoSender(&sender, 1, 10, 1000, 10000); + // 1499ms, nothing. + TestVideoSender(&sender, 999, 0, 0, 0); + // 1999ms, second frame. + TestVideoSender(&sender, 500, 10, 1000, 10000); + // 2499ms, nothing. + TestVideoSender(&sender, 500, 0, 0, 0); + // 2500ms, third frame. + TestVideoSender(&sender, 1, 10, 1000, 10000); + // 3499ms, nothing. + TestVideoSender(&sender, 999, 0, 0, 0); +} + +TEST(BweTestFramework_VideoSenderTest, Fps50Kpbs80_11s) { + // 50 fps, 80 kbps. + VideoSender sender(50.0f, 80, 0x1234, 0); + EXPECT_EQ(10000u, sender.bytes_per_second()); + // 9998ms, should see 500 frames, 200 byte payloads, total 100000 bytes. + TestVideoSender(&sender, 9998, 500, 200, 100000); + // 9999ms, nothing. + TestVideoSender(&sender, 1, 0, 0, 0); + // 10000ms, 501st frame as a single packet. + TestVideoSender(&sender, 1, 1, 200, 200); + // 10998ms, 49 more frames. + TestVideoSender(&sender, 998, 49, 200, 9800); + // 10999ms, nothing. + TestVideoSender(&sender, 1, 0, 0, 0); +} + +TEST(BweTestFramework_VideoSenderTest, Fps10Kpbs120_1s) { + // 20 fps, 120 kbps. + VideoSender sender(20.0f, 120, 0x1234, 0); + EXPECT_EQ(15000u, sender.bytes_per_second()); + // 498ms, 10 frames with 750 byte payloads, total 7500 bytes. + TestVideoSender(&sender, 498, 10, 750, 7500); + // 499ms, nothing. + TestVideoSender(&sender, 1, 0, 0, 0); + // 500ms, one more frame. + TestVideoSender(&sender, 1, 1, 750, 750); + // 998ms, 9 more frames. + TestVideoSender(&sender, 498, 9, 750, 6750); + // 999ms, nothing. + TestVideoSender(&sender, 1, 0, 0, 0); +} + +TEST(BweTestFramework_VideoSenderTest, Fps30Kpbs800_20s) { + // 20 fps, 820 kbps. + VideoSender sender(25.0f, 820, 0x1234, 0); + EXPECT_EQ(102500u, sender.bytes_per_second()); + // 9998ms, 250 frames. 820 kbps = 102500 bytes/s, so total should be 1025000. + // Each frame is 102500/25=4100 bytes, or 5 packets (4 @1000 bytes, 1 @100), + // so packet count should be 5*250=1250 and last packet of each frame has + // 100 bytes of payload. + TestVideoSender(&sender, 9998, 1250, 100, 1025000); + // 9999ms, nothing. + TestVideoSender(&sender, 1, 0, 0, 0); + // 19998ms, 250 more frames. + TestVideoSender(&sender, 9999, 1250, 100, 1025000); + // 19999ms, nothing. + TestVideoSender(&sender, 1, 0, 0, 0); + // 20038ms, one more frame, as described above (25fps == 40ms/frame). + TestVideoSender(&sender, 39, 5, 100, 4100); + // 20039ms, nothing. + TestVideoSender(&sender, 1, 0, 0, 0); +} + +TEST(BweTestFramework_VideoSenderTest, TestAppendInOrder) { + // 1 fps, 80 kbps, 250ms offset. + VideoSender sender1(1.0f, 80, 0x1234, 0.25f); + EXPECT_EQ(10000u, sender1.bytes_per_second()); + Packets packets; + // Generate some packets, verify they are sorted. + sender1.RunFor(999, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + ASSERT_TRUE(IsSequenceNumberSorted(packets)); + EXPECT_EQ(10u, packets.size()); + // Generate some more packets and verify they are appended to end of list. + sender1.RunFor(1000, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + ASSERT_TRUE(IsSequenceNumberSorted(packets)); + EXPECT_EQ(20u, packets.size()); + + // Another sender, 2 fps, 160 kpbs, 150ms offset + VideoSender sender2(2.0f, 160, 0x2234, 0.30f); + EXPECT_EQ(20000u, sender2.bytes_per_second()); + // Generate some packets, verify that they are merged with the packets already + // on the list. + sender2.RunFor(999, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + EXPECT_EQ(40u, packets.size()); + // Generate some more. + sender2.RunFor(1000, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + EXPECT_EQ(60u, packets.size()); +} + +class BweTestFramework_RateCounterFilterTest : public ::testing::Test { + public: + BweTestFramework_RateCounterFilterTest() + : filter_(), + now_ms_(0) { + } + virtual ~BweTestFramework_RateCounterFilterTest() {} + + protected: + void TestRateCounter(int64_t run_for_ms, uint32_t payload_bits, + uint32_t expected_pps, uint32_t expected_bps) { + Packets packets; + RTPHeader header = {0}; + // "Send" a packet every 10 ms. + for (int64_t i = 0; i < run_for_ms; i += 10, now_ms_ += 10) { + packets.push_back(BwePacket(now_ms_ * 1000, payload_bits / 8, header)); + } + filter_.RunFor(run_for_ms, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + EXPECT_EQ(expected_pps, filter_.packets_per_second()); + EXPECT_EQ(expected_bps, filter_.bits_per_second()); + } + + private: + RateCounterFilter filter_; + int64_t now_ms_; + + DISALLOW_COPY_AND_ASSIGN(BweTestFramework_RateCounterFilterTest); +}; + +TEST_F(BweTestFramework_RateCounterFilterTest, Short) { + // 100ms, 100 bytes per packet, should result in 10 pps and 8 kbps. We're + // generating one packet every 10 ms ; 10 * 800 = 8k + TestRateCounter(100, 800, 10, 8000); +} + +TEST_F(BweTestFramework_RateCounterFilterTest, Medium) { + // 100ms, like above. + TestRateCounter(100, 800, 10, 8000); + // 1000ms, 100 bpp, should result in 100 pps and 80 kbps. We're still + // generating packets every 10 ms. + TestRateCounter(900, 800, 100, 80000); +} + +TEST_F(BweTestFramework_RateCounterFilterTest, Long) { + // 100ms, 1000ms, like above. + TestRateCounter(100, 800, 10, 8000); + TestRateCounter(900, 800, 100, 80000); + // 2000ms, should only see rate of last second, so 100 pps, and 40 kbps now. + TestRateCounter(1000, 400, 100, 40000); + // 2500ms, half a second with zero payload size. We should get same pps as + // before, but kbps should drop to half of previous rate. + TestRateCounter(500, 0, 100, 20000); + // Another half second with zero payload size. Now the kbps rate should drop + // to zero. + TestRateCounter(500, 0, 100, 0); + // Increate payload size again. 200 * 100 * 0.5 = 10 kbps. + TestRateCounter(500, 200, 100, 10000); +} + +static void TestLossFilter(float loss_percent, bool zero_tolerance) { + LossFilter filter; + filter.SetLoss(loss_percent); + Packets::size_type sent_packets = 0; + Packets::size_type remaining_packets = 0; + + // No input should yield no output + { + Packets packets; + sent_packets += packets.size(); + filter.RunFor(0, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + ASSERT_TRUE(IsSequenceNumberSorted(packets)); + remaining_packets += packets.size(); + EXPECT_EQ(0u, sent_packets); + EXPECT_EQ(0u, remaining_packets); + } + + // Generate and process 10000 packets in different batch sizes (some empty) + for (int i = 0; i < 2225; ++i) { + Packets packets; + packets.insert(packets.end(), i % 10, BwePacket()); + sent_packets += packets.size(); + filter.RunFor(0, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + ASSERT_TRUE(IsSequenceNumberSorted(packets)); + remaining_packets += packets.size(); + } + + float loss_fraction = 0.01f * (100.0f - loss_percent); + Packets::size_type expected_packets = loss_fraction * sent_packets; + if (zero_tolerance) { + EXPECT_EQ(expected_packets, remaining_packets); + } else { + // Require within 1% of expected + EXPECT_NEAR(expected_packets, remaining_packets, 100); + } +} + +TEST(BweTestFramework_LossFilterTest, Loss0) { + // With 0% loss, the result should be exact (no loss). + TestLossFilter(0.0f, true); +} + +TEST(BweTestFramework_LossFilterTest, Loss10) { + TestLossFilter(10.0f, false); +} + +TEST(BweTestFramework_LossFilterTest, Loss50) { + TestLossFilter(50.0f, false); +} + +TEST(BweTestFramework_LossFilterTest, Loss100) { + // With 100% loss, the result should be exact (no packets out). + TestLossFilter(100.0f, true); +} + +class BweTestFramework_DelayFilterTest : public ::testing::Test { + public: + BweTestFramework_DelayFilterTest() + : filter_(), + now_ms_(0), + sequence_number_(0) { + } + virtual ~BweTestFramework_DelayFilterTest() {} + + protected: + void TestDelayFilter(int64_t run_for_ms, uint32_t in_packets, + uint32_t out_packets) { + Packets packets; + for (uint32_t i = 0; i < in_packets; ++i) { + packets.push_back(BwePacket(now_ms_ * 1000 + (sequence_number_ >> 4), + sequence_number_)); + sequence_number_++; + } + filter_.RunFor(run_for_ms, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + ASSERT_TRUE(IsSequenceNumberSorted(packets)); + for (PacketsConstIt it = packets.begin(); it != packets.end(); ++it) { + EXPECT_LE(now_ms_ * 1000, it->send_time_us()); + } + EXPECT_EQ(out_packets, packets.size()); + accumulated_packets_.splice(accumulated_packets_.end(), packets); + now_ms_ += run_for_ms; + } + + void TestDelayFilter(int64_t delay_ms) { + filter_.SetDelay(delay_ms); + TestDelayFilter(1, 0, 0); // No input should yield no output + + // Single packet + TestDelayFilter(0, 1, 1); + TestDelayFilter(delay_ms, 0, 0); + + for (int i = 0; i < delay_ms; ++i) { + filter_.SetDelay(i); + TestDelayFilter(1, 10, 10); + } + TestDelayFilter(0, 0, 0); + TestDelayFilter(delay_ms, 0, 0); + + // Wait a little longer - should still see no output + TestDelayFilter(delay_ms, 0, 0); + + for (int i = 1; i < delay_ms + 1; ++i) { + filter_.SetDelay(i); + TestDelayFilter(1, 5, 5); + } + TestDelayFilter(0, 0, 0); + filter_.SetDelay(2 * delay_ms); + TestDelayFilter(1, 0, 0); + TestDelayFilter(delay_ms, 13, 13); + TestDelayFilter(delay_ms, 0, 0); + + // Wait a little longer - should still see no output + TestDelayFilter(delay_ms, 0, 0); + + for (int i = 0; i < 2 * delay_ms; ++i) { + filter_.SetDelay(2 * delay_ms - i - 1); + TestDelayFilter(1, 5, 5); + } + TestDelayFilter(0, 0, 0); + filter_.SetDelay(0); + TestDelayFilter(0, 7, 7); + + ASSERT_TRUE(IsTimeSorted(accumulated_packets_)); + ASSERT_TRUE(IsSequenceNumberSorted(accumulated_packets_)); + } + + DelayFilter filter_; + Packets accumulated_packets_; + + private: + int64_t now_ms_; + uint32_t sequence_number_; + + DISALLOW_COPY_AND_ASSIGN(BweTestFramework_DelayFilterTest); +}; + +TEST_F(BweTestFramework_DelayFilterTest, Delay0) { + TestDelayFilter(1, 0, 0); // No input should yield no output + TestDelayFilter(1, 10, 10); // Expect no delay (delay time is zero) + TestDelayFilter(1, 0, 0); // Check no packets are still in buffer + filter_.SetDelay(0); + TestDelayFilter(1, 5, 5); // Expect no delay (delay time is zero) + TestDelayFilter(1, 0, 0); // Check no packets are still in buffer +} + +TEST_F(BweTestFramework_DelayFilterTest, Delay1) { + TestDelayFilter(1); +} + +TEST_F(BweTestFramework_DelayFilterTest, Delay2) { + TestDelayFilter(2); +} + +TEST_F(BweTestFramework_DelayFilterTest, Delay20) { + TestDelayFilter(20); +} + +TEST_F(BweTestFramework_DelayFilterTest, Delay100) { + TestDelayFilter(100); +} + +TEST_F(BweTestFramework_DelayFilterTest, JumpToZeroDelay) { + DelayFilter delay; + Packets acc; + Packets packets; + + // Delay a bunch of packets, accumulate them to the 'acc' list. + delay.SetDelay(100.0f); + for (uint32_t i = 0; i < 10; ++i) { + packets.push_back(BwePacket(i * 100, i)); + } + delay.RunFor(1000, &packets); + acc.splice(acc.end(), packets); + ASSERT_TRUE(IsTimeSorted(acc)); + ASSERT_TRUE(IsSequenceNumberSorted(acc)); + + // Drop delay to zero, send a few more packets through the delay, append them + // to the 'acc' list and verify that it is all sorted. + delay.SetDelay(0.0f); + for (uint32_t i = 10; i < 50; ++i) { + packets.push_back(BwePacket(i * 100, i)); + } + delay.RunFor(1000, &packets); + acc.splice(acc.end(), packets); + ASSERT_TRUE(IsTimeSorted(acc)); + ASSERT_TRUE(IsSequenceNumberSorted(acc)); +} + +TEST_F(BweTestFramework_DelayFilterTest, IncreasingDelay) { + // Gradually increase delay. + for (int i = 1; i < 50; i += 4) { + TestDelayFilter(i); + } + // Reach a steady state. + filter_.SetDelay(100); + TestDelayFilter(1, 20, 20); + TestDelayFilter(2, 0, 0); + TestDelayFilter(99, 20, 20); + // Drop delay back down to zero. + filter_.SetDelay(0); + TestDelayFilter(1, 100, 100); + TestDelayFilter(23010, 0, 0); + ASSERT_TRUE(IsTimeSorted(accumulated_packets_)); + ASSERT_TRUE(IsSequenceNumberSorted(accumulated_packets_)); +} + +static void TestJitterFilter(int64_t stddev_jitter_ms) { + JitterFilter filter; + filter.SetJitter(stddev_jitter_ms); + + int64_t now_ms = 0; + uint32_t sequence_number = 0; + + // Generate packets, add jitter to them, accumulate the altered packets. + Packets original; + Packets jittered; + for (uint32_t i = 0; i < 1000; ++i) { + Packets packets; + for (uint32_t j = 0; j < i % 100; ++j) { + packets.push_back(BwePacket(now_ms * 1000, sequence_number++)); + now_ms += 5 * stddev_jitter_ms; + } + original.insert(original.end(), packets.begin(), packets.end()); + filter.RunFor(stddev_jitter_ms, &packets); + jittered.splice(jittered.end(), packets); + } + + // Jittered packets should still be in order. + ASSERT_TRUE(IsTimeSorted(original)); + ASSERT_TRUE(IsTimeSorted(jittered)); + ASSERT_TRUE(IsSequenceNumberSorted(original)); + ASSERT_TRUE(IsSequenceNumberSorted(jittered)); + EXPECT_EQ(original.size(), jittered.size()); + + // Make sure jittered and original packets are in same order. Collect time + // difference (jitter) in stats, then check that mean jitter is close to zero + // and standard deviation of jitter is what we set it to. + Stats jitter_us; + for (PacketsIt it1 = original.begin(), it2 = jittered.begin(); + it1 != original.end() && it2 != jittered.end(); ++it1, ++it2) { + EXPECT_EQ(it1->header().sequenceNumber, it2->header().sequenceNumber); + jitter_us.Push(it2->send_time_us() - it1->send_time_us()); + } + EXPECT_NEAR(0.0, jitter_us.GetMean(), stddev_jitter_ms * 1000.0 * 0.008); + EXPECT_NEAR(stddev_jitter_ms * 1000.0, jitter_us.GetStdDev(), + stddev_jitter_ms * 1000.0 * 0.02); +} + +TEST(BweTestFramework_JitterFilterTest, Jitter0) { + TestJitterFilter(0); +} + +TEST(BweTestFramework_JitterFilterTest, Jitter1) { + TestJitterFilter(1); +} + +TEST(BweTestFramework_JitterFilterTest, Jitter5) { + TestJitterFilter(5); +} + +TEST(BweTestFramework_JitterFilterTest, Jitter10) { + TestJitterFilter(10); +} + +TEST(BweTestFramework_JitterFilterTest, Jitter1031) { + TestJitterFilter(1031); +} + +static void TestReorderFilter(uint32_t reorder_percent, uint32_t near) { + const uint32_t kPacketCount = 10000; + + // Generate packets with 10 ms interval. + Packets packets; + int64_t now_ms = 0; + uint32_t sequence_number = 1; + for (uint32_t i = 0; i < kPacketCount; ++i, now_ms += 10) { + packets.push_back(BwePacket(now_ms * 1000, sequence_number++)); + } + ASSERT_TRUE(IsTimeSorted(packets)); + ASSERT_TRUE(IsSequenceNumberSorted(packets)); + + // Reorder packets, verify that send times are still in order. + ReorderFilter filter; + filter.SetReorder(reorder_percent); + filter.RunFor(now_ms, &packets); + ASSERT_TRUE(IsTimeSorted(packets)); + + // We measure the amount of reordering by summing the distance by which out- + // of-order packets have been moved in the stream. + uint32_t distance = 0; + uint32_t last_sequence_number = 0; + for (PacketsIt it = packets.begin(); it != packets.end(); ++it) { + uint32_t sequence_number = it->header().sequenceNumber; + if (sequence_number < last_sequence_number) { + distance += last_sequence_number - sequence_number; + } + last_sequence_number = sequence_number; + } + + // Because reordering is random, we allow a threshold when comparing. The + // maximum distance a packet can be moved is PacketCount - 1. + EXPECT_NEAR(((kPacketCount - 1) * reorder_percent) / 100, distance, near); +} + +TEST(BweTestFramework_ReorderFilterTest, Reorder0) { + // For 0% reordering, no packets should have been moved, so result is exact. + TestReorderFilter(0, 0); +} + +TEST(BweTestFramework_ReorderFilterTest, Reorder10) { + TestReorderFilter(10, 30); +} + +TEST(BweTestFramework_ReorderFilterTest, Reorder20) { + TestReorderFilter(20, 20); +} + +TEST(BweTestFramework_ReorderFilterTest, Reorder50) { + TestReorderFilter(50, 20); +} + +TEST(BweTestFramework_ReorderFilterTest, Reorder70) { + TestReorderFilter(70, 20); +} + +TEST(BweTestFramework_ReorderFilterTest, Reorder100) { + // Note that because the implementation works by optionally swapping two + // adjacent packets, when the likelihood of a swap is 1.0, a swap will always + // occur, so the stream will be in order except for the first packet, which + // has been moved to the end. Therefore we expect the result to be exact here. + TestReorderFilter(100.0, 0); +} + +class BweTestFramework_ChokeFilterTest : public ::testing::Test { + public: + BweTestFramework_ChokeFilterTest() + : filter_(), + now_ms_(0), + sequence_number_(0), + output_packets_() { + } + virtual ~BweTestFramework_ChokeFilterTest() {} + + protected: + void TestChoke(int64_t run_for_ms, uint32_t packets_to_generate, + uint32_t choke_kbps, uint32_t expected_kbit_transmitted) { + // Generate a bunch of packets, apply choke, verify output is ordered. + Packets packets; + RTPHeader header = {0}; + for (uint32_t i = 0; i < packets_to_generate; ++i) { + int64_t send_time_ms = now_ms_ + (i * run_for_ms) / packets_to_generate; + header.sequenceNumber = sequence_number_++; + // Payload is 1000 bits. + packets.push_back(BwePacket(send_time_ms * 1000, 125, header)); + } + ASSERT_TRUE(IsTimeSorted(packets)); + filter_.SetCapacity(choke_kbps); + filter_.RunFor(run_for_ms, &packets); + now_ms_ += run_for_ms; + output_packets_.splice(output_packets_.end(), packets); + ASSERT_TRUE(IsTimeSorted(output_packets_)); + ASSERT_TRUE(IsSequenceNumberSorted(output_packets_)); + + // Sum up the transmitted bytes up until the current time. + uint32_t bytes_transmitted = 0; + while (!output_packets_.empty()) { + const BwePacket& packet = output_packets_.front(); + if (packet.send_time_us() > now_ms_ * 1000) { + break; + } + bytes_transmitted += packet.payload_size(); + output_packets_.pop_front(); + } + EXPECT_EQ(expected_kbit_transmitted, (bytes_transmitted * 8) / 1000); + } + + private: + ChokeFilter filter_; + int64_t now_ms_; + uint32_t sequence_number_; + Packets output_packets_; + + DISALLOW_COPY_AND_ASSIGN(BweTestFramework_ChokeFilterTest); +}; + +TEST_F(BweTestFramework_ChokeFilterTest, Short) { + // 100ms, 100 packets, 10 kbps choke -> 1 kbit of data should have propagated. + // That is actually just a single packet, since each packet has 1000 bits of + // payload. + TestChoke(100, 100, 10, 1); +} + +TEST_F(BweTestFramework_ChokeFilterTest, Medium) { + // 100ms, 10 packets, 10 kbps choke -> 1 packet through, or 1 kbit. + TestChoke(100, 10, 10, 1); + // 200ms, no new packets, same choke -> another packet through. + TestChoke(100, 0, 10, 1); + // 1000ms, no new packets, same choke -> 8 more packets. + TestChoke(800, 0, 10, 8); + // 2000ms, no new packets, same choke -> queue is empty so no output. + TestChoke(1000, 0, 10, 0); +} + +TEST_F(BweTestFramework_ChokeFilterTest, Long) { + // 100ms, 100 packets in queue, 10 kbps choke -> 1 packet through, or 1 kbit. + TestChoke(100, 100, 10, 1); + // 200ms, no input, another packet through. + TestChoke(100, 0, 10, 1); + // 1000ms, no input, 8 packets through. + TestChoke(800, 0, 10, 8); + // 10000ms, no input, raise choke to 100 kbps. Remaining 90 packets in queue + // should be propagated, for a total of 90 kbps. + TestChoke(9000, 0, 100, 90); + // 10100ms, 20 more packets, 100 kbps choke -> 10 packets or 10 kbit through. + TestChoke(100, 20, 100, 10); + // 10300ms, 10 more packets, same choke -> 20 packets out. + TestChoke(200, 10, 100, 20); + // 11300ms, no input, queue should be empty. + TestChoke(1000, 0, 10, 0); +} + +} // namespace bwe +} // namespace testing +} // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h b/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h index a2f6bcfd60..5bf8af972d 100644 --- a/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h +++ b/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h @@ -72,7 +72,8 @@ struct RemoteBitrateEstimatorFactory { Clock* clock) const; }; -struct AbsoluteSendTimeRemoteBitrateEstimatorFactory { +struct AbsoluteSendTimeRemoteBitrateEstimatorFactory + : public RemoteBitrateEstimatorFactory { AbsoluteSendTimeRemoteBitrateEstimatorFactory() {} virtual ~AbsoluteSendTimeRemoteBitrateEstimatorFactory() {} diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc new file mode 100644 index 0000000000..29c9ab114a --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "gtest/gtest.h" +#include "webrtc/modules/remote_bitrate_estimator/bwe_test_framework.h" +#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "webrtc/system_wrappers/interface/clock.h" +#include "webrtc/system_wrappers/interface/constructor_magic.h" +#include "webrtc/system_wrappers/interface/scoped_ptr.h" + +#define ENABLE_1_SENDER 1 +#define ENABLE_3_SENDERS 1 +#define ENABLE_10_SENDERS 1 +#define ENABLE_BASIC_TESTS 1 +#define ENABLE_LOSS_TESTS 0 +#define ENABLE_DELAY_TESTS 0 +#define ENABLE_JITTER_TESTS 0 +#define ENABLE_REORDER_TESTS 0 +#define ENABLE_CHOKE_TESTS 0 +#define ENABLE_MULTI_TESTS 0 + +#define ENABLE_TOF_ESTIMATOR 1 +#define ENABLE_AST_ESTIMATOR 1 + +using std::vector; + +namespace webrtc { +namespace testing { +namespace bwe { + +const int64_t kSimulationIntervalMs = 1000; + +namespace stl_helpers { +template void DeleteElements(T* container) { + if (!container) return; + for (typename T::iterator it = container->begin(); it != container->end(); + ++it) { + delete *it; + } + container->clear(); +} +} // namespace stl_helpers + +class TestedEstimator : public RemoteBitrateObserver { + public: + TestedEstimator(const std::string& debug_name, + const RemoteBitrateEstimatorFactory& factory) + : debug_name_(debug_name), + clock_(0), + stats_(), + relative_estimator_stats_(), + latest_estimate_kbps_(-1.0), + estimator_(factory.Create(this, &clock_)), + relative_estimator_(NULL) { + assert(estimator_.get()); + // Default RTT in RemoteRateControl is 200 ms ; 50 ms is more realistic. + estimator_->OnRttUpdate(50); + } + + void SetRelativeEstimator(TestedEstimator* relative_estimator) { + relative_estimator_ = relative_estimator; + } + + void EatPacket(const BwePacket& packet) { + latest_estimate_kbps_ = -1.0; + + // We're treating the send time (from previous filter) as the arrival + // time once packet reaches the estimator. + int64_t packet_time_ms = (packet.send_time_us() + 500) / 1000; + + int64_t step_ms = estimator_->TimeUntilNextProcess(); + while ((clock_.TimeInMilliseconds() + step_ms) < packet_time_ms) { + clock_.AdvanceTimeMilliseconds(step_ms); + estimator_->Process(); + step_ms = estimator_->TimeUntilNextProcess(); + } + + estimator_->IncomingPacket(packet_time_ms, packet.payload_size(), + packet.header()); + clock_.AdvanceTimeMilliseconds(packet_time_ms - + clock_.TimeInMilliseconds()); + ASSERT_TRUE(packet_time_ms == clock_.TimeInMilliseconds()); + } + + void CheckEstimate() { + double estimated_kbps = 0.0; + if (LatestEstimate(&estimated_kbps)) { + stats_.Push(estimated_kbps); + double relative_estimate_kbps = 0.0; + if (relative_estimator_ && + relative_estimator_->LatestEstimate(&relative_estimate_kbps)) { + relative_estimator_stats_.Push(estimated_kbps - relative_estimate_kbps); + } + } + } + + void LogStats() { + printf("%s Mean ", debug_name_.c_str()); + stats_.Log("kbps"); + printf("\n"); + if (relative_estimator_) { + printf("%s Diff ", debug_name_.c_str()); + relative_estimator_stats_.Log("kbps"); + printf("\n"); + } + } + + virtual void OnReceiveBitrateChanged(const vector& ssrcs, + unsigned int bitrate) { + } + + private: + bool LatestEstimate(double* estimate_kbps) { + if (latest_estimate_kbps_ < 0.0) { + vector ssrcs; + unsigned int bps = 0; + if (!estimator_->LatestEstimate(&ssrcs, &bps)) { + return false; + } + latest_estimate_kbps_ = bps / 1000.0; + } + *estimate_kbps = latest_estimate_kbps_; + return true; + } + + std::string debug_name_; + bool log_estimates_; + SimulatedClock clock_; + Stats stats_; + Stats relative_estimator_stats_; + double latest_estimate_kbps_; + scoped_ptr estimator_; + TestedEstimator* relative_estimator_; + + DISALLOW_IMPLICIT_CONSTRUCTORS(TestedEstimator); +}; + +class RemoteBitrateEstimatorsTest : public ::testing::Test { + public: + RemoteBitrateEstimatorsTest() + : run_time_ms_(0), + estimators_(), + previous_packets_(), + processors_(), + video_senders_() { + } + virtual ~RemoteBitrateEstimatorsTest() { + stl_helpers::DeleteElements(&estimators_); + stl_helpers::DeleteElements(&video_senders_); + } + + virtual void SetUp() { +#if ENABLE_TOF_ESTIMATOR + estimators_.push_back(new TestedEstimator("TOF", + RemoteBitrateEstimatorFactory())); +#endif +#if ENABLE_AST_ESTIMATOR + estimators_.push_back(new TestedEstimator("AST", + AbsoluteSendTimeRemoteBitrateEstimatorFactory())); +#endif + // Set all estimators as relative to the first one. + for (uint32_t i = 1; i < estimators_.size(); ++i) { + estimators_[i]->SetRelativeEstimator(estimators_[0]); + } + } + + protected: + void RunFor(int64_t time_ms) { + for (run_time_ms_ += time_ms; run_time_ms_ >= kSimulationIntervalMs; + run_time_ms_ -= kSimulationIntervalMs) { + Packets packets; + for (vector::const_iterator it = + processors_.begin(); it != processors_.end(); ++it) { + (*it)->RunFor(kSimulationIntervalMs, &packets); + } + + // Verify packets are in order between batches. + if (!packets.empty() && !previous_packets_.empty()) { + packets.splice(packets.begin(), previous_packets_, + --previous_packets_.end()); + ASSERT_TRUE(IsTimeSorted(packets)); + packets.erase(packets.begin()); + } else { + ASSERT_TRUE(IsTimeSorted(packets)); + } + + for (PacketsConstIt pit = packets.begin(); pit != packets.end(); ++pit) { + for (vector::iterator eit = estimators_.begin(); + eit != estimators_.end(); ++eit) { + (*eit)->EatPacket(*pit); + } + } + + previous_packets_.swap(packets); + + for (vector::iterator eit = estimators_.begin(); + eit != estimators_.end(); ++eit) { + (*eit)->CheckEstimate(); + } + } + } + + void AddVideoSenders(uint32_t count) { + struct { float fps; uint32_t kbps; uint32_t ssrc; float frame_offset; } + configs[] = { + { 30.00f, 150, 0x1234, 0.13f }, + { 15.00f, 500, 0x2345, 0.16f }, + { 30.00f, 1200, 0x3456, 0.26f }, + { 7.49f, 150, 0x4567, 0.05f }, + { 7.50f, 150, 0x5678, 0.15f }, + { 7.51f, 150, 0x6789, 0.25f }, + { 15.02f, 150, 0x7890, 0.27f }, + { 15.03f, 150, 0x8901, 0.38f }, + { 30.02f, 150, 0x9012, 0.39f }, + { 30.03f, 150, 0x0123, 0.52f } + }; + assert(count <= sizeof(configs) / sizeof(configs[0])); + uint32_t total_capacity = 0; + for (uint32_t i = 0; i < count; ++i) { + video_senders_.push_back(new VideoSender(configs[i].fps, configs[i].kbps, + configs[i].ssrc, configs[i].frame_offset)); + processors_.push_back(video_senders_.back()); + total_capacity += configs[i].kbps; + } + printf("RequiredLinkCapacity %d kbps\n", total_capacity); + } + + void LogStats() { + for (vector::iterator eit = estimators_.begin(); + eit != estimators_.end(); ++eit) { + (*eit)->LogStats(); + } + } + + void UnlimitedSpeedTest() { + RunFor(10 * 60 * 1000); + } + + void SteadyLossTest() { + LossFilter loss; + processors_.push_back(&loss); + loss.SetLoss(20.0); + RunFor(10 * 60 * 1000); + } + void IncreasingLoss1Test() { + LossFilter loss; + processors_.push_back(&loss); + for (int i = 0; i < 76; ++i) { + loss.SetLoss(i); + RunFor(5000); + } + } + + void SteadyDelayTest() { + DelayFilter delay; + processors_.push_back(&delay); + delay.SetDelay(1000); + RunFor(10 * 60 * 1000); + } + void IncreasingDelay1Test() { + DelayFilter delay; + processors_.push_back(&delay); + RunFor(10 * 60 * 1000); + for (int i = 0; i < 30 * 2; ++i) { + delay.SetDelay(i); + RunFor(10 * 1000); + } + RunFor(10 * 60 * 1000); + } + void IncreasingDelay2Test() { + DelayFilter delay; + RateCounterFilter counter; + processors_.push_back(&delay); + processors_.push_back(&counter); + RunFor(1 * 60 * 1000); + for (int i = 1; i < 51; ++i) { + delay.SetDelay(10.0f * i); + RunFor(10 * 1000); + } + delay.SetDelay(0.0f); + RunFor(10 * 60 * 1000); + } + void JumpyDelay1Test() { + DelayFilter delay; + processors_.push_back(&delay); + RunFor(10 * 60 * 1000); + for (int i = 1; i < 200; ++i) { + delay.SetDelay((10 * i) % 500); + RunFor(1000); + delay.SetDelay(1.0f); + RunFor(1000); + } + delay.SetDelay(0.0f); + RunFor(10 * 60 * 1000); + } + + void SteadyJitterTest() { + JitterFilter jitter; + RateCounterFilter counter; + processors_.push_back(&jitter); + processors_.push_back(&counter); + jitter.SetJitter(120); + RunFor(10 * 60 * 1000); + } + void IncreasingJitter1Test() { + JitterFilter jitter; + processors_.push_back(&jitter); + for (int i = 0; i < 2 * 60 * 2; ++i) { + jitter.SetJitter(i); + RunFor(10 * 1000); + } + RunFor(10 * 60 * 1000); + } + void IncreasingJitter2Test() { + JitterFilter jitter; + processors_.push_back(&jitter); + RunFor(30 * 1000); + for (int i = 1; i < 51; ++i) { + jitter.SetJitter(10.0f * i); + RunFor(10 * 1000); + } + jitter.SetJitter(0.0f); + RunFor(10 * 60 * 1000); + } + + void SteadyReorderTest() { + ReorderFilter reorder; + processors_.push_back(&reorder); + reorder.SetReorder(20.0); + RunFor(10 * 60 * 1000); + } + void IncreasingReorder1Test() { + ReorderFilter reorder; + processors_.push_back(&reorder); + for (int i = 0; i < 76; ++i) { + reorder.SetReorder(i); + RunFor(5000); + } + } + + void SteadyChokeTest() { + ChokeFilter choke; + processors_.push_back(&choke); + choke.SetCapacity(140); + RunFor(10 * 60 * 1000); + } + void IncreasingChoke1Test() { + ChokeFilter choke; + processors_.push_back(&choke); + for (int i = 1200; i >= 100; i -= 100) { + choke.SetCapacity(i); + RunFor(5000); + } + } + void IncreasingChoke2Test() { + ChokeFilter choke; + processors_.push_back(&choke); + RunFor(60 * 1000); + for (int i = 1200; i >= 100; i -= 20) { + choke.SetCapacity(i); + RunFor(1000); + } + } + + void Multi1Test() { + DelayFilter delay; + ChokeFilter choke; + RateCounterFilter counter; + processors_.push_back(&delay); + processors_.push_back(&choke); + processors_.push_back(&counter); + choke.SetCapacity(1000); + RunFor(1 * 60 * 1000); + for (int i = 1; i < 51; ++i) { + delay.SetDelay(100.0f * i); + RunFor(10 * 1000); + } + delay.SetDelay(0.0f); + RunFor(5 * 60 * 1000); + } + void Multi2Test() { + ChokeFilter choke; + JitterFilter jitter; + RateCounterFilter counter; + processors_.push_back(&choke); + processors_.push_back(&jitter); + processors_.push_back(&counter); + choke.SetCapacity(2000); + jitter.SetJitter(120); + RunFor(5 * 60 * 1000); + } + + private: + int64_t run_time_ms_; + vector estimators_; + Packets previous_packets_; + vector processors_; + vector video_senders_; + + DISALLOW_COPY_AND_ASSIGN(RemoteBitrateEstimatorsTest); +}; + +#define SINGLE_TEST(enabled, test_name, video_senders)\ + TEST_F(RemoteBitrateEstimatorsTest, test_name##_##video_senders##Sender) {\ + if (enabled) {\ + AddVideoSenders(video_senders);\ + test_name##Test();\ + LogStats();\ + }\ + } + +#define MULTI_TEST(enabled, test_name)\ + SINGLE_TEST((enabled) && ENABLE_1_SENDER, test_name, 1)\ + SINGLE_TEST((enabled) && ENABLE_3_SENDERS, test_name, 3)\ + SINGLE_TEST((enabled) && ENABLE_10_SENDERS, test_name, 10) + +MULTI_TEST(ENABLE_BASIC_TESTS, UnlimitedSpeed) +MULTI_TEST(ENABLE_LOSS_TESTS, SteadyLoss) +MULTI_TEST(ENABLE_LOSS_TESTS, IncreasingLoss1) +MULTI_TEST(ENABLE_DELAY_TESTS, SteadyDelay) +MULTI_TEST(ENABLE_DELAY_TESTS, IncreasingDelay1) +MULTI_TEST(ENABLE_DELAY_TESTS, IncreasingDelay2) +MULTI_TEST(ENABLE_DELAY_TESTS, JumpyDelay1) +MULTI_TEST(ENABLE_JITTER_TESTS, SteadyJitter) +MULTI_TEST(ENABLE_JITTER_TESTS, IncreasingJitter1) +MULTI_TEST(ENABLE_JITTER_TESTS, IncreasingJitter2) +MULTI_TEST(ENABLE_REORDER_TESTS, SteadyReorder) +MULTI_TEST(ENABLE_REORDER_TESTS, IncreasingReorder1) +MULTI_TEST(ENABLE_CHOKE_TESTS, SteadyChoke) +MULTI_TEST(ENABLE_CHOKE_TESTS, IncreasingChoke1) +MULTI_TEST(ENABLE_CHOKE_TESTS, IncreasingChoke2) +MULTI_TEST(ENABLE_MULTI_TESTS, Multi1) +MULTI_TEST(ENABLE_MULTI_TESTS, Multi2) + +} // namespace bwe +} // namespace testing +} // namespace webrtc