diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index b0b3e79c0c..24d6d165a3 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -223,6 +223,7 @@ 'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.cc', 'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h', 'remote_bitrate_estimator/test/bwe_test_framework_unittest.cc', + 'remote_bitrate_estimator/test/bwe_unittest.cc', 'remote_bitrate_estimator/test/estimators/nada_unittest.cc', 'rtp_rtcp/source/mock/mock_rtp_payload_strategy.h', 'rtp_rtcp/source/byte_io_unittest.cc', diff --git a/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc b/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc index af69924f52..2c4cf753b7 100644 --- a/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc +++ b/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc @@ -86,15 +86,15 @@ TEST_P(BweSimulation, Choke1000kbps500kbps1000kbpsBiDirectional) { RateCounterFilter counter2(&downlink_, kFlowIds[1], "receiver_input_1"); PacketReceiver receiver2(&downlink_, kFlowIds[1], GetParam(), true, false); - choke2.SetCapacity(500); - delay.SetDelayMs(0); + choke2.set_capacity_kbps(500); + delay.SetOneWayDelayMs(0); - choke.SetCapacity(1000); - choke.SetMaxDelay(500); + choke.set_capacity_kbps(1000); + choke.set_max_delay_ms(500); RunFor(60 * 1000); - choke.SetCapacity(500); + choke.set_capacity_kbps(500); RunFor(60 * 1000); - choke.SetCapacity(1000); + choke.set_capacity_kbps(1000); RunFor(60 * 1000); } @@ -106,12 +106,12 @@ TEST_P(BweSimulation, Choke1000kbps500kbps1000kbps) { RateCounterFilter counter(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, false); - choke.SetCapacity(1000); - choke.SetMaxDelay(500); + choke.set_capacity_kbps(1000); + choke.set_max_delay_ms(500); RunFor(60 * 1000); - choke.SetCapacity(500); + choke.set_capacity_kbps(500); RunFor(60 * 1000); - choke.SetCapacity(1000); + choke.set_capacity_kbps(1000); RunFor(60 * 1000); } @@ -121,12 +121,12 @@ TEST_P(BweSimulation, PacerChoke1000kbps500kbps1000kbps) { ChokeFilter filter(&uplink_, 0); RateCounterFilter counter(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, true); - filter.SetCapacity(1000); - filter.SetMaxDelay(500); + filter.set_capacity_kbps(1000); + filter.set_max_delay_ms(500); RunFor(60 * 1000); - filter.SetCapacity(500); + filter.set_capacity_kbps(500); RunFor(60 * 1000); - filter.SetCapacity(1000); + filter.set_capacity_kbps(1000); RunFor(60 * 1000); } @@ -136,8 +136,8 @@ TEST_P(BweSimulation, PacerChoke10000kbps) { ChokeFilter filter(&uplink_, 0); RateCounterFilter counter(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, true); - filter.SetCapacity(10000); - filter.SetMaxDelay(500); + filter.set_capacity_kbps(10000); + filter.set_max_delay_ms(500); RunFor(60 * 1000); } @@ -147,12 +147,12 @@ TEST_P(BweSimulation, PacerChoke200kbps30kbps200kbps) { ChokeFilter filter(&uplink_, 0); RateCounterFilter counter(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, true); - filter.SetCapacity(200); - filter.SetMaxDelay(500); + filter.set_capacity_kbps(200); + filter.set_max_delay_ms(500); RunFor(60 * 1000); - filter.SetCapacity(30); + filter.set_capacity_kbps(30); RunFor(60 * 1000); - filter.SetCapacity(200); + filter.set_capacity_kbps(200); RunFor(60 * 1000); } @@ -162,12 +162,12 @@ TEST_P(BweSimulation, Choke200kbps30kbps200kbps) { ChokeFilter filter(&uplink_, 0); RateCounterFilter counter(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, true); - filter.SetCapacity(200); - filter.SetMaxDelay(500); + filter.set_capacity_kbps(200); + filter.set_max_delay_ms(500); RunFor(60 * 1000); - filter.SetCapacity(30); + filter.set_capacity_kbps(30); RunFor(60 * 1000); - filter.SetCapacity(200); + filter.set_capacity_kbps(200); RunFor(60 * 1000); } @@ -176,7 +176,7 @@ TEST_P(BweSimulation, GoogleWifiTrace3Mbps) { VideoSender sender(&uplink_, &source, GetParam()); RateCounterFilter counter1(&uplink_, 0, "sender_output"); TraceBasedDeliveryFilter filter(&uplink_, 0, "link_capacity"); - filter.SetMaxDelay(500); + filter.set_max_delay_ms(500); RateCounterFilter counter2(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, true); ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx"))); @@ -189,7 +189,7 @@ TEST_P(BweSimulation, LinearIncreasingCapacity) { ChokeFilter filter(&uplink_, 0); RateCounterFilter counter(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, true); - filter.SetMaxDelay(500); + filter.set_max_delay_ms(500); const int kStartingCapacityKbps = 150; const int kEndingCapacityKbps = 1500; const int kStepKbps = 5; @@ -197,7 +197,7 @@ TEST_P(BweSimulation, LinearIncreasingCapacity) { for (int i = kStartingCapacityKbps; i <= kEndingCapacityKbps; i += kStepKbps) { - filter.SetCapacity(i); + filter.set_capacity_kbps(i); RunFor(kStepTimeMs); } } @@ -208,7 +208,7 @@ TEST_P(BweSimulation, LinearDecreasingCapacity) { ChokeFilter filter(&uplink_, 0); RateCounterFilter counter(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, true); - filter.SetMaxDelay(500); + filter.set_max_delay_ms(500); const int kStartingCapacityKbps = 1500; const int kEndingCapacityKbps = 150; const int kStepKbps = -5; @@ -216,7 +216,7 @@ TEST_P(BweSimulation, LinearDecreasingCapacity) { for (int i = kStartingCapacityKbps; i >= kEndingCapacityKbps; i += kStepKbps) { - filter.SetCapacity(i); + filter.set_capacity_kbps(i); RunFor(kStepTimeMs); } } @@ -226,7 +226,7 @@ TEST_P(BweSimulation, PacerGoogleWifiTrace3Mbps) { PacedVideoSender sender(&uplink_, &source, GetParam()); RateCounterFilter counter1(&uplink_, 0, "sender_output"); TraceBasedDeliveryFilter filter(&uplink_, 0, "link_capacity"); - filter.SetMaxDelay(500); + filter.set_max_delay_ms(500); RateCounterFilter counter2(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), true, true); ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx"))); @@ -248,7 +248,7 @@ TEST_P(BweSimulation, SelfFairnessTest) { } ChokeFilter choke(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows)); - choke.SetCapacity(1000); + choke.set_capacity_kbps(1000); rtc::scoped_ptr rate_counters[kNumFlows]; for (size_t i = 0; i < kNumFlows; ++i) { diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi index b313d435e5..3fca240d4e 100644 --- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi +++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi @@ -72,6 +72,8 @@ 'test/bwe_test_framework.h', 'test/bwe_test_logging.cc', 'test/bwe_test_logging.h', + 'test/metric_recorder.cc', + 'test/metric_recorder.h', 'test/packet_receiver.cc', 'test/packet_receiver.h', 'test/packet_sender.cc', diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc index 62d8e129c2..5c147d683b 100644 --- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc +++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc @@ -70,7 +70,7 @@ TEST_P(DefaultBweTest, SteadyDelay) { VideoSender sender(&uplink_, &source, GetParam()); DelayFilter delay(&uplink_, 0); PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); - delay.SetDelayMs(1000); + delay.SetOneWayDelayMs(1000); RunFor(10 * 60 * 1000); } @@ -81,7 +81,7 @@ TEST_P(DefaultBweTest, IncreasingDelay1) { PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); RunFor(10 * 60 * 1000); for (int i = 0; i < 30 * 2; ++i) { - delay.SetDelayMs(i); + delay.SetOneWayDelayMs(i); RunFor(10 * 1000); } RunFor(10 * 60 * 1000); @@ -95,10 +95,10 @@ TEST_P(DefaultBweTest, IncreasingDelay2) { PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); RunFor(1 * 60 * 1000); for (int i = 1; i < 51; ++i) { - delay.SetDelayMs(10.0f * i); + delay.SetOneWayDelayMs(10.0f * i); RunFor(10 * 1000); } - delay.SetDelayMs(0.0f); + delay.SetOneWayDelayMs(0.0f); RunFor(10 * 60 * 1000); } @@ -109,12 +109,12 @@ TEST_P(DefaultBweTest, JumpyDelay1) { PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); RunFor(10 * 60 * 1000); for (int i = 1; i < 200; ++i) { - delay.SetDelayMs((10 * i) % 500); + delay.SetOneWayDelayMs((10 * i) % 500); RunFor(1000); - delay.SetDelayMs(1.0f); + delay.SetOneWayDelayMs(1.0f); RunFor(1000); } - delay.SetDelayMs(0.0f); + delay.SetOneWayDelayMs(0.0f); RunFor(10 * 60 * 1000); } @@ -179,7 +179,7 @@ TEST_P(DefaultBweTest, SteadyChoke) { VideoSender sender(&uplink_, &source, GetParam()); ChokeFilter choke(&uplink_, 0); PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); - choke.SetCapacity(140); + choke.set_capacity_kbps(140); RunFor(10 * 60 * 1000); } @@ -189,7 +189,7 @@ TEST_P(DefaultBweTest, IncreasingChoke1) { ChokeFilter choke(&uplink_, 0); PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); for (int i = 1200; i >= 100; i -= 100) { - choke.SetCapacity(i); + choke.set_capacity_kbps(i); RunFor(5000); } } @@ -201,7 +201,7 @@ TEST_P(DefaultBweTest, IncreasingChoke2) { PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); RunFor(60 * 1000); for (int i = 1200; i >= 100; i -= 20) { - choke.SetCapacity(i); + choke.set_capacity_kbps(i); RunFor(1000); } } @@ -213,14 +213,14 @@ TEST_P(DefaultBweTest, Multi1) { ChokeFilter choke(&uplink_, 0); RateCounterFilter counter(&uplink_, 0, ""); PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); - choke.SetCapacity(1000); + choke.set_capacity_kbps(1000); RunFor(1 * 60 * 1000); for (int i = 1; i < 51; ++i) { - delay.SetDelayMs(100.0f * i); + delay.SetOneWayDelayMs(100.0f * i); RunFor(10 * 1000); } RunFor(500 * 1000); - delay.SetDelayMs(0.0f); + delay.SetOneWayDelayMs(0.0f); RunFor(5 * 60 * 1000); } @@ -231,7 +231,7 @@ TEST_P(DefaultBweTest, Multi2) { JitterFilter jitter(&uplink_, 0); RateCounterFilter counter(&uplink_, 0, ""); PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); - choke.SetCapacity(2000); + choke.set_capacity_kbps(2000); jitter.SetJitter(120); RunFor(5 * 60 * 1000); } @@ -271,8 +271,8 @@ TEST_P(BweFeedbackTest, ConstantCapacity) { RateCounterFilter counter(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); const int kCapacityKbps = 1000; - filter.SetCapacity(kCapacityKbps); - filter.SetMaxDelay(500); + filter.set_capacity_kbps(kCapacityKbps); + filter.set_max_delay_ms(500); RunFor(180 * 1000); PrintResults(kCapacityKbps, counter.GetBitrateStats(), 0, receiver.GetDelayStats(), counter.GetBitrateStats()); @@ -286,12 +286,12 @@ TEST_P(BweFeedbackTest, Choke1000kbps500kbps1000kbps) { PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); const int kHighCapacityKbps = 1000; const int kLowCapacityKbps = 500; - filter.SetCapacity(kHighCapacityKbps); - filter.SetMaxDelay(500); + filter.set_capacity_kbps(kHighCapacityKbps); + filter.set_max_delay_ms(500); RunFor(60 * 1000); - filter.SetCapacity(kLowCapacityKbps); + filter.set_capacity_kbps(kLowCapacityKbps); RunFor(60 * 1000); - filter.SetCapacity(kHighCapacityKbps); + filter.set_capacity_kbps(kHighCapacityKbps); RunFor(60 * 1000); PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0, counter.GetBitrateStats(), 0, receiver.GetDelayStats(), @@ -306,12 +306,12 @@ TEST_P(BweFeedbackTest, Choke200kbps30kbps200kbps) { PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); const int kHighCapacityKbps = 200; const int kLowCapacityKbps = 30; - filter.SetCapacity(kHighCapacityKbps); - filter.SetMaxDelay(500); + filter.set_capacity_kbps(kHighCapacityKbps); + filter.set_max_delay_ms(500); RunFor(60 * 1000); - filter.SetCapacity(kLowCapacityKbps); + filter.set_capacity_kbps(kLowCapacityKbps); RunFor(60 * 1000); - filter.SetCapacity(kHighCapacityKbps); + filter.set_capacity_kbps(kHighCapacityKbps); RunFor(60 * 1000); PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0, @@ -338,7 +338,7 @@ TEST_P(BweFeedbackTest, GoogleWifiTrace3Mbps) { VideoSender sender(&uplink_, &source, GetParam()); RateCounterFilter counter1(&uplink_, 0, "sender_output"); TraceBasedDeliveryFilter filter(&uplink_, 0, "link_capacity"); - filter.SetMaxDelay(500); + filter.set_max_delay_ms(500); RateCounterFilter counter2(&uplink_, 0, "receiver_input"); PacketReceiver receiver(&uplink_, 0, GetParam(), false, false); ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx"))); diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe.cc index 5385b9bed8..0e10f045d1 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe.cc @@ -29,7 +29,31 @@ namespace bwe { const int kSetCapacity = 1000; BweReceiver::BweReceiver(int flow_id) - : flow_id_(flow_id), received_packets_(kSetCapacity) { + : flow_id_(flow_id), + received_packets_(kSetCapacity), + rate_counter_(), + loss_account_() { +} + +BweReceiver::BweReceiver(int flow_id, int64_t window_size_ms) + : flow_id_(flow_id), + received_packets_(kSetCapacity), + rate_counter_(window_size_ms), + loss_account_() { +} + +void BweReceiver::ReceivePacket(int64_t arrival_time_ms, + const MediaPacket& media_packet) { + if (received_packets_.size() == kSetCapacity) { + RelieveSetAndUpdateLoss(); + } + + received_packets_.Insert(media_packet.sequence_number(), + media_packet.send_time_ms(), arrival_time_ms, + media_packet.payload_size()); + + rate_counter_.UpdateRates(media_packet.send_time_ms() * 1000, + static_cast(media_packet.payload_size())); } class NullBweSender : public BweSender { @@ -97,24 +121,54 @@ BweReceiver* CreateBweReceiver(BandwidthEstimatorType type, return NULL; } -float BweReceiver::GlobalPacketLossRatio() { - if (received_packets_.empty()) { - return 0.0f; - } - // Possibly there are packets missing. - const uint16_t kMaxGap = 1.5 * kSetCapacity; - uint16_t min = received_packets_.find_min(); - uint16_t max = received_packets_.find_max(); +// Take into account all LinkedSet content. +void BweReceiver::UpdateLoss() { + loss_account_.Add(LinkedSetPacketLossRatio()); +} - int gap; - if (max - min < kMaxGap) { - gap = max - min + 1; - } else { // There was an overflow. - max = received_packets_.upper_bound(kMaxGap); - min = received_packets_.lower_bound(0xFFFF - kMaxGap); - gap = max + (0xFFFF - min) + 2; +// Preserve 10% latest packets and update packet loss based on the oldest +// 90%, that will be removed. +void BweReceiver::RelieveSetAndUpdateLoss() { + // Compute Loss for the whole LinkedSet and updates loss_account_. + UpdateLoss(); + + size_t num_preserved_elements = received_packets_.size() / 10; + PacketNodeIt it = received_packets_.begin(); + std::advance(it, num_preserved_elements); + + while (it != received_packets_.end()) { + received_packets_.Erase(it++); } - return static_cast(received_packets_.size()) / gap; + + // Compute Loss for the preserved elements + loss_account_.Subtract(LinkedSetPacketLossRatio()); +} + +float BweReceiver::GlobalReceiverPacketLossRatio() { + UpdateLoss(); + return loss_account_.LossRatio(); +} + +// This function considers at most kSetCapacity = 1000 packets. +LossAccount BweReceiver::LinkedSetPacketLossRatio() { + if (received_packets_.empty()) { + return LossAccount(); + } + + uint16_t oldest_seq_num = received_packets_.OldestSeqNumber(); + uint16_t newest_seq_num = received_packets_.NewestSeqNumber(); + + size_t set_total_packets = + static_cast(newest_seq_num - oldest_seq_num + 1); + + size_t set_received_packets = received_packets_.size(); + size_t set_lost_packets = set_total_packets - set_received_packets; + + return LossAccount(set_total_packets, set_lost_packets); +} + +uint32_t BweReceiver::RecentKbps() const { + return (rate_counter_.bits_per_second() + 500) / 1000; } // Go through a fixed time window of most recent packets received and @@ -133,26 +187,26 @@ float BweReceiver::RecentPacketLossRatio() { // Lowest timestamp limit, oldest one that should be checked. int64_t time_limit_ms = (*node_it)->arrival_time_ms - kPacketLossTimeWindowMs; // Oldest and newest values found within the given time window. - uint16_t oldest_seq_nb = (*node_it)->sequence_number; - uint16_t newest_seq_nb = oldest_seq_nb; + uint16_t oldest_seq_num = (*node_it)->sequence_number; + uint16_t newest_seq_num = oldest_seq_num; while (node_it != received_packets_.end()) { if ((*node_it)->arrival_time_ms < time_limit_ms) { break; } - uint16_t seq_nb = (*node_it)->sequence_number; - if (IsNewerSequenceNumber(seq_nb, newest_seq_nb)) { - newest_seq_nb = seq_nb; + uint16_t seq_num = (*node_it)->sequence_number; + if (IsNewerSequenceNumber(seq_num, newest_seq_num)) { + newest_seq_num = seq_num; } - if (IsNewerSequenceNumber(oldest_seq_nb, seq_nb)) { - oldest_seq_nb = seq_nb; + if (IsNewerSequenceNumber(oldest_seq_num, seq_num)) { + oldest_seq_num = seq_num; } ++node_it; ++number_packets_received; } // Interval width between oldest and newest sequence number. - // There was an overflow if newest_seq_nb < oldest_seq_nb. - int gap = static_cast(newest_seq_nb - oldest_seq_nb + 1); + // There was an overflow if newest_seq_num < oldest_seq_num. + int gap = static_cast(newest_seq_num - oldest_seq_num + 1); return static_cast(gap - number_packets_received) / gap; } @@ -166,7 +220,7 @@ void LinkedSet::Insert(uint16_t sequence_number, int64_t send_time_ms, int64_t arrival_time_ms, size_t payload_size) { - std::map::iterator it = map_.find(sequence_number); + auto it = map_.find(sequence_number); if (it != map_.end()) { PacketNodeIt node_it = it->second; PacketIdentifierNode* node = *node_it; @@ -184,6 +238,12 @@ void LinkedSet::Insert(uint16_t sequence_number, arrival_time_ms, payload_size)); } } + +void LinkedSet::Insert(PacketIdentifierNode packet_identifier) { + Insert(packet_identifier.sequence_number, packet_identifier.send_time_ms, + packet_identifier.arrival_time_ms, packet_identifier.payload_size); +} + void LinkedSet::RemoveTail() { map_.erase(list_.back()->sequence_number); delete list_.back(); @@ -194,6 +254,27 @@ void LinkedSet::UpdateHead(PacketIdentifierNode* new_head) { map_[new_head->sequence_number] = list_.begin(); } +void LinkedSet::Erase(PacketNodeIt node_it) { + map_.erase((*node_it)->sequence_number); + delete (*node_it); + list_.erase(node_it); +} + +void LossAccount::Add(LossAccount rhs) { + num_total += rhs.num_total; + num_lost += rhs.num_lost; +} +void LossAccount::Subtract(LossAccount rhs) { + num_total -= rhs.num_total; + num_lost -= rhs.num_lost; +} + +float LossAccount::LossRatio() { + if (num_total == 0) + return 0.0f; + return static_cast(num_lost) / num_total; +} + } // namespace bwe } // namespace testing } // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe.h b/webrtc/modules/remote_bitrate_estimator/test/bwe.h index d86331aad7..72e7f73ad9 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe.h +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe.h @@ -13,13 +13,35 @@ #include +#include "webrtc/test/testsupport/gtest_prod_util.h" #include "webrtc/modules/remote_bitrate_estimator/test/packet.h" #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h" +#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h" namespace webrtc { namespace testing { namespace bwe { +// Overload map comparator. +class SequenceNumberOlderThan { + public: + bool operator()(uint16_t seq_num_1, uint16_t seq_num_2) const { + return IsNewerSequenceNumber(seq_num_2, seq_num_1); + } +}; + +// Holds information for computing global packet loss. +struct LossAccount { + LossAccount() : num_total(0), num_lost(0) {} + LossAccount(size_t num_total, size_t num_lost) + : num_total(num_total), num_lost(num_lost) {} + void Add(LossAccount rhs); + void Subtract(LossAccount rhs); + float LossRatio(); + size_t num_total; + size_t num_lost; +}; + // Holds only essential information about packets to be saved for // further use, e.g. for calculating packet loss and receiving rate. struct PacketIdentifierNode { @@ -57,19 +79,21 @@ class LinkedSet { int64_t arrival_time_ms, size_t payload_size); + void Insert(PacketIdentifierNode packet_identifier); + PacketNodeIt begin() { return list_.begin(); } PacketNodeIt end() { return list_.end(); } - bool empty() { return list_.empty(); } - size_t size() { return list_.size(); } - // Gets the latest arrived sequence number. - uint16_t find_max() { return map_.rbegin()->first; } - // Gets the first arrived sequence number still saved in the LinkedSet. - uint16_t find_min() { return map_.begin()->first; } - // Gets the lowest saved sequence number that is >= than the input key. - uint16_t lower_bound(uint16_t key) { return map_.lower_bound(key)->first; } - // Gets the highest saved sequence number that is <= than the input key. - uint16_t upper_bound(uint16_t key) { return map_.upper_bound(key)->first; } - size_t capacity() { return capacity_; } + + bool empty() const { return list_.empty(); } + size_t size() const { return list_.size(); } + size_t capacity() const { return capacity_; } + + uint16_t OldestSeqNumber() const { return empty() ? 0 : map_.begin()->first; } + uint16_t NewestSeqNumber() const { + return empty() ? 0 : map_.rbegin()->first; + } + + void Erase(PacketNodeIt node_it); private: // Pop oldest element from the back of the list and remove it from the map. @@ -77,22 +101,26 @@ class LinkedSet { // Add new element to the front of the list and insert it in the map. void UpdateHead(PacketIdentifierNode* new_head); size_t capacity_; - std::map map_; + std::map map_; std::list list_; }; -const int kMinBitrateKbps = 20; -const int kMaxBitrateKbps = 3000; +const int kMinBitrateKbps = 50; +const int kMaxBitrateKbps = 2500; class BweSender : public Module { public: BweSender() {} + explicit BweSender(int bitrate_kbps) : bitrate_kbps_(bitrate_kbps) {} virtual ~BweSender() {} virtual int GetFeedbackIntervalMs() const = 0; virtual void GiveFeedback(const FeedbackPacket& feedback) = 0; virtual void OnPacketsSent(const Packets& packets) = 0; + protected: + int bitrate_kbps_; + private: DISALLOW_COPY_AND_ASSIGN(BweSender); }; @@ -100,22 +128,44 @@ class BweSender : public Module { class BweReceiver { public: explicit BweReceiver(int flow_id); + BweReceiver(int flow_id, int64_t window_size_ms); + virtual ~BweReceiver() {} virtual void ReceivePacket(int64_t arrival_time_ms, - const MediaPacket& media_packet) {} + const MediaPacket& media_packet); virtual FeedbackPacket* GetFeedback(int64_t now_ms) { return NULL; } - float GlobalPacketLossRatio(); - float RecentPacketLossRatio(); size_t GetSetCapacity() { return received_packets_.capacity(); } + double BitrateWindowS() const { return rate_counter_.BitrateWindowS(); } + uint32_t RecentKbps() const; // Receiving Rate. + + // Computes packet loss during an entire simulation, up to 4 billion packets. + float GlobalReceiverPacketLossRatio(); // Plot histogram. + float RecentPacketLossRatio(); // Plot dynamics. static const int64_t kPacketLossTimeWindowMs = 500; + static const int64_t kReceivingRateTimeWindowMs = 1000; protected: int flow_id_; // Deals with packets sent more than once. LinkedSet received_packets_; + // Used for calculating recent receiving rate. + RateCounter rate_counter_; + + private: + FRIEND_TEST_ALL_PREFIXES(BweReceiverTest, RecentKbps); + FRIEND_TEST_ALL_PREFIXES(BweReceiverTest, Loss); + + void UpdateLoss(); + void RelieveSetAndUpdateLoss(); + // Packet loss for packets stored in the LinkedSet, up to 1000 packets. + // Used to update global loss account whenever the set is filled and cleared. + LossAccount LinkedSetPacketLossRatio(); + + // Used for calculating global packet loss ratio. + LossAccount loss_account_; }; enum BandwidthEstimatorType { @@ -126,6 +176,8 @@ enum BandwidthEstimatorType { kTcpEstimator }; +const std::string bwe_names[] = {"Null", "NADA", "REMB", "GCC", "TCP"}; + int64_t GetAbsSendTimeInMs(uint32_t abs_send_time); BweSender* CreateBweSender(BandwidthEstimatorType estimator, diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc index a2ce34084e..182f93193e 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc @@ -48,7 +48,7 @@ void PacketProcessorRunner::RunFor(int64_t time_ms, processor_->RunFor(time_ms, &to_process); QueuePackets(&to_process, time_now_ms * 1000); if (!to_process.empty()) { - processor_->Plot((to_process.back()->send_time_us() + 500) / 1000); + processor_->Plot(to_process.back()->send_time_ms()); } in_out->merge(to_process, DereferencingComparator); } @@ -269,11 +269,11 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type, senders.push_back(new TcpSender(&uplink_, tcp_flow, kTcpStartOffsetMs)); ChokeFilter choke(&uplink_, all_flow_ids); - choke.SetCapacity(capacity_kbps); - choke.SetMaxDelay(max_delay_ms); + choke.set_capacity_kbps(capacity_kbps); + choke.set_max_delay_ms(max_delay_ms); DelayFilter delay_uplink(&uplink_, all_flow_ids); - delay_uplink.SetDelayMs(25); + delay_uplink.SetOneWayDelayMs(25); std::vector rate_counters; for (int flow : all_flow_ids) { @@ -296,7 +296,7 @@ void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type, } DelayFilter delay_downlink(&downlink_, all_flow_ids); - delay_downlink.SetDelayMs(25); + delay_downlink.SetOneWayDelayMs(25); RunFor(run_time_seconds * 1000); diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc index cac59ae0ad..2d256d49fc 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc @@ -20,9 +20,10 @@ namespace bwe { class DelayCapHelper { public: + // Max delay = 0 stands for +infinite. DelayCapHelper() : max_delay_us_(0), delay_stats_() {} - void SetMaxDelay(int max_delay_ms) { + void set_max_delay_ms(int64_t max_delay_ms) { BWE_TEST_LOGGING_ENABLE(false); BWE_TEST_LOGGING_LOG1("Max Delay", "%d ms", static_cast(max_delay_ms)); assert(max_delay_ms >= 0); @@ -31,7 +32,7 @@ class DelayCapHelper { bool ShouldSendPacket(int64_t send_time_us, int64_t arrival_time_us) { int64_t packet_delay_us = send_time_us - arrival_time_us; - delay_stats_.Push(std::min(packet_delay_us, max_delay_us_) / 1000); + delay_stats_.Push((std::min(packet_delay_us, max_delay_us_) + 500) / 1000); return (max_delay_us_ == 0 || max_delay_us_ >= packet_delay_us); } @@ -51,57 +52,94 @@ const FlowIds CreateFlowIds(const int *flow_ids_array, size_t num_flow_ids) { return flow_ids; } -class RateCounter { - public: - RateCounter() - : kWindowSizeUs(1000000), - packets_per_second_(0), - bytes_per_second_(0), - last_accumulated_us_(0), - window_() {} +const FlowIds CreateFlowIdRange(int initial_value, int last_value) { + int size = last_value - initial_value + 1; + assert(size > 0); + int* flow_ids_array = new int[size]; + for (int i = initial_value; i <= last_value; ++i) { + flow_ids_array[i - initial_value] = i; + } + return CreateFlowIds(flow_ids_array, size); +} - void UpdateRates(int64_t send_time_us, uint32_t payload_size) { - packets_per_second_++; - bytes_per_second_ += payload_size; - last_accumulated_us_ = send_time_us; - window_.push_back(std::make_pair(send_time_us, payload_size)); - while (!window_.empty()) { - const TimeSizePair& packet = window_.front(); - if (packet.first > (last_accumulated_us_ - kWindowSizeUs)) { - break; - } - assert(packets_per_second_ >= 1); - assert(bytes_per_second_ >= packet.second); - packets_per_second_--; - bytes_per_second_ -= packet.second; - window_.pop_front(); +void RateCounter::UpdateRates(int64_t send_time_us, uint32_t payload_size) { + ++recently_received_packets_; + recently_received_bytes_ += payload_size; + last_accumulated_us_ = send_time_us; + window_.push_back(std::make_pair(send_time_us, payload_size)); + while (!window_.empty()) { + const TimeSizePair& packet = window_.front(); + if (packet.first > (last_accumulated_us_ - window_size_us_)) { + break; } + assert(recently_received_packets_ >= 1); + assert(recently_received_bytes_ >= packet.second); + --recently_received_packets_; + recently_received_bytes_ -= packet.second; + window_.pop_front(); } +} - uint32_t bits_per_second() const { - return bytes_per_second_ * 8; - } +uint32_t RateCounter::bits_per_second() const { + return (8 * recently_received_bytes_) / BitrateWindowS(); +} - uint32_t packets_per_second() const { return packets_per_second_; } +uint32_t RateCounter::packets_per_second() const { + return recently_received_packets_ / BitrateWindowS(); +} - private: - typedef std::pair TimeSizePair; +double RateCounter::BitrateWindowS() const { + return static_cast(window_size_us_) / (1000 * 1000); +} - const int64_t kWindowSizeUs; - uint32_t packets_per_second_; - uint32_t bytes_per_second_; - int64_t last_accumulated_us_; - std::list window_; -}; +Random::Random(uint32_t seed) : a_(0x531FDB97 ^ seed), b_(0x6420ECA8 + seed) { +} + +float Random::Rand() { + const float kScale = 1.0f / 0xffffffff; + float result = kScale * b_; + a_ ^= b_; + b_ += a_; + return result; +} + +int Random::Rand(int low, int high) { + float uniform = Rand() * (high - low + 1) + low; + return static_cast(uniform); +} + +int Random::Gaussian(int mean, int standard_deviation) { + // Creating a Normal distribution variable from two independent uniform + // variables based on the Box-Muller transform, which is defined on the + // interval (0, 1], hence the mask+add below. + const double kPi = 3.14159265358979323846; + const double kScale = 1.0 / 0x80000000ul; + double u1 = kScale * ((a_ & 0x7ffffffful) + 1); + double u2 = kScale * ((b_ & 0x7ffffffful) + 1); + a_ ^= b_; + b_ += a_; + return static_cast( + mean + standard_deviation * sqrt(-2 * log(u1)) * cos(2 * kPi * u2)); +} + +int Random::Exponential(float lambda) { + float uniform = Rand(); + return static_cast(-log(uniform) / lambda); +} Packet::Packet() - : flow_id_(0), creation_time_us_(-1), send_time_us_(-1), payload_size_(0) { + : flow_id_(0), + creation_time_us_(-1), + send_time_us_(-1), + sender_timestamp_us_(-1), + payload_size_(0) { } Packet::Packet(int flow_id, int64_t send_time_us, size_t payload_size) : flow_id_(flow_id), creation_time_us_(send_time_us), send_time_us_(send_time_us), + sender_timestamp_us_(send_time_us), payload_size_(payload_size) { } @@ -203,14 +241,22 @@ PacketProcessor::~PacketProcessor() { } } +uint32_t PacketProcessor::packets_per_second() const { + return rate_counter_.packets_per_second(); +} + +uint32_t PacketProcessor::bits_per_second() const { + return rate_counter_.bits_per_second(); +} + RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener, int flow_id, const char* name) : PacketProcessor(listener, flow_id, kRegular), - rate_counter_(new RateCounter()), packets_per_second_stats_(), kbps_stats_(), - name_() { + name_(), + start_plotting_time_ms_(0) { std::stringstream ss; ss << name << "_" << flow_id; name_ = ss.str(); @@ -220,10 +266,10 @@ RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener, const FlowIds& flow_ids, const char* name) : PacketProcessor(listener, flow_ids, kRegular), - rate_counter_(new RateCounter()), packets_per_second_stats_(), kbps_stats_(), - name_() { + name_(), + start_plotting_time_ms_(0) { std::stringstream ss; ss << name << "_"; for (int flow_id : flow_ids) { @@ -232,17 +278,18 @@ RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener, name_ = ss.str(); } +RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener, + const FlowIds& flow_ids, + const char* name, + int64_t start_plotting_time_ms) + : RateCounterFilter(listener, flow_ids, name) { + start_plotting_time_ms_ = start_plotting_time_ms; +} + RateCounterFilter::~RateCounterFilter() { LogStats(); } -uint32_t RateCounterFilter::packets_per_second() const { - return rate_counter_->packets_per_second(); -} - -uint32_t RateCounterFilter::bits_per_second() const { - return rate_counter_->bits_per_second(); -} void RateCounterFilter::LogStats() { BWE_TEST_LOGGING_CONTEXT("RateCounterFilter"); @@ -255,19 +302,23 @@ Stats RateCounterFilter::GetBitrateStats() const { } void RateCounterFilter::Plot(int64_t timestamp_ms) { + uint32_t plot_kbps = 0; + if (timestamp_ms >= start_plotting_time_ms_) { + plot_kbps = rate_counter_.bits_per_second() / 1000.0; + } BWE_TEST_LOGGING_CONTEXT(name_.c_str()); - BWE_TEST_LOGGING_PLOT(0, "Throughput_#1", timestamp_ms, - rate_counter_->bits_per_second() / 1000.0); + BWE_TEST_LOGGING_PLOT(0, "Throughput_#1", timestamp_ms, plot_kbps); + RTC_UNUSED(plot_kbps); } void RateCounterFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) { assert(in_out); for (const Packet* packet : *in_out) { - rate_counter_->UpdateRates(packet->send_time_us(), - static_cast(packet->payload_size())); + rate_counter_.UpdateRates(packet->send_time_us(), + static_cast(packet->payload_size())); } - packets_per_second_stats_.Push(rate_counter_->packets_per_second()); - kbps_stats_.Push(rate_counter_->bits_per_second() / 1000.0); + packets_per_second_stats_.Push(rate_counter_.packets_per_second()); + kbps_stats_.Push(rate_counter_.bits_per_second() / 1000.0); } LossFilter::LossFilter(PacketProcessorListener* listener, int flow_id) @@ -303,30 +354,32 @@ void LossFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) { } } +const int64_t kDefaultOneWayDelayUs = 0; + DelayFilter::DelayFilter(PacketProcessorListener* listener, int flow_id) : PacketProcessor(listener, flow_id, kRegular), - delay_us_(0), + one_way_delay_us_(kDefaultOneWayDelayUs), last_send_time_us_(0) { } DelayFilter::DelayFilter(PacketProcessorListener* listener, const FlowIds& flow_ids) : PacketProcessor(listener, flow_ids, kRegular), - delay_us_(0), + one_way_delay_us_(kDefaultOneWayDelayUs), last_send_time_us_(0) { } -void DelayFilter::SetDelayMs(int64_t delay_ms) { +void DelayFilter::SetOneWayDelayMs(int64_t one_way_delay_ms) { BWE_TEST_LOGGING_ENABLE(false); - BWE_TEST_LOGGING_LOG1("Delay", "%d ms", static_cast(delay_ms)); - assert(delay_ms >= 0); - delay_us_ = delay_ms * 1000; + BWE_TEST_LOGGING_LOG1("Delay", "%d ms", static_cast(one_way_delay_ms)); + assert(one_way_delay_ms >= 0); + one_way_delay_us_ = one_way_delay_ms * 1000; } void DelayFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) { assert(in_out); for (Packet* packet : *in_out) { - int64_t new_send_time_us = packet->send_time_us() + delay_us_; + int64_t new_send_time_us = packet->send_time_us() + one_way_delay_us_; last_send_time_us_ = std::max(last_send_time_us_, new_send_time_us); packet->set_send_time_us(last_send_time_us_); } @@ -404,9 +457,11 @@ void ReorderFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) { } } +const uint32_t kDefaultKbps = 1200; + ChokeFilter::ChokeFilter(PacketProcessorListener* listener, int flow_id) : PacketProcessor(listener, flow_id, kRegular), - kbps_(1200), + capacity_kbps_(kDefaultKbps), last_send_time_us_(0), delay_cap_helper_(new DelayCapHelper()) { } @@ -414,17 +469,21 @@ ChokeFilter::ChokeFilter(PacketProcessorListener* listener, int flow_id) ChokeFilter::ChokeFilter(PacketProcessorListener* listener, const FlowIds& flow_ids) : PacketProcessor(listener, flow_ids, kRegular), - kbps_(1200), + capacity_kbps_(kDefaultKbps), last_send_time_us_(0), delay_cap_helper_(new DelayCapHelper()) { } ChokeFilter::~ChokeFilter() {} -void ChokeFilter::SetCapacity(uint32_t kbps) { +void ChokeFilter::set_capacity_kbps(uint32_t kbps) { BWE_TEST_LOGGING_ENABLE(false); BWE_TEST_LOGGING_LOG1("BitrateChoke", "%d kbps", kbps); - kbps_ = kbps; + capacity_kbps_ = kbps; +} + +uint32_t ChokeFilter::capacity_kbps() { + return capacity_kbps_; } void ChokeFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) { @@ -432,9 +491,12 @@ void ChokeFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) { for (PacketsIt it = in_out->begin(); it != in_out->end(); ) { int64_t earliest_send_time_us = std::max(last_send_time_us_, (*it)->send_time_us()); + int64_t new_send_time_us = earliest_send_time_us + - ((*it)->payload_size() * 8 * 1000 + kbps_ / 2) / kbps_; + ((*it)->payload_size() * 8 * 1000 + capacity_kbps_ / 2) / + capacity_kbps_; + if (delay_cap_helper_->ShouldSendPacket(new_send_time_us, (*it)->send_time_us())) { (*it)->set_send_time_us(new_send_time_us); @@ -447,8 +509,8 @@ void ChokeFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) { } } -void ChokeFilter::SetMaxDelay(int max_delay_ms) { - delay_cap_helper_->SetMaxDelay(max_delay_ms); +void ChokeFilter::set_max_delay_ms(int64_t max_delay_ms) { + delay_cap_helper_->set_max_delay_ms(max_delay_ms); } Stats ChokeFilter::GetDelayStats() const { @@ -565,8 +627,8 @@ void TraceBasedDeliveryFilter::RunFor(int64_t time_ms, Packets* in_out) { kbps_stats_.Push(rate_counter_->bits_per_second() / 1000.0); } -void TraceBasedDeliveryFilter::SetMaxDelay(int max_delay_ms) { - delay_cap_helper_->SetMaxDelay(max_delay_ms); +void TraceBasedDeliveryFilter::set_max_delay_ms(int64_t max_delay_ms) { + delay_cap_helper_->set_max_delay_ms(max_delay_ms); } Stats TraceBasedDeliveryFilter::GetDelayStats() const { @@ -606,8 +668,10 @@ VideoSource::VideoSource(int flow_id, frame_size_bytes_(bits_per_second_ / 8 / fps), flow_id_(flow_id), next_frame_ms_(first_frame_offset_ms), + next_frame_rand_ms_(0), now_ms_(0), - prototype_header_() { + prototype_header_(), + start_plotting_ms_(first_frame_offset_ms) { memset(&prototype_header_, 0, sizeof(prototype_header_)); prototype_header_.ssrc = ssrc; prototype_header_.sequenceNumber = 0xf000u; @@ -617,6 +681,10 @@ uint32_t VideoSource::NextFrameSize() { return frame_size_bytes_; } +int64_t VideoSource::GetTimeUntilNextFrameMs() const { + return next_frame_ms_ + next_frame_rand_ms_ - now_ms_; +} + uint32_t VideoSource::NextPacketSize(uint32_t frame_size, uint32_t remaining_payload) { return std::min(kMaxPayloadSizeBytes, remaining_payload); @@ -624,21 +692,33 @@ uint32_t VideoSource::NextPacketSize(uint32_t frame_size, void VideoSource::RunFor(int64_t time_ms, Packets* in_out) { assert(in_out); - std::stringstream ss; - ss << "SendEstimate_" << flow_id_ << "#1"; - BWE_TEST_LOGGING_PLOT(0, ss.str(), now_ms_, bits_per_second_ / 1000); + now_ms_ += time_ms; Packets new_packets; + while (now_ms_ >= next_frame_ms_) { - prototype_header_.timestamp = kTimestampBase + - static_cast(next_frame_ms_ * 90.0); + const int64_t kRandAmplitude = 2; + // A variance picked uniformly from {-1, 0, 1} ms is added to the frame + // timestamp. + next_frame_rand_ms_ = + kRandAmplitude * static_cast(rand()) / RAND_MAX - + kRandAmplitude / 2; + + // Ensure frame will not have a negative timestamp. + int64_t next_frame_ms = + std::max(next_frame_ms_ + next_frame_rand_ms_, 0); + + prototype_header_.timestamp = + kTimestampBase + static_cast(next_frame_ms * 90.0); prototype_header_.extension.transmissionTimeOffset = 0; // Generate new packets for this frame, all with the same timestamp, // but the payload size is capped, so if the whole frame doesn't fit in // one packet, we will see a number of equally sized packets followed by // one smaller at the tail. - int64_t send_time_us = next_frame_ms_ * 1000.0; + + int64_t send_time_us = next_frame_ms * 1000.0; + uint32_t frame_size = NextFrameSize(); uint32_t payload_size = frame_size; @@ -648,13 +728,14 @@ void VideoSource::RunFor(int64_t time_ms, Packets* in_out) { MediaPacket* new_packet = new MediaPacket(flow_id_, send_time_us, size, prototype_header_); new_packets.push_back(new_packet); - new_packet->SetAbsSendTimeMs(next_frame_ms_); + new_packet->SetAbsSendTimeMs(next_frame_ms); new_packet->set_sender_timestamp_us(send_time_us); payload_size -= size; } next_frame_ms_ += frame_period_ms_; } + in_out->merge(new_packets, DereferencingComparator); } diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h index cfbac9c362..b742f7ffd1 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h @@ -21,6 +21,7 @@ #include #include +#include "webrtc/base/common.h" #include "webrtc/base/scoped_ptr.h" #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h" #include "webrtc/modules/interface/module_common_types.h" @@ -40,10 +41,39 @@ namespace testing { namespace bwe { class DelayCapHelper; -class RateCounter; + +class RateCounter { + public: + RateCounter(int64_t window_size_ms) + : window_size_us_(1000 * window_size_ms), + recently_received_packets_(0), + recently_received_bytes_(0), + last_accumulated_us_(0), + window_() {} + + RateCounter() : RateCounter(1000) {} + + void UpdateRates(int64_t send_time_us, uint32_t payload_size); + + int64_t window_size_ms() const { return (window_size_us_ + 500) / 1000; } + uint32_t packets_per_second() const; + uint32_t bits_per_second() const; + + double BitrateWindowS() const; + + private: + typedef std::pair TimeSizePair; + + int64_t window_size_us_; + uint32_t recently_received_packets_; + uint32_t recently_received_bytes_; + int64_t last_accumulated_us_; + std::list window_; +}; typedef std::set FlowIds; const FlowIds CreateFlowIds(const int *flow_ids_array, size_t num_flow_ids); +const FlowIds CreateFlowIdRange(int initial_value, int last_value); template bool DereferencingComparator(const T* const& a, const T* const& b) { @@ -143,6 +173,32 @@ template class Stats { T max_; }; +class Random { + public: + explicit Random(uint32_t seed); + + // Return pseudo random number in the interval [0.0, 1.0]. + float Rand(); + + // Return pseudo rounded random number in interval [low, high]. + int Rand(int low, int high); + + // Normal Distribution. + int Gaussian(int mean, int standard_deviation); + + // Exponential Distribution. + int Exponential(float lambda); + + // TODO(solenberg): Random from histogram. + // template int Distribution(const std::vector histogram) { + + private: + uint32_t a_; + uint32_t b_; + + DISALLOW_IMPLICIT_CONSTRUCTORS(Random); +}; + bool IsTimeSorted(const Packets& packets); class PacketProcessor; @@ -179,6 +235,12 @@ class PacketProcessor { const FlowIds& flow_ids() const { return flow_ids_; } + uint32_t packets_per_second() const; + uint32_t bits_per_second() const; + + protected: + RateCounter rate_counter_; + private: PacketProcessorListener* listener_; const FlowIds flow_ids_; @@ -194,21 +256,22 @@ class RateCounterFilter : public PacketProcessor { RateCounterFilter(PacketProcessorListener* listener, const FlowIds& flow_ids, const char* name); + RateCounterFilter(PacketProcessorListener* listener, + const FlowIds& flow_ids, + const char* name, + int64_t start_plotting_time_ms); virtual ~RateCounterFilter(); - uint32_t packets_per_second() const; - uint32_t bits_per_second() const; - void LogStats(); Stats GetBitrateStats() const; virtual void Plot(int64_t timestamp_ms); virtual void RunFor(int64_t time_ms, Packets* in_out); private: - rtc::scoped_ptr rate_counter_; Stats packets_per_second_stats_; Stats kbps_stats_; std::string name_; + int64_t start_plotting_time_ms_; DISALLOW_IMPLICIT_CONSTRUCTORS(RateCounterFilter); }; @@ -235,11 +298,11 @@ class DelayFilter : public PacketProcessor { DelayFilter(PacketProcessorListener* listener, const FlowIds& flow_ids); virtual ~DelayFilter() {} - void SetDelayMs(int64_t delay_ms); + void SetOneWayDelayMs(int64_t one_way_delay_ms); virtual void RunFor(int64_t time_ms, Packets* in_out); private: - int64_t delay_us_; + int64_t one_way_delay_us_; int64_t last_send_time_us_; DISALLOW_IMPLICIT_CONSTRUCTORS(DelayFilter); @@ -286,16 +349,20 @@ class ChokeFilter : public PacketProcessor { ChokeFilter(PacketProcessorListener* listener, const FlowIds& flow_ids); virtual ~ChokeFilter(); - void SetCapacity(uint32_t kbps); - void SetMaxDelay(int max_delay_ms); + void set_capacity_kbps(uint32_t kbps); + void set_max_delay_ms(int64_t max_queueing_delay_ms); + + uint32_t capacity_kbps(); + virtual void RunFor(int64_t time_ms, Packets* in_out); Stats GetDelayStats() const; private: - uint32_t kbps_; + uint32_t capacity_kbps_; int64_t last_send_time_us_; rtc::scoped_ptr delay_cap_helper_; + int64_t max_delay_us_; DISALLOW_IMPLICIT_CONSTRUCTORS(ChokeFilter); }; @@ -317,7 +384,7 @@ class TraceBasedDeliveryFilter : public PacketProcessor { virtual void Plot(int64_t timestamp_ms); virtual void RunFor(int64_t time_ms, Packets* in_out); - void SetMaxDelay(int max_delay_ms); + void set_max_delay_ms(int64_t max_delay_ms); Stats GetDelayStats() const; Stats GetBitrateStats() const; @@ -353,7 +420,7 @@ class VideoSource { virtual void SetBitrateBps(int bitrate_bps) {} uint32_t bits_per_second() const { return bits_per_second_; } uint32_t max_payload_size_bytes() const { return kMaxPayloadSizeBytes; } - int64_t GetTimeUntilNextFrameMs() const { return next_frame_ms_ - now_ms_; } + int64_t GetTimeUntilNextFrameMs() const; protected: virtual uint32_t NextFrameSize(); @@ -369,8 +436,11 @@ class VideoSource { private: const int flow_id_; int64_t next_frame_ms_; + int64_t next_frame_rand_ms_; int64_t now_ms_; RTPHeader prototype_header_; + int64_t start_plotting_ms_; + uint32_t previous_bitrate_bps_; DISALLOW_IMPLICIT_CONSTRUCTORS(VideoSource); }; diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc index 01cf7cbe3d..875c8f798f 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework_unittest.cc @@ -332,7 +332,7 @@ class BweTestFramework_DelayFilterTest : public ::testing::Test { } void TestDelayFilter(int64_t delay_ms) { - filter_.SetDelayMs(delay_ms); + filter_.SetOneWayDelayMs(delay_ms); TestDelayFilter(1, 0, 0); // No input should yield no output // Single packet @@ -340,7 +340,7 @@ class BweTestFramework_DelayFilterTest : public ::testing::Test { TestDelayFilter(delay_ms, 0, 0); for (int i = 0; i < delay_ms; ++i) { - filter_.SetDelayMs(i); + filter_.SetOneWayDelayMs(i); TestDelayFilter(1, 10, 10); } TestDelayFilter(0, 0, 0); @@ -350,11 +350,11 @@ class BweTestFramework_DelayFilterTest : public ::testing::Test { TestDelayFilter(delay_ms, 0, 0); for (int i = 1; i < delay_ms + 1; ++i) { - filter_.SetDelayMs(i); + filter_.SetOneWayDelayMs(i); TestDelayFilter(1, 5, 5); } TestDelayFilter(0, 0, 0); - filter_.SetDelayMs(2 * delay_ms); + filter_.SetOneWayDelayMs(2 * delay_ms); TestDelayFilter(1, 0, 0); TestDelayFilter(delay_ms, 13, 13); TestDelayFilter(delay_ms, 0, 0); @@ -363,11 +363,11 @@ class BweTestFramework_DelayFilterTest : public ::testing::Test { TestDelayFilter(delay_ms, 0, 0); for (int i = 0; i < 2 * delay_ms; ++i) { - filter_.SetDelayMs(2 * delay_ms - i - 1); + filter_.SetOneWayDelayMs(2 * delay_ms - i - 1); TestDelayFilter(1, 5, 5); } TestDelayFilter(0, 0, 0); - filter_.SetDelayMs(0); + filter_.SetOneWayDelayMs(0); TestDelayFilter(0, 7, 7); ASSERT_TRUE(IsTimeSorted(accumulated_packets_)); @@ -388,7 +388,7 @@ TEST_F(BweTestFramework_DelayFilterTest, Delay0) { TestDelayFilter(1, 0, 0); // No input should yield no output TestDelayFilter(1, 10, 10); // Expect no delay (delay time is zero) TestDelayFilter(1, 0, 0); // Check no packets are still in buffer - filter_.SetDelayMs(0); + filter_.SetOneWayDelayMs(0); TestDelayFilter(1, 5, 5); // Expect no delay (delay time is zero) TestDelayFilter(1, 0, 0); // Check no packets are still in buffer } @@ -415,7 +415,7 @@ TEST_F(BweTestFramework_DelayFilterTest, JumpToZeroDelay) { Packets packets; // Delay a bunch of packets, accumulate them to the 'acc' list. - delay.SetDelayMs(100.0f); + delay.SetOneWayDelayMs(100.0f); for (uint32_t i = 0; i < 10; ++i) { packets.push_back(new MediaPacket(i * 100, i)); } @@ -426,7 +426,7 @@ TEST_F(BweTestFramework_DelayFilterTest, JumpToZeroDelay) { // Drop delay to zero, send a few more packets through the delay, append them // to the 'acc' list and verify that it is all sorted. - delay.SetDelayMs(0.0f); + delay.SetOneWayDelayMs(0.0f); for (uint32_t i = 10; i < 50; ++i) { packets.push_back(new MediaPacket(i * 100, i)); } @@ -445,12 +445,12 @@ TEST_F(BweTestFramework_DelayFilterTest, IncreasingDelay) { TestDelayFilter(i); } // Reach a steady state. - filter_.SetDelayMs(100); + filter_.SetOneWayDelayMs(100); TestDelayFilter(1, 20, 20); TestDelayFilter(2, 0, 0); TestDelayFilter(99, 20, 20); // Drop delay back down to zero. - filter_.SetDelayMs(0); + filter_.SetOneWayDelayMs(0); TestDelayFilter(1, 100, 100); TestDelayFilter(23010, 0, 0); ASSERT_TRUE(IsTimeSorted(accumulated_packets_)); @@ -643,7 +643,7 @@ class BweTestFramework_ChokeFilterTest : public ::testing::Test { delete output_packets_.front(); output_packets_.pop_front(); } - EXPECT_EQ(expected_kbit_transmitted, (bytes_transmitted * 8) / 1000); + EXPECT_EQ(expected_kbit_transmitted, (bytes_transmitted * 8 + 500) / 1000); } void CheckMaxDelay(int64_t max_delay_ms) { @@ -672,7 +672,7 @@ TEST_F(BweTestFramework_ChokeFilterTest, NoQueue) { uint16_t sequence_number = 0; int64_t send_time_us = 0; ChokeFilter filter(NULL, 0); - filter.SetCapacity(10); + filter.set_capacity_kbps(10); Packets packets; RTPHeader header; for (int i = 0; i < 2; ++i) { @@ -699,14 +699,14 @@ TEST_F(BweTestFramework_ChokeFilterTest, Short) { // That is actually just a single packet, since each packet has 1000 bits of // payload. ChokeFilter filter(NULL, 0); - filter.SetCapacity(10); + filter.set_capacity_kbps(10); TestChoke(&filter, 100, 100, 1); } TEST_F(BweTestFramework_ChokeFilterTest, Medium) { // 100ms, 10 packets, 10 kbps choke -> 1 packet through, or 1 kbit. ChokeFilter filter(NULL, 0); - filter.SetCapacity(10); + filter.set_capacity_kbps(10); TestChoke(&filter, 100, 10, 1); // 200ms, no new packets -> another packet through. TestChoke(&filter, 100, 0, 1); @@ -719,7 +719,7 @@ TEST_F(BweTestFramework_ChokeFilterTest, Medium) { TEST_F(BweTestFramework_ChokeFilterTest, Long) { // 100ms, 100 packets in queue, 10 kbps choke -> 1 packet through, or 1 kbit. ChokeFilter filter(NULL, 0); - filter.SetCapacity(10); + filter.set_capacity_kbps(10); TestChoke(&filter, 100, 100, 1); // 200ms, no input, another packet through. TestChoke(&filter, 100, 0, 1); @@ -727,22 +727,22 @@ TEST_F(BweTestFramework_ChokeFilterTest, Long) { TestChoke(&filter, 800, 0, 8); // 10000ms, no input, raise choke to 100 kbps. Remaining 90 packets in queue // should be propagated, for a total of 90 kbps. - filter.SetCapacity(100); + filter.set_capacity_kbps(100); TestChoke(&filter, 9000, 0, 90); // 10100ms, 20 more packets -> 10 packets or 10 kbit through. TestChoke(&filter, 100, 20, 10); // 10300ms, 10 more packets -> 20 packets out. TestChoke(&filter, 200, 10, 20); // 11300ms, no input, queue should be empty. - filter.SetCapacity(10); + filter.set_capacity_kbps(10); TestChoke(&filter, 1000, 0, 0); } TEST_F(BweTestFramework_ChokeFilterTest, MaxDelay) { // 10 kbps choke, 500 ms delay cap ChokeFilter filter(NULL, 0); - filter.SetCapacity(10); - filter.SetMaxDelay(500); + filter.set_capacity_kbps(10); + filter.set_max_delay_ms(500); // 100ms, 100 packets in queue, 10 kbps choke -> 1 packet through, or 1 kbit. TestChoke(&filter, 100, 100, 1); CheckMaxDelay(500); @@ -752,7 +752,7 @@ TEST_F(BweTestFramework_ChokeFilterTest, MaxDelay) { TestChoke(&filter, 9500, 0, 0); // 100 ms delay cap - filter.SetMaxDelay(100); + filter.set_max_delay_ms(100); // 10100ms, 50 more packets -> 1 packets or 1 kbit through. TestChoke(&filter, 100, 50, 1); CheckMaxDelay(100); @@ -760,8 +760,8 @@ TEST_F(BweTestFramework_ChokeFilterTest, MaxDelay) { TestChoke(&filter, 9900, 0, 0); // Reset delay cap (0 is no cap) and verify no packets are dropped. - filter.SetCapacity(10); - filter.SetMaxDelay(0); + filter.set_capacity_kbps(10); + filter.set_max_delay_ms(0); TestChoke(&filter, 100, 100, 1); TestChoke(&filter, 9900, 0, 99); } @@ -784,7 +784,7 @@ TEST_F(BweTestFramework_ChokeFilterTest, ShortTraceTwoWraps) { TEST_F(BweTestFramework_ChokeFilterTest, ShortTraceMaxDelay) { TraceBasedDeliveryFilter filter(NULL, 0); - filter.SetMaxDelay(25); + filter.set_max_delay_ms(25); ASSERT_TRUE(filter.Init(test::ResourcePath("synthetic-trace", "rx"))); // Uses all slots up to 110 ms. Several packets are being dropped. TestChoke(&filter, 110, 20, 9); @@ -805,12 +805,14 @@ void TestVideoSender(VideoSender* sender, ASSERT_TRUE(IsTimeSorted(packets)); ASSERT_TRUE(IsSequenceNumberSorted(packets)); EXPECT_EQ(expected_packets, packets.size()); + int64_t send_time_us = -1; size_t total_payload_size = 0; uint32_t absolute_send_time = 0; uint32_t absolute_send_time_wraps = 0; uint32_t rtp_timestamp = 0; uint32_t rtp_timestamp_wraps = 0; + for (const auto* packet : packets) { const MediaPacket* media_packet = static_cast(packet); EXPECT_LE(send_time_us, media_packet->send_time_us()); @@ -830,6 +832,7 @@ void TestVideoSender(VideoSender* sender, } rtp_timestamp = media_packet->header().timestamp; } + EXPECT_EQ(expected_total_payload_size, total_payload_size); EXPECT_GE(1u, absolute_send_time_wraps); EXPECT_GE(1u, rtp_timestamp_wraps); @@ -838,6 +841,8 @@ void TestVideoSender(VideoSender* sender, delete packet; } +// Random {-1, 0, +1} ms was added to frame timestamps. + TEST(BweTestFramework_VideoSenderTest, Fps1Kbps80_1s) { // 1 fps, 80 kbps VideoSource source(0, 1.0f, 80, 0x1234, 0); @@ -846,14 +851,16 @@ TEST(BweTestFramework_VideoSenderTest, Fps1Kbps80_1s) { // We're at 1 fps, so all packets should be generated on first call, giving 10 // packets of each 1000 bytes, total 10000 bytes. TestVideoSender(&sender, 1, 9, 400, 10000); - // 999ms, should see no output here. - TestVideoSender(&sender, 998, 0, 0, 0); - // 1999ms, should get data for one more frame. - TestVideoSender(&sender, 1000, 9, 400, 10000); - // 2000ms, one more frame. - TestVideoSender(&sender, 1, 9, 400, 10000); - // 2999ms, should see nothing. - TestVideoSender(&sender, 999, 0, 0, 0); + // 998ms, should see no output here. + TestVideoSender(&sender, 997, 0, 0, 0); + // 1001ms, should get data for one more frame. + TestVideoSender(&sender, 3, 9, 400, 10000); + // 1998ms, should see no output here. + TestVideoSender(&sender, 997, 0, 0, 0); + // 2001ms, one more frame. + TestVideoSender(&sender, 3, 9, 400, 10000); + // 2998ms, should see nothing. + TestVideoSender(&sender, 997, 0, 0, 0); } TEST(BweTestFramework_VideoSenderTest, Fps1Kbps80_1s_Offset) { @@ -861,20 +868,20 @@ TEST(BweTestFramework_VideoSenderTest, Fps1Kbps80_1s_Offset) { VideoSource source(0, 1.0f, 80, 0x1234, 500); VideoSender sender(NULL, &source, kNullEstimator); EXPECT_EQ(80000u, source.bits_per_second()); - // 499ms, no output. - TestVideoSender(&sender, 499, 0, 0, 0); - // 500ms, first frame (this is the offset we set), 10 packets of 1000 bytes. - TestVideoSender(&sender, 1, 9, 400, 10000); - // 1499ms, nothing. - TestVideoSender(&sender, 999, 0, 0, 0); - // 1999ms, second frame. - TestVideoSender(&sender, 500, 9, 400, 10000); - // 2499ms, nothing. - TestVideoSender(&sender, 500, 0, 0, 0); - // 2500ms, third frame. - TestVideoSender(&sender, 1, 9, 400, 10000); - // 3499ms, nothing. - TestVideoSender(&sender, 999, 0, 0, 0); + // 498ms, no output. + TestVideoSender(&sender, 498, 0, 0, 0); + // 501ms, first frame (this is the offset we set), 10 packets of 1000 bytes. + TestVideoSender(&sender, 3, 9, 400, 10000); + // 1498ms, nothing. + TestVideoSender(&sender, 997, 0, 0, 0); + // 1501ms, second frame. + TestVideoSender(&sender, 3, 9, 400, 10000); + // 2498ms, nothing. + TestVideoSender(&sender, 997, 0, 0, 0); + // 2501ms, third frame. + TestVideoSender(&sender, 3, 9, 400, 10000); + // 3498ms, nothing. + TestVideoSender(&sender, 997, 0, 0, 0); } TEST(BweTestFramework_VideoSenderTest, Fps50Kpbs80_11s) { @@ -882,55 +889,55 @@ TEST(BweTestFramework_VideoSenderTest, Fps50Kpbs80_11s) { VideoSource source(0, 50.0f, 80, 0x1234, 0); VideoSender sender(NULL, &source, kNullEstimator); EXPECT_EQ(80000u, source.bits_per_second()); - // 9998ms, should see 500 frames, 200 byte payloads, total 100000 bytes. - TestVideoSender(&sender, 9998, 500, 200, 100000); - // 9999ms, nothing. - TestVideoSender(&sender, 1, 0, 0, 0); - // 10000ms, 501st frame as a single packet. - TestVideoSender(&sender, 1, 1, 200, 200); - // 10998ms, 49 more frames. - TestVideoSender(&sender, 998, 49, 200, 9800); - // 10999ms, nothing. - TestVideoSender(&sender, 1, 0, 0, 0); + // 9981, should see 500 frames, 200 byte payloads, total 100000 bytes. + TestVideoSender(&sender, 9981, 500, 200, 100000); + // 9998ms, nothing. + TestVideoSender(&sender, 17, 0, 0, 0); + // 10001ms, 501st frame as a single packet. + TestVideoSender(&sender, 3, 1, 200, 200); + // 10981ms, 49 more frames. + TestVideoSender(&sender, 981, 49, 200, 9800); + // 10998ms, nothing. + TestVideoSender(&sender, 17, 0, 0, 0); } -TEST(BweTestFramework_VideoSenderTest, Fps10Kpbs120_1s) { +TEST(BweTestFramework_VideoSenderTest, Fps20Kpbs120_1s) { // 20 fps, 120 kbps. VideoSource source(0, 20.0f, 120, 0x1234, 0); VideoSender sender(NULL, &source, kNullEstimator); EXPECT_EQ(120000u, source.bits_per_second()); - // 498ms, 10 frames with 750 byte payloads, total 7500 bytes. - TestVideoSender(&sender, 498, 10, 750, 7500); - // 499ms, nothing. - TestVideoSender(&sender, 1, 0, 0, 0); - // 500ms, one more frame. - TestVideoSender(&sender, 1, 1, 750, 750); - // 998ms, 9 more frames. - TestVideoSender(&sender, 498, 9, 750, 6750); - // 999ms, nothing. - TestVideoSender(&sender, 1, 0, 0, 0); + // 451ms, 10 frames with 750 byte payloads, total 7500 bytes. + TestVideoSender(&sender, 451, 10, 750, 7500); + // 498ms, nothing. + TestVideoSender(&sender, 47, 0, 0, 0); + // 501ms, one more frame. + TestVideoSender(&sender, 3, 1, 750, 750); + // 951ms, 9 more frames. + TestVideoSender(&sender, 450, 9, 750, 6750); + // 998ms, nothing. + TestVideoSender(&sender, 47, 0, 0, 0); } -TEST(BweTestFramework_VideoSenderTest, Fps30Kbps800_20s) { - // 20 fps, 820 kbps. +TEST(BweTestFramework_VideoSenderTest, Fps25Kbps820_20s) { + // 25 fps, 820 kbps. VideoSource source(0, 25.0f, 820, 0x1234, 0); VideoSender sender(NULL, &source, kNullEstimator); EXPECT_EQ(820000u, source.bits_per_second()); - // 9998ms, 250 frames. 820 kbps = 102500 bytes/s, so total should be 1025000. + // 9961ms, 250 frames. 820 kbps = 102500 bytes/s, so total should be 1025000. // Each frame is 102500/25=4100 bytes, or 5 packets (4 @1000 bytes, 1 @100), // so packet count should be 5*250=1250 and last packet of each frame has // 100 bytes of payload. - TestVideoSender(&sender, 9998, 1000, 500, 1025000); - // 9999ms, nothing. - TestVideoSender(&sender, 1, 0, 0, 0); - // 19998ms, 250 more frames. - TestVideoSender(&sender, 9999, 1000, 500, 1025000); - // 19999ms, nothing. - TestVideoSender(&sender, 1, 0, 0, 0); - // 20038ms, one more frame, as described above (25fps == 40ms/frame). - TestVideoSender(&sender, 39, 4, 500, 4100); - // 20039ms, nothing. - TestVideoSender(&sender, 1, 0, 0, 0); + TestVideoSender(&sender, 9961, 1000, 500, 1025000); + // 9998ms, nothing. + TestVideoSender(&sender, 37, 0, 0, 0); + // 19961ms, 250 more frames. + TestVideoSender(&sender, 9963, 1000, 500, 1025000); + // 19998ms, nothing. + TestVideoSender(&sender, 37, 0, 0, 0); + // 20001ms, one more frame, as described above (25fps == 40ms/frame). + TestVideoSender(&sender, 3, 4, 500, 4100); + // 20038ms, nothing. + TestVideoSender(&sender, 37, 0, 0, 0); } TEST(BweTestFramework_VideoSenderTest, TestAppendInOrder) { @@ -973,7 +980,7 @@ TEST(BweTestFramework_VideoSenderTest, FeedbackIneffective) { VideoSender sender(NULL, &source, kNullEstimator); EXPECT_EQ(820000u, source.bits_per_second()); - TestVideoSender(&sender, 9998, 1000, 500, 1025000); + TestVideoSender(&sender, 9961, 1000, 500, 1025000); // Make sure feedback has no effect on a regular video sender. RembFeedback* feedback = new RembFeedback(0, 0, 0, 512000, RTCPReportBlock()); @@ -981,14 +988,14 @@ TEST(BweTestFramework_VideoSenderTest, FeedbackIneffective) { packets.push_back(feedback); sender.RunFor(0, &packets); EXPECT_EQ(820000u, source.bits_per_second()); - TestVideoSender(&sender, 9998, 1000, 500, 1025000); + TestVideoSender(&sender, 10000, 1000, 500, 1025000); } TEST(BweTestFramework_AdaptiveVideoSenderTest, FeedbackChangesBitrate) { AdaptiveVideoSource source(0, 25.0f, 820, 0x1234, 0); VideoSender sender(NULL, &source, kRembEstimator); EXPECT_EQ(820000u, source.bits_per_second()); - TestVideoSender(&sender, 9998, 1000, 500, 1025000); + TestVideoSender(&sender, 9961, 1000, 500, 1025000); // Make sure we can reduce the bitrate. RembFeedback* feedback = new RembFeedback(0, 0, 0, 512000, RTCPReportBlock()); @@ -996,7 +1003,7 @@ TEST(BweTestFramework_AdaptiveVideoSenderTest, FeedbackChangesBitrate) { packets.push_back(feedback); sender.RunFor(0, &packets); EXPECT_EQ(512000u, source.bits_per_second()); - TestVideoSender(&sender, 9998, 750, 160, 640000); + TestVideoSender(&sender, 10000, 750, 160, 640000); // Increase the bitrate to the initial bitrate and verify that the output is // the same. @@ -1021,7 +1028,7 @@ TEST(BweTestFramework_AdaptiveVideoSenderTest, Paced_FeedbackChangesBitrate) { packets.push_back(feedback); sender.RunFor(10000, &packets); ASSERT_EQ(512000u, source.bits_per_second()); - TestVideoSender(&sender, 9998, 750, 160, 640000); + TestVideoSender(&sender, 10000, 750, 160, 640000); // Increase the bitrate to the initial bitrate and verify that the output is // the same. diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.cc index de13023b26..bcf7e13378 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.cc @@ -91,13 +91,103 @@ void Logging::Log(const char format[], ...) { } void Logging::Plot(int figure, double value) { + Plot(figure, value, ""); +} + +void Logging::Plot(int figure, double value, const std::string& alg_name) { + CriticalSectionScoped cs(crit_sect_.get()); + ThreadMap::iterator it = thread_map_.find(ThreadWrapper::GetThreadId()); + assert(it != thread_map_.end()); + const State& state = it->second.stack.top(); + std::string label = state.tag + '@' + alg_name; + std::string prefix("Available"); + if (alg_name.compare(0, prefix.length(), prefix) == 0) { + std::string receiver("Receiver"); + size_t start_pos = label.find(receiver); + if (start_pos != std::string::npos) { + label.replace(start_pos, receiver.length(), "Sender"); + } + } + if (state.enabled) { + printf("PLOT\t%d\t%s\t%f\t%f\n", figure, label.c_str(), + state.timestamp_ms * 0.001, value); + } +} + +void Logging::PlotBar(int figure, + const std::string& name, + double value, + int flow_id) { CriticalSectionScoped cs(crit_sect_.get()); ThreadMap::iterator it = thread_map_.find(ThreadWrapper::GetThreadId()); assert(it != thread_map_.end()); const State& state = it->second.stack.top(); if (state.enabled) { - printf("PLOT\t%d\t%s\t%f\t%f\n", figure, state.tag.c_str(), - state.timestamp_ms * 0.001, value); + printf("BAR\t%d\t%s_%d\t%f\n", figure, name.c_str(), flow_id, value); + } +} + +void Logging::PlotBaselineBar(int figure, + const std::string& name, + double value, + int flow_id) { + CriticalSectionScoped cs(crit_sect_.get()); + ThreadMap::iterator it = thread_map_.find(ThreadWrapper::GetThreadId()); + assert(it != thread_map_.end()); + const State& state = it->second.stack.top(); + if (state.enabled) { + printf("BASELINE\t%d\t%s_%d\t%f\n", figure, name.c_str(), flow_id, value); + } +} + +void Logging::PlotErrorBar(int figure, + const std::string& name, + double value, + double ylow, + double yhigh, + const std::string& error_title, + int flow_id) { + CriticalSectionScoped cs(crit_sect_.get()); + ThreadMap::iterator it = thread_map_.find(ThreadWrapper::GetThreadId()); + assert(it != thread_map_.end()); + const State& state = it->second.stack.top(); + if (state.enabled) { + printf("ERRORBAR\t%d\t%s_%d\t%f\t%f\t%f\t%s\n", figure, name.c_str(), + flow_id, value, ylow, yhigh, error_title.c_str()); + } +} + +void Logging::PlotLimitErrorBar(int figure, + const std::string& name, + double value, + double ylow, + double yhigh, + const std::string& error_title, + double ymax, + const std::string& limit_title, + int flow_id) { + CriticalSectionScoped cs(crit_sect_.get()); + ThreadMap::iterator it = thread_map_.find(ThreadWrapper::GetThreadId()); + assert(it != thread_map_.end()); + const State& state = it->second.stack.top(); + if (state.enabled) { + printf("LIMITERRORBAR\t%d\t%s_%d\t%f\t%f\t%f\t%s\t%f\t%s\n", figure, + name.c_str(), flow_id, value, ylow, yhigh, error_title.c_str(), ymax, + limit_title.c_str()); + } +} + +void Logging::PlotLabel(int figure, + const std::string& title, + const std::string& y_label, + int num_flows) { + CriticalSectionScoped cs(crit_sect_.get()); + ThreadMap::iterator it = thread_map_.find(ThreadWrapper::GetThreadId()); + assert(it != thread_map_.end()); + const State& state = it->second.stack.top(); + if (state.enabled) { + printf("LABEL\t%d\t%s\t%s\t%d\n", figure, title.c_str(), y_label.c_str(), + num_flows); } } diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h index c214d70c27..23fce00d8c 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h @@ -91,7 +91,31 @@ // |name| is a char*, std::string or uint32_t to name the plotted value. // |time| is an int64_t time in ms, or -1 to inherit time from previous context. // |value| is a double precision float to be plotted. +// |alg_name| is an optional argument, a string #define BWE_TEST_LOGGING_PLOT(figure, name, time, value) +#define BWE_TEST_LOGGING_PLOT_WITH_NAME(figure, name, time, value, alg_name) + +// Print to stdout in tab-separated format suitable for plotting, e.g.: +// BAR figure Context1_Context2_Name x_left width value +// |figure| is a figure id. Different figures are plotted in different windows. +// |name| is a char*, std::string or uint32_t to name the plotted value. +// |value| is a double precision float to be plotted. +// |ylow| and |yhigh| are double precision float for the error line. +// |title| is a string and refers to the error label. +// |ymax| is a double precision float for the limit horizontal line. +// |limit_title| is a string and refers to the limit label. +#define BWE_TEST_LOGGING_BAR(figure, name, value, flow_id) +#define BWE_TEST_LOGGING_ERRORBAR(figure, name, value, ylow, yhigh, \ + error_title, flow_id) +#define BWE_TEST_LOGGING_LIMITERRORBAR( \ + figure, name, value, ylow, yhigh, error_title, ymax, limit_title, flow_id) + +#define BWE_TEST_LOGGING_BASELINEBAR(figure, name, value, flow_id) + +// |num_flows| is an integer refering to the number of RMCAT flows in the +// scenario. +// Define |x_label| and |y_label| for plots. +#define BWE_TEST_LOGGING_LABEL(figure, x_label, y_label, num_flows) #else // BWE_TEST_LOGGING_COMPILE_TIME_ENABLE @@ -157,11 +181,57 @@ #define BWE_TEST_LOGGING_PLOT(figure, name, time, value) \ do { \ - __BWE_TEST_LOGGING_CONTEXT_DECLARE(__bwe_log_, __LINE__, name, \ + __BWE_TEST_LOGGING_CONTEXT_DECLARE(__bwe_log_, __PLOT__, name, \ static_cast(time), true); \ webrtc::testing::bwe::Logging::GetInstance()->Plot(figure, value); \ } while (0); +#define BWE_TEST_LOGGING_PLOT_WITH_NAME(figure, name, time, value, alg_name) \ + do { \ + __BWE_TEST_LOGGING_CONTEXT_DECLARE(__bwe_log_, __PLOT__, name, \ + static_cast(time), true); \ + webrtc::testing::bwe::Logging::GetInstance()->Plot(figure, value, \ + alg_name); \ + } while (0); + +#define BWE_TEST_LOGGING_BAR(figure, name, value, flow_id) \ + do { \ + BWE_TEST_LOGGING_CONTEXT(name); \ + webrtc::testing::bwe::Logging::GetInstance()->PlotBar(figure, name, value, \ + flow_id); \ + } while (0); + +#define BWE_TEST_LOGGING_BASELINEBAR(figure, name, value, flow_id) \ + do { \ + BWE_TEST_LOGGING_CONTEXT(name); \ + webrtc::testing::bwe::Logging::GetInstance()->PlotBaselineBar( \ + figure, name, value, flow_id); \ + } while (0); + +#define BWE_TEST_LOGGING_ERRORBAR(figure, name, value, ylow, yhigh, title, \ + flow_id) \ + do { \ + BWE_TEST_LOGGING_CONTEXT(name); \ + webrtc::testing::bwe::Logging::GetInstance()->PlotErrorBar( \ + figure, name, value, ylow, yhigh, title, flow_id); \ + } while (0); + +#define BWE_TEST_LOGGING_LIMITERRORBAR( \ + figure, name, value, ylow, yhigh, error_title, ymax, limit_title, flow_id) \ + do { \ + BWE_TEST_LOGGING_CONTEXT(name); \ + webrtc::testing::bwe::Logging::GetInstance()->PlotLimitErrorBar( \ + figure, name, value, ylow, yhigh, error_title, ymax, limit_title, \ + flow_id); \ + } while (0); + +#define BWE_TEST_LOGGING_LABEL(figure, title, y_label, num_flows) \ + do { \ + BWE_TEST_LOGGING_CONTEXT(title); \ + webrtc::testing::bwe::Logging::GetInstance()->PlotLabel( \ + figure, title, y_label, num_flows); \ + } while (0); + namespace webrtc { class CriticalSectionWrapper; @@ -190,6 +260,33 @@ class Logging { void Log(const char format[], ...); void Plot(int figure, double value); + void Plot(int figure, double value, const std::string& alg_name); + void PlotBar(int figure, const std::string& name, double value, int flow_id); + void PlotBaselineBar(int figure, + const std::string& name, + double value, + int flow_id); + void PlotErrorBar(int figure, + const std::string& name, + double value, + double ylow, + double yhigh, + const std::string& error_title, + int flow_id); + + void PlotLimitErrorBar(int figure, + const std::string& name, + double value, + double ylow, + double yhigh, + const std::string& error_title, + double ymax, + const std::string& limit_title, + int flow_id); + void PlotLabel(int figure, + const std::string& title, + const std::string& y_label, + int num_flows); private: struct State { diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_unittest.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_unittest.cc new file mode 100644 index 0000000000..6b3ce4847c --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_unittest.cc @@ -0,0 +1,393 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/remote_bitrate_estimator/test/bwe.h" + +#include + +#include "testing/gtest/include/gtest/gtest.h" + +namespace webrtc { +namespace testing { +namespace bwe { + +const int kSetCapacity = 1000; + +class LinkedSetTest : public ::testing::Test { + public: + LinkedSetTest() : linked_set_(kSetCapacity) {} + + ~LinkedSetTest() {} + + protected: + LinkedSet linked_set_; +}; + +TEST_F(LinkedSetTest, EmptySet) { + EXPECT_EQ(linked_set_.OldestSeqNumber(), 0); + EXPECT_EQ(linked_set_.NewestSeqNumber(), 0); +} + +TEST_F(LinkedSetTest, SinglePacket) { + const uint16_t kSeqNumber = 1; // Arbitrary. + // Other parameters don't matter here. + linked_set_.Insert(kSeqNumber, 0, 0, 0); + + EXPECT_EQ(linked_set_.OldestSeqNumber(), kSeqNumber); + EXPECT_EQ(linked_set_.NewestSeqNumber(), kSeqNumber); +} + +TEST_F(LinkedSetTest, MultiplePackets) { + const uint16_t kNumberPackets = 100; + + std::vector sequence_numbers; + for (size_t i = 0; i < kNumberPackets; ++i) { + sequence_numbers.push_back(static_cast(i + 1)); + } + random_shuffle(sequence_numbers.begin(), sequence_numbers.end()); + + for (size_t i = 0; i < kNumberPackets; ++i) { + // Other parameters don't matter here. + linked_set_.Insert(static_cast(i), 0, 0, 0); + } + + // Packets arriving out of order should not affect the following values: + EXPECT_EQ(linked_set_.OldestSeqNumber(), 0); + EXPECT_EQ(linked_set_.NewestSeqNumber(), kNumberPackets - 1); +} + +TEST_F(LinkedSetTest, Overflow) { + const int kFirstSeqNumber = -100; + const int kLastSeqNumber = 100; + + for (int i = kFirstSeqNumber; i <= kLastSeqNumber; ++i) { + // Other parameters don't matter here. + linked_set_.Insert(static_cast(i), 0, 0, 0); + } + + // Packets arriving out of order should not affect the following values: + EXPECT_EQ(linked_set_.OldestSeqNumber(), + static_cast(kFirstSeqNumber)); + EXPECT_EQ(linked_set_.NewestSeqNumber(), + static_cast(kLastSeqNumber)); +} + +class SequenceNumberOlderThanTest : public ::testing::Test { + public: + SequenceNumberOlderThanTest() {} + ~SequenceNumberOlderThanTest() {} + + protected: + SequenceNumberOlderThan comparator_; +}; + +TEST_F(SequenceNumberOlderThanTest, Operator) { + // Operator()(x, y) returns true <==> y is newer than x. + EXPECT_TRUE(comparator_.operator()(0x0000, 0x0001)); + EXPECT_TRUE(comparator_.operator()(0x0001, 0x1000)); + EXPECT_FALSE(comparator_.operator()(0x0001, 0x0000)); + EXPECT_FALSE(comparator_.operator()(0x0002, 0x0002)); + EXPECT_TRUE(comparator_.operator()(0xFFF6, 0x000A)); + EXPECT_FALSE(comparator_.operator()(0x000A, 0xFFF6)); + EXPECT_TRUE(comparator_.operator()(0x0000, 0x8000)); + EXPECT_FALSE(comparator_.operator()(0x8000, 0x0000)); +} + +class LossAccountTest : public ::testing::Test { + public: + LossAccountTest() {} + ~LossAccountTest() {} + + protected: + LossAccount loss_account_; +}; + +TEST_F(LossAccountTest, Operations) { + const size_t kTotal = 100; // Arbitrary values. + const size_t kLost = 10; + + LossAccount rhs(kTotal, kLost); + + loss_account_.Add(rhs); + EXPECT_EQ(loss_account_.num_total, kTotal); + EXPECT_EQ(loss_account_.num_lost, kLost); + EXPECT_NEAR(loss_account_.LossRatio(), static_cast(kLost) / kTotal, + 0.001f); + + loss_account_.Subtract(rhs); + EXPECT_EQ(loss_account_.num_total, 0UL); + EXPECT_EQ(loss_account_.num_lost, 0UL); + EXPECT_NEAR(loss_account_.LossRatio(), 0.0f, 0.001f); +} + +class BweReceiverTest : public ::testing::Test { + public: + BweReceiverTest() : bwe_receiver_(kFlowId) {} + ~BweReceiverTest() {} + + protected: + const int kFlowId = 1; // Arbitrary. + BweReceiver bwe_receiver_; +}; + +TEST_F(BweReceiverTest, ReceivingRateNoPackets) { + EXPECT_EQ(bwe_receiver_.RecentKbps(), static_cast(0)); +} + +TEST_F(BweReceiverTest, ReceivingRateSinglePacket) { + const size_t kPayloadSizeBytes = 500 * 1000; + const int64_t kSendTimeUs = 300 * 1000; + const int64_t kArrivalTimeMs = kSendTimeUs / 1000 + 100; + const uint16_t kSequenceNumber = 1; + const int64_t kTimeWindowMs = BweReceiver::kReceivingRateTimeWindowMs; + + const MediaPacket media_packet(kFlowId, kSendTimeUs, kPayloadSizeBytes, + kSequenceNumber); + bwe_receiver_.ReceivePacket(kArrivalTimeMs, media_packet); + + const size_t kReceivingRateKbps = 8 * kPayloadSizeBytes / kTimeWindowMs; + + EXPECT_NEAR(bwe_receiver_.RecentKbps(), kReceivingRateKbps, + static_cast(kReceivingRateKbps) / 100.0f); +} + +TEST_F(BweReceiverTest, ReceivingRateSmallPackets) { + const size_t kPayloadSizeBytes = 100 * 1000; + const int64_t kTimeGapMs = 50; // Between each packet. + const int64_t kOneWayDelayMs = 50; + + for (int i = 1; i < 50; ++i) { + int64_t send_time_us = i * kTimeGapMs * 1000; + int64_t arrival_time_ms = send_time_us / 1000 + kOneWayDelayMs; + uint16_t sequence_number = i; + const MediaPacket media_packet(kFlowId, send_time_us, kPayloadSizeBytes, + sequence_number); + bwe_receiver_.ReceivePacket(arrival_time_ms, media_packet); + } + + const size_t kReceivingRateKbps = 8 * kPayloadSizeBytes / kTimeGapMs; + EXPECT_NEAR(bwe_receiver_.RecentKbps(), kReceivingRateKbps, + static_cast(kReceivingRateKbps) / 100.0f); +} + +TEST_F(BweReceiverTest, PacketLossNoPackets) { + EXPECT_EQ(bwe_receiver_.RecentPacketLossRatio(), 0.0f); +} + +TEST_F(BweReceiverTest, PacketLossSinglePacket) { + const MediaPacket media_packet(kFlowId, 0, 0, 0); + bwe_receiver_.ReceivePacket(0, media_packet); + EXPECT_EQ(bwe_receiver_.RecentPacketLossRatio(), 0.0f); +} + +TEST_F(BweReceiverTest, PacketLossContiguousPackets) { + const int64_t kTimeWindowMs = BweReceiver::kPacketLossTimeWindowMs; + size_t set_capacity = bwe_receiver_.GetSetCapacity(); + + for (int i = 0; i < 10; ++i) { + uint16_t sequence_number = static_cast(i); + // Sequence_number and flow_id are the only members that matter here. + const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); + // Arrival time = 0, all packets will be considered. + bwe_receiver_.ReceivePacket(0, media_packet); + } + EXPECT_EQ(bwe_receiver_.RecentPacketLossRatio(), 0.0f); + + for (int i = 30; i > 20; i--) { + uint16_t sequence_number = static_cast(i); + // Sequence_number and flow_id are the only members that matter here. + const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); + // Only the packets sent in this for loop will be considered. + bwe_receiver_.ReceivePacket(2 * kTimeWindowMs, media_packet); + } + EXPECT_EQ(bwe_receiver_.RecentPacketLossRatio(), 0.0f); + + // Should handle uint16_t overflow. + for (int i = 0xFFFF - 10; i < 0xFFFF + 10; ++i) { + uint16_t sequence_number = static_cast(i); + const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); + // Only the packets sent in this for loop will be considered. + bwe_receiver_.ReceivePacket(4 * kTimeWindowMs, media_packet); + } + EXPECT_EQ(bwe_receiver_.RecentPacketLossRatio(), 0.0f); + + // Should handle set overflow. + for (int i = 0; i < set_capacity * 1.5; ++i) { + uint16_t sequence_number = static_cast(i); + const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); + // Only the packets sent in this for loop will be considered. + bwe_receiver_.ReceivePacket(6 * kTimeWindowMs, media_packet); + } + EXPECT_EQ(bwe_receiver_.RecentPacketLossRatio(), 0.0f); +} + +// Should handle duplicates. +TEST_F(BweReceiverTest, PacketLossDuplicatedPackets) { + const int64_t kTimeWindowMs = BweReceiver::kPacketLossTimeWindowMs; + + for (int i = 0; i < 10; ++i) { + const MediaPacket media_packet(kFlowId, 0, 0, 0); + // Arrival time = 0, all packets will be considered. + bwe_receiver_.ReceivePacket(0, media_packet); + } + EXPECT_EQ(bwe_receiver_.RecentPacketLossRatio(), 0.0f); + + // Missing the element 5. + const uint16_t kSequenceNumbers[] = {1, 2, 3, 4, 6, 7, 8}; + const int kNumPackets = ARRAY_SIZE(kSequenceNumbers); + + // Insert each sequence number twice. + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < kNumPackets; j++) { + const MediaPacket media_packet(kFlowId, 0, 0, kSequenceNumbers[j]); + // Only the packets sent in this for loop will be considered. + bwe_receiver_.ReceivePacket(2 * kTimeWindowMs, media_packet); + } + } + + EXPECT_NEAR(bwe_receiver_.RecentPacketLossRatio(), 1.0f / (kNumPackets + 1), + 0.1f / (kNumPackets + 1)); +} + +TEST_F(BweReceiverTest, PacketLossLakingPackets) { + size_t set_capacity = bwe_receiver_.GetSetCapacity(); + EXPECT_LT(set_capacity, static_cast(0xFFFF)); + + // Missing every other packet. + for (size_t i = 0; i < set_capacity; ++i) { + if ((i & 1) == 0) { // Only even sequence numbers. + uint16_t sequence_number = static_cast(i); + const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); + // Arrival time = 0, all packets will be considered. + bwe_receiver_.ReceivePacket(0, media_packet); + } + } + EXPECT_NEAR(bwe_receiver_.RecentPacketLossRatio(), 0.5f, 0.01f); +} + +TEST_F(BweReceiverTest, PacketLossLakingFewPackets) { + size_t set_capacity = bwe_receiver_.GetSetCapacity(); + EXPECT_LT(set_capacity, static_cast(0xFFFF)); + + const int kPeriod = 100; + // Missing one for each kPeriod packets. + for (size_t i = 0; i < set_capacity; ++i) { + if ((i % kPeriod) != 0) { + uint16_t sequence_number = static_cast(i); + const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); + // Arrival time = 0, all packets will be considered. + bwe_receiver_.ReceivePacket(0, media_packet); + } + } + EXPECT_NEAR(bwe_receiver_.RecentPacketLossRatio(), 1.0f / kPeriod, + 0.1f / kPeriod); +} + +// Packet's sequence numbers greatly apart, expect high loss. +TEST_F(BweReceiverTest, PacketLossWideGap) { + const int64_t kTimeWindowMs = BweReceiver::kPacketLossTimeWindowMs; + + const MediaPacket media_packet1(0, 0, 0, 1); + const MediaPacket media_packet2(0, 0, 0, 1000); + // Only these two packets will be considered. + bwe_receiver_.ReceivePacket(0, media_packet1); + bwe_receiver_.ReceivePacket(0, media_packet2); + EXPECT_NEAR(bwe_receiver_.RecentPacketLossRatio(), 0.998f, 0.0001f); + + const MediaPacket media_packet3(0, 0, 0, 0); + const MediaPacket media_packet4(0, 0, 0, 0x8000); + // Only these two packets will be considered. + bwe_receiver_.ReceivePacket(2 * kTimeWindowMs, media_packet3); + bwe_receiver_.ReceivePacket(2 * kTimeWindowMs, media_packet4); + EXPECT_NEAR(bwe_receiver_.RecentPacketLossRatio(), 0.99994f, 0.00001f); +} + +// Packets arriving unordered should not be counted as losted. +TEST_F(BweReceiverTest, PacketLossUnorderedPackets) { + size_t num_packets = bwe_receiver_.GetSetCapacity() / 2; + std::vector sequence_numbers; + + for (size_t i = 0; i < num_packets; ++i) { + sequence_numbers.push_back(static_cast(i + 1)); + } + + random_shuffle(sequence_numbers.begin(), sequence_numbers.end()); + + for (size_t i = 0; i < num_packets; ++i) { + const MediaPacket media_packet(kFlowId, 0, 0, sequence_numbers[i]); + // Arrival time = 0, all packets will be considered. + bwe_receiver_.ReceivePacket(0, media_packet); + } + + EXPECT_EQ(bwe_receiver_.RecentPacketLossRatio(), 0.0f); +} + +TEST_F(BweReceiverTest, RecentKbps) { + EXPECT_EQ(bwe_receiver_.RecentKbps(), 0U); + + const size_t kPacketSizeBytes = 1200; + const int kNumPackets = 100; + + double window_size_s = bwe_receiver_.BitrateWindowS(); + + // Receive packets at the same time. + for (int i = 0; i < kNumPackets; ++i) { + MediaPacket packet(kFlowId, 0L, kPacketSizeBytes, static_cast(i)); + bwe_receiver_.ReceivePacket(0, packet); + } + + EXPECT_NEAR(bwe_receiver_.RecentKbps(), + (8 * kNumPackets * kPacketSizeBytes) / (1000 * window_size_s), + 10); + + int64_t time_gap_ms = + 2 * 1000 * window_size_s; // Larger than rate_counter time window. + + MediaPacket packet(kFlowId, time_gap_ms * 1000, kPacketSizeBytes, + static_cast(kNumPackets)); + bwe_receiver_.ReceivePacket(time_gap_ms, packet); + + EXPECT_NEAR(bwe_receiver_.RecentKbps(), + (8 * kPacketSizeBytes) / (1000 * window_size_s), 10); +} + +TEST_F(BweReceiverTest, Loss) { + EXPECT_NEAR(bwe_receiver_.GlobalReceiverPacketLossRatio(), 0.0f, 0.001f); + + LossAccount loss_account = bwe_receiver_.LinkedSetPacketLossRatio(); + EXPECT_NEAR(loss_account.LossRatio(), 0.0f, 0.001f); + + // Insert packets 1-50 and 151-200; + for (int i = 1; i <= 200; ++i) { + // Packet size and timestamp do not matter here. + MediaPacket packet(kFlowId, 0L, 0UL, static_cast(i)); + bwe_receiver_.ReceivePacket(0, packet); + if (i == 50) { + i += 100; + } + } + + loss_account = bwe_receiver_.LinkedSetPacketLossRatio(); + EXPECT_NEAR(loss_account.LossRatio(), 0.5f, 0.001f); + + bwe_receiver_.RelieveSetAndUpdateLoss(); + EXPECT_EQ(bwe_receiver_.received_packets_.size(), 100U / 10); + + // No packet loss within the preserved packets. + loss_account = bwe_receiver_.LinkedSetPacketLossRatio(); + EXPECT_NEAR(loss_account.LossRatio(), 0.0f, 0.001f); + + // RelieveSetAndUpdateLoss automatically updates loss account. + EXPECT_NEAR(bwe_receiver_.GlobalReceiverPacketLossRatio(), 0.5f, 0.001f); +} + +} // namespace bwe +} // namespace testing +} // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.cc b/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.cc index ccbc522b7e..2500679343 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include "webrtc/base/common.h" #include "webrtc/modules/remote_bitrate_estimator/test/estimators/nada.h" @@ -28,16 +27,14 @@ namespace webrtc { namespace testing { namespace bwe { -const int NadaBweSender::kMinRefRateKbps = 150; -const int NadaBweSender::kMaxRefRateKbps = 1500; const int64_t NadaBweReceiver::kReceivingRateTimeWindowMs = 500; NadaBweReceiver::NadaBweReceiver(int flow_id) - : BweReceiver(flow_id), + : BweReceiver(flow_id, kReceivingRateTimeWindowMs), clock_(0), last_feedback_ms_(0), recv_stats_(ReceiveStatistics::Create(&clock_)), - baseline_delay_ms_(0), + baseline_delay_ms_(10000), // Initialized as an upper bound. delay_signal_ms_(0), last_congestion_signal_ms_(0), last_delays_index_(0), @@ -57,16 +54,19 @@ void NadaBweReceiver::ReceivePacket(int64_t arrival_time_ms, clock_.AdvanceTimeMilliseconds(arrival_time_ms - clock_.TimeInMilliseconds()); recv_stats_->IncomingPacket(media_packet.header(), media_packet.payload_size(), false); - int64_t delay_ms = arrival_time_ms - - media_packet.creation_time_us() / 1000; // Refered as x_n. + // Refered as x_n. + int64_t delay_ms = arrival_time_ms - media_packet.sender_timestamp_ms(); + // The min should be updated within the first 10 minutes. if (clock_.TimeInMilliseconds() < 10 * 60 * 1000) { baseline_delay_ms_ = std::min(baseline_delay_ms_, delay_ms); } + delay_signal_ms_ = delay_ms - baseline_delay_ms_; // Refered as d_n. const int kMedian = ARRAY_SIZE(last_delays_ms_); last_delays_ms_[(last_delays_index_++) % kMedian] = delay_signal_ms_; int size = std::min(last_delays_index_, kMedian); + int64_t median_filtered_delay_ms_ = MedianFilter(last_delays_ms_, size); exp_smoothed_delay_ms_ = ExponentialSmoothingFilter( median_filtered_delay_ms_, exp_smoothed_delay_ms_, kAlpha); @@ -84,9 +84,8 @@ void NadaBweReceiver::ReceivePacket(int64_t arrival_time_ms, est_queuing_delay_signal_ms_ = 0; } - received_packets_.Insert(media_packet.sequence_number(), - media_packet.send_time_ms(), arrival_time_ms, - media_packet.payload_size()); + // Log received packet information. + BweReceiver::ReceivePacket(arrival_time_ms, media_packet); } FeedbackPacket* NadaBweReceiver::GetFeedback(int64_t now_ms) { @@ -110,67 +109,19 @@ FeedbackPacket* NadaBweReceiver::GetFeedback(int64_t now_ms) { last_feedback_ms_ = now_ms; last_congestion_signal_ms_ = congestion_signal_ms; - PacketIdentifierNode* latest = *(received_packets_.begin()); - int64_t corrected_send_time_ms = - latest->send_time_ms + now_ms - latest->arrival_time_ms; + int64_t corrected_send_time_ms = 0L; + + if (!received_packets_.empty()) { + PacketIdentifierNode* latest = *(received_packets_.begin()); + corrected_send_time_ms = + latest->send_time_ms + now_ms - latest->arrival_time_ms; + } // Sends a tuple containing latest values of and additional information. - return new NadaFeedback(flow_id_, now_ms, exp_smoothed_delay_ms_, + return new NadaFeedback(flow_id_, now_ms * 1000, exp_smoothed_delay_ms_, est_queuing_delay_signal_ms_, congestion_signal_ms, - derivative, RecentReceivingRate(), - corrected_send_time_ms); -} - -// For a given time window, compute the receiving speed rate in kbps. -// As described below, three cases are considered depending on the number of -// packets received. -size_t NadaBweReceiver::RecentReceivingRate() { - // If the receiver didn't receive any packet, return 0. - if (received_packets_.empty()) { - return 0.0f; - } - size_t total_size = 0; - int number_packets = 0; - - PacketNodeIt node_it = received_packets_.begin(); - - int64_t last_time_ms = (*node_it)->arrival_time_ms; - int64_t start_time_ms = last_time_ms; - PacketNodeIt end = received_packets_.end(); - - // Stops after including the first packet out of the timeWindow. - // Ameliorates results when there are wide gaps between packets. - // E.g. Large packets : p1(0ms), p2(3000ms). - while (node_it != end) { - total_size += (*node_it)->payload_size; - last_time_ms = (*node_it)->arrival_time_ms; - ++number_packets; - if ((*node_it)->arrival_time_ms < - start_time_ms - kReceivingRateTimeWindowMs) { - break; - } - ++node_it; - } - - int64_t corrected_time_ms; - // If the receiver received a single packet, return its size*8/timeWindow. - if (number_packets == 1) { - corrected_time_ms = kReceivingRateTimeWindowMs; - } - // If the receiver received multiple packets, use as time interval the gap - // between first and last packet falling in the timeWindow corrected by the - // factor number_packets/(number_packets-1). - // E.g: Let timeWindow = 500ms, payload_size = 500 bytes, number_packets = 2, - // packets received at t1(0ms) and t2(499 or 501ms). This prevent the function - // from returning ~2*8, sending instead a more likely ~1*8 kbps. - else { - corrected_time_ms = (number_packets * (start_time_ms - last_time_ms)) / - (number_packets - 1); - } - - // Converting from bytes/ms to kbits/s. - return static_cast(8 * total_size / corrected_time_ms); + derivative, RecentKbps(), corrected_send_time_ms); } int64_t NadaBweReceiver::MedianFilter(int64_t* last_delays_ms, int size) { @@ -193,16 +144,16 @@ int64_t NadaBweReceiver::ExponentialSmoothingFilter(int64_t new_value, // Implementation according to Cisco's proposal by default. NadaBweSender::NadaBweSender(int kbps, BitrateObserver* observer, Clock* clock) - : clock_(clock), + : BweSender(kbps), // Referred as "Reference Rate" = R_n., + clock_(clock), observer_(observer), - bitrate_kbps_(kbps), original_operating_mode_(true) { } NadaBweSender::NadaBweSender(BitrateObserver* observer, Clock* clock) - : clock_(clock), + : BweSender(kMinBitrateKbps), // Referred as "Reference Rate" = R_n. + clock_(clock), observer_(observer), - bitrate_kbps_(kMinRefRateKbps), original_operating_mode_(true) { } @@ -252,23 +203,23 @@ void NadaBweSender::GiveFeedback(const FeedbackPacket& feedback) { if (fb.congestion_signal() == fb.est_queuing_delay_signal_ms() && fb.est_queuing_delay_signal_ms() < kQueuingDelayUpperBoundMs && fb.exp_smoothed_delay_ms() < - kMinRefRateKbps / kProportionalityDelayBits && + kMinBitrateKbps / kProportionalityDelayBits && fb.derivative() < kDerivativeUpperBound && - fb.receiving_rate() > kMinRefRateKbps) { + fb.receiving_rate() > kMinBitrateKbps) { AcceleratedRampUp(fb); } else if (fb.congestion_signal() > kMaxCongestionSignalMs || fb.exp_smoothed_delay_ms() > kMaxCongestionSignalMs) { AcceleratedRampDown(fb); } else { double bitrate_reference = - (2.0 * bitrate_kbps_) / (kMaxRefRateKbps + kMinRefRateKbps); + (2.0 * bitrate_kbps_) / (kMaxBitrateKbps + kMinBitrateKbps); double smoothing_factor = pow(bitrate_reference, 0.75); GradualRateUpdate(fb, delta_s, smoothing_factor); } } - bitrate_kbps_ = std::min(bitrate_kbps_, kMaxRefRateKbps); - bitrate_kbps_ = std::max(bitrate_kbps_, kMinRefRateKbps); + bitrate_kbps_ = std::min(bitrate_kbps_, kMaxBitrateKbps); + bitrate_kbps_ = std::max(bitrate_kbps_, kMinBitrateKbps); observer_->OnNetworkChanged(1000 * bitrate_kbps_, 0, rtt_ms); } @@ -312,11 +263,11 @@ void NadaBweSender::GradualRateUpdate(const NadaFeedback& fb, float x_hat = fb.congestion_signal() + kEta * kTauOMs * fb.derivative(); float kTheta = - kPriorityWeight * (kMaxRefRateKbps - kMinRefRateKbps) * kReferenceDelayMs; + kPriorityWeight * (kMaxBitrateKbps - kMinBitrateKbps) * kReferenceDelayMs; int original_increase = static_cast((kKappa * delta_s * - (kTheta - (bitrate_kbps_ - kMinRefRateKbps) * x_hat)) / + (kTheta - (bitrate_kbps_ - kMinBitrateKbps) * x_hat)) / (kTauOMs * kTauOMs) + 0.5f); diff --git a/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.h b/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.h index 75dc94d0a9..36c3ecaf6f 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.h +++ b/webrtc/modules/remote_bitrate_estimator/test/estimators/nada.h @@ -40,7 +40,6 @@ class NadaBweReceiver : public BweReceiver { const MediaPacket& media_packet) override; FeedbackPacket* GetFeedback(int64_t now_ms) override; - size_t RecentReceivingRate(); static int64_t MedianFilter(int64_t* v, int size); static int64_t ExponentialSmoothingFilter(int64_t new_value, int64_t last_smoothed_value, @@ -87,16 +86,12 @@ class NadaBweSender : public BweSender { } int64_t NowMs() const { return clock_->TimeInMilliseconds(); } - static const int kMinRefRateKbps; // Referred as R_min. - static const int kMaxRefRateKbps; // Referred as R_max. - private: Clock* const clock_; BitrateObserver* const observer_; // Used as an upper bound for calling AcceleratedRampDown. - const float kMaxCongestionSignalMs = 40.0f + kMinRefRateKbps / 15; + const float kMaxCongestionSignalMs = 40.0f + kMinBitrateKbps / 15; // Referred as R_min, default initialization for bitrate R_n. - int bitrate_kbps_; // Referred as "Reference Rate" = R_n. int64_t last_feedback_ms_ = 0; // Referred as delta_0, initialized as an upper bound. int64_t min_feedback_delay_ms_ = 200; diff --git a/webrtc/modules/remote_bitrate_estimator/test/estimators/nada_unittest.cc b/webrtc/modules/remote_bitrate_estimator/test/estimators/nada_unittest.cc index 967c2b2b5e..80768966b5 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/estimators/nada_unittest.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/estimators/nada_unittest.cc @@ -14,6 +14,7 @@ #include #include "webrtc/base/common.h" +#include "webrtc/base/scoped_ptr.h" #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h" #include "webrtc/modules/remote_bitrate_estimator/test/packet.h" #include "testing/gtest/include/gtest/gtest.h" @@ -112,11 +113,12 @@ class NadaSenderSideTest : public ::testing::Test { }; class NadaReceiverSideTest : public ::testing::Test { - protected: + public: NadaReceiverSideTest() : nada_receiver_(kFlowId) {} ~NadaReceiverSideTest() {} - const int kFlowId = 0; + protected: + const int kFlowId = 1; // Arbitrary. NadaBweReceiver nada_receiver_; }; @@ -165,9 +167,9 @@ class NadaFbGenerator { // Verify if AcceleratedRampUp is called and that bitrate increases. TEST_F(NadaSenderSideTest, AcceleratedRampUp) { - const int64_t kRefSignalMs = 3; + const int64_t kRefSignalMs = 1; const int64_t kOneWayDelayMs = 50; - int original_bitrate = 2 * NadaBweSender::kMinRefRateKbps; + int original_bitrate = 2 * kMinBitrateKbps; size_t receiving_rate = static_cast(original_bitrate); int64_t send_time_ms = nada_sender_.NowMs() - kOneWayDelayMs; @@ -199,7 +201,7 @@ TEST_F(NadaSenderSideTest, AcceleratedRampUp) { // Verify if AcceleratedRampDown is called and if bitrate decreases. TEST_F(NadaSenderSideTest, AcceleratedRampDown) { const int64_t kOneWayDelayMs = 50; - int original_bitrate = 3 * NadaBweSender::kMinRefRateKbps; + int original_bitrate = 3 * kMinBitrateKbps; size_t receiving_rate = static_cast(original_bitrate); int64_t send_time_ms = nada_sender_.NowMs() - kOneWayDelayMs; @@ -216,8 +218,7 @@ TEST_F(NadaSenderSideTest, AcceleratedRampDown) { // Updates the bitrate according to the receiving rate and other constant // parameters. nada_sender_.AcceleratedRampDown(congested_fb); - int bitrate_2_kbps = - std::max(nada_sender_.bitrate_kbps(), NadaBweSender::kMinRefRateKbps); + int bitrate_2_kbps = std::max(nada_sender_.bitrate_kbps(), kMinBitrateKbps); EXPECT_EQ(bitrate_2_kbps, bitrate_1_kbps); } @@ -225,7 +226,7 @@ TEST_F(NadaSenderSideTest, GradualRateUpdate) { const int64_t kDeltaSMs = 20; const int64_t kRefSignalMs = 20; const int64_t kOneWayDelayMs = 50; - int original_bitrate = 2 * NadaBweSender::kMinRefRateKbps; + int original_bitrate = 2 * kMinBitrateKbps; size_t receiving_rate = static_cast(original_bitrate); int64_t send_time_ms = nada_sender_.NowMs() - kOneWayDelayMs; @@ -251,8 +252,8 @@ TEST_F(NadaSenderSideTest, GradualRateUpdate) { // Sending bitrate should decrease and reach its Min bound. TEST_F(NadaSenderSideTest, VeryLowBandwith) { const int64_t kOneWayDelayMs = 50; - const int kMin = NadaBweSender::kMinRefRateKbps; - size_t receiving_rate = static_cast(kMin); + + size_t receiving_rate = static_cast(kMinBitrateKbps); int64_t send_time_ms = nada_sender_.NowMs() - kOneWayDelayMs; NadaFeedback extremely_congested_fb = @@ -260,7 +261,7 @@ TEST_F(NadaSenderSideTest, VeryLowBandwith) { NadaFeedback congested_fb = NadaFbGenerator::CongestedFb(receiving_rate, send_time_ms); - nada_sender_.set_bitrate_kbps(5 * kMin); + nada_sender_.set_bitrate_kbps(5 * kMinBitrateKbps); nada_sender_.set_original_operating_mode(true); for (int i = 0; i < 100; ++i) { // Trigger GradualRateUpdate mode. @@ -268,26 +269,25 @@ TEST_F(NadaSenderSideTest, VeryLowBandwith) { } // The original implementation doesn't allow the bitrate to stay at kMin, // even if the congestion signal is very high. - EXPECT_GE(nada_sender_.bitrate_kbps(), kMin); + EXPECT_GE(nada_sender_.bitrate_kbps(), kMinBitrateKbps); nada_sender_.set_original_operating_mode(false); - nada_sender_.set_bitrate_kbps(5 * kMin); + nada_sender_.set_bitrate_kbps(5 * kMinBitrateKbps); - for (int i = 0; i < 100; ++i) { + for (int i = 0; i < 1000; ++i) { int previous_bitrate = nada_sender_.bitrate_kbps(); // Trigger AcceleratedRampDown mode. nada_sender_.GiveFeedback(congested_fb); EXPECT_LE(nada_sender_.bitrate_kbps(), previous_bitrate); } - EXPECT_EQ(nada_sender_.bitrate_kbps(), kMin); + EXPECT_EQ(nada_sender_.bitrate_kbps(), kMinBitrateKbps); } // Sending bitrate should increase and reach its Max bound. TEST_F(NadaSenderSideTest, VeryHighBandwith) { const int64_t kOneWayDelayMs = 50; - const int kMax = NadaBweSender::kMaxRefRateKbps; - const size_t kRecentReceivingRate = static_cast(kMax); - const int64_t kRefSignalMs = 5; + const size_t kRecentReceivingRate = static_cast(kMaxBitrateKbps); + const int64_t kRefSignalMs = 1; int64_t send_time_ms = nada_sender_.NowMs() - kOneWayDelayMs; NadaFeedback not_congested_fb = NadaFbGenerator::NotCongestedFb( @@ -299,280 +299,164 @@ TEST_F(NadaSenderSideTest, VeryHighBandwith) { nada_sender_.GiveFeedback(not_congested_fb); EXPECT_GE(nada_sender_.bitrate_kbps(), previous_bitrate); } - EXPECT_EQ(nada_sender_.bitrate_kbps(), kMax); + EXPECT_EQ(nada_sender_.bitrate_kbps(), kMaxBitrateKbps); nada_sender_.set_original_operating_mode(false); - nada_sender_.set_bitrate_kbps(NadaBweSender::kMinRefRateKbps); + nada_sender_.set_bitrate_kbps(kMinBitrateKbps); for (int i = 0; i < 100; ++i) { int previous_bitrate = nada_sender_.bitrate_kbps(); nada_sender_.GiveFeedback(not_congested_fb); EXPECT_GE(nada_sender_.bitrate_kbps(), previous_bitrate); } - EXPECT_EQ(nada_sender_.bitrate_kbps(), kMax); + EXPECT_EQ(nada_sender_.bitrate_kbps(), kMaxBitrateKbps); } -TEST_F(NadaReceiverSideTest, ReceivingRateNoPackets) { - EXPECT_EQ(nada_receiver_.RecentReceivingRate(), static_cast(0)); +TEST_F(NadaReceiverSideTest, FeedbackInitialCases) { + rtc::scoped_ptr nada_feedback( + static_cast(nada_receiver_.GetFeedback(0))); + EXPECT_EQ(nada_feedback, nullptr); + + nada_feedback.reset( + static_cast(nada_receiver_.GetFeedback(100))); + EXPECT_EQ(nada_feedback->exp_smoothed_delay_ms(), -1); + EXPECT_EQ(nada_feedback->est_queuing_delay_signal_ms(), 0L); + EXPECT_EQ(nada_feedback->congestion_signal(), 0L); + EXPECT_EQ(nada_feedback->derivative(), 0.0f); + EXPECT_EQ(nada_feedback->receiving_rate(), 0.0f); } -TEST_F(NadaReceiverSideTest, ReceivingRateSinglePacket) { - const size_t kPayloadSizeBytes = 500 * 1000; - const int64_t kSendTimeUs = 300 * 1000; - const int64_t kArrivalTimeMs = kSendTimeUs / 1000 + 100; - const uint16_t kSequenceNumber = 1; - const int64_t kTimeWindowMs = NadaBweReceiver::kReceivingRateTimeWindowMs; - - const MediaPacket media_packet(kFlowId, kSendTimeUs, kPayloadSizeBytes, - kSequenceNumber); - nada_receiver_.ReceivePacket(kArrivalTimeMs, media_packet); - - const size_t kReceivingRateKbps = 8 * kPayloadSizeBytes / kTimeWindowMs; - - EXPECT_EQ(nada_receiver_.RecentReceivingRate(), kReceivingRateKbps); -} - -TEST_F(NadaReceiverSideTest, ReceivingRateLargePackets) { - const size_t kPayloadSizeBytes = 3000 * 1000; - const int64_t kTimeGapMs = 3000; // Between each packet. - const int64_t kOneWayDelayMs = 1000; - - for (int i = 1; i < 5; ++i) { - int64_t send_time_us = i * kTimeGapMs * 1000; - int64_t arrival_time_ms = send_time_us / 1000 + kOneWayDelayMs; - uint16_t sequence_number = i; - const MediaPacket media_packet(kFlowId, send_time_us, kPayloadSizeBytes, - sequence_number); - nada_receiver_.ReceivePacket(arrival_time_ms, media_packet); - } - - const size_t kReceivingRateKbps = 8 * kPayloadSizeBytes / kTimeGapMs; - EXPECT_EQ(nada_receiver_.RecentReceivingRate(), kReceivingRateKbps); -} - -TEST_F(NadaReceiverSideTest, ReceivingRateSmallPackets) { - const size_t kPayloadSizeBytes = 100 * 1000; +TEST_F(NadaReceiverSideTest, FeedbackEmptyQueues) { const int64_t kTimeGapMs = 50; // Between each packet. const int64_t kOneWayDelayMs = 50; - for (int i = 1; i < 50; ++i) { + // No added latency, delay = kOneWayDelayMs. + for (int i = 1; i < 10; ++i) { int64_t send_time_us = i * kTimeGapMs * 1000; int64_t arrival_time_ms = send_time_us / 1000 + kOneWayDelayMs; - uint16_t sequence_number = i; - const MediaPacket media_packet(kFlowId, send_time_us, kPayloadSizeBytes, - sequence_number); + uint16_t sequence_number = static_cast(i); + // Payload sizes are not important here. + const MediaPacket media_packet(kFlowId, send_time_us, 0, sequence_number); nada_receiver_.ReceivePacket(arrival_time_ms, media_packet); } - const size_t kReceivingRateKbps = 8 * kPayloadSizeBytes / kTimeGapMs; - EXPECT_EQ(nada_receiver_.RecentReceivingRate(), kReceivingRateKbps); + // Baseline delay will be equal kOneWayDelayMs. + rtc::scoped_ptr nada_feedback( + static_cast(nada_receiver_.GetFeedback(500))); + EXPECT_EQ(nada_feedback->exp_smoothed_delay_ms(), 0L); + EXPECT_EQ(nada_feedback->est_queuing_delay_signal_ms(), 0L); + EXPECT_EQ(nada_feedback->congestion_signal(), 0L); + EXPECT_EQ(nada_feedback->derivative(), 0.0f); } -TEST_F(NadaReceiverSideTest, ReceivingRateIntermittentPackets) { - const size_t kPayloadSizeBytes = 100 * 1000; - const int64_t kTimeGapMs = 50; // Between each packet. - const int64_t kFirstSendTimeMs = 0; - const int64_t kOneWayDelayMs = 50; +TEST_F(NadaReceiverSideTest, FeedbackIncreasingDelay) { + // Since packets are 100ms apart, each one corresponds to a feedback. + const int64_t kTimeGapMs = 100; // Between each packet. - // Gap between first and other packets - const MediaPacket media_packet(kFlowId, kFirstSendTimeMs, kPayloadSizeBytes, - 1); - nada_receiver_.ReceivePacket(kFirstSendTimeMs + kOneWayDelayMs, media_packet); + // Raw delays are = [10 20 30 40 50 60 70 80] ms. + // Baseline delay will be 50 ms. + // Delay signals should be: [0 10 20 30 40 50 60 70] ms. + const int64_t kMedianFilteredDelaysMs[] = {0, 10, 10, 20, 20, 30, 40, 50}; + const int kNumPackets = ARRAY_SIZE(kMedianFilteredDelaysMs); + const float kAlpha = 0.1f; // Used for exponential smoothing. - const int64_t kDelayAfterFirstPacketMs = 1000; - const int kNumPackets = 5; // Small enough so that all packets are covered. - EXPECT_LT((kNumPackets - 2) * kTimeGapMs, - NadaBweReceiver::kReceivingRateTimeWindowMs); - const int64_t kTimeWindowMs = - kDelayAfterFirstPacketMs + (kNumPackets - 2) * kTimeGapMs; + int64_t exp_smoothed_delays_ms[kNumPackets]; + exp_smoothed_delays_ms[0] = kMedianFilteredDelaysMs[0]; - for (int i = 2; i <= kNumPackets; ++i) { - int64_t send_time_us = - ((i - 2) * kTimeGapMs + kFirstSendTimeMs + kDelayAfterFirstPacketMs) * - 1000; - int64_t arrival_time_ms = send_time_us / 1000 + kOneWayDelayMs; - uint16_t sequence_number = i; - const MediaPacket media_packet(kFlowId, send_time_us, kPayloadSizeBytes, - sequence_number); + for (int i = 1; i < kNumPackets; ++i) { + exp_smoothed_delays_ms[i] = static_cast( + kAlpha * kMedianFilteredDelaysMs[i] + + (1.0f - kAlpha) * exp_smoothed_delays_ms[i - 1] + 0.5f); + } + + for (int i = 0; i < kNumPackets; ++i) { + int64_t send_time_us = (i + 1) * kTimeGapMs * 1000; + int64_t arrival_time_ms = send_time_us / 1000 + 10 * (i + 1); + uint16_t sequence_number = static_cast(i + 1); + // Payload sizes are not important here. + const MediaPacket media_packet(kFlowId, send_time_us, 0, sequence_number); nada_receiver_.ReceivePacket(arrival_time_ms, media_packet); - } - const size_t kTotalReceivedKb = 8 * kNumPackets * kPayloadSizeBytes; - const int64_t kCorrectedTimeWindowMs = - (kTimeWindowMs * kNumPackets) / (kNumPackets - 1); - EXPECT_EQ(nada_receiver_.RecentReceivingRate(), - kTotalReceivedKb / kCorrectedTimeWindowMs); -} - -TEST_F(NadaReceiverSideTest, ReceivingRateDuplicatedPackets) { - const size_t kPayloadSizeBytes = 500 * 1000; - const int64_t kSendTimeUs = 300 * 1000; - const int64_t kArrivalTimeMs = kSendTimeUs / 1000 + 100; - const uint16_t kSequenceNumber = 1; - const int64_t kTimeWindowMs = NadaBweReceiver::kReceivingRateTimeWindowMs; - - // Insert the same packet twice. - for (int i = 0; i < 2; ++i) { - const MediaPacket media_packet(kFlowId, kSendTimeUs + 50 * i, - kPayloadSizeBytes, kSequenceNumber); - nada_receiver_.ReceivePacket(kArrivalTimeMs + 50 * i, media_packet); - } - // Should be counted only once. - const size_t kReceivingRateKbps = 8 * kPayloadSizeBytes / kTimeWindowMs; - - EXPECT_EQ(nada_receiver_.RecentReceivingRate(), kReceivingRateKbps); -} - -TEST_F(NadaReceiverSideTest, PacketLossNoPackets) { - EXPECT_EQ(nada_receiver_.RecentPacketLossRatio(), 0.0f); -} - -TEST_F(NadaReceiverSideTest, PacketLossSinglePacket) { - const MediaPacket media_packet(kFlowId, 0, 0, 0); - nada_receiver_.ReceivePacket(0, media_packet); - EXPECT_EQ(nada_receiver_.RecentPacketLossRatio(), 0.0f); -} - -TEST_F(NadaReceiverSideTest, PacketLossContiguousPackets) { - const int64_t kTimeWindowMs = NadaBweReceiver::kPacketLossTimeWindowMs; - size_t set_capacity = nada_receiver_.GetSetCapacity(); - - for (int i = 0; i < 10; ++i) { - uint16_t sequence_number = static_cast(i); - // Sequence_number and flow_id are the only members that matter here. - const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); - // Arrival time = 0, all packets will be considered. - nada_receiver_.ReceivePacket(0, media_packet); - } - EXPECT_EQ(nada_receiver_.RecentPacketLossRatio(), 0.0f); - - for (int i = 30; i > 20; i--) { - uint16_t sequence_number = static_cast(i); - // Sequence_number and flow_id are the only members that matter here. - const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); - // Only the packets sent in this for loop will be considered. - nada_receiver_.ReceivePacket(2 * kTimeWindowMs, media_packet); - } - EXPECT_EQ(nada_receiver_.RecentPacketLossRatio(), 0.0f); - - // Should handle uint16_t overflow. - for (int i = 0xFFFF - 10; i < 0xFFFF + 10; ++i) { - uint16_t sequence_number = static_cast(i); - const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); - // Only the packets sent in this for loop will be considered. - nada_receiver_.ReceivePacket(4 * kTimeWindowMs, media_packet); - } - EXPECT_EQ(nada_receiver_.RecentPacketLossRatio(), 0.0f); - - // Should handle set overflow. - for (int i = 0; i < set_capacity * 1.5; ++i) { - uint16_t sequence_number = static_cast(i); - const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); - // Only the packets sent in this for loop will be considered. - nada_receiver_.ReceivePacket(6 * kTimeWindowMs, media_packet); - } - EXPECT_EQ(nada_receiver_.RecentPacketLossRatio(), 0.0f); -} - -// Should handle duplicates. -TEST_F(NadaReceiverSideTest, PacketLossDuplicatedPackets) { - const int64_t kTimeWindowMs = NadaBweReceiver::kPacketLossTimeWindowMs; - - for (int i = 0; i < 10; ++i) { - const MediaPacket media_packet(kFlowId, 0, 0, 0); - // Arrival time = 0, all packets will be considered. - nada_receiver_.ReceivePacket(0, media_packet); - } - EXPECT_EQ(nada_receiver_.RecentPacketLossRatio(), 0.0f); - - // Missing the element 5. - const uint16_t kSequenceNumbers[] = {1, 2, 3, 4, 6, 7, 8}; - const int kNumPackets = ARRAY_SIZE(kSequenceNumbers); - - // Insert each sequence number twice. - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < kNumPackets; j++) { - const MediaPacket media_packet(kFlowId, 0, 0, kSequenceNumbers[j]); - // Only the packets sent in this for loop will be considered. - nada_receiver_.ReceivePacket(2 * kTimeWindowMs, media_packet); + rtc::scoped_ptr nada_feedback(static_cast( + nada_receiver_.GetFeedback(arrival_time_ms))); + EXPECT_EQ(nada_feedback->exp_smoothed_delay_ms(), + exp_smoothed_delays_ms[i]); + // Since delay signals are lower than 50ms, they will not be non-linearly + // warped. + EXPECT_EQ(nada_feedback->est_queuing_delay_signal_ms(), + exp_smoothed_delays_ms[i]); + // Zero loss, congestion signal = queuing_delay + EXPECT_EQ(nada_feedback->congestion_signal(), exp_smoothed_delays_ms[i]); + if (i == 0) { + EXPECT_NEAR(nada_feedback->derivative(), + static_cast(exp_smoothed_delays_ms[i]) / kTimeGapMs, + 0.005f); + } else { + EXPECT_NEAR(nada_feedback->derivative(), + static_cast(exp_smoothed_delays_ms[i] - + exp_smoothed_delays_ms[i - 1]) / + kTimeGapMs, + 0.005f); } } - - EXPECT_NEAR(nada_receiver_.RecentPacketLossRatio(), 1.0f / (kNumPackets + 1), - 0.1f / (kNumPackets + 1)); } -TEST_F(NadaReceiverSideTest, PacketLossLakingPackets) { - size_t set_capacity = nada_receiver_.GetSetCapacity(); - EXPECT_LT(set_capacity, static_cast(0xFFFF)); - - // Missing every other packet. - for (size_t i = 0; i < set_capacity; ++i) { - if ((i & 1) == 0) { // Only even sequence numbers. - uint16_t sequence_number = static_cast(i); - const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); - // Arrival time = 0, all packets will be considered. - nada_receiver_.ReceivePacket(0, media_packet); - } +int64_t Warp(int64_t input) { + const int64_t kMinThreshold = 50; // Referred as d_th. + const int64_t kMaxThreshold = 400; // Referred as d_max. + if (input < kMinThreshold) { + return input; + } else if (input < kMaxThreshold) { + return static_cast( + pow((static_cast(kMaxThreshold - input)) / + (kMaxThreshold - kMinThreshold), + 4.0) * + kMinThreshold); + } else { + return 0L; } - EXPECT_NEAR(nada_receiver_.RecentPacketLossRatio(), 0.5f, 0.01f); } -TEST_F(NadaReceiverSideTest, PacketLossLakingFewPackets) { - size_t set_capacity = nada_receiver_.GetSetCapacity(); - EXPECT_LT(set_capacity, static_cast(0xFFFF)); +TEST_F(NadaReceiverSideTest, FeedbackWarpedDelay) { + // Since packets are 100ms apart, each one corresponds to a feedback. + const int64_t kTimeGapMs = 100; // Between each packet. - const int kPeriod = 100; - // Missing one for each kPeriod packets. - for (size_t i = 0; i < set_capacity; ++i) { - if ((i % kPeriod) != 0) { - uint16_t sequence_number = static_cast(i); - const MediaPacket media_packet(kFlowId, 0, 0, sequence_number); - // Arrival time = 0, all packets will be considered. - nada_receiver_.ReceivePacket(0, media_packet); - } - } - EXPECT_NEAR(nada_receiver_.RecentPacketLossRatio(), 1.0f / kPeriod, - 0.1f / kPeriod); -} + // Raw delays are = [50 250 450 650 850 1050 1250 1450] ms. + // Baseline delay will be 50 ms. + // Delay signals should be: [0 200 400 600 800 1000 1200 1400] ms. + const int64_t kMedianFilteredDelaysMs[] = { + 0, 200, 200, 400, 400, 600, 800, 1000}; + const int kNumPackets = ARRAY_SIZE(kMedianFilteredDelaysMs); + const float kAlpha = 0.1f; // Used for exponential smoothing. -// Packet's sequence numbers greatly apart, expect high loss. -TEST_F(NadaReceiverSideTest, PacketLossWideGap) { - const int64_t kTimeWindowMs = NadaBweReceiver::kPacketLossTimeWindowMs; + int64_t exp_smoothed_delays_ms[kNumPackets]; + exp_smoothed_delays_ms[0] = kMedianFilteredDelaysMs[0]; - const MediaPacket media_packet1(0, 0, 0, 1); - const MediaPacket media_packet2(0, 0, 0, 1000); - // Only these two packets will be considered. - nada_receiver_.ReceivePacket(0, media_packet1); - nada_receiver_.ReceivePacket(0, media_packet2); - EXPECT_NEAR(nada_receiver_.RecentPacketLossRatio(), 0.998f, 0.0001f); - - const MediaPacket media_packet3(0, 0, 0, 0); - const MediaPacket media_packet4(0, 0, 0, 0x8000); - // Only these two packets will be considered. - nada_receiver_.ReceivePacket(2 * kTimeWindowMs, media_packet3); - nada_receiver_.ReceivePacket(2 * kTimeWindowMs, media_packet4); - EXPECT_NEAR(nada_receiver_.RecentPacketLossRatio(), 0.99994f, 0.00001f); -} - -// Packets arriving unordered should not be counted as losted. -TEST_F(NadaReceiverSideTest, PacketLossUnorderedPackets) { - size_t num_packets = nada_receiver_.GetSetCapacity() / 2; - std::vector sequence_numbers; - - for (size_t i = 0; i < num_packets; ++i) { - sequence_numbers.push_back(static_cast(i + 1)); + for (int i = 1; i < kNumPackets; ++i) { + exp_smoothed_delays_ms[i] = static_cast( + kAlpha * kMedianFilteredDelaysMs[i] + + (1.0f - kAlpha) * exp_smoothed_delays_ms[i - 1] + 0.5f); } - random_shuffle(sequence_numbers.begin(), sequence_numbers.end()); + for (int i = 0; i < kNumPackets; ++i) { + int64_t send_time_us = (i + 1) * kTimeGapMs * 1000; + int64_t arrival_time_ms = send_time_us / 1000 + 50 + 200 * i; + uint16_t sequence_number = static_cast(i + 1); + // Payload sizes are not important here. + const MediaPacket media_packet(kFlowId, send_time_us, 0, sequence_number); + nada_receiver_.ReceivePacket(arrival_time_ms, media_packet); - for (size_t i = 0; i < num_packets; ++i) { - const MediaPacket media_packet(kFlowId, 0, 0, sequence_numbers[i]); - // Arrival time = 0, all packets will be considered. - nada_receiver_.ReceivePacket(0, media_packet); + rtc::scoped_ptr nada_feedback(static_cast( + nada_receiver_.GetFeedback(arrival_time_ms))); + EXPECT_EQ(nada_feedback->exp_smoothed_delay_ms(), + exp_smoothed_delays_ms[i]); + // Delays can be non-linearly warped. + EXPECT_EQ(nada_feedback->est_queuing_delay_signal_ms(), + Warp(exp_smoothed_delays_ms[i])); + // Zero loss, congestion signal = queuing_delay + EXPECT_EQ(nada_feedback->congestion_signal(), + Warp(exp_smoothed_delays_ms[i])); } - - EXPECT_EQ(nada_receiver_.RecentPacketLossRatio(), 0.0f); } TEST_F(FilterTest, MedianConstantArray) { diff --git a/webrtc/modules/remote_bitrate_estimator/test/estimators/remb.cc b/webrtc/modules/remote_bitrate_estimator/test/estimators/remb.cc index 393df8e871..f0d57ad9bd 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/estimators/remb.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/estimators/remb.cc @@ -101,9 +101,8 @@ void RembReceiver::ReceivePacket(int64_t arrival_time_ms, clock_.AdvanceTimeMilliseconds(arrival_time_ms - clock_.TimeInMilliseconds()); ASSERT_TRUE(arrival_time_ms == clock_.TimeInMilliseconds()); - received_packets_.Insert(media_packet.sequence_number(), - media_packet.send_time_ms(), arrival_time_ms, - media_packet.payload_size()); + // Log received packet information. + BweReceiver::ReceivePacket(arrival_time_ms, media_packet); } FeedbackPacket* RembReceiver::GetFeedback(int64_t now_ms) { diff --git a/webrtc/modules/remote_bitrate_estimator/test/estimators/send_side.cc b/webrtc/modules/remote_bitrate_estimator/test/estimators/send_side.cc index 9caa8b1422..b003d4538c 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/estimators/send_side.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/estimators/send_side.cc @@ -127,12 +127,11 @@ SendSideBweReceiver::~SendSideBweReceiver() { void SendSideBweReceiver::ReceivePacket(int64_t arrival_time_ms, const MediaPacket& media_packet) { packet_feedback_vector_.push_back(PacketInfo( - arrival_time_ms, media_packet.sender_timestamp_us() / 1000, + arrival_time_ms, media_packet.sender_timestamp_ms(), media_packet.header().sequenceNumber, media_packet.payload_size(), true)); - received_packets_.Insert(media_packet.sequence_number(), - media_packet.send_time_ms(), arrival_time_ms, - media_packet.payload_size()); + // Log received packet information. + BweReceiver::ReceivePacket(arrival_time_ms, media_packet); } FeedbackPacket* SendSideBweReceiver::GetFeedback(int64_t now_ms) { diff --git a/webrtc/modules/remote_bitrate_estimator/test/estimators/tcp.cc b/webrtc/modules/remote_bitrate_estimator/test/estimators/tcp.cc index 154d68c520..a02abc6ab8 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/estimators/tcp.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/estimators/tcp.cc @@ -33,12 +33,11 @@ TcpBweReceiver::~TcpBweReceiver() { void TcpBweReceiver::ReceivePacket(int64_t arrival_time_ms, const MediaPacket& media_packet) { - latest_owd_ms_ = arrival_time_ms - media_packet.sender_timestamp_us() / 1000; + latest_owd_ms_ = arrival_time_ms - media_packet.sender_timestamp_ms() / 1000; acks_.push_back(media_packet.header().sequenceNumber); - received_packets_.Insert(media_packet.sequence_number(), - media_packet.send_time_ms(), arrival_time_ms, - media_packet.payload_size()); + // Log received packet information. + BweReceiver::ReceivePacket(arrival_time_ms, media_packet); } FeedbackPacket* TcpBweReceiver::GetFeedback(int64_t now_ms) { diff --git a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc new file mode 100644 index 0000000000..c9033c564f --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" + +#include + +namespace webrtc { +namespace testing { +namespace bwe { + +namespace { + +template +T Sum(const std::vector& input) { + T total = 0; + for (T val : input) { + total += val; + } + return total; +} + +template +double Average(const std::vector& array, size_t size) { + return static_cast(Sum(array)) / size; +} + +template +std::vector Abs(const std::vector& input) { + std::vector output(input); + for (T val : output) { + val = std::abs(val); + } + return output; +} + +template +std::vector Pow(const std::vector& input, double p) { + std::vector output; + for (T val : input) { + output.push_back(pow(static_cast(val), p)); + } + return output; +} + +template +double StandardDeviation(const std::vector& array, size_t size) { + double mean = Average(array, size); + std::vector square_values = Pow(array, 2.0); + double var = Average(square_values, size) - mean * mean; + return sqrt(var); +} + +// Holder mean, Manhattan distance for p=1, EuclidianNorm/sqrt(n) for p=2. +template +double NormLp(const std::vector& array, size_t size, double p) { + std::vector abs_values = Abs(array); + std::vector pow_values = Pow(abs_values, p); + return pow(Sum(pow_values) / size, 1.0 / p); +} + +template +std::vector PositiveFilter(const std::vector& input) { + std::vector output(input); + for (T val : output) { + val = val > 0 ? val : 0; + } + return output; +} + +template +std::vector NegativeFilter(const std::vector& input) { + std::vector output(input); + for (T val : output) { + val = val < 0 ? -val : 0; + } + return output; +} +} // namespace + +LinkShare::LinkShare(ChokeFilter* choke_filter) + : choke_filter_(choke_filter), running_flows_(choke_filter->flow_ids()) { +} + +void LinkShare::PauseFlow(int flow_id) { + running_flows_.erase(flow_id); +} + +void LinkShare::ResumeFlow(int flow_id) { + running_flows_.insert(flow_id); +} + +uint32_t LinkShare::TotalAvailableKbps() { + return choke_filter_->capacity_kbps(); +} + +uint32_t LinkShare::AvailablePerFlowKbps(int flow_id) { + uint32_t available_capacity_per_flow_kbps = 0; + if (running_flows_.find(flow_id) != running_flows_.end()) { + available_capacity_per_flow_kbps = + TotalAvailableKbps() / static_cast(running_flows_.size()); + } + return available_capacity_per_flow_kbps; +} + +MetricRecorder::MetricRecorder(const std::string algorithm_name, + int flow_id, + PacketSender* packet_sender, + LinkShare* link_share) + : algorithm_name_(algorithm_name), + flow_id_(flow_id), + packet_sender_(packet_sender), + link_share_(link_share), + now_ms_(0), + delays_ms_(), + throughput_bytes_(), + weighted_estimate_error_(), + last_unweighted_estimate_error_(0), + optimal_throughput_bits_(0), + last_available_bitrate_per_flow_kbps_(0), + start_computing_metrics_ms_(0), + started_computing_metrics_(false) { +} + +void MetricRecorder::SetPlotInformation( + const std::vector& prefixes) { + assert(prefixes.size() == kNumMetrics); + for (size_t i = 0; i < kNumMetrics; ++i) { + plot_information_[i].prefix = prefixes[i]; + } + plot_information_[kThroughput].plot_interval_ms = 100; + plot_information_[kDelay].plot_interval_ms = 100; + plot_information_[kLoss].plot_interval_ms = 500; + plot_information_[kObjective].plot_interval_ms = 1000; + plot_information_[kTotalAvailable].plot_interval_ms = 1000; + plot_information_[kAvailablePerFlow].plot_interval_ms = 1000; + + for (int i = kThroughput; i < kNumMetrics; ++i) { + plot_information_[i].last_plot_ms = 0; + if (i == kObjective || i == kAvailablePerFlow) { + plot_information_[i].plot = false; + } else { + plot_information_[i].plot = true; + } + } +} + +void MetricRecorder::PlotAllDynamics() { + for (int i = kThroughput; i < kNumMetrics; ++i) { + if (plot_information_[i].plot && + now_ms_ - plot_information_[i].last_plot_ms >= + plot_information_[i].plot_interval_ms) { + PlotDynamics(i); + } + } +} + +void MetricRecorder::PlotDynamics(int metric) { + if (metric == kTotalAvailable) { + BWE_TEST_LOGGING_PLOT_WITH_NAME( + 0, plot_information_[kTotalAvailable].prefix, now_ms_, + GetTotalAvailableKbps(), "Available"); + } else if (metric == kAvailablePerFlow) { + BWE_TEST_LOGGING_PLOT_WITH_NAME( + 0, plot_information_[kAvailablePerFlow].prefix, now_ms_, + GetAvailablePerFlowKbps(), "Available_per_flow"); + } else { + PlotLine(metric, plot_information_[metric].prefix, + plot_information_[metric].time_ms, + plot_information_[metric].value); + } + plot_information_[metric].last_plot_ms = now_ms_; +} + +template +void MetricRecorder::PlotLine(int windows_id, + const std::string& prefix, + int64_t time_ms, + T y) { + BWE_TEST_LOGGING_PLOT_WITH_NAME(windows_id, prefix, time_ms, + static_cast(y), algorithm_name_); +} + +void MetricRecorder::UpdateTime(int64_t time_ms) { + now_ms_ = std::max(now_ms_, time_ms); +} + +void MetricRecorder::UpdateThroughput(int64_t bitrate_kbps, + size_t payload_size) { + // Total throughput should be computed before updating the time. + PushThroughputBytes(payload_size, now_ms_); + plot_information_[kThroughput].Update(now_ms_, bitrate_kbps); +} + +void MetricRecorder::UpdateDelay(int64_t delay_ms) { + PushDelayMs(delay_ms, now_ms_); + plot_information_[kDelay].Update(now_ms_, delay_ms); +} + +void MetricRecorder::UpdateLoss(float loss_ratio) { + plot_information_[kLoss].Update(now_ms_, loss_ratio); +} + +void MetricRecorder::UpdateObjective() { + plot_information_[kObjective].Update(now_ms_, ObjectiveFunction()); +} + +uint32_t MetricRecorder::GetTotalAvailableKbps() { + return link_share_->TotalAvailableKbps(); +} + +uint32_t MetricRecorder::GetAvailablePerFlowKbps() { + return link_share_->AvailablePerFlowKbps(flow_id_); +} + +uint32_t MetricRecorder::GetSendingEstimateKbps() { + return packet_sender_->TargetBitrateKbps(); +} + +void MetricRecorder::PushDelayMs(int64_t delay_ms, int64_t arrival_time_ms) { + if (ShouldRecord(arrival_time_ms)) { + delays_ms_.push_back(delay_ms); + } +} + +void MetricRecorder::PushThroughputBytes(size_t payload_size, + int64_t arrival_time_ms) { + if (ShouldRecord(arrival_time_ms)) { + throughput_bytes_.push_back(payload_size); + + int64_t current_available_per_flow_kbps = + static_cast(GetAvailablePerFlowKbps()); + + int64_t current_bitrate_diff_kbps = + static_cast(GetSendingEstimateKbps()) - + current_available_per_flow_kbps; + + weighted_estimate_error_.push_back( + ((current_bitrate_diff_kbps + last_unweighted_estimate_error_) * + (arrival_time_ms - plot_information_[kThroughput].time_ms)) / + 2); + + optimal_throughput_bits_ += + ((current_available_per_flow_kbps + + last_available_bitrate_per_flow_kbps_) * + (arrival_time_ms - plot_information_[kThroughput].time_ms)) / + 2; + + last_available_bitrate_per_flow_kbps_ = current_available_per_flow_kbps; + } +} + +bool MetricRecorder::ShouldRecord(int64_t arrival_time_ms) { + if (arrival_time_ms >= start_computing_metrics_ms_) { + if (!started_computing_metrics_) { + start_computing_metrics_ms_ = arrival_time_ms; + now_ms_ = arrival_time_ms; + started_computing_metrics_ = true; + } + return true; + } else { + return false; + } +} + +// The weighted_estimate_error_ was weighted based on time windows. +// This function scales back the result before plotting. +double MetricRecorder::Renormalize(double x) { + size_t num_packets_received = delays_ms_.size(); + return (x * num_packets_received) / now_ms_; +} + +inline double U(int64_t x, double alpha) { + if (alpha == 1.0) { + return log(static_cast(x)); + } + return pow(static_cast(x), 1.0 - alpha) / (1.0 - alpha); +} + +inline double U(size_t x, double alpha) { + return U(static_cast(x), alpha); +} + +// TODO(magalhaesc): Update ObjectiveFunction. +double MetricRecorder::ObjectiveFunction() { + const double kDelta = 0.15; // Delay penalty factor. + const double kAlpha = 1.0; + const double kBeta = 1.0; + + double throughput_metric = U(Sum(throughput_bytes_), kAlpha); + double delay_penalty = kDelta * U(Sum(delays_ms_), kBeta); + + return throughput_metric - delay_penalty; +} + +void MetricRecorder::PlotThroughputHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows, + int64_t extra_offset_ms, + const std::string optimum_id) { + size_t num_packets_received = delays_ms_.size(); + + int64_t duration_ms = now_ms_ - start_computing_metrics_ms_ - extra_offset_ms; + + double average_bitrate_kbps = + static_cast(8 * Sum(throughput_bytes_) / duration_ms); + + double optimal_bitrate_per_flow_kbps = + static_cast(optimal_throughput_bits_ / duration_ms); + + std::vector positive = PositiveFilter(weighted_estimate_error_); + std::vector negative = NegativeFilter(weighted_estimate_error_); + + double p_error = Renormalize(NormLp(positive, num_packets_received, 1.0)); + double n_error = Renormalize(NormLp(negative, num_packets_received, 1.0)); + + // Prevent the error to be too close to zero (plotting issue). + double extra_error = average_bitrate_kbps / 500; + + std::string optimum_title = + optimum_id.empty() ? "optimal_bitrate" : "optimal_bitrates#" + optimum_id; + + BWE_TEST_LOGGING_LABEL(4, title, "average_bitrate_(kbps)", num_flows); + BWE_TEST_LOGGING_LIMITERRORBAR( + 4, bwe_name, average_bitrate_kbps, + average_bitrate_kbps - n_error - extra_error, + average_bitrate_kbps + p_error + extra_error, "estimate_error", + optimal_bitrate_per_flow_kbps, optimum_title, flow_id_); + + BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Channel utilization : ", + "%lf %%", + 100.0 * static_cast(average_bitrate_kbps) / + optimal_bitrate_per_flow_kbps); + + RTC_UNUSED(p_error); + RTC_UNUSED(n_error); + RTC_UNUSED(extra_error); + RTC_UNUSED(optimal_bitrate_per_flow_kbps); +} + +void MetricRecorder::PlotThroughputHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows, + int64_t extra_offset_ms) { + PlotThroughputHistogram(title, bwe_name, num_flows, extra_offset_ms, ""); +} + +void MetricRecorder::PlotDelayHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows, + int64_t one_way_path_delay_ms) { + size_t num_packets_received = delays_ms_.size(); + double average_delay_ms = Average(delays_ms_, num_packets_received); + + // Prevent the error to be too close to zero (plotting issue). + double extra_error = average_delay_ms / 500; + + double tenth_sigma_ms = + StandardDeviation(delays_ms_, num_packets_received) / 10.0 + extra_error; + + size_t per_5_index = (num_packets_received - 1) / 20; + std::nth_element(delays_ms_.begin(), delays_ms_.begin() + per_5_index, + delays_ms_.end()); + int64_t percentile_5_ms = delays_ms_[per_5_index]; + + size_t per_95_index = num_packets_received - 1 - per_5_index; + std::nth_element(delays_ms_.begin(), delays_ms_.begin() + per_95_index, + delays_ms_.end()); + int64_t percentile_95_ms = delays_ms_[per_95_index]; + + BWE_TEST_LOGGING_LABEL(5, title, "average_delay_(ms)", num_flows) + BWE_TEST_LOGGING_ERRORBAR(5, bwe_name, average_delay_ms, percentile_5_ms, + percentile_95_ms, "5th and 95th percentiles", + flow_id_); + + // Log added latency, disregard baseline path delay. + BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Delay average : ", + "%lf ms", average_delay_ms - one_way_path_delay_ms); + BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Delay 5th percentile : ", + "%ld ms", percentile_5_ms - one_way_path_delay_ms); + BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Delay 95th percentile : ", + "%ld ms", percentile_95_ms - one_way_path_delay_ms); + + RTC_UNUSED(tenth_sigma_ms); + RTC_UNUSED(percentile_5_ms); + RTC_UNUSED(percentile_95_ms); +} + +void MetricRecorder::PlotLossHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows, + float global_loss_ratio) { + BWE_TEST_LOGGING_LABEL(6, title, "packet_loss_ratio_(%)", num_flows) + BWE_TEST_LOGGING_BAR(6, bwe_name, 100.0f * global_loss_ratio, flow_id_); + + BWE_TEST_LOGGING_LOG1("RESULTS >>> " + bwe_name + " Loss Ratio : ", "%f %%", + 100.0f * global_loss_ratio); +} + +void MetricRecorder::PlotObjectiveHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows) { + BWE_TEST_LOGGING_LABEL(7, title, "objective_function", num_flows) + BWE_TEST_LOGGING_BAR(7, bwe_name, ObjectiveFunction(), flow_id_); +} + +void MetricRecorder::PlotZero() { + for (int i = kThroughput; i <= kLoss; ++i) { + if (plot_information_[i].plot) { + std::stringstream prefix; + prefix << "Receiver_" << flow_id_ << "_" + plot_information_[i].prefix; + PlotLine(i, prefix.str(), now_ms_, 0); + plot_information_[i].last_plot_ms = now_ms_; + } + } +} + +void MetricRecorder::PauseFlow() { + PlotZero(); + link_share_->PauseFlow(flow_id_); +} + +void MetricRecorder::ResumeFlow(int64_t paused_time_ms) { + UpdateTime(now_ms_ + paused_time_ms); + PlotZero(); + link_share_->ResumeFlow(flow_id_); +} + +} // namespace bwe +} // namespace testing +} // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h new file mode 100644 index 0000000000..3a5bb056e1 --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_METRIC_RECORDER_H_ +#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_METRIC_RECORDER_H_ + +#include +#include +#include + +#include "webrtc/base/common.h" +#include "webrtc/modules/remote_bitrate_estimator/test/packet_sender.h" + +namespace webrtc { +namespace testing { +namespace bwe { + +class LinkShare { + public: + explicit LinkShare(ChokeFilter* choke_filter); + + void PauseFlow(int flow_id); // Increases available capacity per flow. + void ResumeFlow(int flow_id); // Decreases available capacity per flow. + + uint32_t TotalAvailableKbps(); + // If the given flow is paused, its output is zero. + uint32_t AvailablePerFlowKbps(int flow_id); + + private: + ChokeFilter* choke_filter_; + std::set running_flows_; +}; + +struct PlotInformation { + PlotInformation() + : prefix(), + last_plot_ms(0), + time_ms(0), + value(0.0), + plot_interval_ms(0) {} + template + void Update(int64_t now_ms, T new_value) { + time_ms = now_ms; + value = static_cast(new_value); + } + std::string prefix; + bool plot; + int64_t last_plot_ms; + int64_t time_ms; + double value; + int64_t plot_interval_ms; +}; + +class MetricRecorder { + public: + MetricRecorder(const std::string algorithm_name, + int flow_id, + PacketSender* packet_sender, + LinkShare* link_share); + + void SetPlotInformation(const std::vector& prefixes); + + template + void PlotLine(int windows_id, + const std::string& prefix, + int64_t time_ms, + T y); + + void PlotDynamics(int metric); + void PlotAllDynamics(); + + void UpdateTime(int64_t time_ms); + void UpdateThroughput(int64_t bitrate_kbps, size_t payload_size); + void UpdateDelay(int64_t delay_ms); + void UpdateLoss(float loss_ratio); + void UpdateObjective(); + + void PlotThroughputHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows, + int64_t extra_offset_ms, + const std::string optimum_id); + + void PlotThroughputHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows, + int64_t extra_offset_ms); + + void PlotDelayHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows, + int64_t one_way_path_delay_ms); + + void PlotLossHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows, + float global_loss_ratio); + + void PlotObjectiveHistogram(const std::string& title, + const std::string& bwe_name, + int num_flows); + + void set_start_computing_metrics_ms(int64_t start_computing_metrics_ms) { + start_computing_metrics_ms_ = start_computing_metrics_ms; + } + + void set_plot_available_capacity(bool plot) { + plot_information_[kTotalAvailable].plot = plot; + } + + void PauseFlow(); // Plot zero. + void ResumeFlow(int64_t paused_time_ms); // Plot zero. + void PlotZero(); + + private: + uint32_t GetTotalAvailableKbps(); + uint32_t GetAvailablePerFlowKbps(); + uint32_t GetSendingEstimateKbps(); + double ObjectiveFunction(); + + double Renormalize(double x); + bool ShouldRecord(int64_t arrival_time_ms); + + void PushDelayMs(int64_t delay_ms, int64_t arrival_time_ms); + void PushThroughputBytes(size_t throughput_bytes, int64_t arrival_time_ms); + + enum Metrics { + kThroughput = 0, + kDelay, + kLoss, + kObjective, + kTotalAvailable, + kAvailablePerFlow, + kNumMetrics + }; + + std::string algorithm_name_; + int flow_id_; + PacketSender* packet_sender_; + LinkShare* link_share_; + + int64_t now_ms_; + + PlotInformation plot_information_[kNumMetrics]; + + std::vector delays_ms_; + std::vector throughput_bytes_; + // (Receiving rate - available bitrate per flow) * time window. + std::vector weighted_estimate_error_; + int64_t last_unweighted_estimate_error_; + int64_t optimal_throughput_bits_; + int64_t last_available_bitrate_per_flow_kbps_; + int64_t start_computing_metrics_ms_; + bool started_computing_metrics_; +}; + +} // namespace bwe +} // namespace testing +} // namespace webrtc +#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_METRIC_RECORDER_H_ diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet.h b/webrtc/modules/remote_bitrate_estimator/test/packet.h index 647e357f9b..e565e9f5db 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet.h +++ b/webrtc/modules/remote_bitrate_estimator/test/packet.h @@ -34,15 +34,20 @@ class Packet { virtual bool operator<(const Packet& rhs) const; virtual int flow_id() const { return flow_id_; } - virtual int64_t creation_time_us() const { return creation_time_us_; } virtual void set_send_time_us(int64_t send_time_us); virtual int64_t send_time_us() const { return send_time_us_; } virtual size_t payload_size() const { return payload_size_; } virtual Packet::Type GetPacketType() const = 0; - void set_sender_timestamp_us(int64_t sender_timestamp_us) { + virtual void set_sender_timestamp_us(int64_t sender_timestamp_us) { sender_timestamp_us_ = sender_timestamp_us; } - int64_t sender_timestamp_us() const { return sender_timestamp_us_; } + virtual int64_t creation_time_ms() const { + return (creation_time_us_ + 500) / 1000; + } + virtual int64_t sender_timestamp_ms() const { + return (sender_timestamp_us_ + 500) / 1000; + } + virtual int64_t send_time_ms() const { return (send_time_us_ + 500) / 1000; } protected: int flow_id_; @@ -64,6 +69,7 @@ class MediaPacket : public Packet { size_t payload_size, const RTPHeader& header); MediaPacket(int64_t send_time_us, uint32_t sequence_number); + virtual ~MediaPacket() {} int64_t GetAbsSendTimeInMs() const { @@ -75,7 +81,6 @@ class MediaPacket : public Packet { const RTPHeader& header() const { return header_; } virtual Packet::Type GetPacketType() const { return kMedia; } uint16_t sequence_number() const { return header_.sequenceNumber; } - int64_t send_time_ms() const { return send_time_us_ / 1000; } private: static const int kAbsSendTimeFraction = 18; @@ -99,7 +104,7 @@ class FeedbackPacket : public Packet { int64_t latest_send_time_ms() const { return latest_send_time_ms_; } private: - int64_t latest_send_time_ms_; // Time stamp for the latest sent packet. + int64_t latest_send_time_ms_; // Time stamp for the latest sent FbPacket. }; class RembFeedback : public FeedbackPacket { diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc index c13f14437a..acae76e797 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc @@ -10,7 +10,6 @@ #include "webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h" -#include #include #include "testing/gtest/include/gtest/gtest.h" @@ -29,35 +28,52 @@ PacketReceiver::PacketReceiver(PacketProcessorListener* listener, int flow_id, BandwidthEstimatorType bwe_type, bool plot_delay, - bool plot_bwe) + bool plot_bwe, + MetricRecorder* metric_recorder) : PacketProcessor(listener, flow_id, kReceiver), - delay_log_prefix_(), - metric_log_prefix_(), - packet_loss_log_prefix_(), - last_delay_plot_ms_(0), - last_metric_plot_ms_(0), - last_packet_loss_plot_ms_(0), - plot_delay_(plot_delay), - // TODO(magalhaesc) Add separated plot_objective_function and - // plot_packet_loss parameters to the constructor. - plot_objective_function_(plot_delay), - plot_packet_loss_(plot_delay), bwe_receiver_(CreateBweReceiver(bwe_type, flow_id, plot_bwe)), - total_delay_ms_(0), - total_throughput_(0), - number_packets_(0) { - // Setup the prefix ststd::rings used when logging. - std::stringstream ss1; - ss1 << "Delay_" << flow_id << "#2"; - delay_log_prefix_ = ss1.str(); + metric_recorder_(metric_recorder) { + if (metric_recorder_ != nullptr) { + // Setup the prefix ststd::rings used when logging. + std::vector prefixes; - std::stringstream ss2; - ss2 << "Objective_function_" << flow_id << "#2"; - metric_log_prefix_ = ss2.str(); + std::stringstream ss1; + ss1 << "Throughput_kbps_" << flow_id << "#2"; + prefixes.push_back(ss1.str()); // Throughput. - std::stringstream ss3; - ss3 << "Packet_Loss_" << flow_id << "#2"; - packet_loss_log_prefix_ = ss3.str(); + std::stringstream ss2; + ss2 << "Delay_ms_" << flow_id << "#2"; + prefixes.push_back(ss2.str()); // Delay. + + std::stringstream ss3; + ss3 << "Packet_Loss_" << flow_id << "#2"; + prefixes.push_back(ss3.str()); // Loss. + + std::stringstream ss4; + ss4 << "Objective_function_" << flow_id << "#2"; + prefixes.push_back(ss4.str()); // Objective. + + // Plot Total/PerFlow Available capacity together with throughputs. + std::stringstream ss5; + ss5 << "Throughput_kbps" << flow_id << "#1"; + prefixes.push_back(ss5.str()); // Total Available. + prefixes.push_back(ss5.str()); // Available per flow. + + metric_recorder_->SetPlotInformation(prefixes); + } +} + +PacketReceiver::PacketReceiver(PacketProcessorListener* listener, + int flow_id, + BandwidthEstimatorType bwe_type, + bool plot_delay, + bool plot_bwe) + : PacketReceiver(listener, + flow_id, + bwe_type, + plot_delay, + plot_bwe, + nullptr) { } PacketReceiver::~PacketReceiver() { @@ -77,16 +93,16 @@ void PacketReceiver::RunFor(int64_t time_ms, Packets* in_out) { const MediaPacket* media_packet = static_cast(*it); // We're treating the send time (from previous filter) as the arrival // time once packet reaches the estimator. - int64_t arrival_time_ms = (media_packet->send_time_us() + 500) / 1000; - int64_t send_time_ms = (media_packet->creation_time_us() + 500) / 1000; + int64_t arrival_time_ms = media_packet->send_time_ms(); + int64_t send_time_ms = media_packet->creation_time_ms(); delay_stats_.Push(arrival_time_ms - send_time_ms); - PlotDelay(arrival_time_ms, send_time_ms); - PlotObjectiveFunction(arrival_time_ms); - PlotPacketLoss(arrival_time_ms); - total_delay_ms_ += arrival_time_ms - send_time_ms; - total_throughput_ += media_packet->payload_size(); - ++number_packets_; + if (metric_recorder_ != nullptr) { + metric_recorder_->UpdateTime(arrival_time_ms); + UpdateMetrics(arrival_time_ms, send_time_ms, + media_packet->payload_size()); + metric_recorder_->PlotAllDynamics(); + } bwe_receiver_->ReceivePacket(arrival_time_ms, *media_packet); FeedbackPacket* fb = bwe_receiver_->GetFeedback(arrival_time_ms); @@ -102,46 +118,17 @@ void PacketReceiver::RunFor(int64_t time_ms, Packets* in_out) { in_out->merge(feedback, DereferencingComparator); } -void PacketReceiver::PlotDelay(int64_t arrival_time_ms, int64_t send_time_ms) { - static const int kDelayPlotIntervalMs = 100; - if (!plot_delay_) - return; - if (arrival_time_ms - last_delay_plot_ms_ > kDelayPlotIntervalMs) { - BWE_TEST_LOGGING_PLOT(0, delay_log_prefix_, arrival_time_ms, - arrival_time_ms - send_time_ms); - last_delay_plot_ms_ = arrival_time_ms; - } +void PacketReceiver::UpdateMetrics(int64_t arrival_time_ms, + int64_t send_time_ms, + size_t payload_size) { + metric_recorder_->UpdateThroughput(bwe_receiver_->RecentKbps(), payload_size); + metric_recorder_->UpdateDelay(arrival_time_ms - send_time_ms); + metric_recorder_->UpdateLoss(bwe_receiver_->RecentPacketLossRatio()); + metric_recorder_->UpdateObjective(); } -double PacketReceiver::ObjectiveFunction() { - const double kDelta = 1.0; // Delay penalty factor. - double throughput_metric = log(static_cast(total_throughput_)); - double delay_penalty = kDelta * log(static_cast(total_delay_ms_)); - return throughput_metric - delay_penalty; -} - -void PacketReceiver::PlotObjectiveFunction(int64_t arrival_time_ms) { - static const int kMetricPlotIntervalMs = 1000; - if (!plot_objective_function_) { - return; - } - if (arrival_time_ms - last_metric_plot_ms_ > kMetricPlotIntervalMs) { - BWE_TEST_LOGGING_PLOT(1, metric_log_prefix_, arrival_time_ms, - ObjectiveFunction()); - last_metric_plot_ms_ = arrival_time_ms; - } -} - -void PacketReceiver::PlotPacketLoss(int64_t arrival_time_ms) { - static const int kPacketLossPlotIntervalMs = 500; - if (!plot_packet_loss_) { - return; - } - if (arrival_time_ms - last_packet_loss_plot_ms_ > kPacketLossPlotIntervalMs) { - BWE_TEST_LOGGING_PLOT(2, packet_loss_log_prefix_, arrival_time_ms, - bwe_receiver_->RecentPacketLossRatio()); - last_packet_loss_plot_ms_ = arrival_time_ms; - } +float PacketReceiver::GlobalPacketLoss() { + return bwe_receiver_->GlobalReceiverPacketLossRatio(); } Stats PacketReceiver::GetDelayStats() const { diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h index a6838269c6..30192358cc 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h @@ -17,6 +17,7 @@ #include "webrtc/base/scoped_ptr.h" #include "webrtc/modules/remote_bitrate_estimator/test/bwe.h" #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h" +#include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" namespace webrtc { namespace testing { @@ -29,6 +30,12 @@ class PacketReceiver : public PacketProcessor { BandwidthEstimatorType bwe_type, bool plot_delay, bool plot_bwe); + PacketReceiver(PacketProcessorListener* listener, + int flow_id, + BandwidthEstimatorType bwe_type, + bool plot_delay, + bool plot_bwe, + MetricRecorder* metric_recorder); ~PacketReceiver(); // Implements PacketProcessor. @@ -38,30 +45,19 @@ class PacketReceiver : public PacketProcessor { Stats GetDelayStats() const; - protected: - void PlotDelay(int64_t arrival_time_ms, int64_t send_time_ms); - void PlotObjectiveFunction(int64_t arrival_time_ms); - void PlotPacketLoss(int64_t arrival_time_ms); - double ObjectiveFunction(); + float GlobalPacketLoss(); + + protected: + void UpdateMetrics(int64_t arrival_time_ms, + int64_t send_time_ms, + size_t payload_size); - int64_t now_ms_; - std::string delay_log_prefix_; - std::string metric_log_prefix_; - std::string packet_loss_log_prefix_; - int64_t last_delay_plot_ms_; - int64_t last_metric_plot_ms_; - int64_t last_packet_loss_plot_ms_; - bool plot_delay_; - bool plot_objective_function_; - bool plot_packet_loss_; Stats delay_stats_; rtc::scoped_ptr bwe_receiver_; - int64_t total_delay_ms_; - size_t total_throughput_; - int number_packets_; - private: + MetricRecorder* metric_recorder_; + DISALLOW_IMPLICIT_CONSTRUCTORS(PacketReceiver); }; } // namespace bwe diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc b/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc index eeaec86589..a7a68aa28b 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc @@ -44,11 +44,13 @@ VideoSender::VideoSender(PacketProcessorListener* listener, VideoSource* source, BandwidthEstimatorType estimator_type) : PacketSender(listener, source->flow_id()), + running_(true), source_(source), bwe_(CreateBweSender(estimator_type, source_->bits_per_second() / 1000, this, - &clock_)) { + &clock_)), + previous_sending_bitrate_(0) { modules_.push_back(bwe_.get()); } @@ -70,22 +72,30 @@ void VideoSender::ProcessFeedbackAndGeneratePackets( int64_t time_to_run_ms = std::min(time_ms, 100); if (!feedbacks->empty()) { int64_t time_until_feedback_ms = - feedbacks->front()->send_time_us() / 1000 - - clock_.TimeInMilliseconds(); + feedbacks->front()->send_time_ms() - clock_.TimeInMilliseconds(); time_to_run_ms = std::max(std::min(time_ms, time_until_feedback_ms), 0); } + + if (!running_) { + source_->SetBitrateBps(0); + } + Packets generated; source_->RunFor(time_to_run_ms, &generated); bwe_->OnPacketsSent(generated); packets->merge(generated, DereferencingComparator); + clock_.AdvanceTimeMilliseconds(time_to_run_ms); + if (!feedbacks->empty()) { bwe_->GiveFeedback(*feedbacks->front()); delete feedbacks->front(); feedbacks->pop_front(); } + bwe_->Process(); + time_ms -= time_to_run_ms; } while (time_ms > 0); assert(feedbacks->empty()); @@ -101,6 +111,20 @@ void VideoSender::OnNetworkChanged(uint32_t target_bitrate_bps, source_->SetBitrateBps(target_bitrate_bps); } +void VideoSender::Pause() { + running_ = false; + previous_sending_bitrate_ = TargetBitrateKbps(); +} + +void VideoSender::Resume() { + running_ = true; + source_->SetBitrateBps(previous_sending_bitrate_); +} + +uint32_t VideoSender::TargetBitrateKbps() { + return (source_->bits_per_second() + 500) / 1000; +} + PacedVideoSender::PacedVideoSender(PacketProcessorListener* listener, VideoSource* source, BandwidthEstimatorType estimator) @@ -133,10 +157,8 @@ void PacedVideoSender::RunFor(int64_t time_ms, Packets* in_out) { int64_t time_until_process_ms = TimeUntilNextProcess(modules_); int64_t time_until_feedback_ms = time_ms; if (!feedbacks.empty()) - time_until_feedback_ms = - std::max(feedbacks.front()->send_time_us() / 1000 - - clock_.TimeInMilliseconds(), - 0); + time_until_feedback_ms = std::max( + feedbacks.front()->send_time_ms() - clock_.TimeInMilliseconds(), 0); int64_t time_until_next_event_ms = std::min(time_until_feedback_ms, time_until_process_ms); @@ -159,11 +181,10 @@ void PacedVideoSender::RunFor(int64_t time_ms, Packets* in_out) { if (!generated_packets.empty()) { for (Packet* packet : generated_packets) { MediaPacket* media_packet = static_cast(packet); - pacer_.SendPacket(PacedSender::kNormalPriority, - media_packet->header().ssrc, - media_packet->header().sequenceNumber, - (media_packet->send_time_us() + 500) / 1000, - media_packet->payload_size(), false); + pacer_.SendPacket( + PacedSender::kNormalPriority, media_packet->header().ssrc, + media_packet->header().sequenceNumber, media_packet->send_time_ms(), + media_packet->payload_size(), false); pacer_queue_.push_back(packet); assert(pacer_queue_.size() < 10000); } @@ -235,9 +256,11 @@ bool PacedVideoSender::TimeToSendPacket(uint32_t ssrc, MediaPacket* media_packet = static_cast(*it); if (media_packet->header().sequenceNumber == sequence_number) { int64_t pace_out_time_ms = clock_.TimeInMilliseconds(); + // Make sure a packet is never paced out earlier than when it was put into // the pacer. - assert(pace_out_time_ms >= (media_packet->send_time_us() + 500) / 1000); + assert(pace_out_time_ms >= media_packet->send_time_ms()); + media_packet->SetAbsSendTimeMs(pace_out_time_ms); media_packet->set_send_time_us(1000 * pace_out_time_ms); media_packet->set_sender_timestamp_us(1000 * pace_out_time_ms); @@ -262,11 +285,49 @@ void PacedVideoSender::OnNetworkChanged(uint32_t target_bitrate_bps, PacedSender::kDefaultPaceMultiplier * target_bitrate_bps / 1000, 0); } +const int kNoLimit = std::numeric_limits::max(); +const int kPacketSizeBytes = 1200; + +TcpSender::TcpSender(PacketProcessorListener* listener, + int flow_id, + int64_t offset_ms) + : TcpSender(listener, flow_id, offset_ms, kNoLimit) { +} + +TcpSender::TcpSender(PacketProcessorListener* listener, + int flow_id, + int64_t offset_ms, + int send_limit_bytes) + : PacketSender(listener, flow_id), + cwnd_(10), + ssthresh_(kNoLimit), + ack_received_(false), + last_acked_seq_num_(0), + next_sequence_number_(0), + offset_ms_(offset_ms), + last_reduction_time_ms_(-1), + last_rtt_ms_(0), + total_sent_bytes_(0), + send_limit_bytes_(send_limit_bytes), + running_(true), + last_generated_packets_ms_(0), + num_recent_sent_packets_(0), + bitrate_kbps_(0) { +} + void TcpSender::RunFor(int64_t time_ms, Packets* in_out) { if (clock_.TimeInMilliseconds() + time_ms < offset_ms_) { clock_.AdvanceTimeMilliseconds(time_ms); + if (running_) { + running_ = false; + } return; } + + if (!running_) { + running_ = true; + } + int64_t start_time_ms = clock_.TimeInMilliseconds(); BWE_TEST_LOGGING_CONTEXT("Sender"); BWE_TEST_LOGGING_CONTEXT(*flow_ids().begin()); @@ -277,9 +338,9 @@ void TcpSender::RunFor(int64_t time_ms, Packets* in_out) { // number of packets in_flight_ and the max number of packets in flight // (cwnd_). Therefore SendPackets() isn't directly dependent on time_ms. for (FeedbackPacket* fb : feedbacks) { - clock_.AdvanceTimeMilliseconds(fb->send_time_us() / 1000 - + clock_.AdvanceTimeMilliseconds(fb->send_time_ms() - clock_.TimeInMilliseconds()); - last_rtt_ms_ = fb->send_time_us() / 1000 - fb->latest_send_time_ms(); + last_rtt_ms_ = fb->send_time_ms() - fb->latest_send_time_ms(); UpdateCongestionControl(fb); SendPackets(in_out); } @@ -359,15 +420,46 @@ void TcpSender::HandleLoss() { Packets TcpSender::GeneratePackets(size_t num_packets) { Packets generated; + + UpdateSendBitrateEstimate(num_packets); + for (size_t i = 0; i < num_packets; ++i) { - generated.push_back(new MediaPacket(*flow_ids().begin(), - 1000 * clock_.TimeInMilliseconds(), - 1200, next_sequence_number_++)); + if ((total_sent_bytes_ + kPacketSizeBytes) > send_limit_bytes_) { + if (running_) { + running_ = false; + } + break; + } + generated.push_back( + new MediaPacket(*flow_ids().begin(), 1000 * clock_.TimeInMilliseconds(), + kPacketSizeBytes, next_sequence_number_++)); generated.back()->set_sender_timestamp_us( 1000 * clock_.TimeInMilliseconds()); + + total_sent_bytes_ += kPacketSizeBytes; } + return generated; } + +void TcpSender::UpdateSendBitrateEstimate(size_t num_packets) { + const int kTimeWindowMs = 500; + num_recent_sent_packets_ += num_packets; + + int64_t delta_ms = clock_.TimeInMilliseconds() - last_generated_packets_ms_; + if (delta_ms >= kTimeWindowMs) { + bitrate_kbps_ = + static_cast(8 * num_recent_sent_packets_ * kPacketSizeBytes) / + delta_ms; + last_generated_packets_ms_ = clock_.TimeInMilliseconds(); + num_recent_sent_packets_ = 0; + } +} + +uint32_t TcpSender::TargetBitrateKbps() { + return bitrate_kbps_; +} + } // namespace bwe } // namespace testing } // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h b/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h index 2e690c8377..8f0b3cdfcb 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h @@ -42,6 +42,8 @@ class PacketSender : public PacketProcessor { virtual int GetFeedbackIntervalMs() const = 0; void SetSenderTimestamps(Packets* in_out); + virtual uint32_t TargetBitrateKbps() { return 0; } + protected: SimulatedClock clock_; }; @@ -58,22 +60,29 @@ class VideoSender : public PacketSender, public BitrateObserver { virtual VideoSource* source() const { return source_; } + uint32_t TargetBitrateKbps() override; + // Implements BitrateObserver. void OnNetworkChanged(uint32_t target_bitrate_bps, uint8_t fraction_lost, int64_t rtt) override; + void Pause(); + void Resume(); + protected: void ProcessFeedbackAndGeneratePackets(int64_t time_ms, std::list* feedbacks, Packets* generated); + bool running_; VideoSource* source_; rtc::scoped_ptr bwe_; int64_t start_of_run_ms_; std::list modules_; private: + uint32_t previous_sending_bitrate_; DISALLOW_COPY_AND_ASSIGN(VideoSender); }; @@ -112,28 +121,24 @@ class PacedVideoSender : public VideoSender, public PacedSender::Callback { class TcpSender : public PacketSender { public: - TcpSender(PacketProcessorListener* listener, int flow_id, int64_t offset_ms) - : PacketSender(listener, flow_id), - cwnd_(10), - ssthresh_(std::numeric_limits::max()), - ack_received_(false), - last_acked_seq_num_(0), - next_sequence_number_(0), - offset_ms_(offset_ms), - last_reduction_time_ms_(-1), - last_rtt_ms_(0) {} - + TcpSender(PacketProcessorListener* listener, int flow_id, int64_t offset_ms); + TcpSender(PacketProcessorListener* listener, + int flow_id, + int64_t offset_ms, + int send_limit_bytes); virtual ~TcpSender() {} void RunFor(int64_t time_ms, Packets* in_out) override; int GetFeedbackIntervalMs() const override { return 10; } + uint32_t TargetBitrateKbps() override; + private: struct InFlight { public: InFlight(const MediaPacket& packet) : sequence_number(packet.header().sequenceNumber), - time_ms(packet.send_time_us() / 1000) {} + time_ms(packet.send_time_ms()) {} InFlight(uint16_t seq_num, int64_t now_ms) : sequence_number(seq_num), time_ms(now_ms) {} @@ -153,6 +158,7 @@ class TcpSender : public PacketSender { int TriggerTimeouts(); void HandleLoss(); Packets GeneratePackets(size_t num_packets); + void UpdateSendBitrateEstimate(size_t num_packets); float cwnd_; int ssthresh_; @@ -163,6 +169,12 @@ class TcpSender : public PacketSender { int64_t offset_ms_; int64_t last_reduction_time_ms_; int64_t last_rtt_ms_; + int total_sent_bytes_; + int send_limit_bytes_; // Initialized by default as kNoLimit. + bool running_; // Initialized by default as true. + int64_t last_generated_packets_ms_; + size_t num_recent_sent_packets_; + uint32_t bitrate_kbps_; }; } // namespace bwe } // namespace testing