diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index 983e89fa94..4422c33ee7 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -228,6 +228,7 @@ 'remote_bitrate_estimator/test/bwe_unittest.cc', 'remote_bitrate_estimator/test/metric_recorder_unittest.cc', 'remote_bitrate_estimator/test/estimators/nada_unittest.cc', + 'remote_bitrate_estimator/transport_feedback_adapter_unittest.cc', 'rtp_rtcp/source/mock/mock_rtp_payload_strategy.h', 'rtp_rtcp/source/byte_io_unittest.cc', 'rtp_rtcp/source/fec_receiver_unittest.cc', diff --git a/webrtc/modules/remote_bitrate_estimator/include/mock/mock_remote_bitrate_estimator.h b/webrtc/modules/remote_bitrate_estimator/include/mock/mock_remote_bitrate_estimator.h new file mode 100644 index 0000000000..7890fb72aa --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/include/mock/mock_remote_bitrate_estimator.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_MOCK_MOCK_REMOTE_BITRATE_ESTIMATOR_H_ +#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_MOCK_MOCK_REMOTE_BITRATE_ESTIMATOR_H_ + +#include + +#include "testing/gmock/include/gmock/gmock.h" +#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" + +namespace webrtc { + +class MockRemoteBitrateEstimator : public RemoteBitrateEstimator { + public: + MOCK_METHOD1(IncomingPacketFeedbackVector, + void(const std::vector&)); + MOCK_METHOD4(IncomingPacket, void(int64_t, size_t, const RTPHeader&, bool)); + MOCK_METHOD1(RemoveStream, void(unsigned int)); + MOCK_CONST_METHOD2(LatestEstimate, + bool(std::vector*, unsigned int*)); + MOCK_CONST_METHOD1(GetStats, bool(ReceiveBandwidthEstimatorStats*)); + + // From CallStatsObserver; + MOCK_METHOD2(OnRttUpdate, void(int64_t, int64_t)); + + // From Module. + MOCK_METHOD0(TimeUntilNextProcess, int64_t()); + MOCK_METHOD0(Process, int32_t()); +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_INCLUDE_MOCK_MOCK_REMOTE_BITRATE_ESTIMATOR_H_ diff --git a/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h b/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h index 057dfb8e97..f201aeb188 100644 --- a/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h +++ b/webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h @@ -19,6 +19,7 @@ #include "webrtc/common_types.h" #include "webrtc/modules/interface/module.h" #include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h" #include "webrtc/typedefs.h" namespace webrtc { @@ -58,34 +59,9 @@ struct ReceiveBandwidthEstimatorStats { std::vector recent_arrival_time_ms; }; -struct PacketInfo { - PacketInfo(int64_t arrival_time_ms, - int64_t send_time_ms, - uint16_t sequence_number, - size_t payload_size, - bool was_paced) - : arrival_time_ms(arrival_time_ms), - send_time_ms(send_time_ms), - sequence_number(sequence_number), - payload_size(payload_size), - was_paced(was_paced) {} - // Time corresponding to when the packet was received. Timestamped with the - // receiver's clock. - int64_t arrival_time_ms; - // Time corresponding to when the packet was sent, timestamped with the - // sender's clock. - int64_t send_time_ms; - // Packet identifier, incremented with 1 for every packet generated by the - // sender. - uint16_t sequence_number; - // Size of the packet excluding RTP headers. - size_t payload_size; - // True if the packet was paced out by the pacer. - bool was_paced; -}; - class RemoteBitrateEstimator : public CallStatsObserver, public Module { public: + static const int kDefaultMinBitrateBps = 30000; virtual ~RemoteBitrateEstimator() {} virtual void IncomingPacketFeedbackVector( diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi index 5f4ed85bd6..dbc5882456 100644 --- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi +++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator.gypi @@ -39,6 +39,8 @@ 'remote_estimator_proxy.cc', 'remote_estimator_proxy.h', 'send_time_history.cc', + 'transport_feedback_adapter.cc', + 'transport_feedback_adapter.h', 'test/bwe_test_logging.cc', 'test/bwe_test_logging.h', ], # source diff --git a/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.cc b/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.cc new file mode 100644 index 0000000000..c6e34f212d --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "webrtc/base/checks.h" +#include "webrtc/base/logging.h" +#include "webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_abs_send_time.h" +#include "webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.h" +#include "webrtc/modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" +#include "webrtc/modules/utility/interface/process_thread.h" + +namespace webrtc { + +const int64_t kNoTimestamp = -1; +const int64_t kSendTimeHistoryWindowMs = 10000; +const int64_t kBaseTimestampScaleFactor = + rtcp::TransportFeedback::kDeltaScaleFactor * (1 << 8); +const int64_t kBaseTimestampRangeSizeUs = kBaseTimestampScaleFactor * (1 << 24); + +TransportFeedbackAdapter::TransportFeedbackAdapter( + RtcpBandwidthObserver* bandwidth_observer, + Clock* clock, + ProcessThread* process_thread) + : send_time_history_(kSendTimeHistoryWindowMs), + rtcp_bandwidth_observer_(bandwidth_observer), + process_thread_(process_thread), + clock_(clock), + current_offset_ms_(kNoTimestamp), + last_timestamp_us_(kNoTimestamp) {} + +TransportFeedbackAdapter::~TransportFeedbackAdapter() { + if (bitrate_estimator_.get()) + process_thread_->DeRegisterModule(bitrate_estimator_.get()); +} + +void TransportFeedbackAdapter::SetBitrateEstimator( + RemoteBitrateEstimator* rbe) { + if (bitrate_estimator_.get() != rbe) { + bitrate_estimator_.reset(rbe); + process_thread_->RegisterModule(rbe); + } +} + +void TransportFeedbackAdapter::OnPacketSent(const PacketInfo& info) { + rtc::CritScope cs(&lock_); + send_time_history_.AddAndRemoveOld(info); +} + +void TransportFeedbackAdapter::OnTransportFeedback( + const rtcp::TransportFeedback& feedback) { + int64_t timestamp_us = feedback.GetBaseTimeUs(); + // Add timestamp deltas to a local time base selected on first packet arrival. + // This won't be the true time base, but makes it easier to manually inspect + // time stamps. + if (last_timestamp_us_ == kNoTimestamp) { + current_offset_ms_ = clock_->TimeInMilliseconds(); + } else { + int64_t delta = timestamp_us - last_timestamp_us_; + + // Detect and compensate for wrap-arounds in base time. + if (std::abs(delta - kBaseTimestampRangeSizeUs) < std::abs(delta)) { + delta -= kBaseTimestampRangeSizeUs; // Wrap backwards. + } else if (std::abs(delta + kBaseTimestampRangeSizeUs) < std::abs(delta)) { + delta += kBaseTimestampRangeSizeUs; // Wrap forwards. + } + + current_offset_ms_ += delta / 1000; + } + last_timestamp_us_ = timestamp_us; + + uint16_t sequence_number = feedback.GetBaseSequence(); + std::vector delta_vec = feedback.GetReceiveDeltasUs(); + auto delta_it = delta_vec.begin(); + std::vector packet_feedback_vector; + packet_feedback_vector.reserve(delta_vec.size()); + + { + rtc::CritScope cs(&lock_); + size_t failed_lookups = 0; + int64_t offset_us = 0; + for (auto symbol : feedback.GetStatusVector()) { + if (symbol != rtcp::TransportFeedback::StatusSymbol::kNotReceived) { + DCHECK(delta_it != delta_vec.end()); + offset_us += *(delta_it++); + int64_t timestamp_ms = current_offset_ms_ + (offset_us / 1000); + PacketInfo info = {timestamp_ms, 0, sequence_number, 0, false}; + if (send_time_history_.GetInfo(&info, true)) { + packet_feedback_vector.push_back(info); + } else { + ++failed_lookups; + } + } + ++sequence_number; + } + DCHECK(delta_it == delta_vec.end()); + if (failed_lookups > 0) { + LOG(LS_WARNING) << "Failed to lookup send time for " << failed_lookups + << " packet" << (failed_lookups > 1 ? "s" : "") + << ". Send time history too small?"; + } + } + DCHECK(bitrate_estimator_.get() != nullptr); + bitrate_estimator_->IncomingPacketFeedbackVector(packet_feedback_vector); +} + +void TransportFeedbackAdapter::OnReceiveBitrateChanged( + const std::vector& ssrcs, + unsigned int bitrate) { + rtcp_bandwidth_observer_->OnReceivedEstimatedBitrate(bitrate); +} + +void TransportFeedbackAdapter::OnRttUpdate(int64_t avg_rtt_ms, + int64_t max_rtt_ms) { + DCHECK(bitrate_estimator_.get() != nullptr); + bitrate_estimator_->OnRttUpdate(avg_rtt_ms, max_rtt_ms); +} + +} // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.h b/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.h new file mode 100644 index 0000000000..2969c29046 --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TRANSPORT_FEEDBACK_ADAPTER_H_ +#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TRANSPORT_FEEDBACK_ADAPTER_H_ + +#include + +#include "webrtc/base/criticalsection.h" +#include "webrtc/base/thread_annotations.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h" +#include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "webrtc/modules/remote_bitrate_estimator/include/send_time_history.h" + +namespace webrtc { + +class ProcessThread; + +class TransportFeedbackAdapter : public TransportFeedbackObserver, + public CallStatsObserver, + public RemoteBitrateObserver { + public: + TransportFeedbackAdapter(RtcpBandwidthObserver* bandwidth_observer, + Clock* clock, + ProcessThread* process_thread); + virtual ~TransportFeedbackAdapter(); + + void OnPacketSent(const PacketInfo& info) override; + + void OnTransportFeedback(const rtcp::TransportFeedback& feedback) override; + + void SetBitrateEstimator(RemoteBitrateEstimator* rbe); + + private: + void OnReceiveBitrateChanged(const std::vector& ssrcs, + unsigned int bitrate) override; + void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; + + rtc::CriticalSection lock_; + SendTimeHistory send_time_history_ GUARDED_BY(&lock_); + rtc::scoped_ptr rtcp_bandwidth_observer_; + rtc::scoped_ptr bitrate_estimator_; + ProcessThread* const process_thread_; + Clock* const clock_; + int64_t current_offset_ms_; + int64_t last_timestamp_us_; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TRANSPORT_FEEDBACK_ADAPTER_H_ diff --git a/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter_unittest.cc b/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter_unittest.cc new file mode 100644 index 0000000000..1bf4b1ec3e --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter_unittest.cc @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +#include "webrtc/base/checks.h" +#include "webrtc/base/scoped_ptr.h" +#include "webrtc/modules/remote_bitrate_estimator/include/mock/mock_remote_bitrate_estimator.h" +#include "webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.h" +#include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h" +#include "webrtc/modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" +#include "webrtc/modules/utility/interface/mock/mock_process_thread.h" +#include "webrtc/system_wrappers/interface/clock.h" + +using ::testing::_; +using ::testing::Invoke; + +namespace webrtc { +namespace test { + +class TransportFeedbackAdapterTest : public ::testing::Test { + public: + TransportFeedbackAdapterTest() + : clock_(0), + bitrate_estimator_(nullptr), + receiver_estimated_bitrate_(0) {} + + virtual ~TransportFeedbackAdapterTest() {} + + virtual void SetUp() { + adapter_.reset(new TransportFeedbackAdapter( + new RtcpBandwidthObserverAdapter(this), &clock_, &process_thread_)); + + bitrate_estimator_ = new MockRemoteBitrateEstimator(); + EXPECT_CALL(process_thread_, RegisterModule(bitrate_estimator_)).Times(1); + adapter_->SetBitrateEstimator(bitrate_estimator_); + } + + virtual void TearDown() { + EXPECT_CALL(process_thread_, DeRegisterModule(bitrate_estimator_)).Times(1); + adapter_.reset(); + } + + protected: + // Proxy class used since TransportFeedbackAdapter will own the instance + // passed at construction. + class RtcpBandwidthObserverAdapter : public RtcpBandwidthObserver { + public: + explicit RtcpBandwidthObserverAdapter(TransportFeedbackAdapterTest* owner) + : owner_(owner) {} + + void OnReceivedEstimatedBitrate(uint32_t bitrate) override { + owner_->receiver_estimated_bitrate_ = bitrate; + } + + void OnReceivedRtcpReceiverReport(const ReportBlockList& report_blocks, + int64_t rtt, + int64_t now_ms) override { + RTC_NOTREACHED(); + } + + TransportFeedbackAdapterTest* const owner_; + }; + + void OnReceivedEstimatedBitrate(uint32_t bitrate) {} + + void OnReceivedRtcpReceiverReport(const ReportBlockList& report_blocks, + int64_t rtt, + int64_t now_ms) {} + + void ComparePacketVectors(const std::vector& truth, + const std::vector& input) { + ASSERT_EQ(truth.size(), input.size()); + size_t len = truth.size(); + // truth contains the input data for the test, and input is what will be + // sent to the bandwidth estimator. truth.arrival_tims_ms is used to + // populate the transport feedback messages. As these times may be changed + // (because of resolution limits in the packets, and because of the time + // base adjustment performed by the TransportFeedbackAdapter at the first + // packet, the truth[x].arrival_time and input[x].arrival_time may not be + // equal. However, the difference must be the same for all x. + int64_t arrival_time_delta = + truth[0].arrival_time_ms - input[0].arrival_time_ms; + for (size_t i = 0; i < len; ++i) { + EXPECT_EQ(truth[i].arrival_time_ms, + input[i].arrival_time_ms + arrival_time_delta); + EXPECT_EQ(truth[i].send_time_ms, input[i].send_time_ms); + EXPECT_EQ(truth[i].sequence_number, input[i].sequence_number); + EXPECT_EQ(truth[i].payload_size, input[i].payload_size); + EXPECT_EQ(truth[i].was_paced, input[i].was_paced); + } + } + + // Utility method, to reset arrival_time_ms before adding send time. + void OnPacketSent(PacketInfo info) { + info.arrival_time_ms = 0; + adapter_->OnPacketSent(info); + } + + SimulatedClock clock_; + MockProcessThread process_thread_; + MockRemoteBitrateEstimator* bitrate_estimator_; + rtc::scoped_ptr adapter_; + + uint32_t receiver_estimated_bitrate_; +}; + +TEST_F(TransportFeedbackAdapterTest, AdaptsFeedbackAndPopulatesSendTimes) { + std::vector packets; + packets.push_back(PacketInfo(100, 200, 0, 1500, true)); + packets.push_back(PacketInfo(110, 210, 1, 1500, true)); + packets.push_back(PacketInfo(120, 220, 2, 1500, true)); + packets.push_back(PacketInfo(130, 230, 3, 1500, true)); + packets.push_back(PacketInfo(140, 240, 4, 1500, true)); + + for (const PacketInfo& packet : packets) + OnPacketSent(packet); + + rtcp::TransportFeedback feedback; + feedback.WithBase(packets[0].sequence_number, + packets[0].arrival_time_ms * 1000); + + for (const PacketInfo& packet : packets) { + EXPECT_TRUE(feedback.WithReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + + feedback.Build(); + + EXPECT_CALL(*bitrate_estimator_, IncomingPacketFeedbackVector(_)) + .Times(1) + .WillOnce(Invoke( + [packets, this](const std::vector& feedback_vector) { + ComparePacketVectors(packets, feedback_vector); + })); + adapter_->OnTransportFeedback(feedback); +} + +TEST_F(TransportFeedbackAdapterTest, HandlesDroppedPackets) { + std::vector packets; + packets.push_back(PacketInfo(100, 200, 0, 1500, true)); + packets.push_back(PacketInfo(110, 210, 1, 1500, true)); + packets.push_back(PacketInfo(120, 220, 2, 1500, true)); + packets.push_back(PacketInfo(130, 230, 3, 1500, true)); + packets.push_back(PacketInfo(140, 240, 4, 1500, true)); + + const uint16_t kSendSideDropBefore = 1; + const uint16_t kReceiveSideDropAfter = 3; + + for (const PacketInfo& packet : packets) { + if (packet.sequence_number >= kSendSideDropBefore) + OnPacketSent(packet); + } + + rtcp::TransportFeedback feedback; + feedback.WithBase(packets[0].sequence_number, + packets[0].arrival_time_ms * 1000); + + for (const PacketInfo& packet : packets) { + if (packet.sequence_number <= kReceiveSideDropAfter) { + EXPECT_TRUE(feedback.WithReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + } + + feedback.Build(); + + std::vector expected_packets( + packets.begin() + kSendSideDropBefore, + packets.begin() + kReceiveSideDropAfter + 1); + + EXPECT_CALL(*bitrate_estimator_, IncomingPacketFeedbackVector(_)) + .Times(1) + .WillOnce(Invoke([expected_packets, + this](const std::vector& feedback_vector) { + ComparePacketVectors(expected_packets, feedback_vector); + })); + adapter_->OnTransportFeedback(feedback); +} + +TEST_F(TransportFeedbackAdapterTest, SendTimeWrapsBothWays) { + int64_t kHighArrivalTimeMs = rtcp::TransportFeedback::kDeltaScaleFactor * + static_cast(1 << 8) * + static_cast((1 << 23) - 1) / 1000; + std::vector packets; + packets.push_back(PacketInfo(kHighArrivalTimeMs - 64, 200, 0, 1500, true)); + packets.push_back(PacketInfo(kHighArrivalTimeMs + 64, 210, 1, 1500, true)); + packets.push_back(PacketInfo(kHighArrivalTimeMs, 220, 2, 1500, true)); + + for (const PacketInfo& packet : packets) + OnPacketSent(packet); + + for (size_t i = 0; i < packets.size(); ++i) { + rtc::scoped_ptr feedback( + new rtcp::TransportFeedback()); + feedback->WithBase(packets[i].sequence_number, + packets[i].arrival_time_ms * 1000); + + EXPECT_TRUE(feedback->WithReceivedPacket( + packets[i].sequence_number, packets[i].arrival_time_ms * 1000)); + + rtc::scoped_ptr raw_packet = feedback->Build(); + feedback = rtcp::TransportFeedback::ParseFrom(raw_packet->Buffer(), + raw_packet->Length()); + + std::vector expected_packets; + expected_packets.push_back(packets[i]); + + EXPECT_CALL(*bitrate_estimator_, IncomingPacketFeedbackVector(_)) + .Times(1) + .WillOnce(Invoke([expected_packets, this]( + const std::vector& feedback_vector) { + ComparePacketVectors(expected_packets, feedback_vector); + })); + adapter_->OnTransportFeedback(*feedback.get()); + } +} + +TEST_F(TransportFeedbackAdapterTest, TimestampDeltas) { + std::vector sent_packets; + const int64_t kSmallDeltaUs = + rtcp::TransportFeedback::kDeltaScaleFactor * ((1 << 8) - 1); + const int64_t kLargePositiveDeltaUs = + rtcp::TransportFeedback::kDeltaScaleFactor * + std::numeric_limits::max(); + const int64_t kLargeNegativeDeltaUs = + rtcp::TransportFeedback::kDeltaScaleFactor * + std::numeric_limits::min(); + + PacketInfo info(100, 200, 0, 1500, true); + sent_packets.push_back(info); + + info.send_time_ms += kSmallDeltaUs / 1000; + info.arrival_time_ms += kSmallDeltaUs / 1000; + ++info.sequence_number; + sent_packets.push_back(info); + + info.send_time_ms += kLargePositiveDeltaUs / 1000; + info.arrival_time_ms += kLargePositiveDeltaUs / 1000; + ++info.sequence_number; + sent_packets.push_back(info); + + info.send_time_ms += kLargeNegativeDeltaUs / 1000; + info.arrival_time_ms += kLargeNegativeDeltaUs / 1000; + ++info.sequence_number; + sent_packets.push_back(info); + + // Too large, delta - will need two feedback messages. + info.send_time_ms += (kLargePositiveDeltaUs + 1000) / 1000; + info.arrival_time_ms += (kLargePositiveDeltaUs + 1000) / 1000; + ++info.sequence_number; + + // Packets will be added to send history. + for (const PacketInfo& packet : sent_packets) + OnPacketSent(packet); + OnPacketSent(info); + + // Create expected feedback and send into adapter. + rtc::scoped_ptr feedback( + new rtcp::TransportFeedback()); + feedback->WithBase(sent_packets[0].sequence_number, + sent_packets[0].arrival_time_ms * 1000); + + for (const PacketInfo& packet : sent_packets) { + EXPECT_TRUE(feedback->WithReceivedPacket(packet.sequence_number, + packet.arrival_time_ms * 1000)); + } + EXPECT_FALSE(feedback->WithReceivedPacket(info.sequence_number, + info.arrival_time_ms * 1000)); + + rtc::scoped_ptr raw_packet = feedback->Build(); + feedback = rtcp::TransportFeedback::ParseFrom(raw_packet->Buffer(), + raw_packet->Length()); + + std::vector received_feedback; + + EXPECT_TRUE(feedback.get() != nullptr); + EXPECT_CALL(*bitrate_estimator_, IncomingPacketFeedbackVector(_)) + .Times(1) + .WillOnce(Invoke([sent_packets, &received_feedback]( + const std::vector& feedback_vector) { + EXPECT_EQ(sent_packets.size(), feedback_vector.size()); + received_feedback = feedback_vector; + })); + adapter_->OnTransportFeedback(*feedback.get()); + + // Create a new feedback message and add the trailing item. + feedback.reset(new rtcp::TransportFeedback()); + feedback->WithBase(info.sequence_number, info.arrival_time_ms * 1000); + EXPECT_TRUE(feedback->WithReceivedPacket(info.sequence_number, + info.arrival_time_ms * 1000)); + raw_packet = feedback->Build(); + feedback = rtcp::TransportFeedback::ParseFrom(raw_packet->Buffer(), + raw_packet->Length()); + + EXPECT_TRUE(feedback.get() != nullptr); + EXPECT_CALL(*bitrate_estimator_, IncomingPacketFeedbackVector(_)) + .Times(1) + .WillOnce(Invoke( + [&received_feedback](const std::vector& feedback_vector) { + EXPECT_EQ(1u, feedback_vector.size()); + received_feedback.push_back(feedback_vector[0]); + })); + adapter_->OnTransportFeedback(*feedback.get()); + + sent_packets.push_back(info); + + ComparePacketVectors(sent_packets, received_feedback); +} + +} // namespace test +} // namespace webrtc diff --git a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h index 6aa4687f8e..15e0ffce8e 100644 --- a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h +++ b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h @@ -65,7 +65,7 @@ class RtpRtcp : public Module { Transport* outgoing_transport; RtcpIntraFrameObserver* intra_frame_callback; RtcpBandwidthObserver* bandwidth_callback; - SendTimeObserver* send_time_callback; + TransportFeedbackObserver* transport_feedback_callback; RtcpRttStats* rtt_stats; RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer; RtpAudioFeedback* audio_messages; diff --git a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h index 7642285caf..8f3500ef1a 100644 --- a/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h +++ b/webrtc/modules/rtp_rtcp/interface/rtp_rtcp_defines.h @@ -24,6 +24,9 @@ #define TIMEOUT_SEI_MESSAGES_MS 30000 // in milliseconds namespace webrtc { +namespace rtcp { +class TransportFeedback; +} const int kVideoPayloadTypeFrequency = 90000; @@ -293,15 +296,42 @@ class RtcpBandwidthObserver { virtual ~RtcpBandwidthObserver() {} }; -class SendTimeObserver { - public: - SendTimeObserver() {} - virtual ~SendTimeObserver() {} +struct PacketInfo { + PacketInfo(int64_t arrival_time_ms, + int64_t send_time_ms, + uint16_t sequence_number, + size_t payload_size, + bool was_paced) + : arrival_time_ms(arrival_time_ms), + send_time_ms(send_time_ms), + sequence_number(sequence_number), + payload_size(payload_size), + was_paced(was_paced) {} + // Time corresponding to when the packet was received. Timestamped with the + // receiver's clock. + int64_t arrival_time_ms; + // Time corresponding to when the packet was sent, timestamped with the + // sender's clock. + int64_t send_time_ms; + // Packet identifier, incremented with 1 for every packet generated by the + // sender. + uint16_t sequence_number; + // Size of the packet excluding RTP headers. + size_t payload_size; + // True if the packet was paced out by the pacer. + bool was_paced; +}; - // Transport-wide sequence number and timestamp (system time in milliseconds), - // of when the packet was put on the wire. - virtual void OnPacketSent(uint16_t transport_sequence_number, - int64_t send_time) = 0; +class TransportFeedbackObserver { + public: + TransportFeedbackObserver() {} + virtual ~TransportFeedbackObserver() {} + + // Note: Transport-wide sequence number as sequence number. Arrival time + // must be set to 0. + virtual void OnPacketSent(const PacketInfo& info) = 0; + + virtual void OnTransportFeedback(const rtcp::TransportFeedback& feedback) = 0; }; class RtcpRttStats { diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 523d000e6c..5d15195cde 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -35,6 +35,7 @@ RtpRtcp::Configuration::Configuration() outgoing_transport(nullptr), intra_frame_callback(nullptr), bandwidth_callback(nullptr), + transport_feedback_callback(nullptr), rtt_stats(nullptr), rtcp_packet_type_counter_observer(nullptr), audio_messages(NullObjectRtpAudioFeedback()), @@ -43,8 +44,7 @@ RtpRtcp::Configuration::Configuration() packet_router(nullptr), send_bitrate_observer(nullptr), send_frame_count_observer(nullptr), - send_side_delay_observer(nullptr) { -} + send_side_delay_observer(nullptr) {} RtpRtcp* RtpRtcp::CreateRtpRtcp(const RtpRtcp::Configuration& configuration) { if (configuration.clock) { @@ -67,7 +67,7 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) configuration.audio_messages, configuration.paced_sender, configuration.packet_router, - configuration.send_time_callback, + configuration.transport_feedback_callback, configuration.send_bitrate_observer, configuration.send_frame_count_observer, configuration.send_side_delay_observer), diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc index bc31212553..0b050b76e6 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc @@ -104,7 +104,7 @@ RTPSender::RTPSender(int32_t id, RtpAudioFeedback* audio_feedback, PacedSender* paced_sender, PacketRouter* packet_router, - SendTimeObserver* send_time_observer, + TransportFeedbackObserver* transport_feedback_observer, BitrateStatisticsObserver* bitrate_callback, FrameCountObserver* frame_count_observer, SendSideDelayObserver* send_side_delay_observer) @@ -122,7 +122,7 @@ RTPSender::RTPSender(int32_t id, video_(audio ? nullptr : new RTPSenderVideo(clock, this)), paced_sender_(paced_sender), packet_router_(packet_router), - send_time_observer_(send_time_observer), + transport_feedback_observer_(transport_feedback_observer), last_capture_time_ms_sent_(0), send_critsect_(CriticalSectionWrapper::CreateCriticalSection()), transport_(transport), @@ -676,8 +676,10 @@ size_t RTPSender::SendPadData(uint32_t timestamp, if (!SendPacketToNetwork(padding_packet, length)) break; - if (using_transport_seq) - send_time_observer_->OnPacketSent(transport_seq, now_ms); + if (using_transport_seq && transport_feedback_observer_) { + transport_feedback_observer_->OnPacketSent( + PacketInfo(0, now_ms, transport_seq, length, true)); + } bytes_sent += padding_bytes_in_packet; UpdateRtpStats(padding_packet, length, rtp_header, over_rtx, false); @@ -919,9 +921,10 @@ bool RTPSender::PrepareAndSendPacket(uint8_t* buffer, UpdateAbsoluteSendTime(buffer_to_send_ptr, length, rtp_header, now_ms); uint16_t transport_seq = 0; + // TODO(sprang): Potentially too much overhead in IsRegistered()? bool using_transport_seq = rtp_header_extension_map_.IsRegistered( kRtpExtensionTransportSequenceNumber) && - packet_router_; + packet_router_ && !is_retransmit; if (using_transport_seq) { transport_seq = UpdateTransportSequenceNumber(buffer_to_send_ptr, length, rtp_header); @@ -932,8 +935,10 @@ bool RTPSender::PrepareAndSendPacket(uint8_t* buffer, CriticalSectionScoped lock(send_critsect_.get()); media_has_been_sent_ = true; } - if (using_transport_seq) - send_time_observer_->OnPacketSent(transport_seq, now_ms); + if (using_transport_seq && transport_feedback_observer_) { + transport_feedback_observer_->OnPacketSent( + PacketInfo(0, now_ms, transport_seq, length, true)); + } UpdateRtpStats(buffer_to_send_ptr, length, rtp_header, send_over_rtx, is_retransmit); return ret; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.h b/webrtc/modules/rtp_rtcp/source/rtp_sender.h index 3b93ae40b6..6d11e8044d 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.h @@ -93,7 +93,7 @@ class RTPSender : public RTPSenderInterface { RtpAudioFeedback* audio_feedback, PacedSender* paced_sender, PacketRouter* packet_router, - SendTimeObserver* send_time_observer, + TransportFeedbackObserver* transport_feedback_callback, BitrateStatisticsObserver* bitrate_callback, FrameCountObserver* frame_count_observer, SendSideDelayObserver* send_side_delay_observer); @@ -396,7 +396,7 @@ class RTPSender : public RTPSenderInterface { PacedSender* const paced_sender_; PacketRouter* const packet_router_; - SendTimeObserver* const send_time_observer_; + TransportFeedbackObserver* const transport_feedback_observer_; int64_t last_capture_time_ms_sent_; rtc::scoped_ptr send_critsect_; diff --git a/webrtc/video_engine/vie_channel.cc b/webrtc/video_engine/vie_channel.cc index 29b9efe6d3..70c447622a 100644 --- a/webrtc/video_engine/vie_channel.cc +++ b/webrtc/video_engine/vie_channel.cc @@ -84,7 +84,7 @@ ViEChannel::ViEChannel(int32_t channel_id, ProcessThread* module_process_thread, RtcpIntraFrameObserver* intra_frame_observer, RtcpBandwidthObserver* bandwidth_observer, - SendTimeObserver* send_time_observer, + TransportFeedbackObserver* transport_feedback_observer, RemoteBitrateEstimator* remote_bitrate_estimator, RtcpRttStats* rtt_stats, PacedSender* paced_sender, @@ -112,7 +112,7 @@ ViEChannel::ViEChannel(int32_t channel_id, paced_sender_(paced_sender), packet_router_(packet_router), bandwidth_observer_(bandwidth_observer), - send_time_observer_(send_time_observer), + transport_feedback_observer_(transport_feedback_observer), nack_history_size_sender_(kSendSidePacketHistorySize), max_nack_reordering_threshold_(kMaxPacketAgeToNack), pre_render_callback_(NULL), @@ -127,12 +127,12 @@ ViEChannel::ViEChannel(int32_t channel_id, transport, sender ? intra_frame_observer_ : nullptr, sender ? bandwidth_observer_.get() : nullptr, - sender ? send_time_observer_ : nullptr, + transport_feedback_observer_, rtt_stats_, &rtcp_packet_type_counter_observer_, remote_bitrate_estimator, paced_sender_, - sender_ ? packet_router_ : nullptr, + packet_router_, &send_bitrate_observer_, &send_frame_count_observer_, &send_side_delay_observer_, @@ -153,8 +153,8 @@ int32_t ViEChannel::Init() { for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) rtp_rtcp->SetStorePacketsStatus(true, nack_history_size_sender_); } + packet_router_->AddRtpModule(rtp_rtcp_modules_[0]); if (sender_) { - packet_router_->AddRtpModule(rtp_rtcp_modules_[0]); std::list send_rtp_modules(1, rtp_rtcp_modules_[0]); send_payload_router_->SetSendingRtpModules(send_rtp_modules); DCHECK(!send_payload_router_->active()); @@ -181,10 +181,8 @@ ViEChannel::~ViEChannel() { module_process_thread_->DeRegisterModule(vcm_); module_process_thread_->DeRegisterModule(&vie_sync_); send_payload_router_->SetSendingRtpModules(std::list()); - if (sender_ && packet_router_) { - for (size_t i = 0; i < num_active_rtp_rtcp_modules_; ++i) - packet_router_->RemoveRtpModule(rtp_rtcp_modules_[i]); - } + for (size_t i = 0; i < num_active_rtp_rtcp_modules_; ++i) + packet_router_->RemoveRtpModule(rtp_rtcp_modules_[i]); for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) { module_process_thread_->DeRegisterModule(rtp_rtcp); delete rtp_rtcp; @@ -406,14 +404,12 @@ int32_t ViEChannel::SetSendCodec(const VideoCodec& video_codec, // Deregister previously registered modules. for (size_t i = num_active_modules; i < num_prev_active_modules; ++i) { module_process_thread_->DeRegisterModule(rtp_rtcp_modules_[i]); - if (sender_ && packet_router_) - packet_router_->RemoveRtpModule(rtp_rtcp_modules_[i]); + packet_router_->RemoveRtpModule(rtp_rtcp_modules_[i]); } // Register new active modules. for (size_t i = num_prev_active_modules; i < num_active_modules; ++i) { module_process_thread_->RegisterModule(rtp_rtcp_modules_[i]); - if (sender_ && packet_router_) - packet_router_->AddRtpModule(rtp_rtcp_modules_[i]); + packet_router_->AddRtpModule(rtp_rtcp_modules_[i]); } return 0; } @@ -435,6 +431,7 @@ int32_t ViEChannel::SetReceiveCodec(const VideoCodec& video_codec) { return 0; } + int32_t ViEChannel::RegisterExternalDecoder(const uint8_t pl_type, VideoDecoder* decoder, bool buffered_rendering, @@ -1026,6 +1023,7 @@ CallStatsObserver* ViEChannel::GetStatsObserver() { int32_t ViEChannel::FrameToRender(VideoFrame& video_frame) { // NOLINT CriticalSectionScoped cs(crit_.get()); + if (pre_render_callback_ != NULL) pre_render_callback_->FrameCallback(&video_frame); @@ -1142,7 +1140,7 @@ std::vector ViEChannel::CreateRtpRtcpModules( Transport* outgoing_transport, RtcpIntraFrameObserver* intra_frame_callback, RtcpBandwidthObserver* bandwidth_callback, - SendTimeObserver* send_time_callback, + TransportFeedbackObserver* transport_feedback_callback, RtcpRttStats* rtt_stats, RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, RemoteBitrateEstimator* remote_bitrate_estimator, @@ -1170,7 +1168,7 @@ std::vector ViEChannel::CreateRtpRtcpModules( configuration.send_frame_count_observer = send_frame_count_observer; configuration.send_side_delay_observer = send_side_delay_observer; configuration.bandwidth_callback = bandwidth_callback; - configuration.send_time_callback = send_time_callback; + configuration.transport_feedback_callback = transport_feedback_callback; std::vector modules; for (size_t i = 0; i < num_modules; ++i) { diff --git a/webrtc/video_engine/vie_channel.h b/webrtc/video_engine/vie_channel.h index b93dc0478c..85f86d09fa 100644 --- a/webrtc/video_engine/vie_channel.h +++ b/webrtc/video_engine/vie_channel.h @@ -72,7 +72,7 @@ class ViEChannel : public VCMFrameTypeCallback, ProcessThread* module_process_thread, RtcpIntraFrameObserver* intra_frame_observer, RtcpBandwidthObserver* bandwidth_observer, - SendTimeObserver* send_time_observer, + TransportFeedbackObserver* transport_feedback_observer, RemoteBitrateEstimator* remote_bitrate_estimator, RtcpRttStats* rtt_stats, PacedSender* paced_sender, @@ -302,7 +302,7 @@ class ViEChannel : public VCMFrameTypeCallback, Transport* outgoing_transport, RtcpIntraFrameObserver* intra_frame_callback, RtcpBandwidthObserver* bandwidth_callback, - SendTimeObserver* send_time_observer, + TransportFeedbackObserver* transport_feedback_callback, RtcpRttStats* rtt_stats, RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, RemoteBitrateEstimator* remote_bitrate_estimator, @@ -441,7 +441,7 @@ class ViEChannel : public VCMFrameTypeCallback, PacketRouter* const packet_router_; const rtc::scoped_ptr bandwidth_observer_; - SendTimeObserver* const send_time_observer_; + TransportFeedbackObserver* const transport_feedback_observer_; rtc::scoped_ptr decode_thread_; diff --git a/webrtc/video_engine/vie_channel_group.cc b/webrtc/video_engine/vie_channel_group.cc index 183e08fed4..25ef7dd3e5 100644 --- a/webrtc/video_engine/vie_channel_group.cc +++ b/webrtc/video_engine/vie_channel_group.cc @@ -18,6 +18,7 @@ #include "webrtc/modules/remote_bitrate_estimator/include/send_time_history.h" #include "webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_abs_send_time.h" #include "webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimator_single_stream.h" +#include "webrtc/modules/remote_bitrate_estimator/transport_feedback_adapter.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h" #include "webrtc/modules/utility/interface/process_thread.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" @@ -34,7 +35,6 @@ namespace webrtc { namespace { static const uint32_t kTimeOffsetSwitchThreshold = 30; -static const uint32_t kMinBitrateBps = 30000; class WrappingBitrateEstimator : public RemoteBitrateEstimator { public: @@ -42,7 +42,7 @@ class WrappingBitrateEstimator : public RemoteBitrateEstimator { : observer_(observer), clock_(clock), crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), - min_bitrate_bps_(kMinBitrateBps), + min_bitrate_bps_(RemoteBitrateEstimator::kDefaultMinBitrateBps), rbe_(new RemoteBitrateEstimatorSingleStream(observer_, clock_, min_bitrate_bps_)), @@ -139,21 +139,8 @@ class WrappingBitrateEstimator : public RemoteBitrateEstimator { DISALLOW_IMPLICIT_CONSTRUCTORS(WrappingBitrateEstimator); }; -static const int64_t kSendTimeHistoryWindowMs = 2000; - } // namespace -class AdaptedSendTimeHistory : public SendTimeHistory, public SendTimeObserver { - public: - AdaptedSendTimeHistory() : SendTimeHistory(kSendTimeHistoryWindowMs) {} - virtual ~AdaptedSendTimeHistory() {} - - void OnPacketSent(uint16_t sequence_number, int64_t send_time) override { - PacketInfo info(0, send_time, sequence_number, 0, false); - SendTimeHistory::AddAndRemoveOld(info); - } -}; - ChannelGroup::ChannelGroup(ProcessThread* process_thread) : remb_(new VieRemb()), bitrate_allocator_(new BitrateAllocator()), @@ -172,8 +159,7 @@ ChannelGroup::ChannelGroup(ProcessThread* process_thread) // construction. bitrate_controller_( BitrateController::CreateBitrateController(Clock::GetRealTimeClock(), - this)), - send_time_history_(new AdaptedSendTimeHistory()) { + this)) { remote_bitrate_estimator_.reset(new WrappingBitrateEstimator( remb_.get(), Clock::GetRealTimeClock())); @@ -250,10 +236,9 @@ bool ChannelGroup::CreateChannel(int channel_id, rtc::scoped_ptr channel(new ViEChannel( channel_id, engine_id, number_of_cores, transport, process_thread_, encoder_state_feedback_->GetRtcpIntraFrameObserver(), - bitrate_controller_->CreateRtcpBandwidthObserver(), - send_time_history_.get(), remote_bitrate_estimator_.get(), - call_stats_->rtcp_rtt_stats(), pacer_.get(), packet_router_.get(), - max_rtp_streams, sender)); + bitrate_controller_->CreateRtcpBandwidthObserver(), nullptr, + remote_bitrate_estimator_.get(), call_stats_->rtcp_rtt_stats(), + pacer_.get(), packet_router_.get(), max_rtp_streams, sender)); if (channel->Init() != 0) { return false; } diff --git a/webrtc/video_engine/vie_channel_group.h b/webrtc/video_engine/vie_channel_group.h index 3c8008fdb1..f1faa80cc6 100644 --- a/webrtc/video_engine/vie_channel_group.h +++ b/webrtc/video_engine/vie_channel_group.h @@ -22,7 +22,6 @@ namespace webrtc { -class AdaptedSendTimeHistory; class BitrateAllocator; class CallStats; class Config; @@ -97,11 +96,10 @@ class ChannelGroup : public BitrateObserver { EncoderMap vie_encoder_map_ GUARDED_BY(encoder_map_crit_); // Registered at construct time and assumed to outlive this class. - ProcessThread* process_thread_; + ProcessThread* const process_thread_; rtc::scoped_ptr pacer_thread_; rtc::scoped_ptr bitrate_controller_; - rtc::scoped_ptr send_time_history_; }; } // namespace webrtc