diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 028b0d5940..b084577243 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -74,6 +74,8 @@ rtc_library("nack_module") { "histogram.h", "nack_module.cc", "nack_module.h", + "nack_module2.cc", + "nack_module2.h", ] deps = [ @@ -876,6 +878,7 @@ if (rtc_include_tests) { "jitter_buffer_unittest.cc", "jitter_estimator_tests.cc", "loss_notification_controller_unittest.cc", + "nack_module2_unittest.cc", "nack_module_unittest.cc", "packet_buffer_unittest.cc", "receiver_unittest.cc", diff --git a/modules/video_coding/nack_module2.cc b/modules/video_coding/nack_module2.cc new file mode 100644 index 0000000000..267eaebb7a --- /dev/null +++ b/modules/video_coding/nack_module2.cc @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/nack_module2.h" + +#include +#include + +#include "api/units/timestamp.h" +#include "modules/utility/include/process_thread.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { +const int kMaxPacketAge = 10000; +const int kMaxNackPackets = 1000; +const int kDefaultRttMs = 100; +const int kMaxNackRetries = 10; +const int kProcessFrequency = 50; +const int kProcessIntervalMs = 1000 / kProcessFrequency; +const int kMaxReorderedPackets = 128; +const int kNumReorderingBuckets = 10; +const int kDefaultSendNackDelayMs = 0; + +int64_t GetSendNackDelay() { + int64_t delay_ms = strtol( + webrtc::field_trial::FindFullName("WebRTC-SendNackDelayMs").c_str(), + nullptr, 10); + if (delay_ms > 0 && delay_ms <= 20) { + RTC_LOG(LS_INFO) << "SendNackDelay is set to " << delay_ms; + return delay_ms; + } + return kDefaultSendNackDelayMs; +} +} // namespace + +NackModule2::NackInfo::NackInfo() + : seq_num(0), send_at_seq_num(0), sent_at_time(-1), retries(0) {} + +NackModule2::NackInfo::NackInfo(uint16_t seq_num, + uint16_t send_at_seq_num, + int64_t created_at_time) + : seq_num(seq_num), + send_at_seq_num(send_at_seq_num), + created_at_time(created_at_time), + sent_at_time(-1), + retries(0) {} + +NackModule2::BackoffSettings::BackoffSettings(TimeDelta min_retry, + TimeDelta max_rtt, + double base) + : min_retry_interval(min_retry), max_rtt(max_rtt), base(base) {} + +absl::optional +NackModule2::BackoffSettings::ParseFromFieldTrials() { + // Matches magic number in RTPSender::OnReceivedNack(). + const TimeDelta kDefaultMinRetryInterval = TimeDelta::Millis(5); + // Upper bound on link-delay considered for exponential backoff. + // Selected so that cumulative delay with 1.25 base and 10 retries ends up + // below 3s, since above that there will be a FIR generated instead. + const TimeDelta kDefaultMaxRtt = TimeDelta::Millis(160); + // Default base for exponential backoff, adds 25% RTT delay for each retry. + const double kDefaultBase = 1.25; + + FieldTrialParameter enabled("enabled", false); + FieldTrialParameter min_retry("min_retry", + kDefaultMinRetryInterval); + FieldTrialParameter max_rtt("max_rtt", kDefaultMaxRtt); + FieldTrialParameter base("base", kDefaultBase); + ParseFieldTrial({&enabled, &min_retry, &max_rtt, &base}, + field_trial::FindFullName("WebRTC-ExponentialNackBackoff")); + + if (enabled) { + return NackModule2::BackoffSettings(min_retry.Get(), max_rtt.Get(), + base.Get()); + } + return absl::nullopt; +} + +NackModule2::NackModule2(Clock* clock, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender) + : clock_(clock), + nack_sender_(nack_sender), + keyframe_request_sender_(keyframe_request_sender), + reordering_histogram_(kNumReorderingBuckets, kMaxReorderedPackets), + initialized_(false), + rtt_ms_(kDefaultRttMs), + newest_seq_num_(0), + next_process_time_ms_(-1), + send_nack_delay_ms_(GetSendNackDelay()), + backoff_settings_(BackoffSettings::ParseFromFieldTrials()) { + RTC_DCHECK(clock_); + RTC_DCHECK(nack_sender_); + RTC_DCHECK(keyframe_request_sender_); +} + +int NackModule2::OnReceivedPacket(uint16_t seq_num, bool is_keyframe) { + return OnReceivedPacket(seq_num, is_keyframe, false); +} + +int NackModule2::OnReceivedPacket(uint16_t seq_num, + bool is_keyframe, + bool is_recovered) { + rtc::CritScope lock(&crit_); + // TODO(philipel): When the packet includes information whether it is + // retransmitted or not, use that value instead. For + // now set it to true, which will cause the reordering + // statistics to never be updated. + bool is_retransmitted = true; + + if (!initialized_) { + newest_seq_num_ = seq_num; + if (is_keyframe) + keyframe_list_.insert(seq_num); + initialized_ = true; + return 0; + } + + // Since the |newest_seq_num_| is a packet we have actually received we know + // that packet has never been Nacked. + if (seq_num == newest_seq_num_) + return 0; + + if (AheadOf(newest_seq_num_, seq_num)) { + // An out of order packet has been received. + auto nack_list_it = nack_list_.find(seq_num); + int nacks_sent_for_packet = 0; + if (nack_list_it != nack_list_.end()) { + nacks_sent_for_packet = nack_list_it->second.retries; + nack_list_.erase(nack_list_it); + } + if (!is_retransmitted) + UpdateReorderingStatistics(seq_num); + return nacks_sent_for_packet; + } + + // Keep track of new keyframes. + if (is_keyframe) + keyframe_list_.insert(seq_num); + + // And remove old ones so we don't accumulate keyframes. + auto it = keyframe_list_.lower_bound(seq_num - kMaxPacketAge); + if (it != keyframe_list_.begin()) + keyframe_list_.erase(keyframe_list_.begin(), it); + + if (is_recovered) { + recovered_list_.insert(seq_num); + + // Remove old ones so we don't accumulate recovered packets. + auto it = recovered_list_.lower_bound(seq_num - kMaxPacketAge); + if (it != recovered_list_.begin()) + recovered_list_.erase(recovered_list_.begin(), it); + + // Do not send nack for packets recovered by FEC or RTX. + return 0; + } + + AddPacketsToNack(newest_seq_num_ + 1, seq_num); + newest_seq_num_ = seq_num; + + // Are there any nacks that are waiting for this seq_num. + std::vector nack_batch = GetNackBatch(kSeqNumOnly); + if (!nack_batch.empty()) { + // This batch of NACKs is triggered externally; the initiator can + // batch them with other feedback messages. + nack_sender_->SendNack(nack_batch, /*buffering_allowed=*/true); + } + + return 0; +} + +void NackModule2::ClearUpTo(uint16_t seq_num) { + rtc::CritScope lock(&crit_); + nack_list_.erase(nack_list_.begin(), nack_list_.lower_bound(seq_num)); + keyframe_list_.erase(keyframe_list_.begin(), + keyframe_list_.lower_bound(seq_num)); + recovered_list_.erase(recovered_list_.begin(), + recovered_list_.lower_bound(seq_num)); +} + +void NackModule2::UpdateRtt(int64_t rtt_ms) { + rtc::CritScope lock(&crit_); + rtt_ms_ = rtt_ms; +} + +void NackModule2::Clear() { + rtc::CritScope lock(&crit_); + nack_list_.clear(); + keyframe_list_.clear(); + recovered_list_.clear(); +} + +int64_t NackModule2::TimeUntilNextProcess() { + return std::max(next_process_time_ms_ - clock_->TimeInMilliseconds(), + 0); +} + +void NackModule2::Process() { + if (nack_sender_) { + std::vector nack_batch; + { + rtc::CritScope lock(&crit_); + nack_batch = GetNackBatch(kTimeOnly); + } + + if (!nack_batch.empty()) { + // This batch of NACKs is triggered externally; there is no external + // initiator who can batch them with other feedback messages. + nack_sender_->SendNack(nack_batch, /*buffering_allowed=*/false); + } + } + + // Update the next_process_time_ms_ in intervals to achieve + // the targeted frequency over time. Also add multiple intervals + // in case of a skip in time as to not make uneccessary + // calls to Process in order to catch up. + int64_t now_ms = clock_->TimeInMilliseconds(); + if (next_process_time_ms_ == -1) { + next_process_time_ms_ = now_ms + kProcessIntervalMs; + } else { + next_process_time_ms_ = next_process_time_ms_ + kProcessIntervalMs + + (now_ms - next_process_time_ms_) / + kProcessIntervalMs * kProcessIntervalMs; + } +} + +bool NackModule2::RemovePacketsUntilKeyFrame() { + while (!keyframe_list_.empty()) { + auto it = nack_list_.lower_bound(*keyframe_list_.begin()); + + if (it != nack_list_.begin()) { + // We have found a keyframe that actually is newer than at least one + // packet in the nack list. + nack_list_.erase(nack_list_.begin(), it); + return true; + } + + // If this keyframe is so old it does not remove any packets from the list, + // remove it from the list of keyframes and try the next keyframe. + keyframe_list_.erase(keyframe_list_.begin()); + } + return false; +} + +void NackModule2::AddPacketsToNack(uint16_t seq_num_start, + uint16_t seq_num_end) { + // Remove old packets. + auto it = nack_list_.lower_bound(seq_num_end - kMaxPacketAge); + nack_list_.erase(nack_list_.begin(), it); + + // If the nack list is too large, remove packets from the nack list until + // the latest first packet of a keyframe. If the list is still too large, + // clear it and request a keyframe. + uint16_t num_new_nacks = ForwardDiff(seq_num_start, seq_num_end); + if (nack_list_.size() + num_new_nacks > kMaxNackPackets) { + while (RemovePacketsUntilKeyFrame() && + nack_list_.size() + num_new_nacks > kMaxNackPackets) { + } + + if (nack_list_.size() + num_new_nacks > kMaxNackPackets) { + nack_list_.clear(); + RTC_LOG(LS_WARNING) << "NACK list full, clearing NACK" + " list and requesting keyframe."; + keyframe_request_sender_->RequestKeyFrame(); + return; + } + } + + for (uint16_t seq_num = seq_num_start; seq_num != seq_num_end; ++seq_num) { + // Do not send nack for packets that are already recovered by FEC or RTX + if (recovered_list_.find(seq_num) != recovered_list_.end()) + continue; + NackInfo nack_info(seq_num, seq_num + WaitNumberOfPackets(0.5), + clock_->TimeInMilliseconds()); + RTC_DCHECK(nack_list_.find(seq_num) == nack_list_.end()); + nack_list_[seq_num] = nack_info; + } +} + +std::vector NackModule2::GetNackBatch(NackFilterOptions options) { + bool consider_seq_num = options != kTimeOnly; + bool consider_timestamp = options != kSeqNumOnly; + Timestamp now = clock_->CurrentTime(); + std::vector nack_batch; + auto it = nack_list_.begin(); + while (it != nack_list_.end()) { + TimeDelta resend_delay = TimeDelta::Millis(rtt_ms_); + if (backoff_settings_) { + resend_delay = + std::max(resend_delay, backoff_settings_->min_retry_interval); + if (it->second.retries > 1) { + TimeDelta exponential_backoff = + std::min(TimeDelta::Millis(rtt_ms_), backoff_settings_->max_rtt) * + std::pow(backoff_settings_->base, it->second.retries - 1); + resend_delay = std::max(resend_delay, exponential_backoff); + } + } + + bool delay_timed_out = + now.ms() - it->second.created_at_time >= send_nack_delay_ms_; + bool nack_on_rtt_passed = + now.ms() - it->second.sent_at_time >= resend_delay.ms(); + bool nack_on_seq_num_passed = + it->second.sent_at_time == -1 && + AheadOrAt(newest_seq_num_, it->second.send_at_seq_num); + if (delay_timed_out && ((consider_seq_num && nack_on_seq_num_passed) || + (consider_timestamp && nack_on_rtt_passed))) { + nack_batch.emplace_back(it->second.seq_num); + ++it->second.retries; + it->second.sent_at_time = now.ms(); + if (it->second.retries >= kMaxNackRetries) { + RTC_LOG(LS_WARNING) << "Sequence number " << it->second.seq_num + << " removed from NACK list due to max retries."; + it = nack_list_.erase(it); + } else { + ++it; + } + continue; + } + ++it; + } + return nack_batch; +} + +void NackModule2::UpdateReorderingStatistics(uint16_t seq_num) { + RTC_DCHECK(AheadOf(newest_seq_num_, seq_num)); + uint16_t diff = ReverseDiff(newest_seq_num_, seq_num); + reordering_histogram_.Add(diff); +} + +int NackModule2::WaitNumberOfPackets(float probability) const { + if (reordering_histogram_.NumValues() == 0) + return 0; + return reordering_histogram_.InverseCdf(probability); +} + +} // namespace webrtc diff --git a/modules/video_coding/nack_module2.h b/modules/video_coding/nack_module2.h new file mode 100644 index 0000000000..6518f32bb6 --- /dev/null +++ b/modules/video_coding/nack_module2.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_NACK_MODULE2_H_ +#define MODULES_VIDEO_CODING_NACK_MODULE2_H_ + +#include + +#include +#include +#include + +#include "api/units/time_delta.h" +#include "modules/include/module.h" +#include "modules/include/module_common_types.h" +#include "modules/video_coding/histogram.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/numerics/sequence_number_util.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +class NackModule2 final : public Module { + public: + NackModule2(Clock* clock, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender); + + int OnReceivedPacket(uint16_t seq_num, bool is_keyframe); + int OnReceivedPacket(uint16_t seq_num, bool is_keyframe, bool is_recovered); + + void ClearUpTo(uint16_t seq_num); + void UpdateRtt(int64_t rtt_ms); + void Clear(); + + // Module implementation + int64_t TimeUntilNextProcess() override; + void Process() override; + + private: + // Which fields to consider when deciding which packet to nack in + // GetNackBatch. + enum NackFilterOptions { kSeqNumOnly, kTimeOnly, kSeqNumAndTime }; + + // This class holds the sequence number of the packet that is in the nack list + // as well as the meta data about when it should be nacked and how many times + // we have tried to nack this packet. + struct NackInfo { + NackInfo(); + NackInfo(uint16_t seq_num, + uint16_t send_at_seq_num, + int64_t created_at_time); + + uint16_t seq_num; + uint16_t send_at_seq_num; + int64_t created_at_time; + int64_t sent_at_time; + int retries; + }; + + struct BackoffSettings { + BackoffSettings(TimeDelta min_retry, TimeDelta max_rtt, double base); + static absl::optional ParseFromFieldTrials(); + + // Min time between nacks. + const TimeDelta min_retry_interval; + // Upper bound on link-delay considered for exponential backoff. + const TimeDelta max_rtt; + // Base for the exponential backoff. + const double base; + }; + + void AddPacketsToNack(uint16_t seq_num_start, uint16_t seq_num_end) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + + // Removes packets from the nack list until the next keyframe. Returns true + // if packets were removed. + bool RemovePacketsUntilKeyFrame() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + std::vector GetNackBatch(NackFilterOptions options) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + + // Update the reordering distribution. + void UpdateReorderingStatistics(uint16_t seq_num) + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + + // Returns how many packets we have to wait in order to receive the packet + // with probability |probabilty| or higher. + int WaitNumberOfPackets(float probability) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + + rtc::CriticalSection crit_; + Clock* const clock_; + NackSender* const nack_sender_; + KeyFrameRequestSender* const keyframe_request_sender_; + + // TODO(philipel): Some of the variables below are consistently used on a + // known thread (e.g. see |initialized_|). Those probably do not need + // synchronized access. + std::map> nack_list_ + RTC_GUARDED_BY(crit_); + std::set> keyframe_list_ + RTC_GUARDED_BY(crit_); + std::set> recovered_list_ + RTC_GUARDED_BY(crit_); + video_coding::Histogram reordering_histogram_ RTC_GUARDED_BY(crit_); + bool initialized_ RTC_GUARDED_BY(crit_); + int64_t rtt_ms_ RTC_GUARDED_BY(crit_); + uint16_t newest_seq_num_ RTC_GUARDED_BY(crit_); + + // Only touched on the process thread. + int64_t next_process_time_ms_; + + // Adds a delay before send nack on packet received. + const int64_t send_nack_delay_ms_; + + const absl::optional backoff_settings_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_NACK_MODULE2_H_ diff --git a/modules/video_coding/nack_module2_unittest.cc b/modules/video_coding/nack_module2_unittest.cc new file mode 100644 index 0000000000..ebc28ecb5a --- /dev/null +++ b/modules/video_coding/nack_module2_unittest.cc @@ -0,0 +1,371 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/nack_module2.h" + +#include +#include +#include +#include + +#include "system_wrappers/include/clock.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +class TestNackModule2 : public ::testing::TestWithParam, + public NackSender, + public KeyFrameRequestSender { + protected: + TestNackModule2() + : clock_(new SimulatedClock(0)), + field_trial_(GetParam() + ? "WebRTC-ExponentialNackBackoff/enabled:true/" + : "WebRTC-ExponentialNackBackoff/enabled:false/"), + nack_module_(clock_.get(), this, this), + keyframes_requested_(0) {} + + void SetUp() override { nack_module_.UpdateRtt(kDefaultRttMs); } + + void SendNack(const std::vector& sequence_numbers, + bool buffering_allowed) override { + sent_nacks_.insert(sent_nacks_.end(), sequence_numbers.begin(), + sequence_numbers.end()); + } + + void RequestKeyFrame() override { ++keyframes_requested_; } + + static constexpr int64_t kDefaultRttMs = 20; + std::unique_ptr clock_; + test::ScopedFieldTrials field_trial_; + NackModule2 nack_module_; + std::vector sent_nacks_; + int keyframes_requested_; +}; + +TEST_P(TestNackModule2, NackOnePacket) { + nack_module_.OnReceivedPacket(1, false, false); + nack_module_.OnReceivedPacket(3, false, false); + EXPECT_EQ(1u, sent_nacks_.size()); + EXPECT_EQ(2, sent_nacks_[0]); +} + +TEST_P(TestNackModule2, WrappingSeqNum) { + nack_module_.OnReceivedPacket(0xfffe, false, false); + nack_module_.OnReceivedPacket(1, false, false); + EXPECT_EQ(2u, sent_nacks_.size()); + EXPECT_EQ(0xffff, sent_nacks_[0]); + EXPECT_EQ(0, sent_nacks_[1]); +} + +TEST_P(TestNackModule2, WrappingSeqNumClearToKeyframe) { + nack_module_.OnReceivedPacket(0xfffe, false, false); + nack_module_.OnReceivedPacket(1, false, false); + EXPECT_EQ(2u, sent_nacks_.size()); + EXPECT_EQ(0xffff, sent_nacks_[0]); + EXPECT_EQ(0, sent_nacks_[1]); + + sent_nacks_.clear(); + nack_module_.OnReceivedPacket(2, true, false); + EXPECT_EQ(0u, sent_nacks_.size()); + + nack_module_.OnReceivedPacket(501, true, false); + EXPECT_EQ(498u, sent_nacks_.size()); + for (int seq_num = 3; seq_num < 501; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 3]); + + sent_nacks_.clear(); + nack_module_.OnReceivedPacket(1001, false, false); + EXPECT_EQ(499u, sent_nacks_.size()); + for (int seq_num = 502; seq_num < 1001; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 502]); + + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + nack_module_.Process(); + EXPECT_EQ(999u, sent_nacks_.size()); + EXPECT_EQ(0xffff, sent_nacks_[0]); + EXPECT_EQ(0, sent_nacks_[1]); + for (int seq_num = 3; seq_num < 501; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 1]); + for (int seq_num = 502; seq_num < 1001; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 2]); + + // Adding packet 1004 will cause the nack list to reach it's max limit. + // It will then clear all nacks up to the next keyframe (seq num 2), + // thus removing 0xffff and 0 from the nack list. + sent_nacks_.clear(); + nack_module_.OnReceivedPacket(1004, false, false); + EXPECT_EQ(2u, sent_nacks_.size()); + EXPECT_EQ(1002, sent_nacks_[0]); + EXPECT_EQ(1003, sent_nacks_[1]); + + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + nack_module_.Process(); + EXPECT_EQ(999u, sent_nacks_.size()); + for (int seq_num = 3; seq_num < 501; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 3]); + for (int seq_num = 502; seq_num < 1001; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 4]); + + // Adding packet 1007 will cause the nack module to overflow again, thus + // clearing everything up to 501 which is the next keyframe. + nack_module_.OnReceivedPacket(1007, false, false); + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + nack_module_.Process(); + EXPECT_EQ(503u, sent_nacks_.size()); + for (int seq_num = 502; seq_num < 1001; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 502]); + EXPECT_EQ(1005, sent_nacks_[501]); + EXPECT_EQ(1006, sent_nacks_[502]); +} + +TEST_P(TestNackModule2, DontBurstOnTimeSkip) { + nack_module_.Process(); + clock_->AdvanceTimeMilliseconds(20); + EXPECT_EQ(0, nack_module_.TimeUntilNextProcess()); + nack_module_.Process(); + + clock_->AdvanceTimeMilliseconds(100); + EXPECT_EQ(0, nack_module_.TimeUntilNextProcess()); + nack_module_.Process(); + EXPECT_EQ(20, nack_module_.TimeUntilNextProcess()); + + clock_->AdvanceTimeMilliseconds(19); + EXPECT_EQ(1, nack_module_.TimeUntilNextProcess()); + clock_->AdvanceTimeMilliseconds(2); + nack_module_.Process(); + EXPECT_EQ(19, nack_module_.TimeUntilNextProcess()); + + clock_->AdvanceTimeMilliseconds(19); + EXPECT_EQ(0, nack_module_.TimeUntilNextProcess()); + nack_module_.Process(); + + clock_->AdvanceTimeMilliseconds(21); + EXPECT_EQ(0, nack_module_.TimeUntilNextProcess()); + nack_module_.Process(); + EXPECT_EQ(19, nack_module_.TimeUntilNextProcess()); +} + +TEST_P(TestNackModule2, ResendNack) { + nack_module_.OnReceivedPacket(1, false, false); + nack_module_.OnReceivedPacket(3, false, false); + size_t expected_nacks_sent = 1; + EXPECT_EQ(expected_nacks_sent, sent_nacks_.size()); + EXPECT_EQ(2, sent_nacks_[0]); + + if (GetParam()) { + // Retry has to wait at least 5ms by default. + nack_module_.UpdateRtt(1); + clock_->AdvanceTimeMilliseconds(4); + nack_module_.Process(); // Too early. + EXPECT_EQ(expected_nacks_sent, sent_nacks_.size()); + + clock_->AdvanceTimeMilliseconds(1); + nack_module_.Process(); // Now allowed. + EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size()); + } else { + nack_module_.UpdateRtt(1); + clock_->AdvanceTimeMilliseconds(1); + nack_module_.Process(); // Fast retransmit allowed. + EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size()); + } + + // N:th try has to wait b^(N-1) * rtt by default. + const double b = GetParam() ? 1.25 : 1.0; + for (int i = 2; i < 10; ++i) { + // Change RTT, above the 40ms max for exponential backoff. + TimeDelta rtt = TimeDelta::Millis(160); // + (i * 10 - 40) + nack_module_.UpdateRtt(rtt.ms()); + + // RTT gets capped at 160ms in backoff calculations. + TimeDelta expected_backoff_delay = + std::pow(b, i - 1) * std::min(rtt, TimeDelta::Millis(160)); + + // Move to one millisecond before next allowed NACK. + clock_->AdvanceTimeMilliseconds(expected_backoff_delay.ms() - 1); + nack_module_.Process(); + EXPECT_EQ(expected_nacks_sent, sent_nacks_.size()); + + // Move to one millisecond after next allowed NACK. + // After rather than on to avoid rounding errors. + clock_->AdvanceTimeMilliseconds(2); + nack_module_.Process(); // Now allowed. + EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size()); + } + + // Giving up after 10 tries. + clock_->AdvanceTimeMilliseconds(3000); + nack_module_.Process(); + EXPECT_EQ(expected_nacks_sent, sent_nacks_.size()); +} + +TEST_P(TestNackModule2, ResendPacketMaxRetries) { + nack_module_.OnReceivedPacket(1, false, false); + nack_module_.OnReceivedPacket(3, false, false); + EXPECT_EQ(1u, sent_nacks_.size()); + EXPECT_EQ(2, sent_nacks_[0]); + + int backoff_factor = 1; + for (size_t retries = 1; retries < 10; ++retries) { + // Exponential backoff, so that we don't reject NACK because of time. + clock_->AdvanceTimeMilliseconds(backoff_factor * kDefaultRttMs); + backoff_factor *= 2; + nack_module_.Process(); + EXPECT_EQ(retries + 1, sent_nacks_.size()); + } + + clock_->AdvanceTimeMilliseconds(backoff_factor * kDefaultRttMs); + nack_module_.Process(); + EXPECT_EQ(10u, sent_nacks_.size()); +} + +TEST_P(TestNackModule2, TooLargeNackList) { + nack_module_.OnReceivedPacket(0, false, false); + nack_module_.OnReceivedPacket(1001, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(0, keyframes_requested_); + nack_module_.OnReceivedPacket(1003, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(1, keyframes_requested_); + nack_module_.OnReceivedPacket(1004, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(1, keyframes_requested_); +} + +TEST_P(TestNackModule2, TooLargeNackListWithKeyFrame) { + nack_module_.OnReceivedPacket(0, false, false); + nack_module_.OnReceivedPacket(1, true, false); + nack_module_.OnReceivedPacket(1001, false, false); + EXPECT_EQ(999u, sent_nacks_.size()); + EXPECT_EQ(0, keyframes_requested_); + nack_module_.OnReceivedPacket(1003, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(0, keyframes_requested_); + nack_module_.OnReceivedPacket(1005, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(1, keyframes_requested_); +} + +TEST_P(TestNackModule2, ClearUpTo) { + nack_module_.OnReceivedPacket(0, false, false); + nack_module_.OnReceivedPacket(100, false, false); + EXPECT_EQ(99u, sent_nacks_.size()); + + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + nack_module_.ClearUpTo(50); + nack_module_.Process(); + EXPECT_EQ(50u, sent_nacks_.size()); + EXPECT_EQ(50, sent_nacks_[0]); +} + +TEST_P(TestNackModule2, ClearUpToWrap) { + nack_module_.OnReceivedPacket(0xfff0, false, false); + nack_module_.OnReceivedPacket(0xf, false, false); + EXPECT_EQ(30u, sent_nacks_.size()); + + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + nack_module_.ClearUpTo(0); + nack_module_.Process(); + EXPECT_EQ(15u, sent_nacks_.size()); + EXPECT_EQ(0, sent_nacks_[0]); +} + +TEST_P(TestNackModule2, PacketNackCount) { + EXPECT_EQ(0, nack_module_.OnReceivedPacket(0, false, false)); + EXPECT_EQ(0, nack_module_.OnReceivedPacket(2, false, false)); + EXPECT_EQ(1, nack_module_.OnReceivedPacket(1, false, false)); + + sent_nacks_.clear(); + nack_module_.UpdateRtt(100); + EXPECT_EQ(0, nack_module_.OnReceivedPacket(5, false, false)); + clock_->AdvanceTimeMilliseconds(100); + nack_module_.Process(); + clock_->AdvanceTimeMilliseconds(125); + nack_module_.Process(); + EXPECT_EQ(3, nack_module_.OnReceivedPacket(3, false, false)); + EXPECT_EQ(3, nack_module_.OnReceivedPacket(4, false, false)); + EXPECT_EQ(0, nack_module_.OnReceivedPacket(4, false, false)); +} + +TEST_P(TestNackModule2, NackListFullAndNoOverlapWithKeyframes) { + const int kMaxNackPackets = 1000; + const unsigned int kFirstGap = kMaxNackPackets - 20; + const unsigned int kSecondGap = 200; + uint16_t seq_num = 0; + nack_module_.OnReceivedPacket(seq_num++, true, false); + seq_num += kFirstGap; + nack_module_.OnReceivedPacket(seq_num++, true, false); + EXPECT_EQ(kFirstGap, sent_nacks_.size()); + sent_nacks_.clear(); + seq_num += kSecondGap; + nack_module_.OnReceivedPacket(seq_num, true, false); + EXPECT_EQ(kSecondGap, sent_nacks_.size()); +} + +TEST_P(TestNackModule2, HandleFecRecoveredPacket) { + nack_module_.OnReceivedPacket(1, false, false); + nack_module_.OnReceivedPacket(4, false, true); + EXPECT_EQ(0u, sent_nacks_.size()); + nack_module_.OnReceivedPacket(5, false, false); + EXPECT_EQ(2u, sent_nacks_.size()); +} + +TEST_P(TestNackModule2, SendNackWithoutDelay) { + nack_module_.OnReceivedPacket(0, false, false); + nack_module_.OnReceivedPacket(100, false, false); + EXPECT_EQ(99u, sent_nacks_.size()); +} + +INSTANTIATE_TEST_SUITE_P(WithAndWithoutBackoff, + TestNackModule2, + ::testing::Values(true, false)); + +class TestNackModule2WithFieldTrial : public ::testing::Test, + public NackSender, + public KeyFrameRequestSender { + protected: + TestNackModule2WithFieldTrial() + : nack_delay_field_trial_("WebRTC-SendNackDelayMs/10/"), + clock_(new SimulatedClock(0)), + nack_module_(clock_.get(), this, this), + keyframes_requested_(0) {} + + void SendNack(const std::vector& sequence_numbers, + bool buffering_allowed) override { + sent_nacks_.insert(sent_nacks_.end(), sequence_numbers.begin(), + sequence_numbers.end()); + } + + void RequestKeyFrame() override { ++keyframes_requested_; } + + test::ScopedFieldTrials nack_delay_field_trial_; + std::unique_ptr clock_; + NackModule2 nack_module_; + std::vector sent_nacks_; + int keyframes_requested_; +}; + +TEST_F(TestNackModule2WithFieldTrial, SendNackWithDelay) { + nack_module_.OnReceivedPacket(0, false, false); + nack_module_.OnReceivedPacket(100, false, false); + EXPECT_EQ(0u, sent_nacks_.size()); + clock_->AdvanceTimeMilliseconds(10); + nack_module_.OnReceivedPacket(106, false, false); + EXPECT_EQ(99u, sent_nacks_.size()); + clock_->AdvanceTimeMilliseconds(10); + nack_module_.OnReceivedPacket(109, false, false); + EXPECT_EQ(104u, sent_nacks_.size()); +} +} // namespace webrtc diff --git a/video/BUILD.gn b/video/BUILD.gn index 25df5085f3..07b03eebc1 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -34,6 +34,8 @@ rtc_library("video") { "rtp_streams_synchronizer2.h", "rtp_video_stream_receiver.cc", "rtp_video_stream_receiver.h", + "rtp_video_stream_receiver2.cc", + "rtp_video_stream_receiver2.h", "rtp_video_stream_receiver_frame_transformer_delegate.cc", "rtp_video_stream_receiver_frame_transformer_delegate.h", "send_delay_stats.cc", @@ -528,6 +530,7 @@ if (rtc_include_tests) { "receive_statistics_proxy2_unittest.cc", "receive_statistics_proxy_unittest.cc", "report_block_stats_unittest.cc", + "rtp_video_stream_receiver2_unittest.cc", "rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc", "rtp_video_stream_receiver_unittest.cc", "send_delay_stats_unittest.cc", diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 87691d4171..53fd637119 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -77,8 +77,6 @@ int PacketBufferMaxSize() { return packet_buffer_max_size; } -} // namespace - std::unique_ptr CreateRtpRtcpModule( Clock* clock, ReceiveStatistics* receive_statistics, @@ -107,6 +105,8 @@ std::unique_ptr CreateRtpRtcpModule( static const int kPacketLogIntervalMs = 10000; +} // namespace + RtpVideoStreamReceiver::RtcpFeedbackBuffer::RtcpFeedbackBuffer( KeyFrameRequestSender* key_frame_request_sender, NackSender* nack_sender, diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h index 32d675c307..902c118b8c 100644 --- a/video/rtp_video_stream_receiver.h +++ b/video/rtp_video_stream_receiver.h @@ -69,7 +69,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender, public KeyFrameRequestSender, public video_coding::OnCompleteFrameCallback, public OnDecryptedFrameCallback, - public OnDecryptionStatusChangeCallback { + public OnDecryptionStatusChangeCallback, + public RtpVideoFrameReceiver { public: // DEPRECATED due to dependency on ReceiveStatisticsProxy. RtpVideoStreamReceiver( @@ -205,9 +206,11 @@ class RtpVideoStreamReceiver : public LossNotificationSender, void AddSecondarySink(RtpPacketSinkInterface* sink); void RemoveSecondarySink(const RtpPacketSinkInterface* sink); - virtual void ManageFrame(std::unique_ptr frame); - private: + // Implements RtpVideoFrameReceiver. + void ManageFrame( + std::unique_ptr frame) override; + // Used for buffering RTCP feedback messages and sending them all together. // Note: // 1. Key frame requests and NACKs are mutually exclusive, with the diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc new file mode 100644 index 0000000000..98351c49cf --- /dev/null +++ b/video/rtp_video_stream_receiver2.cc @@ -0,0 +1,1152 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/rtp_video_stream_receiver2.h" + +#include +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/base/macros.h" +#include "absl/memory/memory.h" +#include "absl/types/optional.h" +#include "media/base/media_constants.h" +#include "modules/pacing/packet_router.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/include/rtp_cvo.h" +#include "modules/rtp_rtcp/include/rtp_rtcp.h" +#include "modules/rtp_rtcp/include/ulpfec_receiver.h" +#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h" +#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_format.h" +#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" +#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_header_extensions.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_config.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h" +#include "modules/utility/include/process_thread.h" +#include "modules/video_coding/frame_object.h" +#include "modules/video_coding/h264_sprop_parameter_sets.h" +#include "modules/video_coding/h264_sps_pps_tracker.h" +#include "modules/video_coding/nack_module2.h" +#include "modules/video_coding/packet_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/location.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/ntp_time.h" +#include "video/receive_statistics_proxy2.h" + +namespace webrtc { + +namespace { +// TODO(philipel): Change kPacketBufferStartSize back to 32 in M63 see: +// crbug.com/752886 +constexpr int kPacketBufferStartSize = 512; +constexpr int kPacketBufferMaxSize = 2048; + +int PacketBufferMaxSize() { + // The group here must be a positive power of 2, in which case that is used as + // size. All other values shall result in the default value being used. + const std::string group_name = + webrtc::field_trial::FindFullName("WebRTC-PacketBufferMaxSize"); + int packet_buffer_max_size = kPacketBufferMaxSize; + if (!group_name.empty() && + (sscanf(group_name.c_str(), "%d", &packet_buffer_max_size) != 1 || + packet_buffer_max_size <= 0 || + // Verify that the number is a positive power of 2. + (packet_buffer_max_size & (packet_buffer_max_size - 1)) != 0)) { + RTC_LOG(LS_WARNING) << "Invalid packet buffer max size: " << group_name; + packet_buffer_max_size = kPacketBufferMaxSize; + } + return packet_buffer_max_size; +} + +std::unique_ptr CreateRtpRtcpModule( + Clock* clock, + ReceiveStatistics* receive_statistics, + Transport* outgoing_transport, + RtcpRttStats* rtt_stats, + RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, + RtcpCnameCallback* rtcp_cname_callback, + uint32_t local_ssrc) { + RtpRtcp::Configuration configuration; + configuration.clock = clock; + configuration.audio = false; + configuration.receiver_only = true; + configuration.receive_statistics = receive_statistics; + configuration.outgoing_transport = outgoing_transport; + configuration.rtt_stats = rtt_stats; + configuration.rtcp_packet_type_counter_observer = + rtcp_packet_type_counter_observer; + configuration.rtcp_cname_callback = rtcp_cname_callback; + configuration.local_media_ssrc = local_ssrc; + + std::unique_ptr rtp_rtcp = RtpRtcp::Create(configuration); + rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); + + return rtp_rtcp; +} + +static const int kPacketLogIntervalMs = 10000; +} // namespace + +RtpVideoStreamReceiver2::RtcpFeedbackBuffer::RtcpFeedbackBuffer( + KeyFrameRequestSender* key_frame_request_sender, + NackSender* nack_sender, + LossNotificationSender* loss_notification_sender) + : key_frame_request_sender_(key_frame_request_sender), + nack_sender_(nack_sender), + loss_notification_sender_(loss_notification_sender), + request_key_frame_(false) { + RTC_DCHECK(key_frame_request_sender_); + RTC_DCHECK(nack_sender_); + RTC_DCHECK(loss_notification_sender_); +} + +void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::RequestKeyFrame() { + rtc::CritScope lock(&cs_); + request_key_frame_ = true; +} + +void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendNack( + const std::vector& sequence_numbers, + bool buffering_allowed) { + RTC_DCHECK(!sequence_numbers.empty()); + rtc::CritScope lock(&cs_); + nack_sequence_numbers_.insert(nack_sequence_numbers_.end(), + sequence_numbers.cbegin(), + sequence_numbers.cend()); + if (!buffering_allowed) { + // Note that while *buffering* is not allowed, *batching* is, meaning that + // previously buffered messages may be sent along with the current message. + SendRtcpFeedback(ConsumeRtcpFeedbackLocked()); + } +} + +void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendLossNotification( + uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) { + RTC_DCHECK(buffering_allowed); + rtc::CritScope lock(&cs_); + RTC_DCHECK(!lntf_state_) + << "SendLossNotification() called twice in a row with no call to " + "SendBufferedRtcpFeedback() in between."; + lntf_state_ = absl::make_optional( + last_decoded_seq_num, last_received_seq_num, decodability_flag); +} + +void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendBufferedRtcpFeedback() { + SendRtcpFeedback(ConsumeRtcpFeedback()); +} + +RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ConsumedRtcpFeedback +RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ConsumeRtcpFeedback() { + rtc::CritScope lock(&cs_); + return ConsumeRtcpFeedbackLocked(); +} + +RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ConsumedRtcpFeedback +RtpVideoStreamReceiver2::RtcpFeedbackBuffer::ConsumeRtcpFeedbackLocked() { + ConsumedRtcpFeedback feedback; + std::swap(feedback.request_key_frame, request_key_frame_); + std::swap(feedback.nack_sequence_numbers, nack_sequence_numbers_); + std::swap(feedback.lntf_state, lntf_state_); + return feedback; +} + +void RtpVideoStreamReceiver2::RtcpFeedbackBuffer::SendRtcpFeedback( + ConsumedRtcpFeedback feedback) { + if (feedback.lntf_state) { + // If either a NACK or a key frame request is sent, we should buffer + // the LNTF and wait for them (NACK or key frame request) to trigger + // the compound feedback message. + // Otherwise, the LNTF should be sent out immediately. + const bool buffering_allowed = + feedback.request_key_frame || !feedback.nack_sequence_numbers.empty(); + + loss_notification_sender_->SendLossNotification( + feedback.lntf_state->last_decoded_seq_num, + feedback.lntf_state->last_received_seq_num, + feedback.lntf_state->decodability_flag, buffering_allowed); + } + + if (feedback.request_key_frame) { + key_frame_request_sender_->RequestKeyFrame(); + } else if (!feedback.nack_sequence_numbers.empty()) { + nack_sender_->SendNack(feedback.nack_sequence_numbers, true); + } +} + +RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, + RtcpCnameCallback* rtcp_cname_callback, + ProcessThread* process_thread, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer) + : clock_(clock), + config_(*config), + packet_router_(packet_router), + process_thread_(process_thread), + ntp_estimator_(clock), + rtp_header_extensions_(config_.rtp.extensions), + forced_playout_delay_max_ms_("max_ms", absl::nullopt), + forced_playout_delay_min_ms_("min_ms", absl::nullopt), + rtp_receive_statistics_(rtp_receive_statistics), + ulpfec_receiver_(UlpfecReceiver::Create(config->rtp.remote_ssrc, + this, + config->rtp.extensions)), + receiving_(false), + last_packet_log_ms_(-1), + rtp_rtcp_(CreateRtpRtcpModule(clock, + rtp_receive_statistics_, + transport, + rtt_stats, + rtcp_packet_type_counter_observer, + rtcp_cname_callback, + config_.rtp.local_ssrc)), + complete_frame_callback_(complete_frame_callback), + keyframe_request_sender_(keyframe_request_sender), + // TODO(bugs.webrtc.org/10336): Let |rtcp_feedback_buffer_| communicate + // directly with |rtp_rtcp_|. + rtcp_feedback_buffer_(this, nack_sender, this), + packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()), + has_received_frame_(false), + frames_decryptable_(false), + absolute_capture_time_receiver_(clock) { + constexpr bool remb_candidate = true; + if (packet_router_) + packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); + + RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff) + << "A stream should not be configured with RTCP disabled. This value is " + "reserved for internal usage."; + // TODO(pbos): What's an appropriate local_ssrc for receive-only streams? + RTC_DCHECK(config_.rtp.local_ssrc != 0); + RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc); + + rtp_rtcp_->SetRTCPStatus(config_.rtp.rtcp_mode); + rtp_rtcp_->SetRemoteSSRC(config_.rtp.remote_ssrc); + + static const int kMaxPacketAgeToNack = 450; + const int max_reordering_threshold = (config_.rtp.nack.rtp_history_ms > 0) + ? kMaxPacketAgeToNack + : kDefaultMaxReorderingThreshold; + rtp_receive_statistics_->SetMaxReorderingThreshold(config_.rtp.remote_ssrc, + max_reordering_threshold); + // TODO(nisse): For historic reasons, we applied the above + // max_reordering_threshold also for RTX stats, which makes little sense since + // we don't NACK rtx packets. Consider deleting the below block, and rely on + // the default threshold. + if (config_.rtp.rtx_ssrc) { + rtp_receive_statistics_->SetMaxReorderingThreshold( + config_.rtp.rtx_ssrc, max_reordering_threshold); + } + if (config_.rtp.rtcp_xr.receiver_reference_time_report) + rtp_rtcp_->SetRtcpXrRrtrStatus(true); + + ParseFieldTrial( + {&forced_playout_delay_max_ms_, &forced_playout_delay_min_ms_}, + field_trial::FindFullName("WebRTC-ForcePlayoutDelay")); + + process_thread_->RegisterModule(rtp_rtcp_.get(), RTC_FROM_HERE); + + if (config_.rtp.lntf.enabled) { + loss_notification_controller_ = + std::make_unique(&rtcp_feedback_buffer_, + &rtcp_feedback_buffer_); + } + + if (config_.rtp.nack.rtp_history_ms != 0) { + nack_module_ = std::make_unique(clock_, &rtcp_feedback_buffer_, + &rtcp_feedback_buffer_); + process_thread_->RegisterModule(nack_module_.get(), RTC_FROM_HERE); + } + + reference_finder_ = + std::make_unique(this); + + // Only construct the encrypted receiver if frame encryption is enabled. + if (config_.crypto_options.sframe.require_frame_encryption) { + buffered_frame_decryptor_ = + std::make_unique(this, this); + if (frame_decryptor != nullptr) { + buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor)); + } + } + + if (frame_transformer) { + frame_transformer_delegate_ = new rtc::RefCountedObject< + RtpVideoStreamReceiverFrameTransformerDelegate>( + this, std::move(frame_transformer), rtc::Thread::Current(), + config_.rtp.remote_ssrc); + frame_transformer_delegate_->Init(); + } +} + +RtpVideoStreamReceiver2::~RtpVideoStreamReceiver2() { + RTC_DCHECK(secondary_sinks_.empty()); + + if (nack_module_) { + process_thread_->DeRegisterModule(nack_module_.get()); + } + + process_thread_->DeRegisterModule(rtp_rtcp_.get()); + + if (packet_router_) + packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get()); + UpdateHistograms(); + if (frame_transformer_delegate_) + frame_transformer_delegate_->Reset(); +} + +void RtpVideoStreamReceiver2::AddReceiveCodec( + const VideoCodec& video_codec, + const std::map& codec_params, + bool raw_payload) { + payload_type_map_.emplace( + video_codec.plType, + raw_payload ? std::make_unique() + : CreateVideoRtpDepacketizer(video_codec.codecType)); + pt_codec_params_.emplace(video_codec.plType, codec_params); +} + +absl::optional RtpVideoStreamReceiver2::GetSyncInfo() const { + Syncable::Info info; + if (rtp_rtcp_->RemoteNTP(&info.capture_time_ntp_secs, + &info.capture_time_ntp_frac, nullptr, nullptr, + &info.capture_time_source_clock) != 0) { + return absl::nullopt; + } + { + rtc::CritScope lock(&sync_info_lock_); + if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_ms_) { + return absl::nullopt; + } + info.latest_received_capture_timestamp = *last_received_rtp_timestamp_; + info.latest_receive_time_ms = *last_received_rtp_system_time_ms_; + } + + // Leaves info.current_delay_ms uninitialized. + return info; +} + +RtpVideoStreamReceiver2::ParseGenericDependenciesResult +RtpVideoStreamReceiver2::ParseGenericDependenciesExtension( + const RtpPacketReceived& rtp_packet, + RTPVideoHeader* video_header) { + if (rtp_packet.HasExtension()) { + webrtc::DependencyDescriptor dependency_descriptor; + if (!rtp_packet.GetExtension( + video_structure_.get(), &dependency_descriptor)) { + // Descriptor is there, but failed to parse. Either it is invalid, + // or too old packet (after relevant video_structure_ changed), + // or too new packet (before relevant video_structure_ arrived). + // Drop such packet to be on the safe side. + // TODO(bugs.webrtc.org/10342): Stash too new packet. + RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc() + << " Failed to parse dependency descriptor."; + return kDropPacket; + } + if (dependency_descriptor.attached_structure != nullptr && + !dependency_descriptor.first_packet_in_frame) { + RTC_LOG(LS_WARNING) << "ssrc: " << rtp_packet.Ssrc() + << "Invalid dependency descriptor: structure " + "attached to non first packet of a frame."; + return kDropPacket; + } + video_header->is_first_packet_in_frame = + dependency_descriptor.first_packet_in_frame; + video_header->is_last_packet_in_frame = + dependency_descriptor.last_packet_in_frame; + + int64_t frame_id = + frame_id_unwrapper_.Unwrap(dependency_descriptor.frame_number); + auto& generic_descriptor_info = video_header->generic.emplace(); + generic_descriptor_info.frame_id = frame_id; + generic_descriptor_info.spatial_index = + dependency_descriptor.frame_dependencies.spatial_id; + generic_descriptor_info.temporal_index = + dependency_descriptor.frame_dependencies.temporal_id; + for (int fdiff : dependency_descriptor.frame_dependencies.frame_diffs) { + generic_descriptor_info.dependencies.push_back(frame_id - fdiff); + } + generic_descriptor_info.decode_target_indications = + dependency_descriptor.frame_dependencies.decode_target_indications; + if (dependency_descriptor.resolution) { + video_header->width = dependency_descriptor.resolution->Width(); + video_header->height = dependency_descriptor.resolution->Height(); + } + + // FrameDependencyStructure is sent in dependency descriptor of the first + // packet of a key frame and required for parsed dependency descriptor in + // all the following packets until next key frame. + // Save it if there is a (potentially) new structure. + if (dependency_descriptor.attached_structure) { + RTC_DCHECK(dependency_descriptor.first_packet_in_frame); + if (video_structure_frame_id_ > frame_id) { + RTC_LOG(LS_WARNING) + << "Arrived key frame with id " << frame_id << " and structure id " + << dependency_descriptor.attached_structure->structure_id + << " is older than the latest received key frame with id " + << *video_structure_frame_id_ << " and structure id " + << video_structure_->structure_id; + return kDropPacket; + } + video_structure_ = std::move(dependency_descriptor.attached_structure); + video_structure_frame_id_ = frame_id; + video_header->frame_type = VideoFrameType::kVideoFrameKey; + } else { + video_header->frame_type = VideoFrameType::kVideoFrameDelta; + } + return kHasGenericDescriptor; + } + + RtpGenericFrameDescriptor generic_frame_descriptor; + if (!rtp_packet.GetExtension( + &generic_frame_descriptor)) { + return kNoGenericDescriptor; + } + + video_header->is_first_packet_in_frame = + generic_frame_descriptor.FirstPacketInSubFrame(); + video_header->is_last_packet_in_frame = + generic_frame_descriptor.LastPacketInSubFrame(); + + if (generic_frame_descriptor.FirstPacketInSubFrame()) { + video_header->frame_type = + generic_frame_descriptor.FrameDependenciesDiffs().empty() + ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta; + + auto& generic_descriptor_info = video_header->generic.emplace(); + int64_t frame_id = + frame_id_unwrapper_.Unwrap(generic_frame_descriptor.FrameId()); + generic_descriptor_info.frame_id = frame_id; + generic_descriptor_info.spatial_index = + generic_frame_descriptor.SpatialLayer(); + generic_descriptor_info.temporal_index = + generic_frame_descriptor.TemporalLayer(); + for (uint16_t fdiff : generic_frame_descriptor.FrameDependenciesDiffs()) { + generic_descriptor_info.dependencies.push_back(frame_id - fdiff); + } + } + video_header->width = generic_frame_descriptor.Width(); + video_header->height = generic_frame_descriptor.Height(); + return kHasGenericDescriptor; +} + +void RtpVideoStreamReceiver2::OnReceivedPayloadData( + rtc::CopyOnWriteBuffer codec_payload, + const RtpPacketReceived& rtp_packet, + const RTPVideoHeader& video) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + auto packet = std::make_unique( + rtp_packet, video, ntp_estimator_.Estimate(rtp_packet.Timestamp()), + clock_->TimeInMilliseconds()); + + // Try to extrapolate absolute capture time if it is missing. + packet->packet_info.set_absolute_capture_time( + absolute_capture_time_receiver_.OnReceivePacket( + AbsoluteCaptureTimeReceiver::GetSource(packet->packet_info.ssrc(), + packet->packet_info.csrcs()), + packet->packet_info.rtp_timestamp(), + // Assume frequency is the same one for all video frames. + kVideoPayloadTypeFrequency, + packet->packet_info.absolute_capture_time())); + + RTPVideoHeader& video_header = packet->video_header; + video_header.rotation = kVideoRotation_0; + video_header.content_type = VideoContentType::UNSPECIFIED; + video_header.video_timing.flags = VideoSendTiming::kInvalid; + video_header.is_last_packet_in_frame |= rtp_packet.Marker(); + video_header.frame_marking.temporal_id = kNoTemporalIdx; + + if (const auto* vp9_header = + absl::get_if(&video_header.video_type_header)) { + video_header.is_last_packet_in_frame |= vp9_header->end_of_frame; + video_header.is_first_packet_in_frame |= vp9_header->beginning_of_frame; + } + + rtp_packet.GetExtension(&video_header.rotation); + rtp_packet.GetExtension( + &video_header.content_type); + rtp_packet.GetExtension(&video_header.video_timing); + if (forced_playout_delay_max_ms_ && forced_playout_delay_min_ms_) { + video_header.playout_delay.max_ms = *forced_playout_delay_max_ms_; + video_header.playout_delay.min_ms = *forced_playout_delay_min_ms_; + } else { + rtp_packet.GetExtension(&video_header.playout_delay); + } + rtp_packet.GetExtension(&video_header.frame_marking); + + ParseGenericDependenciesResult generic_descriptor_state = + ParseGenericDependenciesExtension(rtp_packet, &video_header); + if (generic_descriptor_state == kDropPacket) + return; + + // Color space should only be transmitted in the last packet of a frame, + // therefore, neglect it otherwise so that last_color_space_ is not reset by + // mistake. + if (video_header.is_last_packet_in_frame) { + video_header.color_space = rtp_packet.GetExtension(); + if (video_header.color_space || + video_header.frame_type == VideoFrameType::kVideoFrameKey) { + // Store color space since it's only transmitted when changed or for key + // frames. Color space will be cleared if a key frame is transmitted + // without color space information. + last_color_space_ = video_header.color_space; + } else if (last_color_space_) { + video_header.color_space = last_color_space_; + } + } + + if (loss_notification_controller_) { + if (rtp_packet.recovered()) { + // TODO(bugs.webrtc.org/10336): Implement support for reordering. + RTC_LOG(LS_INFO) + << "LossNotificationController does not support reordering."; + } else if (generic_descriptor_state == kNoGenericDescriptor) { + RTC_LOG(LS_WARNING) << "LossNotificationController requires generic " + "frame descriptor, but it is missing."; + } else { + if (video_header.is_first_packet_in_frame) { + RTC_DCHECK(video_header.generic); + LossNotificationController::FrameDetails frame; + frame.is_keyframe = + video_header.frame_type == VideoFrameType::kVideoFrameKey; + frame.frame_id = video_header.generic->frame_id; + frame.frame_dependencies = video_header.generic->dependencies; + loss_notification_controller_->OnReceivedPacket( + rtp_packet.SequenceNumber(), &frame); + } else { + loss_notification_controller_->OnReceivedPacket( + rtp_packet.SequenceNumber(), nullptr); + } + } + } + + if (nack_module_) { + const bool is_keyframe = + video_header.is_first_packet_in_frame && + video_header.frame_type == VideoFrameType::kVideoFrameKey; + + packet->times_nacked = nack_module_->OnReceivedPacket( + rtp_packet.SequenceNumber(), is_keyframe, rtp_packet.recovered()); + } else { + packet->times_nacked = -1; + } + + if (codec_payload.size() == 0) { + NotifyReceiverOfEmptyPacket(packet->seq_num); + rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); + return; + } + + if (packet->codec() == kVideoCodecH264) { + // Only when we start to receive packets will we know what payload type + // that will be used. When we know the payload type insert the correct + // sps/pps into the tracker. + if (packet->payload_type != last_payload_type_) { + last_payload_type_ = packet->payload_type; + InsertSpsPpsIntoTracker(packet->payload_type); + } + + video_coding::H264SpsPpsTracker::FixedBitstream fixed = + tracker_.CopyAndFixBitstream( + rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()), + &packet->video_header); + + switch (fixed.action) { + case video_coding::H264SpsPpsTracker::kRequestKeyframe: + rtcp_feedback_buffer_.RequestKeyFrame(); + rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); + ABSL_FALLTHROUGH_INTENDED; + case video_coding::H264SpsPpsTracker::kDrop: + return; + case video_coding::H264SpsPpsTracker::kInsert: + packet->video_payload = std::move(fixed.bitstream); + break; + } + + } else { + packet->video_payload = std::move(codec_payload); + } + + rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); + frame_counter_.Add(packet->timestamp); + OnInsertedPacket(packet_buffer_.InsertPacket(std::move(packet))); +} + +void RtpVideoStreamReceiver2::OnRecoveredPacket(const uint8_t* rtp_packet, + size_t rtp_packet_length) { + RtpPacketReceived packet; + if (!packet.Parse(rtp_packet, rtp_packet_length)) + return; + if (packet.PayloadType() == config_.rtp.red_payload_type) { + RTC_LOG(LS_WARNING) << "Discarding recovered packet with RED encapsulation"; + return; + } + + packet.IdentifyExtensions(rtp_header_extensions_); + packet.set_payload_type_frequency(kVideoPayloadTypeFrequency); + // TODO(nisse): UlpfecReceiverImpl::ProcessReceivedFec passes both + // original (decapsulated) media packets and recovered packets to + // this callback. We need a way to distinguish, for setting + // packet.recovered() correctly. Ideally, move RED decapsulation out + // of the Ulpfec implementation. + + ReceivePacket(packet); +} + +// This method handles both regular RTP packets and packets recovered +// via FlexFEC. +void RtpVideoStreamReceiver2::OnRtpPacket(const RtpPacketReceived& packet) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + + if (!receiving_) { + return; + } + + if (!packet.recovered()) { + // TODO(nisse): Exclude out-of-order packets? + int64_t now_ms = clock_->TimeInMilliseconds(); + { + rtc::CritScope cs(&sync_info_lock_); + last_received_rtp_timestamp_ = packet.Timestamp(); + last_received_rtp_system_time_ms_ = now_ms; + } + // Periodically log the RTP header of incoming packets. + if (now_ms - last_packet_log_ms_ > kPacketLogIntervalMs) { + rtc::StringBuilder ss; + ss << "Packet received on SSRC: " << packet.Ssrc() + << " with payload type: " << static_cast(packet.PayloadType()) + << ", timestamp: " << packet.Timestamp() + << ", sequence number: " << packet.SequenceNumber() + << ", arrival time: " << packet.arrival_time_ms(); + int32_t time_offset; + if (packet.GetExtension(&time_offset)) { + ss << ", toffset: " << time_offset; + } + uint32_t send_time; + if (packet.GetExtension(&send_time)) { + ss << ", abs send time: " << send_time; + } + RTC_LOG(LS_INFO) << ss.str(); + last_packet_log_ms_ = now_ms; + } + } + + ReceivePacket(packet); + + // Update receive statistics after ReceivePacket. + // Receive statistics will be reset if the payload type changes (make sure + // that the first packet is included in the stats). + if (!packet.recovered()) { + rtp_receive_statistics_->OnRtpPacket(packet); + } + + for (RtpPacketSinkInterface* secondary_sink : secondary_sinks_) { + secondary_sink->OnRtpPacket(packet); + } +} + +void RtpVideoStreamReceiver2::RequestKeyFrame() { + // TODO(bugs.webrtc.org/10336): Allow the sender to ignore key frame requests + // issued by anything other than the LossNotificationController if it (the + // sender) is relying on LNTF alone. + if (keyframe_request_sender_) { + keyframe_request_sender_->RequestKeyFrame(); + } else { + rtp_rtcp_->SendPictureLossIndication(); + } +} + +void RtpVideoStreamReceiver2::SendLossNotification( + uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) { + RTC_DCHECK(config_.rtp.lntf.enabled); + rtp_rtcp_->SendLossNotification(last_decoded_seq_num, last_received_seq_num, + decodability_flag, buffering_allowed); +} + +bool RtpVideoStreamReceiver2::IsUlpfecEnabled() const { + return config_.rtp.ulpfec_payload_type != -1; +} + +bool RtpVideoStreamReceiver2::IsRetransmissionsEnabled() const { + return config_.rtp.nack.rtp_history_ms > 0; +} + +void RtpVideoStreamReceiver2::RequestPacketRetransmit( + const std::vector& sequence_numbers) { + rtp_rtcp_->SendNack(sequence_numbers); +} + +bool RtpVideoStreamReceiver2::IsDecryptable() const { + return frames_decryptable_.load(); +} + +void RtpVideoStreamReceiver2::OnInsertedPacket( + video_coding::PacketBuffer::InsertResult result) { + video_coding::PacketBuffer::Packet* first_packet = nullptr; + int max_nack_count; + int64_t min_recv_time; + int64_t max_recv_time; + std::vector> payloads; + RtpPacketInfos::vector_type packet_infos; + + bool frame_boundary = true; + for (auto& packet : result.packets) { + // PacketBuffer promisses frame boundaries are correctly set on each + // packet. Document that assumption with the DCHECKs. + RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame()); + if (packet->is_first_packet_in_frame()) { + first_packet = packet.get(); + max_nack_count = packet->times_nacked; + min_recv_time = packet->packet_info.receive_time_ms(); + max_recv_time = packet->packet_info.receive_time_ms(); + payloads.clear(); + packet_infos.clear(); + } else { + max_nack_count = std::max(max_nack_count, packet->times_nacked); + min_recv_time = + std::min(min_recv_time, packet->packet_info.receive_time_ms()); + max_recv_time = + std::max(max_recv_time, packet->packet_info.receive_time_ms()); + } + payloads.emplace_back(packet->video_payload); + packet_infos.push_back(packet->packet_info); + + frame_boundary = packet->is_last_packet_in_frame(); + if (packet->is_last_packet_in_frame()) { + auto depacketizer_it = payload_type_map_.find(first_packet->payload_type); + RTC_CHECK(depacketizer_it != payload_type_map_.end()); + + rtc::scoped_refptr bitstream = + depacketizer_it->second->AssembleFrame(payloads); + if (!bitstream) { + // Failed to assemble a frame. Discard and continue. + continue; + } + + const video_coding::PacketBuffer::Packet& last_packet = *packet; + OnAssembledFrame(std::make_unique( + first_packet->seq_num, // + last_packet.seq_num, // + last_packet.marker_bit, // + max_nack_count, // + min_recv_time, // + max_recv_time, // + first_packet->timestamp, // + first_packet->ntp_time_ms, // + last_packet.video_header.video_timing, // + first_packet->payload_type, // + first_packet->codec(), // + last_packet.video_header.rotation, // + last_packet.video_header.content_type, // + first_packet->video_header, // + last_packet.video_header.color_space, // + RtpPacketInfos(std::move(packet_infos)), // + std::move(bitstream))); + } + } + RTC_DCHECK(frame_boundary); + if (result.buffer_cleared) { + RequestKeyFrame(); + } +} + +void RtpVideoStreamReceiver2::OnAssembledFrame( + std::unique_ptr frame) { + RTC_DCHECK_RUN_ON(&network_tc_); + RTC_DCHECK(frame); + + const absl::optional& descriptor = + frame->GetRtpVideoHeader().generic; + + if (loss_notification_controller_ && descriptor) { + loss_notification_controller_->OnAssembledFrame( + frame->first_seq_num(), descriptor->frame_id, + absl::c_linear_search(descriptor->decode_target_indications, + DecodeTargetIndication::kDiscardable), + descriptor->dependencies); + } + + // If frames arrive before a key frame, they would not be decodable. + // In that case, request a key frame ASAP. + if (!has_received_frame_) { + if (frame->FrameType() != VideoFrameType::kVideoFrameKey) { + // |loss_notification_controller_|, if present, would have already + // requested a key frame when the first packet for the non-key frame + // had arrived, so no need to replicate the request. + if (!loss_notification_controller_) { + RequestKeyFrame(); + } + } + has_received_frame_ = true; + } + + rtc::CritScope lock(&reference_finder_lock_); + // Reset |reference_finder_| if |frame| is new and the codec have changed. + if (current_codec_) { + bool frame_is_newer = + AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_); + + if (frame->codec_type() != current_codec_) { + if (frame_is_newer) { + // When we reset the |reference_finder_| we don't want new picture ids + // to overlap with old picture ids. To ensure that doesn't happen we + // start from the |last_completed_picture_id_| and add an offset in case + // of reordering. + reference_finder_ = + std::make_unique( + this, last_completed_picture_id_ + + std::numeric_limits::max()); + current_codec_ = frame->codec_type(); + } else { + // Old frame from before the codec switch, discard it. + return; + } + } + + if (frame_is_newer) { + last_assembled_frame_rtp_timestamp_ = frame->Timestamp(); + } + } else { + current_codec_ = frame->codec_type(); + last_assembled_frame_rtp_timestamp_ = frame->Timestamp(); + } + + if (buffered_frame_decryptor_ != nullptr) { + buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame)); + } else if (frame_transformer_delegate_) { + frame_transformer_delegate_->TransformFrame(std::move(frame)); + } else { + reference_finder_->ManageFrame(std::move(frame)); + } +} + +void RtpVideoStreamReceiver2::OnCompleteFrame( + std::unique_ptr frame) { + { + rtc::CritScope lock(&last_seq_num_cs_); + video_coding::RtpFrameObject* rtp_frame = + static_cast(frame.get()); + last_seq_num_for_pic_id_[rtp_frame->id.picture_id] = + rtp_frame->last_seq_num(); + } + last_completed_picture_id_ = + std::max(last_completed_picture_id_, frame->id.picture_id); + complete_frame_callback_->OnCompleteFrame(std::move(frame)); +} + +void RtpVideoStreamReceiver2::OnDecryptedFrame( + std::unique_ptr frame) { + rtc::CritScope lock(&reference_finder_lock_); + reference_finder_->ManageFrame(std::move(frame)); +} + +void RtpVideoStreamReceiver2::OnDecryptionStatusChange( + FrameDecryptorInterface::Status status) { + frames_decryptable_.store( + (status == FrameDecryptorInterface::Status::kOk) || + (status == FrameDecryptorInterface::Status::kRecoverable)); +} + +void RtpVideoStreamReceiver2::SetFrameDecryptor( + rtc::scoped_refptr frame_decryptor) { + RTC_DCHECK_RUN_ON(&network_tc_); + if (buffered_frame_decryptor_ == nullptr) { + buffered_frame_decryptor_ = + std::make_unique(this, this); + } + buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor)); +} + +void RtpVideoStreamReceiver2::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(&network_tc_); + frame_transformer_delegate_ = + new rtc::RefCountedObject( + this, std::move(frame_transformer), rtc::Thread::Current(), + config_.rtp.remote_ssrc); + frame_transformer_delegate_->Init(); +} + +void RtpVideoStreamReceiver2::UpdateRtt(int64_t max_rtt_ms) { + if (nack_module_) + nack_module_->UpdateRtt(max_rtt_ms); +} + +absl::optional RtpVideoStreamReceiver2::LastReceivedPacketMs() const { + return packet_buffer_.LastReceivedPacketMs(); +} + +absl::optional RtpVideoStreamReceiver2::LastReceivedKeyframePacketMs() + const { + return packet_buffer_.LastReceivedKeyframePacketMs(); +} + +void RtpVideoStreamReceiver2::AddSecondarySink(RtpPacketSinkInterface* sink) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + RTC_DCHECK(!absl::c_linear_search(secondary_sinks_, sink)); + secondary_sinks_.push_back(sink); +} + +void RtpVideoStreamReceiver2::RemoveSecondarySink( + const RtpPacketSinkInterface* sink) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + auto it = absl::c_find(secondary_sinks_, sink); + if (it == secondary_sinks_.end()) { + // We might be rolling-back a call whose setup failed mid-way. In such a + // case, it's simpler to remove "everything" rather than remember what + // has already been added. + RTC_LOG(LS_WARNING) << "Removal of unknown sink."; + return; + } + secondary_sinks_.erase(it); +} + +void RtpVideoStreamReceiver2::ManageFrame( + std::unique_ptr frame) { + rtc::CritScope lock(&reference_finder_lock_); + reference_finder_->ManageFrame(std::move(frame)); +} + +void RtpVideoStreamReceiver2::ReceivePacket(const RtpPacketReceived& packet) { + if (packet.payload_size() == 0) { + // Padding or keep-alive packet. + // TODO(nisse): Could drop empty packets earlier, but need to figure out how + // they should be counted in stats. + NotifyReceiverOfEmptyPacket(packet.SequenceNumber()); + return; + } + if (packet.PayloadType() == config_.rtp.red_payload_type) { + ParseAndHandleEncapsulatingHeader(packet); + return; + } + + const auto type_it = payload_type_map_.find(packet.PayloadType()); + if (type_it == payload_type_map_.end()) { + return; + } + absl::optional parsed_payload = + type_it->second->Parse(packet.PayloadBuffer()); + if (parsed_payload == absl::nullopt) { + RTC_LOG(LS_WARNING) << "Failed parsing payload."; + return; + } + + OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet, + parsed_payload->video_header); +} + +void RtpVideoStreamReceiver2::ParseAndHandleEncapsulatingHeader( + const RtpPacketReceived& packet) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + if (packet.PayloadType() == config_.rtp.red_payload_type && + packet.payload_size() > 0) { + if (packet.payload()[0] == config_.rtp.ulpfec_payload_type) { + // Notify video_receiver about received FEC packets to avoid NACKing these + // packets. + NotifyReceiverOfEmptyPacket(packet.SequenceNumber()); + } + if (!ulpfec_receiver_->AddReceivedRedPacket( + packet, config_.rtp.ulpfec_payload_type)) { + return; + } + ulpfec_receiver_->ProcessReceivedFec(); + } +} + +// In the case of a video stream without picture ids and no rtx the +// RtpFrameReferenceFinder will need to know about padding to +// correctly calculate frame references. +void RtpVideoStreamReceiver2::NotifyReceiverOfEmptyPacket(uint16_t seq_num) { + { + rtc::CritScope lock(&reference_finder_lock_); + reference_finder_->PaddingReceived(seq_num); + } + OnInsertedPacket(packet_buffer_.InsertPadding(seq_num)); + if (nack_module_) { + nack_module_->OnReceivedPacket(seq_num, /* is_keyframe = */ false, + /* is _recovered = */ false); + } + if (loss_notification_controller_) { + // TODO(bugs.webrtc.org/10336): Handle empty packets. + RTC_LOG(LS_WARNING) + << "LossNotificationController does not expect empty packets."; + } +} + +bool RtpVideoStreamReceiver2::DeliverRtcp(const uint8_t* rtcp_packet, + size_t rtcp_packet_length) { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + + if (!receiving_) { + return false; + } + + rtp_rtcp_->IncomingRtcpPacket(rtcp_packet, rtcp_packet_length); + + int64_t rtt = 0; + rtp_rtcp_->RTT(config_.rtp.remote_ssrc, &rtt, nullptr, nullptr, nullptr); + if (rtt == 0) { + // Waiting for valid rtt. + return true; + } + uint32_t ntp_secs = 0; + uint32_t ntp_frac = 0; + uint32_t rtp_timestamp = 0; + uint32_t recieved_ntp_secs = 0; + uint32_t recieved_ntp_frac = 0; + if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, &recieved_ntp_secs, + &recieved_ntp_frac, &rtp_timestamp) != 0) { + // Waiting for RTCP. + return true; + } + NtpTime recieved_ntp(recieved_ntp_secs, recieved_ntp_frac); + int64_t time_since_recieved = + clock_->CurrentNtpInMilliseconds() - recieved_ntp.ToMs(); + // Don't use old SRs to estimate time. + if (time_since_recieved <= 1) { + ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp); + absl::optional remote_to_local_clock_offset_ms = + ntp_estimator_.EstimateRemoteToLocalClockOffsetMs(); + if (remote_to_local_clock_offset_ms.has_value()) { + absolute_capture_time_receiver_.SetRemoteToLocalClockOffset( + Int64MsToQ32x32(*remote_to_local_clock_offset_ms)); + } + } + + return true; +} + +void RtpVideoStreamReceiver2::FrameContinuous(int64_t picture_id) { + if (!nack_module_) + return; + + int seq_num = -1; + { + rtc::CritScope lock(&last_seq_num_cs_); + auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); + if (seq_num_it != last_seq_num_for_pic_id_.end()) + seq_num = seq_num_it->second; + } + if (seq_num != -1) + nack_module_->ClearUpTo(seq_num); +} + +void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) { + int seq_num = -1; + { + rtc::CritScope lock(&last_seq_num_cs_); + auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); + if (seq_num_it != last_seq_num_for_pic_id_.end()) { + seq_num = seq_num_it->second; + last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(), + ++seq_num_it); + } + } + if (seq_num != -1) { + packet_buffer_.ClearTo(seq_num); + rtc::CritScope lock(&reference_finder_lock_); + reference_finder_->ClearTo(seq_num); + } +} + +void RtpVideoStreamReceiver2::SignalNetworkState(NetworkState state) { + rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode + : RtcpMode::kOff); +} + +void RtpVideoStreamReceiver2::StartReceive() { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + receiving_ = true; +} + +void RtpVideoStreamReceiver2::StopReceive() { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + receiving_ = false; +} + +void RtpVideoStreamReceiver2::UpdateHistograms() { + FecPacketCounter counter = ulpfec_receiver_->GetPacketCounter(); + if (counter.first_packet_time_ms == -1) + return; + + int64_t elapsed_sec = + (clock_->TimeInMilliseconds() - counter.first_packet_time_ms) / 1000; + if (elapsed_sec < metrics::kMinRunTimeInSeconds) + return; + + if (counter.num_packets > 0) { + RTC_HISTOGRAM_PERCENTAGE( + "WebRTC.Video.ReceivedFecPacketsInPercent", + static_cast(counter.num_fec_packets * 100 / counter.num_packets)); + } + if (counter.num_fec_packets > 0) { + RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.RecoveredMediaPacketsInPercentOfFec", + static_cast(counter.num_recovered_packets * + 100 / counter.num_fec_packets)); + } + if (config_.rtp.ulpfec_payload_type != -1) { + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.FecBitrateReceivedInKbps", + static_cast(counter.num_bytes * 8 / elapsed_sec / 1000)); + } +} + +void RtpVideoStreamReceiver2::InsertSpsPpsIntoTracker(uint8_t payload_type) { + auto codec_params_it = pt_codec_params_.find(payload_type); + if (codec_params_it == pt_codec_params_.end()) + return; + + RTC_LOG(LS_INFO) << "Found out of band supplied codec parameters for" + " payload type: " + << static_cast(payload_type); + + H264SpropParameterSets sprop_decoder; + auto sprop_base64_it = + codec_params_it->second.find(cricket::kH264FmtpSpropParameterSets); + + if (sprop_base64_it == codec_params_it->second.end()) + return; + + if (!sprop_decoder.DecodeSprop(sprop_base64_it->second.c_str())) + return; + + tracker_.InsertSpsPpsNalus(sprop_decoder.sps_nalu(), + sprop_decoder.pps_nalu()); +} + +} // namespace webrtc diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h new file mode 100644 index 0000000000..3026e1dac3 --- /dev/null +++ b/video/rtp_video_stream_receiver2.h @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ +#define VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ + +#include +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/video/color_space.h" +#include "api/video_codecs/video_codec.h" +#include "call/rtp_packet_sink_interface.h" +#include "call/syncable.h" +#include "call/video_receive_stream.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" +#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "modules/rtp_rtcp/include/rtp_rtcp.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" +#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_video_header.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" +#include "modules/video_coding/h264_sps_pps_tracker.h" +#include "modules/video_coding/loss_notification_controller.h" +#include "modules/video_coding/packet_buffer.h" +#include "modules/video_coding/rtp_frame_reference_finder.h" +#include "modules/video_coding/unique_timestamp_counter.h" +#include "rtc_base/constructor_magic.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/numerics/sequence_number_util.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/thread_checker.h" +#include "video/buffered_frame_decryptor.h" +#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" + +namespace webrtc { + +class NackModule2; +class PacketRouter; +class ProcessThread; +class ReceiveStatistics; +class RtcpRttStats; +class RtpPacketReceived; +class Transport; +class UlpfecReceiver; + +class RtpVideoStreamReceiver2 : public LossNotificationSender, + public RecoveredPacketReceiver, + public RtpPacketSinkInterface, + public KeyFrameRequestSender, + public video_coding::OnCompleteFrameCallback, + public OnDecryptedFrameCallback, + public OnDecryptionStatusChangeCallback, + public RtpVideoFrameReceiver { + public: + RtpVideoStreamReceiver2( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + // The packet router is optional; if provided, the RtpRtcp module for this + // stream is registered as a candidate for sending REMB and transport + // feedback. + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, + RtcpCnameCallback* rtcp_cname_callback, + ProcessThread* process_thread, + NackSender* nack_sender, + // The KeyFrameRequestSender is optional; if not provided, key frame + // requests are sent via the internal RtpRtcp module. + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer); + ~RtpVideoStreamReceiver2() override; + + void AddReceiveCodec(const VideoCodec& video_codec, + const std::map& codec_params, + bool raw_payload); + + void StartReceive(); + void StopReceive(); + + // Produces the transport-related timestamps; current_delay_ms is left unset. + absl::optional GetSyncInfo() const; + + bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); + + void FrameContinuous(int64_t seq_num); + + void FrameDecoded(int64_t seq_num); + + void SignalNetworkState(NetworkState state); + + // Returns number of different frames seen. + int GetUniqueFramesSeen() const { + RTC_DCHECK_RUN_ON(&worker_task_checker_); + return frame_counter_.GetUniqueSeen(); + } + + // Implements RtpPacketSinkInterface. + void OnRtpPacket(const RtpPacketReceived& packet) override; + + // TODO(philipel): Stop using VCMPacket in the new jitter buffer and then + // remove this function. Public only for tests. + void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, + const RtpPacketReceived& rtp_packet, + const RTPVideoHeader& video); + + // Implements RecoveredPacketReceiver. + void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; + + // Send an RTCP keyframe request. + void RequestKeyFrame() override; + + // Implements LossNotificationSender. + void SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) override; + + bool IsUlpfecEnabled() const; + bool IsRetransmissionsEnabled() const; + + // Returns true if a decryptor is attached and frames can be decrypted. + // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame + // Decryption not SRTP. + bool IsDecryptable() const; + + // Don't use, still experimental. + void RequestPacketRetransmit(const std::vector& sequence_numbers); + + // Implements OnCompleteFrameCallback. + void OnCompleteFrame( + std::unique_ptr frame) override; + + // Implements OnDecryptedFrameCallback. + void OnDecryptedFrame( + std::unique_ptr frame) override; + + // Implements OnDecryptionStatusChangeCallback. + void OnDecryptionStatusChange( + FrameDecryptorInterface::Status status) override; + + // Optionally set a frame decryptor after a stream has started. This will not + // reset the decoder state. + void SetFrameDecryptor( + rtc::scoped_refptr frame_decryptor); + + // Sets a frame transformer after a stream has started, if no transformer + // has previously been set. Does not reset the decoder state. + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer); + + // Called by VideoReceiveStream when stats are updated. + void UpdateRtt(int64_t max_rtt_ms); + + absl::optional LastReceivedPacketMs() const; + absl::optional LastReceivedKeyframePacketMs() const; + + // RtpDemuxer only forwards a given RTP packet to one sink. However, some + // sinks, such as FlexFEC, might wish to be informed of all of the packets + // a given sink receives (or any set of sinks). They may do so by registering + // themselves as secondary sinks. + void AddSecondarySink(RtpPacketSinkInterface* sink); + void RemoveSecondarySink(const RtpPacketSinkInterface* sink); + + private: + // Implements RtpVideoFrameReceiver. + void ManageFrame( + std::unique_ptr frame) override; + + // Used for buffering RTCP feedback messages and sending them all together. + // Note: + // 1. Key frame requests and NACKs are mutually exclusive, with the + // former taking precedence over the latter. + // 2. Loss notifications are orthogonal to either. (That is, may be sent + // alongside either.) + class RtcpFeedbackBuffer : public KeyFrameRequestSender, + public NackSender, + public LossNotificationSender { + public: + RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, + NackSender* nack_sender, + LossNotificationSender* loss_notification_sender); + + ~RtcpFeedbackBuffer() override = default; + + // KeyFrameRequestSender implementation. + void RequestKeyFrame() RTC_LOCKS_EXCLUDED(cs_) override; + + // NackSender implementation. + void SendNack(const std::vector& sequence_numbers, + bool buffering_allowed) RTC_LOCKS_EXCLUDED(cs_) override; + + // LossNotificationSender implementation. + void SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) + RTC_LOCKS_EXCLUDED(cs_) override; + + // Send all RTCP feedback messages buffered thus far. + void SendBufferedRtcpFeedback() RTC_LOCKS_EXCLUDED(cs_); + + private: + // LNTF-related state. + struct LossNotificationState { + LossNotificationState(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag) + : last_decoded_seq_num(last_decoded_seq_num), + last_received_seq_num(last_received_seq_num), + decodability_flag(decodability_flag) {} + + uint16_t last_decoded_seq_num; + uint16_t last_received_seq_num; + bool decodability_flag; + }; + struct ConsumedRtcpFeedback { + bool request_key_frame = false; + std::vector nack_sequence_numbers; + absl::optional lntf_state; + }; + + ConsumedRtcpFeedback ConsumeRtcpFeedback() RTC_LOCKS_EXCLUDED(cs_); + ConsumedRtcpFeedback ConsumeRtcpFeedbackLocked() + RTC_EXCLUSIVE_LOCKS_REQUIRED(cs_); + // This method is called both with and without cs_ held. + void SendRtcpFeedback(ConsumedRtcpFeedback feedback); + + KeyFrameRequestSender* const key_frame_request_sender_; + NackSender* const nack_sender_; + LossNotificationSender* const loss_notification_sender_; + + // NACKs are accessible from two threads due to nack_module_ being a module. + rtc::CriticalSection cs_; + + // Key-frame-request-related state. + bool request_key_frame_ RTC_GUARDED_BY(cs_); + + // NACK-related state. + std::vector nack_sequence_numbers_ RTC_GUARDED_BY(cs_); + + absl::optional lntf_state_ RTC_GUARDED_BY(cs_); + }; + enum ParseGenericDependenciesResult { + kDropPacket, + kHasGenericDescriptor, + kNoGenericDescriptor + }; + + // Entry point doing non-stats work for a received packet. Called + // for the same packet both before and after RED decapsulation. + void ReceivePacket(const RtpPacketReceived& packet); + // Parses and handles RED headers. + // This function assumes that it's being called from only one thread. + void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet); + void NotifyReceiverOfEmptyPacket(uint16_t seq_num); + void UpdateHistograms(); + bool IsRedEnabled() const; + void InsertSpsPpsIntoTracker(uint8_t payload_type); + void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result); + ParseGenericDependenciesResult ParseGenericDependenciesExtension( + const RtpPacketReceived& rtp_packet, + RTPVideoHeader* video_header) RTC_RUN_ON(worker_task_checker_); + void OnAssembledFrame(std::unique_ptr frame); + + Clock* const clock_; + // Ownership of this object lies with VideoReceiveStream, which owns |this|. + const VideoReceiveStream::Config& config_; + PacketRouter* const packet_router_; + ProcessThread* const process_thread_; + + RemoteNtpTimeEstimator ntp_estimator_; + + RtpHeaderExtensionMap rtp_header_extensions_; + // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout + // delay that is specified in the received packets. + FieldTrialOptional forced_playout_delay_max_ms_; + FieldTrialOptional forced_playout_delay_min_ms_; + ReceiveStatistics* const rtp_receive_statistics_; + std::unique_ptr ulpfec_receiver_; + + SequenceChecker worker_task_checker_; + bool receiving_ RTC_GUARDED_BY(worker_task_checker_); + int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); + + const std::unique_ptr rtp_rtcp_; + + video_coding::OnCompleteFrameCallback* complete_frame_callback_; + KeyFrameRequestSender* const keyframe_request_sender_; + + RtcpFeedbackBuffer rtcp_feedback_buffer_; + std::unique_ptr nack_module_; + std::unique_ptr loss_notification_controller_; + + video_coding::PacketBuffer packet_buffer_; + UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(worker_task_checker_); + SeqNumUnwrapper frame_id_unwrapper_ + RTC_GUARDED_BY(worker_task_checker_); + + // Video structure provided in the dependency descriptor in a first packet + // of a key frame. It is required to parse dependency descriptor in the + // following delta packets. + std::unique_ptr video_structure_ + RTC_GUARDED_BY(worker_task_checker_); + // Frame id of the last frame with the attached video structure. + // absl::nullopt when `video_structure_ == nullptr`; + absl::optional video_structure_frame_id_ + RTC_GUARDED_BY(worker_task_checker_); + + rtc::CriticalSection reference_finder_lock_; + std::unique_ptr reference_finder_ + RTC_GUARDED_BY(reference_finder_lock_); + absl::optional current_codec_; + uint32_t last_assembled_frame_rtp_timestamp_; + + rtc::CriticalSection last_seq_num_cs_; + std::map last_seq_num_for_pic_id_ + RTC_GUARDED_BY(last_seq_num_cs_); + video_coding::H264SpsPpsTracker tracker_; + + // Maps payload id to the depacketizer. + std::map> payload_type_map_; + + // TODO(johan): Remove pt_codec_params_ once + // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. + // Maps a payload type to a map of out-of-band supplied codec parameters. + std::map> pt_codec_params_; + int16_t last_payload_type_ = -1; + + bool has_received_frame_; + + std::vector secondary_sinks_ + RTC_GUARDED_BY(worker_task_checker_); + + // Info for GetSyncInfo is updated on network or worker thread, and queried on + // the worker thread. + rtc::CriticalSection sync_info_lock_; + absl::optional last_received_rtp_timestamp_ + RTC_GUARDED_BY(sync_info_lock_); + absl::optional last_received_rtp_system_time_ms_ + RTC_GUARDED_BY(sync_info_lock_); + + // Used to validate the buffered frame decryptor is always run on the correct + // thread. + rtc::ThreadChecker network_tc_; + // Handles incoming encrypted frames and forwards them to the + // rtp_reference_finder if they are decryptable. + std::unique_ptr buffered_frame_decryptor_ + RTC_PT_GUARDED_BY(network_tc_); + std::atomic frames_decryptable_; + absl::optional last_color_space_; + + AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_ + RTC_GUARDED_BY(worker_task_checker_); + + int64_t last_completed_picture_id_ = 0; + + rtc::scoped_refptr + frame_transformer_delegate_; +}; + +} // namespace webrtc + +#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ diff --git a/video/rtp_video_stream_receiver2_unittest.cc b/video/rtp_video_stream_receiver2_unittest.cc new file mode 100644 index 0000000000..c8584fcd55 --- /dev/null +++ b/video/rtp_video_stream_receiver2_unittest.cc @@ -0,0 +1,1215 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/rtp_video_stream_receiver2.h" + +#include +#include + +#include "api/video/video_codec_type.h" +#include "api/video/video_frame_type.h" +#include "common_video/h264/h264_common.h" +#include "media/base/media_constants.h" +#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" +#include "modules/rtp_rtcp/source/rtp_format.h" +#include "modules/rtp_rtcp/source/rtp_format_vp9.h" +#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" +#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_header_extensions.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/utility/include/process_thread.h" +#include "modules/video_coding/frame_object.h" +#include "modules/video_coding/include/video_coding_defines.h" +#include "modules/video_coding/rtp_frame_reference_finder.h" +#include "rtc_base/byte_buffer.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/mock_frame_transformer.h" + +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::Invoke; +using ::testing::SizeIs; +using ::testing::Values; + +namespace webrtc { + +namespace { + +const uint8_t kH264StartCode[] = {0x00, 0x00, 0x00, 0x01}; + +std::vector GetAbsoluteCaptureTimestamps( + const video_coding::EncodedFrame* frame) { + std::vector result; + for (const auto& packet_info : frame->PacketInfos()) { + if (packet_info.absolute_capture_time()) { + result.push_back( + packet_info.absolute_capture_time()->absolute_capture_timestamp); + } + } + return result; +} + +RTPVideoHeader GetGenericVideoHeader(VideoFrameType frame_type) { + RTPVideoHeader video_header; + video_header.is_first_packet_in_frame = true; + video_header.is_last_packet_in_frame = true; + video_header.codec = kVideoCodecGeneric; + video_header.frame_type = frame_type; + return video_header; +} + +class MockTransport : public Transport { + public: + MOCK_METHOD(bool, + SendRtp, + (const uint8_t*, size_t length, const PacketOptions& options), + (override)); + MOCK_METHOD(bool, SendRtcp, (const uint8_t*, size_t length), (override)); +}; + +class MockNackSender : public NackSender { + public: + MOCK_METHOD(void, + SendNack, + (const std::vector& sequence_numbers, + bool buffering_allowed), + (override)); +}; + +class MockKeyFrameRequestSender : public KeyFrameRequestSender { + public: + MOCK_METHOD(void, RequestKeyFrame, (), (override)); +}; + +class MockOnCompleteFrameCallback + : public video_coding::OnCompleteFrameCallback { + public: + MOCK_METHOD(void, DoOnCompleteFrame, (video_coding::EncodedFrame*), ()); + MOCK_METHOD(void, + DoOnCompleteFrameFailNullptr, + (video_coding::EncodedFrame*), + ()); + MOCK_METHOD(void, + DoOnCompleteFrameFailLength, + (video_coding::EncodedFrame*), + ()); + MOCK_METHOD(void, + DoOnCompleteFrameFailBitstream, + (video_coding::EncodedFrame*), + ()); + void OnCompleteFrame( + std::unique_ptr frame) override { + if (!frame) { + DoOnCompleteFrameFailNullptr(nullptr); + return; + } + EXPECT_EQ(buffer_.Length(), frame->size()); + if (buffer_.Length() != frame->size()) { + DoOnCompleteFrameFailLength(frame.get()); + return; + } + if (frame->size() != buffer_.Length() || + memcmp(buffer_.Data(), frame->data(), buffer_.Length()) != 0) { + DoOnCompleteFrameFailBitstream(frame.get()); + return; + } + DoOnCompleteFrame(frame.get()); + } + + void ClearExpectedBitstream() { buffer_.Clear(); } + + void AppendExpectedBitstream(const uint8_t data[], size_t size_in_bytes) { + // TODO(Johan): Let rtc::ByteBuffer handle uint8_t* instead of char*. + buffer_.WriteBytes(reinterpret_cast(data), size_in_bytes); + } + rtc::ByteBufferWriter buffer_; +}; + +class MockRtpPacketSink : public RtpPacketSinkInterface { + public: + MOCK_METHOD(void, OnRtpPacket, (const RtpPacketReceived&), (override)); +}; + +constexpr uint32_t kSsrc = 111; +constexpr uint16_t kSequenceNumber = 222; +constexpr int kPayloadType = 100; +constexpr int kRedPayloadType = 125; + +std::unique_ptr CreateRtpPacketReceived() { + auto packet = std::make_unique(); + packet->SetSsrc(kSsrc); + packet->SetSequenceNumber(kSequenceNumber); + packet->SetPayloadType(kPayloadType); + return packet; +} + +MATCHER_P(SamePacketAs, other, "") { + return arg.Ssrc() == other.Ssrc() && + arg.SequenceNumber() == other.SequenceNumber(); +} + +} // namespace + +class RtpVideoStreamReceiver2Test : public ::testing::Test { + public: + RtpVideoStreamReceiver2Test() : RtpVideoStreamReceiver2Test("") {} + explicit RtpVideoStreamReceiver2Test(std::string field_trials) + : override_field_trials_(field_trials), + config_(CreateConfig()), + process_thread_(ProcessThread::Create("TestThread")) { + rtp_receive_statistics_ = + ReceiveStatistics::Create(Clock::GetRealTimeClock()); + rtp_video_stream_receiver_ = std::make_unique( + Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, + rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(), + &mock_nack_sender_, &mock_key_frame_request_sender_, + &mock_on_complete_frame_callback_, nullptr, nullptr); + VideoCodec codec; + codec.plType = kPayloadType; + codec.codecType = kVideoCodecGeneric; + rtp_video_stream_receiver_->AddReceiveCodec(codec, {}, + /*raw_payload=*/false); + } + + RTPVideoHeader GetDefaultH264VideoHeader() { + RTPVideoHeader video_header; + video_header.codec = kVideoCodecH264; + video_header.video_type_header.emplace(); + return video_header; + } + + // TODO(Johan): refactor h264_sps_pps_tracker_unittests.cc to avoid duplicate + // code. + void AddSps(RTPVideoHeader* video_header, + uint8_t sps_id, + rtc::CopyOnWriteBuffer* data) { + NaluInfo info; + info.type = H264::NaluType::kSps; + info.sps_id = sps_id; + info.pps_id = -1; + data->AppendData({H264::NaluType::kSps, sps_id}); + auto& h264 = absl::get(video_header->video_type_header); + h264.nalus[h264.nalus_length++] = info; + } + + void AddPps(RTPVideoHeader* video_header, + uint8_t sps_id, + uint8_t pps_id, + rtc::CopyOnWriteBuffer* data) { + NaluInfo info; + info.type = H264::NaluType::kPps; + info.sps_id = sps_id; + info.pps_id = pps_id; + data->AppendData({H264::NaluType::kPps, pps_id}); + auto& h264 = absl::get(video_header->video_type_header); + h264.nalus[h264.nalus_length++] = info; + } + + void AddIdr(RTPVideoHeader* video_header, int pps_id) { + NaluInfo info; + info.type = H264::NaluType::kIdr; + info.sps_id = -1; + info.pps_id = pps_id; + auto& h264 = absl::get(video_header->video_type_header); + h264.nalus[h264.nalus_length++] = info; + } + + protected: + static VideoReceiveStream::Config CreateConfig() { + VideoReceiveStream::Config config(nullptr); + config.rtp.remote_ssrc = 1111; + config.rtp.local_ssrc = 2222; + config.rtp.red_payload_type = kRedPayloadType; + return config; + } + + const webrtc::test::ScopedFieldTrials override_field_trials_; + VideoReceiveStream::Config config_; + MockNackSender mock_nack_sender_; + MockKeyFrameRequestSender mock_key_frame_request_sender_; + MockTransport mock_transport_; + MockOnCompleteFrameCallback mock_on_complete_frame_callback_; + std::unique_ptr process_thread_; + std::unique_ptr rtp_receive_statistics_; + std::unique_ptr rtp_video_stream_receiver_; +}; + +TEST_F(RtpVideoStreamReceiver2Test, CacheColorSpaceFromLastPacketOfKeyframe) { + // Test that color space is cached from the last packet of a key frame and + // that it's not reset by padding packets without color space. + constexpr int kVp9PayloadType = 99; + const ColorSpace kColorSpace( + ColorSpace::PrimaryID::kFILM, ColorSpace::TransferID::kBT2020_12, + ColorSpace::MatrixID::kBT2020_NCL, ColorSpace::RangeID::kFull); + const std::vector kKeyFramePayload = {0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10}; + const std::vector kDeltaFramePayload = {0, 1, 2, 3, 4}; + + // Anonymous helper class that generates received packets. + class { + public: + void SetPayload(const std::vector& payload, + VideoFrameType video_frame_type) { + video_frame_type_ = video_frame_type; + RtpPacketizer::PayloadSizeLimits pay_load_size_limits; + // Reduce max payload length to make sure the key frame generates two + // packets. + pay_load_size_limits.max_payload_len = 8; + RTPVideoHeaderVP9 rtp_video_header_vp9; + rtp_video_header_vp9.InitRTPVideoHeaderVP9(); + rtp_video_header_vp9.inter_pic_predicted = + (video_frame_type == VideoFrameType::kVideoFrameDelta); + rtp_packetizer_ = std::make_unique( + payload, pay_load_size_limits, rtp_video_header_vp9); + } + + size_t NumPackets() { return rtp_packetizer_->NumPackets(); } + void SetColorSpace(const ColorSpace& color_space) { + color_space_ = color_space; + } + + RtpPacketReceived NextPacket() { + RtpHeaderExtensionMap extension_map; + extension_map.Register(1); + RtpPacketToSend packet_to_send(&extension_map); + packet_to_send.SetSequenceNumber(sequence_number_++); + packet_to_send.SetSsrc(kSsrc); + packet_to_send.SetPayloadType(kVp9PayloadType); + bool include_color_space = + (rtp_packetizer_->NumPackets() == 1u && + video_frame_type_ == VideoFrameType::kVideoFrameKey); + if (include_color_space) { + EXPECT_TRUE( + packet_to_send.SetExtension(color_space_)); + } + rtp_packetizer_->NextPacket(&packet_to_send); + + RtpPacketReceived received_packet(&extension_map); + received_packet.Parse(packet_to_send.data(), packet_to_send.size()); + return received_packet; + } + + private: + uint16_t sequence_number_ = 0; + VideoFrameType video_frame_type_; + ColorSpace color_space_; + std::unique_ptr rtp_packetizer_; + } received_packet_generator; + received_packet_generator.SetColorSpace(kColorSpace); + + // Prepare the receiver for VP9. + VideoCodec codec; + codec.plType = kVp9PayloadType; + codec.codecType = kVideoCodecVP9; + std::map codec_params; + rtp_video_stream_receiver_->AddReceiveCodec(codec, codec_params, + /*raw_payload=*/false); + + // Generate key frame packets. + received_packet_generator.SetPayload(kKeyFramePayload, + VideoFrameType::kVideoFrameKey); + EXPECT_EQ(received_packet_generator.NumPackets(), 2u); + RtpPacketReceived key_frame_packet1 = received_packet_generator.NextPacket(); + RtpPacketReceived key_frame_packet2 = received_packet_generator.NextPacket(); + + // Generate delta frame packet. + received_packet_generator.SetPayload(kDeltaFramePayload, + VideoFrameType::kVideoFrameDelta); + EXPECT_EQ(received_packet_generator.NumPackets(), 1u); + RtpPacketReceived delta_frame_packet = received_packet_generator.NextPacket(); + + rtp_video_stream_receiver_->StartReceive(); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kKeyFramePayload.data(), kKeyFramePayload.size()); + + // Send the key frame and expect a callback with color space information. + EXPECT_FALSE(key_frame_packet1.GetExtension()); + EXPECT_TRUE(key_frame_packet2.GetExtension()); + rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([kColorSpace](video_coding::EncodedFrame* frame) { + ASSERT_TRUE(frame->EncodedImage().ColorSpace()); + EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace); + })); + rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet2); + // Resend the first key frame packet to simulate padding for example. + rtp_video_stream_receiver_->OnRtpPacket(key_frame_packet1); + + mock_on_complete_frame_callback_.ClearExpectedBitstream(); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kDeltaFramePayload.data(), kDeltaFramePayload.size()); + + // Expect delta frame to have color space set even though color space not + // included in the RTP packet. + EXPECT_FALSE(delta_frame_packet.GetExtension()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([kColorSpace](video_coding::EncodedFrame* frame) { + ASSERT_TRUE(frame->EncodedImage().ColorSpace()); + EXPECT_EQ(*frame->EncodedImage().ColorSpace(), kColorSpace); + })); + rtp_video_stream_receiver_->OnRtpPacket(delta_frame_packet); +} + +TEST_F(RtpVideoStreamReceiver2Test, GenericKeyFrame) { + RtpPacketReceived rtp_packet; + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(1); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiver2Test, PacketInfoIsPropagatedIntoVideoFrames) { + constexpr uint64_t kAbsoluteCaptureTimestamp = 12; + constexpr int kId0 = 1; + + RtpHeaderExtensionMap extension_map; + extension_map.Register(kId0); + RtpPacketReceived rtp_packet(&extension_map); + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); + rtp_packet.SetSequenceNumber(1); + rtp_packet.SetTimestamp(1); + rtp_packet.SetSsrc(kSsrc); + rtp_packet.SetExtension( + AbsoluteCaptureTime{kAbsoluteCaptureTimestamp, + /*estimated_capture_clock_offset=*/absl::nullopt}); + + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke( + [kAbsoluteCaptureTimestamp](video_coding::EncodedFrame* frame) { + EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), + ElementsAre(kAbsoluteCaptureTimestamp)); + })); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiver2Test, + MissingAbsoluteCaptureTimeIsFilledWithExtrapolatedValue) { + constexpr uint64_t kAbsoluteCaptureTimestamp = 12; + constexpr int kId0 = 1; + + RtpHeaderExtensionMap extension_map; + extension_map.Register(kId0); + RtpPacketReceived rtp_packet(&extension_map); + rtp_packet.SetPayloadType(kPayloadType); + + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); + uint16_t sequence_number = 1; + uint32_t rtp_timestamp = 1; + rtp_packet.SetSequenceNumber(sequence_number); + rtp_packet.SetTimestamp(rtp_timestamp); + rtp_packet.SetSsrc(kSsrc); + rtp_packet.SetExtension( + AbsoluteCaptureTime{kAbsoluteCaptureTimestamp, + /*estimated_capture_clock_offset=*/absl::nullopt}); + + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + + // Rtp packet without absolute capture time. + rtp_packet = RtpPacketReceived(&extension_map); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(++sequence_number); + rtp_packet.SetTimestamp(++rtp_timestamp); + rtp_packet.SetSsrc(kSsrc); + + // There is no absolute capture time in the second packet. + // Expect rtp video stream receiver to extrapolate it for the resulting video + // frame using absolute capture time from the previous packet. + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([](video_coding::EncodedFrame* frame) { + EXPECT_THAT(GetAbsoluteCaptureTimestamps(frame), SizeIs(1)); + })); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiver2Test, + NoInfiniteRecursionOnEncapsulatedRedPacket) { + const std::vector data({ + 0x80, // RTP version. + kRedPayloadType, // Payload type. + 0, 0, 0, 0, 0, 0, // Don't care. + 0, 0, 0x4, 0x57, // SSRC + kRedPayloadType, // RED header. + 0, 0, 0, 0, 0 // Don't care. + }); + RtpPacketReceived packet; + EXPECT_TRUE(packet.Parse(data.data(), data.size())); + rtp_video_stream_receiver_->StartReceive(); + rtp_video_stream_receiver_->OnRtpPacket(packet); +} + +TEST_F(RtpVideoStreamReceiver2Test, + DropsPacketWithRedPayloadTypeAndEmptyPayload) { + const uint8_t kRedPayloadType = 125; + config_.rtp.red_payload_type = kRedPayloadType; + SetUp(); // re-create rtp_video_stream_receiver with red payload type. + // clang-format off + const uint8_t data[] = { + 0x80, // RTP version. + kRedPayloadType, // Payload type. + 0, 0, 0, 0, 0, 0, // Don't care. + 0, 0, 0x4, 0x57, // SSRC + // Empty rtp payload. + }; + // clang-format on + RtpPacketReceived packet; + // Manually convert to CopyOnWriteBuffer to be sure capacity == size + // and asan bot can catch read buffer overflow. + EXPECT_TRUE(packet.Parse(rtc::CopyOnWriteBuffer(data))); + rtp_video_stream_receiver_->StartReceive(); + rtp_video_stream_receiver_->OnRtpPacket(packet); + // Expect asan doesn't find anything. +} + +TEST_F(RtpVideoStreamReceiver2Test, GenericKeyFrameBitstreamError) { + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); + rtp_packet.SetSequenceNumber(1); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + constexpr uint8_t expected_bitsteam[] = {1, 2, 3, 0xff}; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + expected_bitsteam, sizeof(expected_bitsteam)); + EXPECT_CALL(mock_on_complete_frame_callback_, + DoOnCompleteFrameFailBitstream(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +class RtpVideoStreamReceiver2TestH264 + : public RtpVideoStreamReceiver2Test, + public ::testing::WithParamInterface { + protected: + RtpVideoStreamReceiver2TestH264() : RtpVideoStreamReceiver2Test(GetParam()) {} +}; + +INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe, + RtpVideoStreamReceiver2TestH264, + Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/")); + +// Fails on MSAN: https://bugs.chromium.org/p/webrtc/issues/detail?id=11376. +#if defined(MEMORY_SANITIZER) +#define MAYBE_InBandSpsPps DISABLED_InBandSpsPps +#else +#define MAYBE_InBandSpsPps InBandSpsPps +#endif +TEST_P(RtpVideoStreamReceiver2TestH264, MAYBE_InBandSpsPps) { + rtc::CopyOnWriteBuffer sps_data; + RtpPacketReceived rtp_packet; + RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader(); + AddSps(&sps_video_header, 0, &sps_data); + rtp_packet.SetSequenceNumber(0); + rtp_packet.SetPayloadType(kPayloadType); + sps_video_header.is_first_packet_in_frame = true; + sps_video_header.frame_type = VideoFrameType::kEmptyFrame; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(sps_data.data(), + sps_data.size()); + rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet, + sps_video_header); + + rtc::CopyOnWriteBuffer pps_data; + RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader(); + AddPps(&pps_video_header, 0, 1, &pps_data); + rtp_packet.SetSequenceNumber(1); + pps_video_header.is_first_packet_in_frame = true; + pps_video_header.frame_type = VideoFrameType::kEmptyFrame; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(pps_data.data(), + pps_data.size()); + rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet, + pps_video_header); + + rtc::CopyOnWriteBuffer idr_data; + RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader(); + AddIdr(&idr_video_header, 1); + rtp_packet.SetSequenceNumber(2); + idr_video_header.is_first_packet_in_frame = true; + idr_video_header.is_last_packet_in_frame = true; + idr_video_header.frame_type = VideoFrameType::kVideoFrameKey; + const uint8_t idr[] = {0x65, 1, 2, 3}; + idr_data.AppendData(idr); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), + idr_data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet, + idr_video_header); +} + +TEST_P(RtpVideoStreamReceiver2TestH264, OutOfBandFmtpSpsPps) { + constexpr int kPayloadType = 99; + VideoCodec codec; + codec.plType = kPayloadType; + std::map codec_params; + // Example parameter sets from https://tools.ietf.org/html/rfc3984#section-8.2 + // . + codec_params.insert( + {cricket::kH264FmtpSpropParameterSets, "Z0IACpZTBYmI,aMljiA=="}); + rtp_video_stream_receiver_->AddReceiveCodec(codec, codec_params, + /*raw_payload=*/false); + const uint8_t binary_sps[] = {0x67, 0x42, 0x00, 0x0a, 0x96, + 0x53, 0x05, 0x89, 0x88}; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_sps, + sizeof(binary_sps)); + const uint8_t binary_pps[] = {0x68, 0xc9, 0x63, 0x88}; + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_pps, + sizeof(binary_pps)); + + RtpPacketReceived rtp_packet; + RTPVideoHeader video_header = GetDefaultH264VideoHeader(); + AddIdr(&video_header, 0); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(2); + video_header.is_first_packet_in_frame = true; + video_header.is_last_packet_in_frame = true; + video_header.codec = kVideoCodecH264; + video_header.frame_type = VideoFrameType::kVideoFrameKey; + rtc::CopyOnWriteBuffer data({1, 2, 3}); + mock_on_complete_frame_callback_.AppendExpectedBitstream( + kH264StartCode, sizeof(kH264StartCode)); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiver2Test, PaddingInMediaStream) { + RtpPacketReceived rtp_packet; + RTPVideoHeader video_header = GetDefaultH264VideoHeader(); + rtc::CopyOnWriteBuffer data({1, 2, 3}); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(2); + video_header.is_first_packet_in_frame = true; + video_header.is_last_packet_in_frame = true; + video_header.codec = kVideoCodecGeneric; + video_header.frame_type = VideoFrameType::kVideoFrameKey; + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + + rtp_packet.SetSequenceNumber(3); + rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet, + video_header); + + rtp_packet.SetSequenceNumber(4); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + video_header.frame_type = VideoFrameType::kVideoFrameDelta; + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + + rtp_packet.SetSequenceNumber(6); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_packet.SetSequenceNumber(5); + rtp_video_stream_receiver_->OnReceivedPayloadData({}, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiver2Test, RequestKeyframeIfFirstFrameIsDelta) { + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); + rtp_packet.SetSequenceNumber(1); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta); + EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame()); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiver2Test, RequestKeyframeWhenPacketBufferGetsFull) { + constexpr int kPacketBufferMaxSize = 2048; + + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameDelta); + // Incomplete frames so that the packet buffer is filling up. + video_header.is_last_packet_in_frame = false; + uint16_t start_sequence_number = 1234; + rtp_packet.SetSequenceNumber(start_sequence_number); + while (rtp_packet.SequenceNumber() - start_sequence_number < + kPacketBufferMaxSize) { + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); + rtp_packet.SetSequenceNumber(rtp_packet.SequenceNumber() + 2); + } + + EXPECT_CALL(mock_key_frame_request_sender_, RequestKeyFrame()); + rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, + video_header); +} + +TEST_F(RtpVideoStreamReceiver2Test, SecondarySinksGetRtpNotifications) { + rtp_video_stream_receiver_->StartReceive(); + + MockRtpPacketSink secondary_sink_1; + MockRtpPacketSink secondary_sink_2; + + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_1); + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink_2); + + auto rtp_packet = CreateRtpPacketReceived(); + EXPECT_CALL(secondary_sink_1, OnRtpPacket(SamePacketAs(*rtp_packet))); + EXPECT_CALL(secondary_sink_2, OnRtpPacket(SamePacketAs(*rtp_packet))); + + rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); + + // Test tear-down. + rtp_video_stream_receiver_->StopReceive(); + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_1); + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink_2); +} + +TEST_F(RtpVideoStreamReceiver2Test, + RemovedSecondarySinksGetNoRtpNotifications) { + rtp_video_stream_receiver_->StartReceive(); + + MockRtpPacketSink secondary_sink; + + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); + + auto rtp_packet = CreateRtpPacketReceived(); + + EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0); + + rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); + + // Test tear-down. + rtp_video_stream_receiver_->StopReceive(); +} + +TEST_F(RtpVideoStreamReceiver2Test, + OnlyRemovedSecondarySinksExcludedFromNotifications) { + rtp_video_stream_receiver_->StartReceive(); + + MockRtpPacketSink kept_secondary_sink; + MockRtpPacketSink removed_secondary_sink; + + rtp_video_stream_receiver_->AddSecondarySink(&kept_secondary_sink); + rtp_video_stream_receiver_->AddSecondarySink(&removed_secondary_sink); + rtp_video_stream_receiver_->RemoveSecondarySink(&removed_secondary_sink); + + auto rtp_packet = CreateRtpPacketReceived(); + EXPECT_CALL(kept_secondary_sink, OnRtpPacket(SamePacketAs(*rtp_packet))); + + rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); + + // Test tear-down. + rtp_video_stream_receiver_->StopReceive(); + rtp_video_stream_receiver_->RemoveSecondarySink(&kept_secondary_sink); +} + +TEST_F(RtpVideoStreamReceiver2Test, + SecondariesOfNonStartedStreamGetNoNotifications) { + // Explicitly showing that the stream is not in the |started| state, + // regardless of whether streams start out |started| or |stopped|. + rtp_video_stream_receiver_->StopReceive(); + + MockRtpPacketSink secondary_sink; + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); + + auto rtp_packet = CreateRtpPacketReceived(); + EXPECT_CALL(secondary_sink, OnRtpPacket(_)).Times(0); + + rtp_video_stream_receiver_->OnRtpPacket(*rtp_packet); + + // Test tear-down. + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); +} + +TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorOnePacket) { + const std::vector data = {0, 1, 2, 3, 4}; + const int kSpatialIndex = 1; + + rtp_video_stream_receiver_->StartReceive(); + + RtpHeaderExtensionMap extension_map; + extension_map.Register(5); + RtpPacketReceived rtp_packet(&extension_map); + rtp_packet.SetPayloadType(kPayloadType); + + RtpGenericFrameDescriptor generic_descriptor; + generic_descriptor.SetFirstPacketInSubFrame(true); + generic_descriptor.SetLastPacketInSubFrame(true); + generic_descriptor.SetFrameId(100); + generic_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex); + generic_descriptor.AddFrameDependencyDiff(90); + generic_descriptor.AddFrameDependencyDiff(80); + ASSERT_TRUE(rtp_packet.SetExtension( + generic_descriptor)); + + uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); + memcpy(payload, data.data(), data.size()); + // The first byte is the header, so we ignore the first byte of |data|. + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, + data.size() - 1); + + rtp_packet.SetMarker(true); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(1); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce(Invoke([kSpatialIndex](video_coding::EncodedFrame* frame) { + EXPECT_EQ(frame->num_references, 2U); + EXPECT_EQ(frame->references[0], frame->id.picture_id - 90); + EXPECT_EQ(frame->references[1], frame->id.picture_id - 80); + EXPECT_EQ(frame->id.spatial_layer, kSpatialIndex); + EXPECT_THAT(frame->PacketInfos(), SizeIs(1)); + })); + + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); +} + +TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorTwoPackets) { + const std::vector data = {0, 1, 2, 3, 4}; + const int kSpatialIndex = 1; + + rtp_video_stream_receiver_->StartReceive(); + + RtpHeaderExtensionMap extension_map; + extension_map.Register(5); + RtpPacketReceived first_packet(&extension_map); + + RtpGenericFrameDescriptor first_packet_descriptor; + first_packet_descriptor.SetFirstPacketInSubFrame(true); + first_packet_descriptor.SetLastPacketInSubFrame(false); + first_packet_descriptor.SetFrameId(100); + first_packet_descriptor.SetSpatialLayersBitmask(1 << kSpatialIndex); + first_packet_descriptor.SetResolution(480, 360); + ASSERT_TRUE(first_packet.SetExtension( + first_packet_descriptor)); + + uint8_t* first_packet_payload = first_packet.SetPayloadSize(data.size()); + memcpy(first_packet_payload, data.data(), data.size()); + // The first byte is the header, so we ignore the first byte of |data|. + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, + data.size() - 1); + + first_packet.SetPayloadType(kPayloadType); + first_packet.SetSequenceNumber(1); + rtp_video_stream_receiver_->OnRtpPacket(first_packet); + + RtpPacketReceived second_packet(&extension_map); + RtpGenericFrameDescriptor second_packet_descriptor; + second_packet_descriptor.SetFirstPacketInSubFrame(false); + second_packet_descriptor.SetLastPacketInSubFrame(true); + ASSERT_TRUE(second_packet.SetExtension( + second_packet_descriptor)); + + second_packet.SetMarker(true); + second_packet.SetPayloadType(kPayloadType); + second_packet.SetSequenceNumber(2); + + uint8_t* second_packet_payload = second_packet.SetPayloadSize(data.size()); + memcpy(second_packet_payload, data.data(), data.size()); + // The first byte is the header, so we ignore the first byte of |data|. + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data() + 1, + data.size() - 1); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce(Invoke([kSpatialIndex](video_coding::EncodedFrame* frame) { + EXPECT_EQ(frame->num_references, 0U); + EXPECT_EQ(frame->id.spatial_layer, kSpatialIndex); + EXPECT_EQ(frame->EncodedImage()._encodedWidth, 480u); + EXPECT_EQ(frame->EncodedImage()._encodedHeight, 360u); + EXPECT_THAT(frame->PacketInfos(), SizeIs(2)); + })); + + rtp_video_stream_receiver_->OnRtpPacket(second_packet); +} + +TEST_F(RtpVideoStreamReceiver2Test, ParseGenericDescriptorRawPayload) { + const std::vector data = {0, 1, 2, 3, 4}; + const int kRawPayloadType = 123; + + VideoCodec codec; + codec.plType = kRawPayloadType; + rtp_video_stream_receiver_->AddReceiveCodec(codec, {}, /*raw_payload=*/true); + rtp_video_stream_receiver_->StartReceive(); + + RtpHeaderExtensionMap extension_map; + extension_map.Register(5); + RtpPacketReceived rtp_packet(&extension_map); + + RtpGenericFrameDescriptor generic_descriptor; + generic_descriptor.SetFirstPacketInSubFrame(true); + generic_descriptor.SetLastPacketInSubFrame(true); + ASSERT_TRUE(rtp_packet.SetExtension( + generic_descriptor)); + + uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); + memcpy(payload, data.data(), data.size()); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + + rtp_packet.SetMarker(true); + rtp_packet.SetPayloadType(kRawPayloadType); + rtp_packet.SetSequenceNumber(1); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); +} + +TEST_F(RtpVideoStreamReceiver2Test, UnwrapsFrameId) { + const std::vector data = {0, 1, 2, 3, 4}; + const int kPayloadType = 123; + + VideoCodec codec; + codec.plType = kPayloadType; + rtp_video_stream_receiver_->AddReceiveCodec(codec, {}, /*raw_payload=*/true); + rtp_video_stream_receiver_->StartReceive(); + RtpHeaderExtensionMap extension_map; + extension_map.Register(5); + + uint16_t rtp_sequence_number = 1; + auto inject_packet = [&](uint16_t wrapped_frame_id) { + RtpPacketReceived rtp_packet(&extension_map); + + RtpGenericFrameDescriptor generic_descriptor; + generic_descriptor.SetFirstPacketInSubFrame(true); + generic_descriptor.SetLastPacketInSubFrame(true); + generic_descriptor.SetFrameId(wrapped_frame_id); + ASSERT_TRUE(rtp_packet.SetExtension( + generic_descriptor)); + + uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); + ASSERT_TRUE(payload); + memcpy(payload, data.data(), data.size()); + mock_on_complete_frame_callback_.ClearExpectedBitstream(); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + rtp_packet.SetMarker(true); + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(++rtp_sequence_number); + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); + }; + + int64_t first_picture_id; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](video_coding::EncodedFrame* frame) { + first_picture_id = frame->id.picture_id; + }); + inject_packet(/*wrapped_frame_id=*/0xffff); + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](video_coding::EncodedFrame* frame) { + EXPECT_EQ(frame->id.picture_id - first_picture_id, 3); + }); + inject_packet(/*wrapped_frame_id=*/0x0002); +} + +class RtpVideoStreamReceiver2DependencyDescriptorTest + : public RtpVideoStreamReceiver2Test { + public: + RtpVideoStreamReceiver2DependencyDescriptorTest() { + VideoCodec codec; + codec.plType = payload_type_; + rtp_video_stream_receiver_->AddReceiveCodec(codec, {}, + /*raw_payload=*/true); + extension_map_.Register(7); + rtp_video_stream_receiver_->StartReceive(); + } + + // Returns some valid structure for the DependencyDescriptors. + // First template of that structure always fit for a key frame. + static FrameDependencyStructure CreateStreamStructure() { + FrameDependencyStructure stream_structure; + stream_structure.num_decode_targets = 1; + stream_structure.templates = { + GenericFrameInfo::Builder().Dtis("S").Build(), + GenericFrameInfo::Builder().Dtis("S").Fdiffs({1}).Build(), + }; + return stream_structure; + } + + void InjectPacketWith(const FrameDependencyStructure& stream_structure, + const DependencyDescriptor& dependency_descriptor) { + const std::vector data = {0, 1, 2, 3, 4}; + RtpPacketReceived rtp_packet(&extension_map_); + ASSERT_TRUE(rtp_packet.SetExtension( + stream_structure, dependency_descriptor)); + uint8_t* payload = rtp_packet.SetPayloadSize(data.size()); + ASSERT_TRUE(payload); + memcpy(payload, data.data(), data.size()); + mock_on_complete_frame_callback_.ClearExpectedBitstream(); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + rtp_packet.SetMarker(true); + rtp_packet.SetPayloadType(payload_type_); + rtp_packet.SetSequenceNumber(++rtp_sequence_number_); + rtp_video_stream_receiver_->OnRtpPacket(rtp_packet); + } + + private: + const int payload_type_ = 123; + RtpHeaderExtensionMap extension_map_; + uint16_t rtp_sequence_number_ = 321; +}; + +TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest, UnwrapsFrameId) { + FrameDependencyStructure stream_structure = CreateStreamStructure(); + + DependencyDescriptor keyframe_descriptor; + keyframe_descriptor.attached_structure = + std::make_unique(stream_structure); + keyframe_descriptor.frame_dependencies = stream_structure.templates[0]; + keyframe_descriptor.frame_number = 0xfff0; + // DependencyDescriptor doesn't support reordering delta frame before + // keyframe. Thus feed a key frame first, then test reodered delta frames. + int64_t first_picture_id; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](video_coding::EncodedFrame* frame) { + first_picture_id = frame->id.picture_id; + }); + InjectPacketWith(stream_structure, keyframe_descriptor); + + DependencyDescriptor deltaframe1_descriptor; + deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1]; + deltaframe1_descriptor.frame_number = 0xfffe; + + DependencyDescriptor deltaframe2_descriptor; + deltaframe1_descriptor.frame_dependencies = stream_structure.templates[1]; + deltaframe2_descriptor.frame_number = 0x0002; + + // Parser should unwrap frame ids correctly even if packets were reordered by + // the network. + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](video_coding::EncodedFrame* frame) { + // 0x0002 - 0xfff0 + EXPECT_EQ(frame->id.picture_id - first_picture_id, 18); + }) + .WillOnce([&](video_coding::EncodedFrame* frame) { + // 0xfffe - 0xfff0 + EXPECT_EQ(frame->id.picture_id - first_picture_id, 14); + }); + InjectPacketWith(stream_structure, deltaframe2_descriptor); + InjectPacketWith(stream_structure, deltaframe1_descriptor); +} + +TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest, + DropsLateDeltaFramePacketWithDependencyDescriptorExtension) { + FrameDependencyStructure stream_structure1 = CreateStreamStructure(); + FrameDependencyStructure stream_structure2 = CreateStreamStructure(); + // Make sure template ids for these two structures do not collide: + // adjust structure_id (that is also used as template id offset). + stream_structure1.structure_id = 13; + stream_structure2.structure_id = + stream_structure1.structure_id + stream_structure1.templates.size(); + + DependencyDescriptor keyframe1_descriptor; + keyframe1_descriptor.attached_structure = + std::make_unique(stream_structure1); + keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0]; + keyframe1_descriptor.frame_number = 1; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); + InjectPacketWith(stream_structure1, keyframe1_descriptor); + + // Pass in 2nd key frame with different structure. + DependencyDescriptor keyframe2_descriptor; + keyframe2_descriptor.attached_structure = + std::make_unique(stream_structure2); + keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0]; + keyframe2_descriptor.frame_number = 3; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame); + InjectPacketWith(stream_structure2, keyframe2_descriptor); + + // Pass in late delta frame that uses structure of the 1st key frame. + DependencyDescriptor deltaframe_descriptor; + deltaframe_descriptor.frame_dependencies = stream_structure1.templates[0]; + deltaframe_descriptor.frame_number = 2; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame).Times(0); + InjectPacketWith(stream_structure1, deltaframe_descriptor); +} + +TEST_F(RtpVideoStreamReceiver2DependencyDescriptorTest, + DropsLateKeyFramePacketWithDependencyDescriptorExtension) { + FrameDependencyStructure stream_structure1 = CreateStreamStructure(); + FrameDependencyStructure stream_structure2 = CreateStreamStructure(); + // Make sure template ids for these two structures do not collide: + // adjust structure_id (that is also used as template id offset). + stream_structure1.structure_id = 13; + stream_structure2.structure_id = + stream_structure1.structure_id + stream_structure1.templates.size(); + + DependencyDescriptor keyframe1_descriptor; + keyframe1_descriptor.attached_structure = + std::make_unique(stream_structure1); + keyframe1_descriptor.frame_dependencies = stream_structure1.templates[0]; + keyframe1_descriptor.frame_number = 1; + + DependencyDescriptor keyframe2_descriptor; + keyframe2_descriptor.attached_structure = + std::make_unique(stream_structure2); + keyframe2_descriptor.frame_dependencies = stream_structure2.templates[0]; + keyframe2_descriptor.frame_number = 3; + + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](video_coding::EncodedFrame* frame) { + EXPECT_EQ(frame->id.picture_id & 0xFFFF, 3); + }); + InjectPacketWith(stream_structure2, keyframe2_descriptor); + InjectPacketWith(stream_structure1, keyframe1_descriptor); + + // Pass in delta frame that uses structure of the 2nd key frame. Late key + // frame shouldn't block it. + DependencyDescriptor deltaframe_descriptor; + deltaframe_descriptor.frame_dependencies = stream_structure2.templates[0]; + deltaframe_descriptor.frame_number = 4; + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce([&](video_coding::EncodedFrame* frame) { + EXPECT_EQ(frame->id.picture_id & 0xFFFF, 4); + }); + InjectPacketWith(stream_structure2, deltaframe_descriptor); +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST_F(RtpVideoStreamReceiver2Test, RepeatedSecondarySinkDisallowed) { + MockRtpPacketSink secondary_sink; + + rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink); + EXPECT_DEATH(rtp_video_stream_receiver_->AddSecondarySink(&secondary_sink), + ""); + + // Test tear-down. + rtp_video_stream_receiver_->RemoveSecondarySink(&secondary_sink); +} +#endif + +TEST_F(RtpVideoStreamReceiver2Test, TransformFrame) { + rtc::scoped_refptr mock_frame_transformer = + new rtc::RefCountedObject>(); + EXPECT_CALL(*mock_frame_transformer, + RegisterTransformedFrameSinkCallback(_, config_.rtp.remote_ssrc)); + auto receiver = std::make_unique( + Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, + rtp_receive_statistics_.get(), nullptr, nullptr, process_thread_.get(), + &mock_nack_sender_, nullptr, &mock_on_complete_frame_callback_, nullptr, + mock_frame_transformer); + VideoCodec video_codec; + video_codec.plType = kPayloadType; + video_codec.codecType = kVideoCodecGeneric; + receiver->AddReceiveCodec(video_codec, {}, /*raw_payload=*/false); + + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); + rtp_packet.SetSequenceNumber(1); + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), + data.size()); + EXPECT_CALL(*mock_frame_transformer, Transform(_)); + receiver->OnReceivedPayloadData(data, rtp_packet, video_header); + + EXPECT_CALL(*mock_frame_transformer, + UnregisterTransformedFrameSinkCallback(config_.rtp.remote_ssrc)); + receiver = nullptr; +} + +// Test default behavior and when playout delay is overridden by field trial. +const PlayoutDelay kTransmittedPlayoutDelay = {100, 200}; +const PlayoutDelay kForcedPlayoutDelay = {70, 90}; +struct PlayoutDelayOptions { + std::string field_trial; + PlayoutDelay expected_delay; +}; +const PlayoutDelayOptions kDefaultBehavior = { + /*field_trial=*/"", /*expected_delay=*/kTransmittedPlayoutDelay}; +const PlayoutDelayOptions kOverridePlayoutDelay = { + /*field_trial=*/"WebRTC-ForcePlayoutDelay/min_ms:70,max_ms:90/", + /*expected_delay=*/kForcedPlayoutDelay}; + +class RtpVideoStreamReceiver2TestPlayoutDelay + : public RtpVideoStreamReceiver2Test, + public ::testing::WithParamInterface { + protected: + RtpVideoStreamReceiver2TestPlayoutDelay() + : RtpVideoStreamReceiver2Test(GetParam().field_trial) {} +}; + +INSTANTIATE_TEST_SUITE_P(PlayoutDelay, + RtpVideoStreamReceiver2TestPlayoutDelay, + Values(kDefaultBehavior, kOverridePlayoutDelay)); + +TEST_P(RtpVideoStreamReceiver2TestPlayoutDelay, PlayoutDelay) { + rtc::CopyOnWriteBuffer payload_data({1, 2, 3, 4}); + RtpHeaderExtensionMap extension_map; + extension_map.Register(1); + RtpPacketToSend packet_to_send(&extension_map); + packet_to_send.SetPayloadType(kPayloadType); + packet_to_send.SetSequenceNumber(1); + + // Set playout delay on outgoing packet. + EXPECT_TRUE(packet_to_send.SetExtension( + kTransmittedPlayoutDelay)); + uint8_t* payload = packet_to_send.AllocatePayload(payload_data.size()); + memcpy(payload, payload_data.data(), payload_data.size()); + + RtpPacketReceived received_packet(&extension_map); + received_packet.Parse(packet_to_send.data(), packet_to_send.size()); + + RTPVideoHeader video_header = + GetGenericVideoHeader(VideoFrameType::kVideoFrameKey); + mock_on_complete_frame_callback_.AppendExpectedBitstream(payload_data.data(), + payload_data.size()); + // Expect the playout delay of encoded frame to be the same as the transmitted + // playout delay unless it was overridden by a field trial. + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)) + .WillOnce(Invoke([expected_playout_delay = GetParam().expected_delay]( + video_coding::EncodedFrame* frame) { + EXPECT_EQ(frame->EncodedImage().playout_delay_, expected_playout_delay); + })); + rtp_video_stream_receiver_->OnReceivedPayloadData( + received_packet.PayloadBuffer(), received_packet, video_header); +} + +} // namespace webrtc diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc index 753dbca2e1..31eb344d5b 100644 --- a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -17,7 +17,6 @@ #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "rtc_base/task_utils/to_queued_task.h" #include "rtc_base/thread.h" -#include "video/rtp_video_stream_receiver.h" namespace webrtc { @@ -69,7 +68,7 @@ class TransformableVideoReceiverFrame RtpVideoStreamReceiverFrameTransformerDelegate:: RtpVideoStreamReceiverFrameTransformerDelegate( - RtpVideoStreamReceiver* receiver, + RtpVideoFrameReceiver* receiver, rtc::scoped_refptr frame_transformer, rtc::Thread* network_thread, uint32_t ssrc) diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.h b/video/rtp_video_stream_receiver_frame_transformer_delegate.h index 0a106c956a..e687e7f47b 100644 --- a/video/rtp_video_stream_receiver_frame_transformer_delegate.h +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.h @@ -20,7 +20,16 @@ namespace webrtc { -class RtpVideoStreamReceiver; +// Called back by RtpVideoStreamReceiverFrameTransformerDelegate on the network +// thread after transformation. +class RtpVideoFrameReceiver { + public: + virtual void ManageFrame( + std::unique_ptr frame) = 0; + + protected: + virtual ~RtpVideoFrameReceiver() = default; +}; // Delegates calls to FrameTransformerInterface to transform frames, and to // RtpVideoStreamReceiver to manage transformed frames on the |network_thread_|. @@ -28,7 +37,7 @@ class RtpVideoStreamReceiverFrameTransformerDelegate : public TransformedFrameCallback { public: RtpVideoStreamReceiverFrameTransformerDelegate( - RtpVideoStreamReceiver* receiver, + RtpVideoFrameReceiver* receiver, rtc::scoped_refptr frame_transformer, rtc::Thread* network_thread, uint32_t ssrc); @@ -44,7 +53,7 @@ class RtpVideoStreamReceiverFrameTransformerDelegate void OnTransformedFrame( std::unique_ptr frame) override; - // Delegates the call to RtpVideoReceiver::ManageFrame on the + // Delegates the call to RtpVideoFrameReceiver::ManageFrame on the // |network_thread_|. void ManageFrame(std::unique_ptr frame); @@ -53,7 +62,7 @@ class RtpVideoStreamReceiverFrameTransformerDelegate private: SequenceChecker network_sequence_checker_; - RtpVideoStreamReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_); + RtpVideoFrameReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_); rtc::scoped_refptr frame_transformer_ RTC_GUARDED_BY(network_sequence_checker_); rtc::Thread* const network_thread_; diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc index b427137373..a411ca6e9a 100644 --- a/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc @@ -21,11 +21,11 @@ #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "modules/utility/include/process_thread.h" #include "rtc_base/event.h" +#include "rtc_base/ref_counted_object.h" #include "rtc_base/task_utils/to_queued_task.h" #include "test/gmock.h" #include "test/gtest.h" #include "test/mock_frame_transformer.h" -#include "video/rtp_video_stream_receiver.h" namespace webrtc { namespace { @@ -47,70 +47,10 @@ std::unique_ptr CreateRtpFrameObject() { return CreateRtpFrameObject(RTPVideoHeader()); } -class FakeTransport : public Transport { +class TestRtpVideoFrameReceiver : public RtpVideoFrameReceiver { public: - bool SendRtp(const uint8_t* packet, - size_t length, - const PacketOptions& options) { - return true; - } - bool SendRtcp(const uint8_t* packet, size_t length) { return true; } -}; - -class FakeNackSender : public NackSender { - public: - void SendNack(const std::vector& sequence_numbers) {} - void SendNack(const std::vector& sequence_numbers, - bool buffering_allowed) {} -}; - -class FakeOnCompleteFrameCallback - : public video_coding::OnCompleteFrameCallback { - public: - void OnCompleteFrame( - std::unique_ptr frame) override {} -}; - -class TestRtpVideoStreamReceiverInitializer { - public: - TestRtpVideoStreamReceiverInitializer() - : test_config_(nullptr), - test_process_thread_(ProcessThread::Create("TestThread")) { - test_config_.rtp.remote_ssrc = 1111; - test_config_.rtp.local_ssrc = 2222; - test_rtp_receive_statistics_ = - ReceiveStatistics::Create(Clock::GetRealTimeClock()); - } - - protected: - VideoReceiveStream::Config test_config_; - FakeTransport fake_transport_; - FakeNackSender fake_nack_sender_; - FakeOnCompleteFrameCallback fake_on_complete_frame_callback_; - std::unique_ptr test_process_thread_; - std::unique_ptr test_rtp_receive_statistics_; -}; - -class TestRtpVideoStreamReceiver : public TestRtpVideoStreamReceiverInitializer, - public RtpVideoStreamReceiver { - public: - TestRtpVideoStreamReceiver() - : TestRtpVideoStreamReceiverInitializer(), - RtpVideoStreamReceiver(Clock::GetRealTimeClock(), - &fake_transport_, - nullptr, - nullptr, - &test_config_, - test_rtp_receive_statistics_.get(), - nullptr, - nullptr, - test_process_thread_.get(), - &fake_nack_sender_, - nullptr, - &fake_on_complete_frame_callback_, - nullptr, - nullptr) {} - ~TestRtpVideoStreamReceiver() override = default; + TestRtpVideoFrameReceiver() {} + ~TestRtpVideoFrameReceiver() override = default; MOCK_METHOD(void, ManageFrame, @@ -120,7 +60,7 @@ class TestRtpVideoStreamReceiver : public TestRtpVideoStreamReceiverInitializer, TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, RegisterTransformedFrameCallbackSinkOnInit) { - TestRtpVideoStreamReceiver receiver; + TestRtpVideoFrameReceiver receiver; rtc::scoped_refptr frame_transformer( new rtc::RefCountedObject()); rtc::scoped_refptr delegate( @@ -134,7 +74,7 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, UnregisterTransformedFrameSinkCallbackOnReset) { - TestRtpVideoStreamReceiver receiver; + TestRtpVideoFrameReceiver receiver; rtc::scoped_refptr frame_transformer( new rtc::RefCountedObject()); rtc::scoped_refptr delegate( @@ -146,7 +86,7 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, } TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformFrame) { - TestRtpVideoStreamReceiver receiver; + TestRtpVideoFrameReceiver receiver; rtc::scoped_refptr frame_transformer( new rtc::RefCountedObject>()); rtc::scoped_refptr delegate( @@ -160,7 +100,7 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformFrame) { TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, ManageFrameOnTransformedFrame) { - TestRtpVideoStreamReceiver receiver; + TestRtpVideoFrameReceiver receiver; rtc::scoped_refptr mock_frame_transformer( new rtc::RefCountedObject>()); rtc::scoped_refptr delegate = @@ -186,7 +126,7 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformableFrameMetadataHasCorrectValue) { - TestRtpVideoStreamReceiver receiver; + TestRtpVideoFrameReceiver receiver; rtc::scoped_refptr mock_frame_transformer = new rtc::RefCountedObject>(); rtc::scoped_refptr delegate = diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index b1b482da29..1470123cd5 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -49,7 +49,7 @@ #include "system_wrappers/include/field_trial.h" #include "video/call_stats2.h" #include "video/frame_dumping_decoder.h" -#include "video/receive_statistics_proxy.h" +#include "video/receive_statistics_proxy2.h" namespace webrtc { diff --git a/video/video_receive_stream2.h b/video/video_receive_stream2.h index f8cd65dc9d..0eab5dd293 100644 --- a/video/video_receive_stream2.h +++ b/video/video_receive_stream2.h @@ -30,7 +30,7 @@ #include "system_wrappers/include/clock.h" #include "video/receive_statistics_proxy2.h" #include "video/rtp_streams_synchronizer2.h" -#include "video/rtp_video_stream_receiver.h" +#include "video/rtp_video_stream_receiver2.h" #include "video/transport_adapter.h" #include "video/video_stream_decoder2.h" @@ -199,7 +199,7 @@ class VideoReceiveStream2 : public webrtc::VideoReceiveStream, std::unique_ptr timing_; // Jitter buffer experiment. VideoReceiver2 video_receiver_; std::unique_ptr> incoming_video_stream_; - RtpVideoStreamReceiver rtp_video_stream_receiver_; + RtpVideoStreamReceiver2 rtp_video_stream_receiver_; std::unique_ptr video_stream_decoder_; RtpStreamsSynchronizer rtp_stream_sync_;