diff --git a/webrtc/modules/video_coding/packet_buffer.cc b/webrtc/modules/video_coding/packet_buffer.cc index 6c3cfacac5..1031042581 100644 --- a/webrtc/modules/video_coding/packet_buffer.cc +++ b/webrtc/modules/video_coding/packet_buffer.cc @@ -17,6 +17,7 @@ #include "webrtc/base/atomicops.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/common_video/h264/h264_common.h" #include "webrtc/modules/video_coding/frame_object.h" #include "webrtc/system_wrappers/include/clock.h" @@ -108,6 +109,8 @@ bool PacketBuffer::InsertPacket(VCMPacket* packet) { data_buffer_[index] = *packet; packet->dataPtr = nullptr; + UpdateMissingPackets(packet->seqNum); + int64_t now_ms = clock_->TimeInMilliseconds(); last_received_packet_ms_ = rtc::Optional(now_ms); if (packet->frameType == kVideoFrameKey) @@ -137,6 +140,9 @@ void PacketBuffer::ClearTo(uint16_t seq_num) { sequence_buffer_[index].used = false; ++first_seq_num_; } + + missing_packets_.erase(missing_packets_.begin(), + missing_packets_.upper_bound(seq_num)); } void PacketBuffer::Clear() { @@ -149,8 +155,22 @@ void PacketBuffer::Clear() { first_packet_received_ = false; is_cleared_to_first_seq_num_ = false; - last_received_packet_ms_ = rtc::Optional(); - last_received_keyframe_packet_ms_ = rtc::Optional(); + last_received_packet_ms_.reset(); + last_received_keyframe_packet_ms_.reset(); + newest_inserted_seq_num_.reset(); + missing_packets_.clear(); +} + +void PacketBuffer::PaddingReceived(uint16_t seq_num) { + std::vector> found_frames; + { + rtc::CritScope lock(&crit_); + UpdateMissingPackets(seq_num); + found_frames = FindFrames(static_cast(seq_num + 1)); + } + + for (std::unique_ptr& frame : found_frames) + received_frame_callback_->OnReceivedFrame(std::move(frame)); } rtc::Optional PacketBuffer::LastReceivedPacketMs() const { @@ -194,6 +214,8 @@ bool PacketBuffer::PotentialNewFrame(uint16_t seq_num) const { if (!sequence_buffer_[index].used) return false; + if (sequence_buffer_[index].seq_num != seq_num) + return false; if (sequence_buffer_[index].frame_created) return false; if (sequence_buffer_[index].frame_begin) @@ -231,6 +253,7 @@ std::vector> PacketBuffer::FindFrames( int start_index = index; bool is_h264 = data_buffer_[start_index].codec == kVideoCodecH264; + bool is_h264_keyframe = false; int64_t frame_timestamp = data_buffer_[start_index].timestamp; // Since packet at |data_buffer_[index]| is already part of the frame @@ -244,6 +267,17 @@ std::vector> PacketBuffer::FindFrames( if (!is_h264 && sequence_buffer_[start_index].frame_begin) break; + if (is_h264 && !is_h264_keyframe) { + const RTPVideoHeaderH264& header = + data_buffer_[start_index].video_header.codecHeader.H264; + for (size_t i = 0; i < header.nalus_length; ++i) { + if (header.nalus[i].type == H264::NaluType::kIdr) { + is_h264_keyframe = true; + break; + } + } + } + start_index = start_index > 0 ? start_index - 1 : size_ - 1; // In the case of H264 we don't have a frame_begin bit (yes, @@ -261,6 +295,23 @@ std::vector> PacketBuffer::FindFrames( --start_seq_num; } + // If this is H264 but not a keyframe, make sure there are no gaps in the + // packet sequence numbers up until this point. + if (is_h264 && !is_h264_keyframe && + missing_packets_.upper_bound(start_seq_num) != + missing_packets_.begin()) { + uint16_t stop_index = (index + 1) % size_; + while (start_index != stop_index) { + sequence_buffer_[start_index].frame_created = false; + start_index = (start_index + 1) % size_; + } + + return found_frames; + } + + missing_packets_.erase(missing_packets_.begin(), + missing_packets_.upper_bound(seq_num)); + found_frames.emplace_back( new RtpFrameObject(this, start_seq_num, seq_num, frame_size, max_nack_count, clock_->TimeInMilliseconds())); @@ -331,5 +382,30 @@ int PacketBuffer::Release() const { return count; } +void PacketBuffer::UpdateMissingPackets(uint16_t seq_num) { + if (!newest_inserted_seq_num_) + newest_inserted_seq_num_ = rtc::Optional(seq_num); + + const int kMaxPaddingAge = 1000; + if (AheadOf(seq_num, *newest_inserted_seq_num_)) { + uint16_t old_seq_num = seq_num - kMaxPaddingAge; + auto erase_to = missing_packets_.lower_bound(old_seq_num); + missing_packets_.erase(missing_packets_.begin(), erase_to); + + // Guard against inserting a large amount of missing packets if there is a + // jump in the sequence number. + if (AheadOf(old_seq_num, *newest_inserted_seq_num_)) + *newest_inserted_seq_num_ = old_seq_num; + + ++*newest_inserted_seq_num_; + while (AheadOf(seq_num, *newest_inserted_seq_num_)) { + missing_packets_.insert(*newest_inserted_seq_num_); + ++*newest_inserted_seq_num_; + } + } else { + missing_packets_.erase(seq_num); + } +} + } // namespace video_coding } // namespace webrtc diff --git a/webrtc/modules/video_coding/packet_buffer.h b/webrtc/modules/video_coding/packet_buffer.h index 3d9eb9cb42..d5011b67c3 100644 --- a/webrtc/modules/video_coding/packet_buffer.h +++ b/webrtc/modules/video_coding/packet_buffer.h @@ -11,8 +11,9 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_PACKET_BUFFER_H_ #define WEBRTC_MODULES_VIDEO_CODING_PACKET_BUFFER_H_ -#include #include +#include +#include #include "webrtc/base/criticalsection.h" #include "webrtc/base/scoped_ref_ptr.h" @@ -54,6 +55,7 @@ class PacketBuffer { virtual bool InsertPacket(VCMPacket* packet); void ClearTo(uint16_t seq_num); void Clear(); + void PaddingReceived(uint16_t seq_num); // Timestamp (not RTP timestamp) of the last received packet/keyframe packet. rtc::Optional LastReceivedPacketMs() const; @@ -121,6 +123,8 @@ class PacketBuffer { // Virtual for testing. virtual void ReturnFrame(RtpFrameObject* frame); + void UpdateMissingPackets(uint16_t seq_num) EXCLUSIVE_LOCKS_REQUIRED(crit_); + rtc::CriticalSection crit_; // Buffer size_ and max_size_ must always be a power of two. @@ -150,6 +154,10 @@ class PacketBuffer { rtc::Optional last_received_packet_ms_ GUARDED_BY(crit_); rtc::Optional last_received_keyframe_packet_ms_ GUARDED_BY(crit_); + rtc::Optional newest_inserted_seq_num_ GUARDED_BY(crit_); + std::set> missing_packets_ + GUARDED_BY(crit_); + mutable volatile int ref_count_ = 0; }; diff --git a/webrtc/modules/video_coding/video_packet_buffer_unittest.cc b/webrtc/modules/video_coding/video_packet_buffer_unittest.cc index c5d1fce81d..300a2a6f1c 100644 --- a/webrtc/modules/video_coding/video_packet_buffer_unittest.cc +++ b/webrtc/modules/video_coding/video_packet_buffer_unittest.cc @@ -14,6 +14,7 @@ #include #include "webrtc/base/random.h" +#include "webrtc/common_video/h264/h264_common.h" #include "webrtc/modules/video_coding/frame_object.h" #include "webrtc/modules/video_coding/packet_buffer.h" #include "webrtc/system_wrappers/include/clock.h" @@ -41,6 +42,7 @@ class TestPacketBuffer : public ::testing::Test, << first_seq_num << "."; return; } + frames_from_callback_.insert( std::make_pair(frame->first_seq_num(), std::move(frame))); } @@ -68,6 +70,27 @@ class TestPacketBuffer : public ::testing::Test, return packet_buffer_->InsertPacket(&packet); } + bool InsertH264(uint16_t seq_num, // packet sequence number + IsKeyFrame keyframe, // is keyframe + IsFirst first, // is first packet of frame + IsLast last, // is last packet of frame + uint32_t timestamp, // rtp timestamp + int data_size = 0, // size of data + uint8_t* data = nullptr) { // data pointer + VCMPacket packet; + packet.codec = kVideoCodecH264; + packet.seqNum = seq_num; + packet.timestamp = timestamp; + packet.video_header.codecHeader.H264.nalus[0].type = H264::NaluType::kIdr; + packet.video_header.codecHeader.H264.nalus_length = keyframe == kKeyFrame; + packet.is_first_packet_in_frame = first == kFirst; + packet.markerBit = last == kLast; + packet.sizeBytes = data_size; + packet.dataPtr = data; + + return packet_buffer_->InsertPacket(&packet); + } + void CheckFrame(uint16_t first_seq_num) { auto frame_it = frames_from_callback_.find(first_seq_num); ASSERT_FALSE(frame_it == frames_from_callback_.end()) @@ -366,6 +389,8 @@ TEST_F(TestPacketBuffer, GetBitstreamH264BufferPadding) { new uint8_t[sizeof(data_data) + EncodedImage::kBufferPaddingBytesH264]); VCMPacket packet; + packet.video_header.codecHeader.H264.nalus_length = 1; + packet.video_header.codecHeader.H264.nalus[0].type = H264::NaluType::kIdr; packet.seqNum = seq_num; packet.codec = kVideoCodecH264; packet.insertStartCode = true; @@ -478,49 +503,6 @@ TEST_F(TestPacketBuffer, ContinuousSeqNumDoubleMarkerBit) { EXPECT_EQ(0UL, frames_from_callback_.size()); } -TEST_F(TestPacketBuffer, OneH264FrameFillBuffer) { - VCMPacket packet; - packet.seqNum = 0; - packet.codec = kVideoCodecH264; - packet.dataPtr = nullptr; - packet.sizeBytes = 0; - packet.is_first_packet_in_frame = true; - packet.markerBit = false; - packet_buffer_->InsertPacket(&packet); - - packet.is_first_packet_in_frame = false; - for (int i = 1; i < kStartSize - 1; ++i) { - packet.seqNum = i; - packet_buffer_->InsertPacket(&packet); - } - - packet.seqNum = kStartSize - 1; - packet.markerBit = true; - packet_buffer_->InsertPacket(&packet); - - EXPECT_EQ(1UL, frames_from_callback_.size()); - CheckFrame(0); -} - -TEST_F(TestPacketBuffer, OneH264FrameMaxSeqNum) { - VCMPacket packet; - packet.seqNum = 65534; - packet.codec = kVideoCodecH264; - packet.dataPtr = nullptr; - packet.sizeBytes = 0; - packet.is_first_packet_in_frame = true; - packet.markerBit = false; - packet_buffer_->InsertPacket(&packet); - - packet.is_first_packet_in_frame = false; - packet.seqNum = 65535; - packet.markerBit = true; - packet_buffer_->InsertPacket(&packet); - - EXPECT_EQ(1UL, frames_from_callback_.size()); - CheckFrame(65534); -} - TEST_F(TestPacketBuffer, PacketTimestamps) { rtc::Optional packet_ms; rtc::Optional packet_keyframe_ms; @@ -556,5 +538,51 @@ TEST_F(TestPacketBuffer, PacketTimestamps) { EXPECT_FALSE(packet_keyframe_ms); } +TEST_F(TestPacketBuffer, OneFrameFillBufferH264) { + InsertH264(0, kKeyFrame, kFirst, kNotLast, 1000); + for (int i = 1; i < kStartSize - 1; ++i) + InsertH264(i, kKeyFrame, kNotFirst, kNotLast, 1000); + InsertH264(kStartSize - 1, kKeyFrame, kNotFirst, kLast, 1000); + + EXPECT_EQ(1UL, frames_from_callback_.size()); + CheckFrame(0); +} + +TEST_F(TestPacketBuffer, OneFrameMaxSeqNumH264) { + InsertH264(65534, kKeyFrame, kFirst, kNotLast, 1000); + InsertH264(65535, kKeyFrame, kNotFirst, kLast, 1000); + + EXPECT_EQ(1UL, frames_from_callback_.size()); + CheckFrame(65534); +} + +TEST_F(TestPacketBuffer, ClearMissingPacketsOnKeyframeH264) { + InsertH264(0, kKeyFrame, kFirst, kLast, 1000); + InsertH264(2, kKeyFrame, kFirst, kLast, 3000); + InsertH264(3, kDeltaFrame, kFirst, kNotLast, 4000); + InsertH264(4, kDeltaFrame, kNotFirst, kLast, 4000); + + ASSERT_EQ(3UL, frames_from_callback_.size()); + + InsertH264(kStartSize + 1, kKeyFrame, kFirst, kLast, 18000); + + ASSERT_EQ(4UL, frames_from_callback_.size()); + CheckFrame(0); + CheckFrame(2); + CheckFrame(3); + CheckFrame(kStartSize + 1); +} + +TEST_F(TestPacketBuffer, FindFramesOnPaddingH264) { + InsertH264(0, kKeyFrame, kFirst, kLast, 1000); + InsertH264(2, kDeltaFrame, kFirst, kLast, 1000); + + ASSERT_EQ(1UL, frames_from_callback_.size()); + packet_buffer_->PaddingReceived(1); + ASSERT_EQ(2UL, frames_from_callback_.size()); + CheckFrame(0); + CheckFrame(2); +} + } // namespace video_coding } // namespace webrtc diff --git a/webrtc/video/rtp_video_stream_receiver.cc b/webrtc/video/rtp_video_stream_receiver.cc index 14e5958c0b..f1e530d3a7 100644 --- a/webrtc/video/rtp_video_stream_receiver.cc +++ b/webrtc/video/rtp_video_stream_receiver.cc @@ -245,6 +245,7 @@ int32_t RtpVideoStreamReceiver::OnReceivedPayloadData( // correctly calculate frame references. if (packet.sizeBytes == 0) { reference_finder_->PaddingReceived(packet.seqNum); + packet_buffer_->PaddingReceived(packet.seqNum); return 0; }