From 4348ce240aaec218fb57d0e2b8701a7539166426 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Fri, 7 Dec 2018 16:26:56 +0100 Subject: [PATCH] Calculate min and max receive timestamps for packets in a video frame Bug: webrtc:10106 Change-Id: I1d3469abb1e7bb7c91a5912d7b781505526abaca Reviewed-on: https://webrtc-review.googlesource.com/c/113507 Reviewed-by: Sebastian Jansson Reviewed-by: Philip Eliasson Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#25935} --- modules/video_coding/frame_object.cc | 11 ++++++----- modules/video_coding/frame_object.h | 5 +++-- modules/video_coding/packet_buffer.cc | 9 ++++++++- .../rtp_frame_reference_finder_unittest.cc | 8 ++++---- test/fuzzers/rtp_frame_reference_finder_fuzzer.cc | 2 +- video/buffered_frame_decryptor_unittest.cc | 2 +- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/modules/video_coding/frame_object.cc b/modules/video_coding/frame_object.cc index 000c36a3e8..750c977fa5 100644 --- a/modules/video_coding/frame_object.cc +++ b/modules/video_coding/frame_object.cc @@ -27,11 +27,12 @@ RtpFrameObject::RtpFrameObject(PacketBuffer* packet_buffer, uint16_t last_seq_num, size_t frame_size, int times_nacked, - int64_t received_time) + int64_t first_packet_received_time, + int64_t last_packet_received_time) : packet_buffer_(packet_buffer), first_seq_num_(first_seq_num), last_seq_num_(last_seq_num), - received_time_(received_time), + last_packet_received_time_(last_packet_received_time), times_nacked_(times_nacked) { VCMPacket* first_packet = packet_buffer_->GetPacket(first_seq_num); RTC_CHECK(first_packet); @@ -100,9 +101,9 @@ RtpFrameObject::RtpFrameObject(PacketBuffer* packet_buffer, ntp_time_ms_ + last_packet->video_header.video_timing.network2_timestamp_delta_ms; } + timing_.receive_start_ms = first_packet_received_time; + timing_.receive_finish_ms = last_packet_received_time; timing_.flags = last_packet->video_header.video_timing.flags; - timing_.receive_start_ms = first_packet->receive_time_ms; - timing_.receive_finish_ms = last_packet->receive_time_ms; is_last_spatial_layer = last_packet->markerBit; } @@ -131,7 +132,7 @@ VideoCodecType RtpFrameObject::codec_type() const { } int64_t RtpFrameObject::ReceivedTime() const { - return received_time_; + return last_packet_received_time_; } int64_t RtpFrameObject::RenderTime() const { diff --git a/modules/video_coding/frame_object.h b/modules/video_coding/frame_object.h index bb4513c107..5a3efd9513 100644 --- a/modules/video_coding/frame_object.h +++ b/modules/video_coding/frame_object.h @@ -29,7 +29,8 @@ class RtpFrameObject : public EncodedFrame { uint16_t last_seq_num, size_t frame_size, int times_nacked, - int64_t received_time); + int64_t first_packet_received_time, + int64_t last_packet_received_time); ~RtpFrameObject() override; uint16_t first_seq_num() const; @@ -53,7 +54,7 @@ class RtpFrameObject : public EncodedFrame { VideoCodecType codec_type_; uint16_t first_seq_num_; uint16_t last_seq_num_; - int64_t received_time_; + int64_t last_packet_received_time_; // Equal to times nacked of the packet with the highet times nacked // belonging to this frame. diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index 0a2b8bf095..80c66661fb 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -285,6 +285,8 @@ std::vector> PacketBuffer::FindFrames( size_t frame_size = 0; int max_nack_count = -1; uint16_t start_seq_num = seq_num; + int64_t min_recv_time = data_buffer_[index].receive_time_ms; + int64_t max_recv_time = data_buffer_[index].receive_time_ms; // Find the start index by searching backward until the packet with // the |frame_begin| flag is set. @@ -306,6 +308,11 @@ std::vector> PacketBuffer::FindFrames( std::max(max_nack_count, data_buffer_[start_index].timesNacked); sequence_buffer_[start_index].frame_created = true; + min_recv_time = + std::min(min_recv_time, data_buffer_[start_index].receive_time_ms); + max_recv_time = + std::max(max_recv_time, data_buffer_[start_index].receive_time_ms); + if (!is_h264 && sequence_buffer_[start_index].frame_begin) break; @@ -393,7 +400,7 @@ std::vector> PacketBuffer::FindFrames( found_frames.emplace_back( new RtpFrameObject(this, start_seq_num, seq_num, frame_size, - max_nack_count, clock_->TimeInMilliseconds())); + max_nack_count, min_recv_time, max_recv_time)); } ++seq_num; } diff --git a/modules/video_coding/rtp_frame_reference_finder_unittest.cc b/modules/video_coding/rtp_frame_reference_finder_unittest.cc index 50a87799d1..1e8a9c5a73 100644 --- a/modules/video_coding/rtp_frame_reference_finder_unittest.cc +++ b/modules/video_coding/rtp_frame_reference_finder_unittest.cc @@ -92,7 +92,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, ref_packet_buffer_->InsertPacket(&packet); std::unique_ptr frame(new RtpFrameObject( - ref_packet_buffer_, seq_num_start, seq_num_end, 0, 0, 0)); + ref_packet_buffer_, seq_num_start, seq_num_end, 0, 0, 0, 0)); reference_finder_->ManageFrame(std::move(frame)); } @@ -123,7 +123,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, } std::unique_ptr frame(new RtpFrameObject( - ref_packet_buffer_, seq_num_start, seq_num_end, 0, 0, 0)); + ref_packet_buffer_, seq_num_start, seq_num_end, 0, 0, 0, 0)); reference_finder_->ManageFrame(std::move(frame)); } @@ -164,7 +164,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, } std::unique_ptr frame(new RtpFrameObject( - ref_packet_buffer_, seq_num_start, seq_num_end, 0, 0, 0)); + ref_packet_buffer_, seq_num_start, seq_num_end, 0, 0, 0, 0)); reference_finder_->ManageFrame(std::move(frame)); } @@ -202,7 +202,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, } std::unique_ptr frame(new RtpFrameObject( - ref_packet_buffer_, seq_num_start, seq_num_end, 0, 0, 0)); + ref_packet_buffer_, seq_num_start, seq_num_end, 0, 0, 0, 0)); reference_finder_->ManageFrame(std::move(frame)); } diff --git a/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc b/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc index 57d7a9623a..b171e3d2f2 100644 --- a/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc +++ b/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc @@ -111,7 +111,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) { while (reader.MoreToRead()) { auto frame = absl::make_unique( - pb, reader.GetNum(), reader.GetNum(), 0, 0, 0); + pb, reader.GetNum(), reader.GetNum(), 0, 0, 0, 0); reference_finder.ManageFrame(std::move(frame)); } } diff --git a/video/buffered_frame_decryptor_unittest.cc b/video/buffered_frame_decryptor_unittest.cc index 95c647e3ff..8a0c951c7c 100644 --- a/video/buffered_frame_decryptor_unittest.cc +++ b/video/buffered_frame_decryptor_unittest.cc @@ -90,7 +90,7 @@ class BufferedFrameDecryptorTest return std::unique_ptr( new video_coding::RtpFrameObject(fake_packet_buffer_.get(), seq_num_, - seq_num_, 0, 0, 0)); + seq_num_, 0, 0, 0, 0)); } protected: