From fd5a20fd683c826bb26b3a03abb068c1ce084a7b Mon Sep 17 00:00:00 2001 From: philipel Date: Tue, 15 Nov 2016 00:57:57 -0800 Subject: [PATCH] New jitter buffer experiment. BUG=webrtc:5514 Review-Url: https://codereview.webrtc.org/2480293002 Cr-Commit-Position: refs/heads/master@{#15077} --- .../video_coding/codecs/vp9/vp9_impl.cc | 8 +- webrtc/modules/video_coding/frame_buffer2.cc | 4 +- .../video_coding/frame_buffer2_unittest.cc | 2 +- webrtc/modules/video_coding/packet_buffer.cc | 96 ++++---- webrtc/modules/video_coding/packet_buffer.h | 8 +- .../rtp_frame_reference_finder.cc | 145 +++++++++++ .../video_coding/rtp_frame_reference_finder.h | 37 ++- .../rtp_frame_reference_finder_unittest.cc | 232 ++++++++++++++++++ webrtc/modules/video_coding/timing.cc | 1 + .../modules/video_coding/video_coding_impl.h | 2 + webrtc/modules/video_coding/video_receiver.cc | 8 + webrtc/video/rtp_stream_receiver.cc | 123 +++++++++- webrtc/video/rtp_stream_receiver.h | 72 ++++-- webrtc/video/video_receive_stream.cc | 67 ++++- webrtc/video/video_receive_stream.h | 16 +- 15 files changed, 737 insertions(+), 84 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 8c798db665..a4f74b7b44 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -256,6 +256,10 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, } if (encoder_ == NULL) { encoder_ = new vpx_codec_ctx_t; + // Only randomize pid/tl0 the first time the encoder is initialized + // in order to not make random jumps mid-stream. + picture_id_ = static_cast(rand()) & 0x7FFF; // NOLINT + tl0_pic_idx_ = static_cast(rand()); // NOLINT } if (config_ == NULL) { config_ = new vpx_codec_enc_cfg_t; @@ -270,8 +274,6 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, if (num_temporal_layers_ == 0) num_temporal_layers_ = 1; - // Random start 16 bits is enough. - picture_id_ = static_cast(rand()) & 0x7FFF; // NOLINT // Allocate memory for encoded image if (encoded_image_._buffer != NULL) { delete[] encoded_image_._buffer; @@ -366,8 +368,6 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - tl0_pic_idx_ = static_cast(rand()); // NOLINT - return InitAndSetControlSettings(inst); } diff --git a/webrtc/modules/video_coding/frame_buffer2.cc b/webrtc/modules/video_coding/frame_buffer2.cc index daac7d5c2b..e02e5b6071 100644 --- a/webrtc/modules/video_coding/frame_buffer2.cc +++ b/webrtc/modules/video_coding/frame_buffer2.cc @@ -28,7 +28,7 @@ namespace { constexpr int kMaxFramesBuffered = 600; // Max number of decoded frame info that will be saved. -constexpr int kMaxFramesHistory = 20; +constexpr int kMaxFramesHistory = 50; } // namespace FrameBuffer::FrameBuffer(Clock* clock, @@ -114,7 +114,7 @@ FrameBuffer::ReturnReason FrameBuffer::NextFrame( if (next_frame_it != frames_.end()) { std::unique_ptr frame = std::move(next_frame_it->second.frame); int64_t received_time = frame->ReceivedTime(); - uint32_t timestamp = frame->Timestamp(); + uint32_t timestamp = frame->timestamp; int64_t frame_delay; if (inter_frame_delay_.CalculateDelay(timestamp, &frame_delay, diff --git a/webrtc/modules/video_coding/frame_buffer2_unittest.cc b/webrtc/modules/video_coding/frame_buffer2_unittest.cc index ebcbeb0254..6079bb95dd 100644 --- a/webrtc/modules/video_coding/frame_buffer2_unittest.cc +++ b/webrtc/modules/video_coding/frame_buffer2_unittest.cc @@ -299,7 +299,7 @@ TEST_F(TestFrameBuffer2, DropTemporalLayerSlowDecoder) { uint32_t ts = Rand(); InsertFrame(pid, 0, ts, false); - InsertFrame(pid + 1, 0, ts + kFps20, false); + InsertFrame(pid + 1, 0, ts + kFps20, false, pid); for (int i = 2; i < 10; i += 2) { uint32_t ts_tl0 = ts + i / 2 * kFps10; InsertFrame(pid + i, 0, ts_tl0, false, pid + i - 2); diff --git a/webrtc/modules/video_coding/packet_buffer.cc b/webrtc/modules/video_coding/packet_buffer.cc index 58ad31e50c..1b6fecc385 100644 --- a/webrtc/modules/video_coding/packet_buffer.cc +++ b/webrtc/modules/video_coding/packet_buffer.cc @@ -57,50 +57,57 @@ PacketBuffer::~PacketBuffer() { } bool PacketBuffer::InsertPacket(const VCMPacket& packet) { - rtc::CritScope lock(&crit_); - uint16_t seq_num = packet.seqNum; - size_t index = seq_num % size_; + std::vector> found_frames; + { + rtc::CritScope lock(&crit_); + uint16_t seq_num = packet.seqNum; + size_t index = seq_num % size_; - if (!first_packet_received_) { - first_seq_num_ = seq_num; - last_seq_num_ = seq_num; - first_packet_received_ = true; - } else if (AheadOf(first_seq_num_, seq_num)) { - // If we have explicitly cleared past this packet then it's old, - // don't insert it. - if (is_cleared_to_first_seq_num_) - return false; + if (!first_packet_received_) { + first_seq_num_ = seq_num; + last_seq_num_ = seq_num; + first_packet_received_ = true; + } else if (AheadOf(first_seq_num_, seq_num)) { + // If we have explicitly cleared past this packet then it's old, + // don't insert it. + if (is_cleared_to_first_seq_num_) + return false; - first_seq_num_ = seq_num; - } - - if (sequence_buffer_[index].used) { - // Duplicate packet, do nothing. - if (data_buffer_[index].seqNum == packet.seqNum) - return true; - - // The packet buffer is full, try to expand the buffer. - while (ExpandBufferSize() && sequence_buffer_[seq_num % size_].used) { + first_seq_num_ = seq_num; } - index = seq_num % size_; - // Packet buffer is still full. - if (sequence_buffer_[index].used) - return false; + if (sequence_buffer_[index].used) { + // Duplicate packet, do nothing. + if (data_buffer_[index].seqNum == packet.seqNum) + return true; + + // The packet buffer is full, try to expand the buffer. + while (ExpandBufferSize() && sequence_buffer_[seq_num % size_].used) { + } + index = seq_num % size_; + + // Packet buffer is still full. + if (sequence_buffer_[index].used) + return false; + } + + if (AheadOf(seq_num, last_seq_num_)) + last_seq_num_ = seq_num; + + sequence_buffer_[index].frame_begin = packet.isFirstPacket; + sequence_buffer_[index].frame_end = packet.markerBit; + sequence_buffer_[index].seq_num = packet.seqNum; + sequence_buffer_[index].continuous = false; + sequence_buffer_[index].frame_created = false; + sequence_buffer_[index].used = true; + data_buffer_[index] = packet; + + found_frames = FindFrames(seq_num); } - if (AheadOf(seq_num, last_seq_num_)) - last_seq_num_ = seq_num; + for (std::unique_ptr& frame : found_frames) + received_frame_callback_->OnReceivedFrame(std::move(frame)); - sequence_buffer_[index].frame_begin = packet.isFirstPacket; - sequence_buffer_[index].frame_end = packet.markerBit; - sequence_buffer_[index].seq_num = packet.seqNum; - sequence_buffer_[index].continuous = false; - sequence_buffer_[index].frame_created = false; - sequence_buffer_[index].used = true; - data_buffer_[index] = packet; - - FindFrames(seq_num); return true; } @@ -187,7 +194,9 @@ bool PacketBuffer::PotentialNewFrame(uint16_t seq_num) const { return false; } -void PacketBuffer::FindFrames(uint16_t seq_num) { +std::vector> PacketBuffer::FindFrames( + uint16_t seq_num) { + std::vector> found_frames; while (PotentialNewFrame(seq_num)) { size_t index = seq_num % size_; sequence_buffer_[index].continuous = true; @@ -204,8 +213,8 @@ void PacketBuffer::FindFrames(uint16_t seq_num) { int start_index = index; while (true) { frame_size += data_buffer_[start_index].sizeBytes; - max_nack_count = std::max( - max_nack_count, data_buffer_[start_index].timesNacked); + max_nack_count = + std::max(max_nack_count, data_buffer_[start_index].timesNacked); sequence_buffer_[start_index].frame_created = true; if (sequence_buffer_[start_index].frame_begin) @@ -215,15 +224,13 @@ void PacketBuffer::FindFrames(uint16_t seq_num) { start_seq_num--; } - std::unique_ptr frame( + found_frames.emplace_back( new RtpFrameObject(this, start_seq_num, seq_num, frame_size, max_nack_count, clock_->TimeInMilliseconds())); - - received_frame_callback_->OnReceivedFrame(std::move(frame)); } - ++seq_num; } + return found_frames; } void PacketBuffer::ReturnFrame(RtpFrameObject* frame) { @@ -267,7 +274,6 @@ bool PacketBuffer::GetBitstream(const RtpFrameObject& frame, } VCMPacket* PacketBuffer::GetPacket(uint16_t seq_num) { - rtc::CritScope lock(&crit_); size_t index = seq_num % size_; if (!sequence_buffer_[index].used || seq_num != sequence_buffer_[index].seq_num) { diff --git a/webrtc/modules/video_coding/packet_buffer.h b/webrtc/modules/video_coding/packet_buffer.h index 15645f607a..36b3027158 100644 --- a/webrtc/modules/video_coding/packet_buffer.h +++ b/webrtc/modules/video_coding/packet_buffer.h @@ -99,8 +99,9 @@ class PacketBuffer { EXCLUSIVE_LOCKS_REQUIRED(crit_); // Test if all packets of a frame has arrived, and if so, creates a frame. - // May create multiple frames per invocation. - void FindFrames(uint16_t seq_num) EXCLUSIVE_LOCKS_REQUIRED(crit_); + // Returns a vector of received frames. + std::vector> FindFrames(uint16_t seq_num) + EXCLUSIVE_LOCKS_REQUIRED(crit_); // Copy the bitstream for |frame| to |destination|. // Virtual for testing. @@ -108,7 +109,8 @@ class PacketBuffer { // Get the packet with sequence number |seq_num|. // Virtual for testing. - virtual VCMPacket* GetPacket(uint16_t seq_num); + virtual VCMPacket* GetPacket(uint16_t seq_num) + EXCLUSIVE_LOCKS_REQUIRED(crit_); // Mark all slots used by |frame| as not used. // Virtual for testing. diff --git a/webrtc/modules/video_coding/rtp_frame_reference_finder.cc b/webrtc/modules/video_coding/rtp_frame_reference_finder.cc index 861a1178c0..5a9420a847 100644 --- a/webrtc/modules/video_coding/rtp_frame_reference_finder.cc +++ b/webrtc/modules/video_coding/rtp_frame_reference_finder.cc @@ -371,6 +371,11 @@ void RtpFrameReferenceFinder::ManageFrameVp9( const RTPVideoHeaderVP9& codec_header = rtp_codec_header->VP9; + bool old_frame = Vp9PidTl0Fix(*frame, &rtp_codec_header->VP9.picture_id, + &rtp_codec_header->VP9.tl0_pic_idx); + if (old_frame) + return; + if (codec_header.picture_id == kNoPictureId || codec_header.temporal_idx == kNoTemporalIdx) { ManageFrameGeneric(std::move(frame), codec_header.picture_id); @@ -585,5 +590,145 @@ uint16_t RtpFrameReferenceFinder::UnwrapPictureId(uint16_t picture_id) { return last_unwrap_; } +bool RtpFrameReferenceFinder::Vp9PidTl0Fix(const RtpFrameObject& frame, + int16_t* picture_id, + int16_t* tl0_pic_idx) { + const int kTl0PicIdLength = 256; + const uint8_t kMaxPidDiff = 128; + + // We are currently receiving VP9 without PID, nothing to fix. + if (*picture_id == kNoPictureId) + return false; + + // If |vp9_fix_jump_timestamp_| != -1 then a jump has occurred recently. + if (vp9_fix_jump_timestamp_ != -1) { + // If this frame has a timestamp older than |vp9_fix_jump_timestamp_| then + // this frame is old (more previous than the frame where we detected the + // jump) and should be dropped. + if (AheadOf(vp9_fix_jump_timestamp_, frame.timestamp)) + return true; + + // After 60 seconds, reset |vp9_fix_jump_timestamp_| in order to not + // discard old frames when the timestamp wraps. + int diff_ms = + ForwardDiff(vp9_fix_jump_timestamp_, frame.timestamp) / 90; + if (diff_ms > 60 * 1000) + vp9_fix_jump_timestamp_ = -1; + } + + // Update |vp9_fix_last_timestamp_| with the most recent timestamp. + if (vp9_fix_last_timestamp_ == -1) + vp9_fix_last_timestamp_ = frame.timestamp; + if (AheadOf(frame.timestamp, vp9_fix_last_timestamp_)) + vp9_fix_last_timestamp_ = frame.timestamp; + + uint16_t fixed_pid = Add(*picture_id, vp9_fix_pid_offset_); + if (vp9_fix_last_picture_id_ == -1) + vp9_fix_last_picture_id_ = *picture_id; + + int16_t fixed_tl0 = kNoTl0PicIdx; + if (*tl0_pic_idx != kNoTl0PicIdx) { + fixed_tl0 = Add(*tl0_pic_idx, vp9_fix_tl0_pic_idx_offset_); + // Update |vp9_fix_last_tl0_pic_idx_| with the most recent tl0 pic index. + if (vp9_fix_last_tl0_pic_idx_ == -1) + vp9_fix_last_tl0_pic_idx_ = *tl0_pic_idx; + if (AheadOf(fixed_tl0, vp9_fix_last_tl0_pic_idx_)) + vp9_fix_last_tl0_pic_idx_ = fixed_tl0; + } + + bool has_jumped = DetectVp9PicIdJump(fixed_pid, fixed_tl0, frame.timestamp); + if (!has_jumped) + has_jumped = DetectVp9Tl0PicIdxJump(fixed_tl0, frame.timestamp); + + if (has_jumped) { + // First we calculate the offset to get to the previous picture id, and then + // we add kMaxPid to avoid accidently referencing any previous + // frames that was inserted into the FrameBuffer. + vp9_fix_pid_offset_ = ForwardDiff( + *picture_id, vp9_fix_last_picture_id_); + vp9_fix_pid_offset_ += kMaxPidDiff; + + fixed_pid = Add(*picture_id, vp9_fix_pid_offset_); + vp9_fix_last_picture_id_ = fixed_pid; + vp9_fix_jump_timestamp_ = frame.timestamp; + gof_info_.clear(); + + vp9_fix_tl0_pic_idx_offset_ = + ForwardDiff(*tl0_pic_idx, vp9_fix_last_tl0_pic_idx_); + vp9_fix_tl0_pic_idx_offset_ += kMaxGofSaved; + fixed_tl0 = Add(*tl0_pic_idx, vp9_fix_tl0_pic_idx_offset_); + vp9_fix_last_tl0_pic_idx_ = fixed_tl0; + } + + // Update |vp9_fix_last_picture_id_| with the most recent picture id. + if (AheadOf(fixed_pid, vp9_fix_last_picture_id_)) + vp9_fix_last_picture_id_ = fixed_pid; + + *picture_id = fixed_pid; + *tl0_pic_idx = fixed_tl0; + + return false; +} + +bool RtpFrameReferenceFinder::DetectVp9PicIdJump(int fixed_pid, + int fixed_tl0, + uint32_t timestamp) const { + // Test if there has been a jump backwards in the picture id. + if (AheadOrAt(timestamp, vp9_fix_last_timestamp_) && + AheadOf(vp9_fix_last_picture_id_, fixed_pid)) { + return true; + } + + // Test if we have jumped forward too much. The reason we have to do this + // is because the FrameBuffer holds history of old frames and inserting + // frames with a much advanced picture id can result in the frame buffer + // holding more than half of the interval of picture ids. + if (AheadOrAt(timestamp, vp9_fix_last_timestamp_) && + ForwardDiff(vp9_fix_last_picture_id_, fixed_pid) > + 128) { + return true; + } + + // Special case where the picture id jump forward but not by much and the + // tl0 jumps to the id of an already saved gof for that id. In order to + // detect this we check if the picture id span over the length of the GOF. + if (fixed_tl0 != kNoTl0PicIdx) { + auto info_it = gof_info_.find(fixed_tl0); + if (info_it != gof_info_.end()) { + int last_pid_gof_idx_0 = + Subtract(info_it->second.last_picture_id, + info_it->second.last_picture_id % + info_it->second.gof->num_frames_in_gof); + int pif_gof_end = Add( + last_pid_gof_idx_0, info_it->second.gof->num_frames_in_gof); + if (AheadOf(fixed_pid, pif_gof_end)) + return true; + } + } + + return false; +} + +bool RtpFrameReferenceFinder::DetectVp9Tl0PicIdxJump(int fixed_tl0, + uint32_t timestamp) const { + if (fixed_tl0 != kNoTl0PicIdx) { + // Test if there has been a jump backwards in tl0 pic index. + if (AheadOrAt(timestamp, vp9_fix_last_timestamp_) && + AheadOf(vp9_fix_last_tl0_pic_idx_, fixed_tl0)) { + return true; + } + + // Test if there has been a jump forward. If the jump forward results + // in the tl0 pic index for this frame to be considered smaller than the + // smallest item in |gof_info_| then we have jumped forward far enough to + // wrap. + if (!gof_info_.empty() && + AheadOf(gof_info_.begin()->first, fixed_tl0)) { + return true; + } + } + return false; +} + } // namespace video_coding } // namespace webrtc diff --git a/webrtc/modules/video_coding/rtp_frame_reference_finder.h b/webrtc/modules/video_coding/rtp_frame_reference_finder.h index e9e7d60a33..db1ddf8072 100644 --- a/webrtc/modules/video_coding/rtp_frame_reference_finder.h +++ b/webrtc/modules/video_coding/rtp_frame_reference_finder.h @@ -57,12 +57,12 @@ class RtpFrameReferenceFinder { void ClearTo(uint16_t seq_num); private: - static const uint16_t kPicIdLength = 1 << 7; + static const uint16_t kPicIdLength = 1 << 15; static const uint8_t kMaxTemporalLayers = 5; - static const int kMaxLayerInfo = 10; - static const int kMaxStashedFrames = 10; - static const int kMaxNotYetReceivedFrames = 20; - static const int kMaxGofSaved = 15; + static const int kMaxLayerInfo = 50; + static const int kMaxStashedFrames = 50; + static const int kMaxNotYetReceivedFrames = 100; + static const int kMaxGofSaved = 50; static const int kMaxPaddingAge = 100; @@ -129,6 +129,24 @@ class RtpFrameReferenceFinder { // All picture ids are unwrapped to 16 bits. uint16_t UnwrapPictureId(uint16_t picture_id) EXCLUSIVE_LOCKS_REQUIRED(crit_); + // Returns true if the frame is old and should be dropped. + // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be + // around M59). + bool Vp9PidTl0Fix(const RtpFrameObject& frame, + int16_t* picture_id, + int16_t* tl0_pic_idx) EXCLUSIVE_LOCKS_REQUIRED(crit_); + + // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be + // around M59). + bool DetectVp9PicIdJump(int fixed_pid, + int fixed_tl0, + uint32_t timestamp) const + EXCLUSIVE_LOCKS_REQUIRED(crit_); + + // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be + // around M59). + bool DetectVp9Tl0PicIdxJump(int fixed_tl0, uint32_t timestamp) const + EXCLUSIVE_LOCKS_REQUIRED(crit_); // For every group of pictures, hold two sequence numbers. The first being // the sequence number of the last packet of the last completed frame, and @@ -196,6 +214,15 @@ class RtpFrameReferenceFinder { int cleared_to_seq_num_ GUARDED_BY(crit_); OnCompleteFrameCallback* frame_callback_; + + // Vp9PidFix variables + // TODO(philipel): Remove when VP9 PID does not jump mid-stream. + int vp9_fix_last_timestamp_ = -1; + int vp9_fix_jump_timestamp_ = -1; + int vp9_fix_last_picture_id_ = -1; + int vp9_fix_pid_offset_ = 0; + int vp9_fix_last_tl0_pic_idx_ = -1; + int vp9_fix_tl0_pic_idx_offset_ = 0; }; } // namespace video_coding diff --git a/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc b/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc index 5fa5ac843b..928785c107 100644 --- a/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc +++ b/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc @@ -122,6 +122,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, bool up_switch = false, GofInfoVP9* ss = nullptr) { VCMPacket packet; + packet.timestamp = pid; packet.codec = kVideoCodecVP9; packet.seqNum = seq_num_start; packet.frameType = keyframe ? kVideoFrameKey : kVideoFrameDelta; @@ -152,6 +153,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, bool inter = false, std::vector refs = std::vector()) { VCMPacket packet; + packet.timestamp = pid; packet.codec = kVideoCodecVP9; packet.seqNum = seq_num_start; packet.frameType = keyframe ? kVideoFrameKey : kVideoFrameDelta; @@ -1221,5 +1223,235 @@ TEST_F(TestRtpFrameReferenceFinder, Vp9FlexibleModeTwoSpatialLayersReordered) { CheckReferencesVp9(pid + 8, 1, pid + 7); } +// TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be +// around M59). +TEST_F(TestRtpFrameReferenceFinder, Vp9PidFix_PidJumpsBackwardThenForward) { + GofInfoVP9 ss; + ss.SetGofInfoVP9(kTemporalStructureMode1); + + VCMPacket packet; + packet.timestamp = 0; + packet.codec = kVideoCodecVP9; + packet.frameType = kVideoFrameKey; + packet.video_header.codecHeader.VP9.flexible_mode = false; + packet.video_header.codecHeader.VP9.picture_id = 1; + packet.video_header.codecHeader.VP9.temporal_idx = 0; + packet.video_header.codecHeader.VP9.spatial_idx = 0; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 0; + packet.video_header.codecHeader.VP9.temporal_up_switch = true; + packet.video_header.codecHeader.VP9.ss_data_available = true; + packet.video_header.codecHeader.VP9.gof = ss; + + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + // Timestamp goes forward but pid goes backwards. + packet.timestamp = 1; + packet.video_header.codecHeader.VP9.picture_id = 0; + + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 2; + packet.video_header.codecHeader.VP9.picture_id = 5000; + + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + ASSERT_EQ(3UL, frames_from_callback_.size()); + CheckReferencesVp9(1, 0); + CheckReferencesVp9(129, 0); + CheckReferencesVp9(257, 0); +} + +// TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be +// around M59). +TEST_F(TestRtpFrameReferenceFinder, Vp9PidFix_Tl0JumpsBackwardThenForward) { + GofInfoVP9 ss; + ss.SetGofInfoVP9(kTemporalStructureMode1); + + VCMPacket packet; + packet.timestamp = 0; + packet.codec = kVideoCodecVP9; + packet.frameType = kVideoFrameKey; + packet.video_header.codecHeader.VP9.flexible_mode = false; + packet.video_header.codecHeader.VP9.picture_id = 0; + packet.video_header.codecHeader.VP9.temporal_idx = 0; + packet.video_header.codecHeader.VP9.spatial_idx = 0; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 1; + packet.video_header.codecHeader.VP9.temporal_up_switch = true; + packet.video_header.codecHeader.VP9.ss_data_available = true; + packet.video_header.codecHeader.VP9.gof = ss; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 1; + packet.video_header.codecHeader.VP9.picture_id = 1; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 0; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 2; + packet.frameType = kVideoFrameDelta; + packet.video_header.codecHeader.VP9.picture_id = 2; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 2; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 3; + packet.frameType = kVideoFrameKey; + packet.video_header.codecHeader.VP9.ss_data_available = true; + packet.video_header.codecHeader.VP9.picture_id = 3; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 129; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + ASSERT_EQ(4UL, frames_from_callback_.size()); + CheckReferencesVp9(0, 0); + CheckReferencesVp9(128, 0); + CheckReferencesVp9(129, 0, 128); + CheckReferencesVp9(257, 0); +} + +// TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be +// around M59). +TEST_F(TestRtpFrameReferenceFinder, Vp9PidFix_PidSmallJumpForward) { + GofInfoVP9 ss; + ss.SetGofInfoVP9(kTemporalStructureMode1); + + VCMPacket packet; + packet.timestamp = 0; + packet.codec = kVideoCodecVP9; + packet.frameType = kVideoFrameKey; + packet.video_header.codecHeader.VP9.flexible_mode = false; + packet.video_header.codecHeader.VP9.picture_id = 1; + packet.video_header.codecHeader.VP9.temporal_idx = 0; + packet.video_header.codecHeader.VP9.spatial_idx = 0; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 1; + packet.video_header.codecHeader.VP9.temporal_up_switch = true; + packet.video_header.codecHeader.VP9.ss_data_available = true; + packet.video_header.codecHeader.VP9.gof = ss; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 1; + packet.video_header.codecHeader.VP9.picture_id = 2; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 2; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 2; + packet.video_header.codecHeader.VP9.picture_id = 3; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 2; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 2; + packet.video_header.codecHeader.VP9.picture_id = 4; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 1; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + ASSERT_EQ(4UL, frames_from_callback_.size()); + CheckReferencesVp9(1, 0); + CheckReferencesVp9(2, 0); + CheckReferencesVp9(3, 0); + CheckReferencesVp9(131, 0); +} + +// TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be +// around M59). +TEST_F(TestRtpFrameReferenceFinder, Vp9PidFix_DropOldFrame) { + GofInfoVP9 ss; + ss.SetGofInfoVP9(kTemporalStructureMode1); + + VCMPacket packet; + packet.timestamp = 0; + packet.codec = kVideoCodecVP9; + packet.frameType = kVideoFrameKey; + packet.video_header.codecHeader.VP9.flexible_mode = false; + packet.video_header.codecHeader.VP9.picture_id = 1; + packet.video_header.codecHeader.VP9.temporal_idx = 0; + packet.video_header.codecHeader.VP9.spatial_idx = 0; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 1; + packet.video_header.codecHeader.VP9.temporal_up_switch = true; + packet.video_header.codecHeader.VP9.ss_data_available = true; + packet.video_header.codecHeader.VP9.gof = ss; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 1; + packet.video_header.codecHeader.VP9.picture_id = 0; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 2; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + packet.timestamp = 0; + packet.video_header.codecHeader.VP9.picture_id = 3; + packet.video_header.codecHeader.VP9.tl0_pic_idx = 2; + { + ref_packet_buffer_->InsertPacket(packet); + std::unique_ptr frame( + new RtpFrameObject(ref_packet_buffer_, 0, 0, 0, 0, 0)); + reference_finder_->ManageFrame(std::move(frame)); + } + + ASSERT_EQ(2UL, frames_from_callback_.size()); + CheckReferencesVp9(1, 0); + CheckReferencesVp9(129, 0); +} + } // namespace video_coding } // namespace webrtc diff --git a/webrtc/modules/video_coding/timing.cc b/webrtc/modules/video_coding/timing.cc index 29b064ea60..d41e53008c 100644 --- a/webrtc/modules/video_coding/timing.cc +++ b/webrtc/modules/video_coding/timing.cc @@ -154,6 +154,7 @@ void VCMTiming::UpdateCurrentDelay(uint32_t frame_timestamp) { max_change_ms = kDelayMaxChangeMsPerS * (frame_timestamp - prev_frame_timestamp_) / 90000; } + if (max_change_ms <= 0) { // Any changes less than 1 ms are truncated and // will be postponed. Negative change will be due diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h index cb24654041..fa19ac47c3 100644 --- a/webrtc/modules/video_coding/video_coding_impl.h +++ b/webrtc/modules/video_coding/video_coding_impl.h @@ -143,6 +143,8 @@ class VideoReceiver : public Module { int32_t Decode(uint16_t maxWaitTimeMs); + int32_t Decode(const webrtc::VCMEncodedFrame* frame); + int32_t ReceiveCodec(VideoCodec* currentReceiveCodec) const; VideoCodecType ReceiveCodec() const; diff --git a/webrtc/modules/video_coding/video_receiver.cc b/webrtc/modules/video_coding/video_receiver.cc index 475f686d23..46373ef6cb 100644 --- a/webrtc/modules/video_coding/video_receiver.cc +++ b/webrtc/modules/video_coding/video_receiver.cc @@ -290,6 +290,14 @@ int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) { return ret; } +// Used for the WebRTC-NewVideoJitterBuffer experiment. +// TODO(philipel): Clean up among the Decode functions as we replace +// VCMEncodedFrame with FrameObject. +int32_t VideoReceiver::Decode(const webrtc::VCMEncodedFrame* frame) { + rtc::CritScope lock(&receive_crit_); + return Decode(*frame); +} + int32_t VideoReceiver::RequestSliceLossIndication( const uint64_t pictureID) const { TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID); diff --git a/webrtc/video/rtp_stream_receiver.cc b/webrtc/video/rtp_stream_receiver.cc index 138ed84583..fc84007610 100644 --- a/webrtc/video/rtp_stream_receiver.cc +++ b/webrtc/video/rtp_stream_receiver.cc @@ -11,6 +11,7 @@ #include "webrtc/video/rtp_stream_receiver.h" #include +#include #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" @@ -24,7 +25,11 @@ #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h" #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" #include "webrtc/modules/rtp_rtcp/include/ulpfec_receiver.h" +#include "webrtc/modules/video_coding/frame_object.h" +#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h" +#include "webrtc/modules/video_coding/packet_buffer.h" #include "webrtc/modules/video_coding/video_coding_impl.h" +#include "webrtc/system_wrappers/include/field_trial.h" #include "webrtc/system_wrappers/include/metrics.h" #include "webrtc/system_wrappers/include/timestamp_extrapolator.h" #include "webrtc/system_wrappers/include/trace.h" @@ -33,6 +38,11 @@ namespace webrtc { +namespace { +constexpr int kPacketBufferStartSize = 32; +constexpr int kPacketBufferMaxSixe = 2048; +} + std::unique_ptr CreateRtpRtcpModule( ReceiveStatistics* receive_statistics, Transport* outgoing_transport, @@ -83,7 +93,11 @@ RtpStreamReceiver::RtpStreamReceiver( const VideoReceiveStream::Config* config, ReceiveStatisticsProxy* receive_stats_proxy, ProcessThread* process_thread, - RateLimiter* retransmission_rate_limiter) + RateLimiter* retransmission_rate_limiter, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + VCMTiming* timing) : clock_(Clock::GetRealTimeClock()), config_(*config), video_receiver_(video_receiver), @@ -110,7 +124,10 @@ RtpStreamReceiver::RtpStreamReceiver( remote_bitrate_estimator_, paced_sender, packet_router, - retransmission_rate_limiter)) { + retransmission_rate_limiter)), + complete_frame_callback_(complete_frame_callback), + keyframe_request_sender_(keyframe_request_sender), + timing_(timing) { packet_router_->AddRtpModule(rtp_rtcp_.get()); rtp_receive_statistics_->RegisterRtpStatisticsCallback(receive_stats_proxy); rtp_receive_statistics_->RegisterRtcpStatisticsCallback(receive_stats_proxy); @@ -180,11 +197,27 @@ RtpStreamReceiver::RtpStreamReceiver( rtp_rtcp_->RegisterRtcpStatisticsCallback(receive_stats_proxy); process_thread_->RegisterModule(rtp_rtcp_.get()); + + jitter_buffer_experiment_ = + field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") == "Enabled"; + + if (jitter_buffer_experiment_) { + nack_module_.reset( + new NackModule(clock_, nack_sender, keyframe_request_sender)); + process_thread_->RegisterModule(nack_module_.get()); + + packet_buffer_ = video_coding::PacketBuffer::Create( + clock_, kPacketBufferStartSize, kPacketBufferMaxSixe, this); + reference_finder_.reset(new video_coding::RtpFrameReferenceFinder(this)); + } } RtpStreamReceiver::~RtpStreamReceiver() { process_thread_->DeRegisterModule(rtp_rtcp_.get()); + if (jitter_buffer_experiment_) + process_thread_->DeRegisterModule(nack_module_.get()); + packet_router_->RemoveRtpModule(rtp_rtcp_.get()); rtp_rtcp_->SetREMBStatus(false); remb_->RemoveReceiveChannel(rtp_rtcp_.get()); @@ -224,10 +257,34 @@ int32_t RtpStreamReceiver::OnReceivedPayloadData( WebRtcRTPHeader rtp_header_with_ntp = *rtp_header; rtp_header_with_ntp.ntp_time_ms = ntp_estimator_.Estimate(rtp_header->header.timestamp); - if (video_receiver_->IncomingPacket(payload_data, payload_size, - rtp_header_with_ntp) != 0) { - // Check this... - return -1; + if (jitter_buffer_experiment_) { + VCMPacket packet(payload_data, payload_size, rtp_header_with_ntp); + timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds()); + packet.timesNacked = nack_module_->OnReceivedPacket(packet); + + if (packet.codec == kVideoCodecH264) { + switch (tracker_.CopyAndFixBitstream(&packet)) { + case video_coding::H264SpsPpsTracker::kRequestKeyframe: + keyframe_request_sender_->RequestKeyFrame(); + FALLTHROUGH(); + case video_coding::H264SpsPpsTracker::kDrop: + return 0; + case video_coding::H264SpsPpsTracker::kInsert: + break; + } + } else { + uint8_t* data = new uint8_t[packet.sizeBytes]; + memcpy(data, packet.dataPtr, packet.sizeBytes); + packet.dataPtr = data; + } + + packet_buffer_->InsertPacket(packet); + } else { + if (video_receiver_->IncomingPacket(payload_data, payload_size, + rtp_header_with_ntp) != 0) { + // Check this... + return -1; + } } return 0; } @@ -348,6 +405,27 @@ int32_t RtpStreamReceiver::ResendPackets(const uint16_t* sequence_numbers, return rtp_rtcp_->SendNACK(sequence_numbers, length); } +void RtpStreamReceiver::OnReceivedFrame( + std::unique_ptr frame) { + reference_finder_->ManageFrame(std::move(frame)); +} + +void RtpStreamReceiver::OnCompleteFrame( + std::unique_ptr frame) { + { + rtc::CritScope lock(&last_seq_num_cs_); + video_coding::RtpFrameObject* rtp_frame = + static_cast(frame.get()); + last_seq_num_for_pic_id_[rtp_frame->picture_id] = rtp_frame->last_seq_num(); + } + complete_frame_callback_->OnCompleteFrame(std::move(frame)); +} + +void RtpStreamReceiver::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) { + if (jitter_buffer_experiment_) + nack_module_->UpdateRtt(max_rtt_ms); +} + bool RtpStreamReceiver::ReceivePacket(const uint8_t* packet, size_t packet_length, const RTPHeader& header, @@ -472,6 +550,39 @@ bool RtpStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet, return true; } +void RtpStreamReceiver::FrameContinuous(uint16_t picture_id) { + if (jitter_buffer_experiment_) { + int seq_num = -1; + { + rtc::CritScope lock(&last_seq_num_cs_); + auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); + if (seq_num_it != last_seq_num_for_pic_id_.end()) + seq_num = seq_num_it->second; + } + if (seq_num != -1) + nack_module_->ClearUpTo(seq_num); + } +} + +void RtpStreamReceiver::FrameDecoded(uint16_t picture_id) { + if (jitter_buffer_experiment_) { + int seq_num = -1; + { + rtc::CritScope lock(&last_seq_num_cs_); + auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id); + if (seq_num_it != last_seq_num_for_pic_id_.end()) { + seq_num = seq_num_it->second; + last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(), + ++seq_num_it); + } + } + if (seq_num != -1) { + packet_buffer_->ClearTo(seq_num); + reference_finder_->ClearTo(seq_num); + } + } +} + void RtpStreamReceiver::SignalNetworkState(NetworkState state) { rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode : RtcpMode::kOff); diff --git a/webrtc/video/rtp_stream_receiver.h b/webrtc/video/rtp_stream_receiver.h index 7f0f3722e9..36689848eb 100644 --- a/webrtc/video/rtp_stream_receiver.h +++ b/webrtc/video/rtp_stream_receiver.h @@ -12,57 +12,74 @@ #define WEBRTC_VIDEO_RTP_STREAM_RECEIVER_H_ #include +#include #include #include #include #include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" +#include "webrtc/modules/include/module_common_types.h" #include "webrtc/modules/rtp_rtcp/include/receive_statistics.h" #include "webrtc/modules/rtp_rtcp/include/remote_ntp_time_estimator.h" #include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h" #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h" #include "webrtc/modules/video_coding/include/video_coding_defines.h" +#include "webrtc/modules/video_coding/packet_buffer.h" +#include "webrtc/modules/video_coding/rtp_frame_reference_finder.h" +#include "webrtc/modules/video_coding/sequence_number_util.h" #include "webrtc/typedefs.h" #include "webrtc/video_receive_stream.h" namespace webrtc { +class NackModule; class PacedSender; class PacketRouter; class ProcessThread; -class RemoteNtpTimeEstimator; class ReceiveStatistics; class ReceiveStatisticsProxy; class RemoteBitrateEstimator; +class RemoteNtpTimeEstimator; class RtcpRttStats; class RtpHeaderParser; class RTPPayloadRegistry; class RtpReceiver; class Transport; class UlpfecReceiver; +class VCMTiming; class VieRemb; namespace vcm { class VideoReceiver; } // namespace vcm -class RtpStreamReceiver : public RtpData, public RtpFeedback, +class RtpStreamReceiver : public RtpData, + public RtpFeedback, public VCMFrameTypeCallback, - public VCMPacketRequestCallback { + public VCMPacketRequestCallback, + public video_coding::OnReceivedFrameCallback, + public video_coding::OnCompleteFrameCallback, + public CallStatsObserver { public: - RtpStreamReceiver(vcm::VideoReceiver* video_receiver, - RemoteBitrateEstimator* remote_bitrate_estimator, - Transport* transport, - RtcpRttStats* rtt_stats, - PacedSender* paced_sender, - PacketRouter* packet_router, - VieRemb* remb, - const VideoReceiveStream::Config* config, - ReceiveStatisticsProxy* receive_stats_proxy, - ProcessThread* process_thread, - RateLimiter* retransmission_rate_limiter); + RtpStreamReceiver( + vcm::VideoReceiver* video_receiver, + RemoteBitrateEstimator* remote_bitrate_estimator, + Transport* transport, + RtcpRttStats* rtt_stats, + PacedSender* paced_sender, + PacketRouter* packet_router, + VieRemb* remb, + const VideoReceiveStream::Config* config, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + RateLimiter* retransmission_rate_limiter, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + VCMTiming* timing); ~RtpStreamReceiver(); bool SetReceiveCodec(const VideoCodec& video_codec); @@ -81,6 +98,10 @@ class RtpStreamReceiver : public RtpData, public RtpFeedback, const PacketTime& packet_time); bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); + void FrameContinuous(uint16_t seq_num); + + void FrameDecoded(uint16_t seq_num); + void SignalNetworkState(NetworkState state); // Implements RtpData. @@ -111,6 +132,16 @@ class RtpStreamReceiver : public RtpData, public RtpFeedback, int32_t ResendPackets(const uint16_t* sequenceNumbers, uint16_t length) override; + // Implements OnReceivedFrameCallback. + void OnReceivedFrame( + std::unique_ptr frame) override; + + // Implements OnCompleteFrameCallback. + void OnCompleteFrame( + std::unique_ptr frame) override; + + void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; + private: bool ReceivePacket(const uint8_t* packet, size_t packet_length, @@ -152,6 +183,19 @@ class RtpStreamReceiver : public RtpData, public RtpFeedback, int64_t last_packet_log_ms_ GUARDED_BY(receive_cs_); const std::unique_ptr rtp_rtcp_; + + // Members for the new jitter buffer experiment. + bool jitter_buffer_experiment_; + video_coding::OnCompleteFrameCallback* complete_frame_callback_; + KeyFrameRequestSender* keyframe_request_sender_; + VCMTiming* timing_; + std::unique_ptr nack_module_; + rtc::scoped_refptr packet_buffer_; + std::unique_ptr reference_finder_; + rtc::CriticalSection last_seq_num_cs_; + std::map> + last_seq_num_for_pic_id_ GUARDED_BY(last_seq_num_cs_); + video_coding::H264SpsPpsTracker tracker_; }; } // namespace webrtc diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index 5173b2d479..ea0289a3d1 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -18,12 +18,17 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/base/optional.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/congestion_controller/include/congestion_controller.h" #include "webrtc/modules/utility/include/process_thread.h" +#include "webrtc/modules/video_coding/frame_object.h" #include "webrtc/modules/video_coding/include/video_coding.h" +#include "webrtc/modules/video_coding/jitter_estimator.h" +#include "webrtc/modules/video_coding/timing.h" #include "webrtc/modules/video_coding/utility/ivf_file_writer.h" #include "webrtc/system_wrappers/include/clock.h" +#include "webrtc/system_wrappers/include/field_trial.h" #include "webrtc/video/call_stats.h" #include "webrtc/video/receive_statistics_proxy.h" #include "webrtc/video_receive_stream.h" @@ -197,6 +202,7 @@ VideoReceiveStream::VideoReceiveStream( call_stats_(call_stats), video_receiver_(clock_, nullptr, this, this, this), stats_proxy_(&config_, clock_), + timing_(new VCMTiming(clock_)), rtp_stream_receiver_( &video_receiver_, congestion_controller_->GetRemoteBitrateEstimator( @@ -209,8 +215,15 @@ VideoReceiveStream::VideoReceiveStream( &config_, &stats_proxy_, process_thread_, - congestion_controller_->GetRetransmissionRateLimiter()), - rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_) { + congestion_controller_->GetRetransmissionRateLimiter(), + this, // NackSender + this, // KeyFrameRequestSender + this, // OnCompleteFrameCallback + timing_.get()), + rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_), + jitter_buffer_experiment_( + field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") == + "Enabled") { LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString(); RTC_DCHECK(process_thread_); @@ -230,6 +243,12 @@ VideoReceiveStream::VideoReceiveStream( video_receiver_.SetRenderDelay(config.render_delay_ms); + if (jitter_buffer_experiment_) { + jitter_estimator_.reset(new VCMJitterEstimator(clock_)); + frame_buffer_.reset(new video_coding::FrameBuffer( + clock_, jitter_estimator_.get(), timing_.get())); + } + process_thread_->RegisterModule(&video_receiver_); process_thread_->RegisterModule(&rtp_stream_sync_); } @@ -268,6 +287,15 @@ bool VideoReceiveStream::OnRecoveredPacket(const uint8_t* packet, void VideoReceiveStream::Start() { if (decode_thread_.IsRunning()) return; + if (jitter_buffer_experiment_) { + frame_buffer_->Start(); + call_stats_->RegisterStatsObserver(&rtp_stream_receiver_); + + if (rtp_stream_receiver_.IsRetransmissionsEnabled() && + rtp_stream_receiver_.IsFecEnabled()) { + frame_buffer_->SetProtectionMode(kProtectionNackFEC); + } + } transport_adapter_.Enable(); rtc::VideoSinkInterface* renderer = nullptr; if (config_.renderer) { @@ -310,6 +338,12 @@ void VideoReceiveStream::Stop() { // stop immediately, instead of waiting for a timeout. Needs to be called // before joining the decoder thread thread. video_receiver_.TriggerDecoderShutdown(); + + if (jitter_buffer_experiment_) { + frame_buffer_->Stop(); + call_stats_->DeregisterStatsObserver(&rtp_stream_receiver_); + } + if (decode_thread_.IsRunning()) { decode_thread_.Stop(); // Deregister external decoders so they are no longer running during @@ -319,6 +353,7 @@ void VideoReceiveStream::Stop() { for (const Decoder& decoder : config_.decoders) video_receiver_.RegisterExternalDecoder(nullptr, decoder.payload_type); } + call_stats_->DeregisterStatsObserver(video_stream_decoder_.get()); video_stream_decoder_.reset(); incoming_video_stream_.reset(); @@ -365,6 +400,13 @@ void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) { stats_proxy_.OnRenderedFrame(video_frame); } +void VideoReceiveStream::OnCompleteFrame( + std::unique_ptr frame) { + int last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame)); + if (last_continuous_pid != -1) + rtp_stream_receiver_.FrameContinuous(last_continuous_pid); +} + // TODO(asapersson): Consider moving callback from video_encoder.h or // creating a different callback. EncodedImageCallback::Result VideoReceiveStream::OnEncodedImage( @@ -397,7 +439,26 @@ bool VideoReceiveStream::DecodeThreadFunction(void* ptr) { void VideoReceiveStream::Decode() { static const int kMaxDecodeWaitTimeMs = 50; - video_receiver_.Decode(kMaxDecodeWaitTimeMs); + if (jitter_buffer_experiment_) { + static const int kMaxWaitForFrameMs = 3000; + std::unique_ptr frame; + video_coding::FrameBuffer::ReturnReason res = + frame_buffer_->NextFrame(kMaxWaitForFrameMs, &frame); + + if (res == video_coding::FrameBuffer::ReturnReason::kStopped) + return; + + if (frame) { + if (video_receiver_.Decode(frame.get()) == VCM_OK) + rtp_stream_receiver_.FrameDecoded(frame->picture_id); + } else { + LOG(LS_WARNING) << "No decodable frame in " << kMaxWaitForFrameMs + << " ms, requesting keyframe."; + RequestKeyFrame(); + } + } else { + video_receiver_.Decode(kMaxDecodeWaitTimeMs); + } } void VideoReceiveStream::SendNack( diff --git a/webrtc/video/video_receive_stream.h b/webrtc/video/video_receive_stream.h index 0efbe75c38..f1efc4b4b8 100644 --- a/webrtc/video/video_receive_stream.h +++ b/webrtc/video/video_receive_stream.h @@ -17,6 +17,7 @@ #include "webrtc/common_video/include/incoming_video_stream.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/rtp_rtcp/include/flexfec_receiver.h" +#include "webrtc/modules/video_coding/frame_buffer2.h" #include "webrtc/modules/video_coding/video_coding_impl.h" #include "webrtc/system_wrappers/include/clock.h" #include "webrtc/video/receive_statistics_proxy.h" @@ -35,6 +36,8 @@ class ProcessThread; class RTPFragmentationHeader; class VoiceEngine; class VieRemb; +class VCMTiming; +class VCMJitterEstimator; namespace internal { @@ -42,7 +45,8 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, public rtc::VideoSinkInterface, public EncodedImageCallback, public NackSender, - public KeyFrameRequestSender { + public KeyFrameRequestSender, + public video_coding::OnCompleteFrameCallback { public: VideoReceiveStream(int num_cpu_cores, CongestionController* congestion_controller, @@ -70,6 +74,10 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, // Overrides rtc::VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; + // Implements video_coding::OnCompleteFrameCallback. + void OnCompleteFrame( + std::unique_ptr frame) override; + // Overrides EncodedImageCallback. EncodedImageCallback::Result OnEncodedImage( const EncodedImage& encoded_image, @@ -112,12 +120,18 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, vcm::VideoReceiver video_receiver_; std::unique_ptr> incoming_video_stream_; ReceiveStatisticsProxy stats_proxy_; + std::unique_ptr timing_; // Jitter buffer experiment. RtpStreamReceiver rtp_stream_receiver_; std::unique_ptr video_stream_decoder_; RtpStreamsSynchronizer rtp_stream_sync_; rtc::CriticalSection ivf_writer_lock_; std::unique_ptr ivf_writer_ GUARDED_BY(ivf_writer_lock_); + + // Members for the new jitter buffer experiment. + const bool jitter_buffer_experiment_; + std::unique_ptr jitter_estimator_; + std::unique_ptr frame_buffer_; }; } // namespace internal } // namespace webrtc