diff --git a/webrtc/modules/video_coding/rtp_frame_reference_finder.cc b/webrtc/modules/video_coding/rtp_frame_reference_finder.cc index 9f372b6685..998d757a7b 100644 --- a/webrtc/modules/video_coding/rtp_frame_reference_finder.cc +++ b/webrtc/modules/video_coding/rtp_frame_reference_finder.cc @@ -368,9 +368,9 @@ void RtpFrameReferenceFinder::ManageFrameVp9( scalability_structures_[current_ss_idx_] = codec_header.gof; scalability_structures_[current_ss_idx_].pid_start = frame->picture_id; - auto pid_and_gof = std::make_pair( - frame->picture_id, &scalability_structures_[current_ss_idx_]); - gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, pid_and_gof)); + GofInfo info(&scalability_structures_[current_ss_idx_], + frame->picture_id); + gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, info)); } } @@ -385,8 +385,8 @@ void RtpFrameReferenceFinder::ManageFrameVp9( LOG(LS_WARNING) << "Received keyframe without scalability structure"; frame->num_references = 0; - GofInfoVP9* gof = gof_info_.find(codec_header.tl0_pic_idx)->second.second; - FrameReceivedVp9(frame->picture_id, *gof); + GofInfo info = gof_info_.find(codec_header.tl0_pic_idx)->second; + FrameReceivedVp9(frame->picture_id, &info); CompletedFrameVp9(std::move(frame)); return; } @@ -402,14 +402,12 @@ void RtpFrameReferenceFinder::ManageFrameVp9( return; } - GofInfoVP9* gof = gof_info_it->second.second; - uint16_t picture_id_tl0 = gof_info_it->second.first; - - FrameReceivedVp9(frame->picture_id, *gof); + GofInfo* info = &gof_info_it->second; + FrameReceivedVp9(frame->picture_id, info); // Make sure we don't miss any frame that could potentially have the // up switch flag set. - if (MissingRequiredFrameVp9(frame->picture_id, *gof)) { + if (MissingRequiredFrameVp9(frame->picture_id, *info)) { stashed_frames_.emplace(std::move(frame)); return; } @@ -424,27 +422,24 @@ void RtpFrameReferenceFinder::ManageFrameVp9( // then gof info has already been inserted earlier, so we only want to // insert if we haven't done so already. if (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) { - auto pid_and_gof = std::make_pair(frame->picture_id, gof); - gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, pid_and_gof)); + GofInfo new_info(info->gof, frame->picture_id); + gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, new_info)); } // Clean out old info about up switch frames. - uint16_t old_picture_id = Subtract(last_picture_id_, 50); + uint16_t old_picture_id = Subtract(frame->picture_id, 50); auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id); up_switch_.erase(up_switch_.begin(), up_switch_erase_to); - RTC_DCHECK( - (AheadOrAt(frame->picture_id, picture_id_tl0))); - - size_t diff = - ForwardDiff(gof->pid_start, frame->picture_id); - size_t gof_idx = diff % gof->num_frames_in_gof; + size_t diff = ForwardDiff(info->gof->pid_start, + frame->picture_id); + size_t gof_idx = diff % info->gof->num_frames_in_gof; // Populate references according to the scalability structure. - frame->num_references = gof->num_ref_pics[gof_idx]; + frame->num_references = info->gof->num_ref_pics[gof_idx]; for (size_t i = 0; i < frame->num_references; ++i) { - frame->references[i] = - Subtract(frame->picture_id, gof->pid_diff[gof_idx][i]); + frame->references[i] = Subtract( + frame->picture_id, info->gof->pid_diff[gof_idx][i]); // If this is a reference to a frame earlier than the last up switch point, // then ignore this reference. @@ -458,18 +453,19 @@ void RtpFrameReferenceFinder::ManageFrameVp9( } bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id, - const GofInfoVP9& gof) { - size_t diff = ForwardDiff(gof.pid_start, picture_id); - size_t gof_idx = diff % gof.num_frames_in_gof; - size_t temporal_idx = gof.temporal_idx[gof_idx]; + const GofInfo& info) { + size_t diff = + ForwardDiff(info.gof->pid_start, picture_id); + size_t gof_idx = diff % info.gof->num_frames_in_gof; + size_t temporal_idx = info.gof->temporal_idx[gof_idx]; // For every reference this frame has, check if there is a frame missing in // the interval (|ref_pid|, |picture_id|) in any of the lower temporal // layers. If so, we are missing a required frame. - uint8_t num_references = gof.num_ref_pics[gof_idx]; + uint8_t num_references = info.gof->num_ref_pics[gof_idx]; for (size_t i = 0; i < num_references; ++i) { uint16_t ref_pid = - Subtract(picture_id, gof.pid_diff[gof_idx][i]); + Subtract(picture_id, info.gof->pid_diff[gof_idx][i]); for (size_t l = 0; l < temporal_idx; ++l) { auto missing_frame_it = missing_frames_for_layer_[l].lower_bound(ref_pid); if (missing_frame_it != missing_frames_for_layer_[l].end() && @@ -482,30 +478,31 @@ bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id, } void RtpFrameReferenceFinder::FrameReceivedVp9(uint16_t picture_id, - const GofInfoVP9& gof) { - RTC_DCHECK_NE(-1, last_picture_id_); + GofInfo* info) { + int last_picture_id = info->last_picture_id; // If there is a gap, find which temporal layer the missing frames // belong to and add the frame as missing for that temporal layer. // Otherwise, remove this frame from the set of missing frames. - if (AheadOf(picture_id, last_picture_id_)) { - size_t diff = - ForwardDiff(gof.pid_start, last_picture_id_); - size_t gof_idx = diff % gof.num_frames_in_gof; + if (AheadOf(picture_id, last_picture_id)) { + size_t diff = ForwardDiff(info->gof->pid_start, + last_picture_id); + size_t gof_idx = diff % info->gof->num_frames_in_gof; - last_picture_id_ = Add(last_picture_id_, 1); - while (last_picture_id_ != picture_id) { + last_picture_id = Add(last_picture_id, 1); + while (last_picture_id != picture_id) { ++gof_idx; - RTC_DCHECK_NE(0ul, gof_idx % gof.num_frames_in_gof); - size_t temporal_idx = gof.temporal_idx[gof_idx]; - missing_frames_for_layer_[temporal_idx].insert(last_picture_id_); - last_picture_id_ = Add(last_picture_id_, 1); + RTC_DCHECK_NE(0ul, gof_idx % info->gof->num_frames_in_gof); + size_t temporal_idx = info->gof->temporal_idx[gof_idx]; + missing_frames_for_layer_[temporal_idx].insert(last_picture_id); + last_picture_id = Add(last_picture_id, 1); } + info->last_picture_id = last_picture_id; } else { size_t diff = - ForwardDiff(gof.pid_start, picture_id); - size_t gof_idx = diff % gof.num_frames_in_gof; - size_t temporal_idx = gof.temporal_idx[gof_idx]; + ForwardDiff(info->gof->pid_start, picture_id); + size_t gof_idx = diff % info->gof->num_frames_in_gof; + size_t temporal_idx = info->gof->temporal_idx[gof_idx]; missing_frames_for_layer_[temporal_idx].erase(picture_id); } } diff --git a/webrtc/modules/video_coding/rtp_frame_reference_finder.h b/webrtc/modules/video_coding/rtp_frame_reference_finder.h index 23c36c061d..a12d6d8fca 100644 --- a/webrtc/modules/video_coding/rtp_frame_reference_finder.h +++ b/webrtc/modules/video_coding/rtp_frame_reference_finder.h @@ -44,6 +44,14 @@ class RtpFrameReferenceFinder { static const int kMaxGofSaved = 15; static const int kMaxPaddingAge = 100; + + struct GofInfo { + GofInfo(GofInfoVP9* gof, uint16_t last_picture_id) + : gof(gof), last_picture_id(last_picture_id) {} + GofInfoVP9* gof; + uint16_t last_picture_id; + }; + rtc::CriticalSection crit_; // Find the relevant group of pictures and update its "last-picture-id-with @@ -82,13 +90,13 @@ class RtpFrameReferenceFinder { // Check if we are missing a frame necessary to determine the references // for this frame. - bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfoVP9& gof) + bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfo& info) EXCLUSIVE_LOCKS_REQUIRED(crit_); // Updates which frames that have been received. If there is a gap, // missing frames will be added to |missing_frames_for_layer_| or // if this is an already missing frame then it will be removed. - void FrameReceivedVp9(uint16_t picture_id, const GofInfoVP9& gof) + void FrameReceivedVp9(uint16_t picture_id, GofInfo* info) EXCLUSIVE_LOCKS_REQUIRED(crit_); // Check if there is a frame with the up-switch flag set in the interval @@ -147,15 +155,14 @@ class RtpFrameReferenceFinder { std::array scalability_structures_ GUARDED_BY(crit_); - // Holds the picture id and the Gof information for a given TL0 picture index. - std::map, - DescendingSeqNumComp> - gof_info_ GUARDED_BY(crit_); + // Holds the the Gof information for a given TL0 picture index. + std::map> gof_info_ + GUARDED_BY(crit_); // Keep track of which picture id and which temporal layer that had the // up switch flag set. - std::map up_switch_ GUARDED_BY(crit_); + std::map> + up_switch_ GUARDED_BY(crit_); // For every temporal layer, keep a set of which frames that are missing. std::array>, diff --git a/webrtc/modules/video_coding/video_packet_buffer_unittest.cc b/webrtc/modules/video_coding/video_packet_buffer_unittest.cc index 3952eb5d56..ee9a853a24 100644 --- a/webrtc/modules/video_coding/video_packet_buffer_unittest.cc +++ b/webrtc/modules/video_coding/video_packet_buffer_unittest.cc @@ -1118,6 +1118,83 @@ TEST_F(TestPacketBuffer, Vp9GofTemporalLayersReordered_0) { CheckReferencesVp9(pid + 19, 0, pid + 18); } +TEST_F(TestPacketBuffer, Vp9GofSkipFramesTemporalLayers_01) { + uint16_t pid = Rand(); + uint16_t sn = Rand(); + GofInfoVP9 ss; + ss.SetGofInfoVP9(kTemporalStructureMode2); // 0101 pattern + + // sn , kf, frst, lst, up, pid , sid, tid, tl0, ss + InsertVp9Gof(sn , kT, kT , kT , kF, pid , 0 , 0 , 0 , &ss); + InsertVp9Gof(sn + 1 , kF, kT , kT , kF, pid + 1 , 0 , 1 , 0); + // Skip GOF with tl0 1 + InsertVp9Gof(sn + 4 , kT, kT , kT , kF, pid + 4 , 0 , 0 , 2 , &ss); + InsertVp9Gof(sn + 5 , kF, kT , kT , kF, pid + 5 , 0 , 1 , 2); + // Skip GOF with tl0 3 + // Skip GOF with tl0 4 + InsertVp9Gof(sn + 10, kF, kT , kT , kF, pid + 10, 0 , 0 , 5 , &ss); + InsertVp9Gof(sn + 11, kF, kT , kT , kF, pid + 11, 0 , 1 , 5); + + ASSERT_EQ(6UL, frames_from_callback_.size()); + CheckReferencesVp9(pid, 0); + CheckReferencesVp9(pid + 1 , 0, pid); + CheckReferencesVp9(pid + 4 , 0); + CheckReferencesVp9(pid + 5 , 0, pid + 4); + CheckReferencesVp9(pid + 10, 0, pid + 8); + CheckReferencesVp9(pid + 11, 0, pid + 10); +} + +TEST_F(TestPacketBuffer, Vp9GofSkipFramesTemporalLayers_0212) { + uint16_t pid = Rand(); + uint16_t sn = Rand(); + GofInfoVP9 ss; + ss.SetGofInfoVP9(kTemporalStructureMode3); // 02120212 pattern + + // sn , kf, frst, lst, up, pid , sid, tid, tl0, ss + InsertVp9Gof(sn , kT, kT , kT , kF, pid , 0 , 0 , 0 , &ss); + InsertVp9Gof(sn + 1 , kF, kT , kT , kF, pid + 1 , 0 , 2 , 0); + InsertVp9Gof(sn + 2 , kF, kT , kT , kF, pid + 2 , 0 , 1 , 0); + InsertVp9Gof(sn + 3 , kF, kT , kT , kF, pid + 3 , 0 , 2 , 0); + + ASSERT_EQ(4UL, frames_from_callback_.size()); + CheckReferencesVp9(pid, 0); + CheckReferencesVp9(pid + 1 , 0, pid); + CheckReferencesVp9(pid + 2 , 0, pid); + CheckReferencesVp9(pid + 3 , 0, pid + 1, pid + 2); + + // Skip frames with tl0 = 1 + + // sn , kf, frst, lst, up, pid , sid, tid, tl0, ss + InsertVp9Gof(sn + 8 , kT, kT , kT , kF, pid + 8 , 0 , 0 , 2 , &ss); + InsertVp9Gof(sn + 9 , kF, kT , kT , kF, pid + 9 , 0 , 2 , 2); + InsertVp9Gof(sn + 10, kF, kT , kT , kF, pid + 10, 0 , 1 , 2); + InsertVp9Gof(sn + 11, kF, kT , kT , kF, pid + 11, 0 , 2 , 2); + + ASSERT_EQ(8UL, frames_from_callback_.size()); + CheckReferencesVp9(pid + 8, 0); + CheckReferencesVp9(pid + 9 , 0, pid + 8); + CheckReferencesVp9(pid + 10, 0, pid + 8); + CheckReferencesVp9(pid + 11, 0, pid + 9, pid + 10); + + // Now insert frames with tl0 = 1 + // sn , kf, frst, lst, up, pid , sid, tid, tl0, ss + InsertVp9Gof(sn + 4 , kT, kT , kT , kF, pid + 4 , 0 , 0 , 1 , &ss); + InsertVp9Gof(sn + 7 , kF, kT , kT , kF, pid + 7 , 0 , 2 , 1); + + ASSERT_EQ(9UL, frames_from_callback_.size()); + CheckReferencesVp9(pid + 4, 0); + + // Rest of frames belonging to tl0 = 1 + // sn , kf, frst, lst, up, pid , sid, tid, tl0, ss + InsertVp9Gof(sn + 5 , kF, kT , kT , kF, pid + 5 , 0 , 2 , 1); + InsertVp9Gof(sn + 6 , kF, kT , kT , kT, pid + 6 , 0 , 1 , 1); // up-switch + + ASSERT_EQ(12UL, frames_from_callback_.size()); + CheckReferencesVp9(pid + 5 , 0, pid + 4); + CheckReferencesVp9(pid + 6 , 0, pid + 4); + CheckReferencesVp9(pid + 7 , 0, pid + 6); +} + TEST_F(TestPacketBuffer, Vp9GofTemporalLayers_01) { uint16_t pid = Rand(); uint16_t sn = Rand();