From 326df690b2a3e1bbb5d1d7f466d86bbec16bad68 Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Mon, 15 Apr 2024 09:11:25 +0800 Subject: [PATCH] Use H26xPacketBuffer for H.264 and H.265 packets. This CL updates RtpVideoStreamReceiver2 to use H26xPacketBuffer for H.264 and H.265 packets. H.264 specific fixes are moved to H26xPacketBuffer as well. H26xPacketBuffer is behind field trial WebRTC-Video-H26xPacketBuffer. Bug: webrtc:13485 Change-Id: I1874c5a624b94c2d75ce607cf10c939619d7b5b9 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/346280 Reviewed-by: Philip Eliasson Commit-Queue: Philip Eliasson Reviewed-by: Sergey Silkin Cr-Commit-Position: refs/heads/main@{#42062} --- experiments/field_trials.py | 3 + video/BUILD.gn | 2 + video/rtp_video_stream_receiver2.cc | 51 ++++---- video/rtp_video_stream_receiver2.h | 7 +- video/rtp_video_stream_receiver2_unittest.cc | 118 +++++++++++++++++-- 5 files changed, 143 insertions(+), 38 deletions(-) diff --git a/experiments/field_trials.py b/experiments/field_trials.py index 58f6a47cea..10c94e62ac 100755 --- a/experiments/field_trials.py +++ b/experiments/field_trials.py @@ -150,6 +150,9 @@ ACTIVE_FIELD_TRIALS: FrozenSet[FieldTrial] = frozenset([ FieldTrial('WebRTC-ZeroHertzQueueOverload', 'webrtc:332381', date(2024, 7, 1)), + FieldTrial('WebRTC-Video-H26xPacketBuffer', + 'webrtc:13485', + date(2024, 6, 1)), # keep-sorted end ]) # yapf: disable diff --git a/video/BUILD.gn b/video/BUILD.gn index 9f20746c1f..cf724b7a72 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -129,6 +129,7 @@ rtc_library("video") { "../modules/rtp_rtcp:rtp_video_header", "../modules/video_coding", "../modules/video_coding:h264_sprop_parameter_sets", + "../modules/video_coding:h26x_packet_buffer", "../modules/video_coding:nack_requester", "../modules/video_coding:packet_buffer", "../modules/video_coding:video_codec_interface", @@ -902,6 +903,7 @@ if (rtc_include_tests) { "../modules/video_coding", "../modules/video_coding:codec_globals_headers", "../modules/video_coding:encoded_frame", + "../modules/video_coding:h26x_packet_buffer", "../modules/video_coding:packet_buffer", "../modules/video_coding:video_codec_interface", "../modules/video_coding:video_coding_utility", diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index 6b41acc1f2..8fef92c294 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -362,6 +362,7 @@ void RtpVideoStreamReceiver2::AddReceiveCodec( if (codec_params.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) > 0 || field_trials_.IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe")) { packet_buffer_.ForceSpsPpsIdrIsH264Keyframe(); + sps_pps_idr_is_h264_keyframe_ = true; } payload_type_map_.emplace( payload_type, raw_payload ? std::make_unique() @@ -369,39 +370,13 @@ void RtpVideoStreamReceiver2::AddReceiveCodec( pt_codec_params_.emplace(payload_type, codec_params); } -void RtpVideoStreamReceiver2::RemoveReceiveCodec(uint8_t payload_type) { - RTC_DCHECK_RUN_ON(&packet_sequence_checker_); - auto codec_params_it = pt_codec_params_.find(payload_type); - if (codec_params_it == pt_codec_params_.end()) - return; - - const bool sps_pps_idr_in_key_frame = - codec_params_it->second.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) > 0; - - pt_codec_params_.erase(codec_params_it); - payload_type_map_.erase(payload_type); - - if (sps_pps_idr_in_key_frame) { - bool reset_setting = true; - for (auto& [unused, codec_params] : pt_codec_params_) { - if (codec_params.count(cricket::kH264FmtpSpsPpsIdrInKeyframe) > 0) { - reset_setting = false; - break; - } - } - - if (reset_setting) { - packet_buffer_.ResetSpsPpsIdrIsH264Keyframe(); - } - } -} - void RtpVideoStreamReceiver2::RemoveReceiveCodecs() { RTC_DCHECK_RUN_ON(&packet_sequence_checker_); pt_codec_params_.clear(); payload_type_map_.clear(); packet_buffer_.ResetSpsPpsIdrIsH264Keyframe(); + h26x_packet_buffer_.reset(); } absl::optional RtpVideoStreamReceiver2::GetSyncInfo() const { @@ -676,7 +651,9 @@ bool RtpVideoStreamReceiver2::OnReceivedPayloadData( last_payload_type_ = packet->payload_type; InsertSpsPpsIntoTracker(packet->payload_type); } + } + if (packet->codec() == kVideoCodecH264 && !h26x_packet_buffer_) { video_coding::H264SpsPpsTracker::FixedBitstream fixed = tracker_.CopyAndFixBitstream( rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()), @@ -700,7 +677,14 @@ bool RtpVideoStreamReceiver2::OnReceivedPayloadData( rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); frame_counter_.Add(packet->timestamp); - OnInsertedPacket(packet_buffer_.InsertPacket(std::move(packet))); + + if ((packet->codec() == kVideoCodecH264 || + packet->codec() == kVideoCodecH265) && + h26x_packet_buffer_) { + OnInsertedPacket(h26x_packet_buffer_->InsertPacket(std::move(packet))); + } else { + OnInsertedPacket(packet_buffer_.InsertPacket(std::move(packet))); + } return false; } @@ -1254,6 +1238,13 @@ void RtpVideoStreamReceiver2::SignalNetworkState(NetworkState state) { void RtpVideoStreamReceiver2::StartReceive() { RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + // |h26x_packet_buffer_| is created here instead of in the ctor because we + // need to know the value of |sps_pps_id_is_h264_keyframe_|. + if (field_trials_.IsEnabled("WebRTC-Video-H26xPacketBuffer") && + !h26x_packet_buffer_) { + h26x_packet_buffer_ = + std::make_unique(!sps_pps_idr_is_h264_keyframe_); + } if (!receiving_ && packet_router_) { // Change REMB candidate egibility. packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get()); @@ -1298,6 +1289,10 @@ void RtpVideoStreamReceiver2::InsertSpsPpsIntoTracker(uint8_t payload_type) { tracker_.InsertSpsPpsNalus(sprop_decoder.sps_nalu(), sprop_decoder.pps_nalu()); + + if (h26x_packet_buffer_) { + h26x_packet_buffer_->SetSpropParameterSets(sprop_base64_it->second); + } } void RtpVideoStreamReceiver2::UpdatePacketReceiveTimestamps( diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h index d436aa38a7..16212832bf 100644 --- a/video/rtp_video_stream_receiver2.h +++ b/video/rtp_video_stream_receiver2.h @@ -39,6 +39,7 @@ #include "modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h" #include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" #include "modules/video_coding/h264_sps_pps_tracker.h" +#include "modules/video_coding/h26x_packet_buffer.h" #include "modules/video_coding/loss_notification_controller.h" #include "modules/video_coding/nack_requester.h" #include "modules/video_coding/packet_buffer.h" @@ -104,7 +105,6 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, VideoCodecType video_codec, const webrtc::CodecParameterMap& codec_params, bool raw_payload); - void RemoveReceiveCodec(uint8_t payload_type); // Clears state for all receive codecs added via `AddReceiveCodec`. void RemoveReceiveCodecs(); @@ -360,6 +360,10 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, video_coding::PacketBuffer packet_buffer_ RTC_GUARDED_BY(packet_sequence_checker_); + // h26x_packet_buffer_ is nullptr if codec list doens't contain H.264 or + // H.265, or field trial WebRTC-Video-H26xPacketBuffer is not enabled. + std::unique_ptr h26x_packet_buffer_ + RTC_GUARDED_BY(packet_sequence_checker_); UniqueTimestampCounter frame_counter_ RTC_GUARDED_BY(packet_sequence_checker_); SeqNumUnwrapper frame_id_unwrapper_ @@ -438,6 +442,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, Timestamp next_keyframe_request_for_missing_video_structure_ = Timestamp::MinusInfinity(); + bool sps_pps_idr_is_h264_keyframe_ = false; }; } // namespace webrtc diff --git a/video/rtp_video_stream_receiver2_unittest.cc b/video/rtp_video_stream_receiver2_unittest.cc index 6962540152..2d5956abd3 100644 --- a/video/rtp_video_stream_receiver2_unittest.cc +++ b/video/rtp_video_stream_receiver2_unittest.cc @@ -518,17 +518,28 @@ class RtpVideoStreamReceiver2TestH264 RtpVideoStreamReceiver2TestH264() : RtpVideoStreamReceiver2Test(GetParam()) {} }; -INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe, +INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframeAndH26xPacketBuffer, RtpVideoStreamReceiver2TestH264, - Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/")); + Values("", + "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/", + "WebRTC-Video-H26xPacketBuffer/Enabled/", + "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/" + "WebRTC-Video-H26xPacketBuffer/Enabled/")); TEST_P(RtpVideoStreamReceiver2TestH264, InBandSpsPps) { + constexpr int kH264PayloadType = 98; + webrtc::CodecParameterMap codec_params; + rtp_video_stream_receiver_->AddReceiveCodec(kH264PayloadType, kVideoCodecH264, + codec_params, + /*raw_payload=*/false); + rtp_video_stream_receiver_->StartReceive(); + rtc::CopyOnWriteBuffer sps_data; RtpPacketReceived rtp_packet; RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader(); AddSps(&sps_video_header, 0, &sps_data); rtp_packet.SetSequenceNumber(0); - rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetPayloadType(kH264PayloadType); sps_video_header.is_first_packet_in_frame = true; sps_video_header.frame_type = VideoFrameType::kEmptyFrame; mock_on_complete_frame_callback_.AppendExpectedBitstream( @@ -542,6 +553,7 @@ TEST_P(RtpVideoStreamReceiver2TestH264, InBandSpsPps) { RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader(); AddPps(&pps_video_header, 0, 1, &pps_data); rtp_packet.SetSequenceNumber(1); + rtp_packet.SetPayloadType(kH264PayloadType); pps_video_header.is_first_packet_in_frame = true; pps_video_header.frame_type = VideoFrameType::kEmptyFrame; mock_on_complete_frame_callback_.AppendExpectedBitstream( @@ -555,6 +567,8 @@ TEST_P(RtpVideoStreamReceiver2TestH264, InBandSpsPps) { RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader(); AddIdr(&idr_video_header, 1); rtp_packet.SetSequenceNumber(2); + rtp_packet.SetPayloadType(kH264PayloadType); + rtp_packet.SetMarker(true); idr_video_header.is_first_packet_in_frame = true; idr_video_header.is_last_packet_in_frame = true; idr_video_header.frame_type = VideoFrameType::kVideoFrameKey; @@ -579,6 +593,7 @@ TEST_P(RtpVideoStreamReceiver2TestH264, OutOfBandFmtpSpsPps) { rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecH264, codec_params, /*raw_payload=*/false); + rtp_video_stream_receiver_->StartReceive(); const uint8_t binary_sps[] = {0x67, 0x42, 0x00, 0x0a, 0x96, 0x53, 0x05, 0x89, 0x88}; mock_on_complete_frame_callback_.AppendExpectedBitstream( @@ -596,6 +611,7 @@ TEST_P(RtpVideoStreamReceiver2TestH264, OutOfBandFmtpSpsPps) { AddIdr(&video_header, 0); rtp_packet.SetPayloadType(kPayloadType); rtp_packet.SetSequenceNumber(2); + rtp_packet.SetMarker(true); video_header.is_first_packet_in_frame = true; video_header.is_last_packet_in_frame = true; video_header.codec = kVideoCodecH264; @@ -605,7 +621,13 @@ TEST_P(RtpVideoStreamReceiver2TestH264, OutOfBandFmtpSpsPps) { kH264StartCode, sizeof(kH264StartCode)); mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), data.size()); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + // IDR frames without SPS/PPS are not returned by + // |H26xPacketBuffer.InsertPacket| until SPS and PPS are received when + // WebRTC-SpsPpsIdrIsH264Keyframe is enabled. + if (!field_trials_.IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe") || + !field_trials_.IsEnabled("WebRTC-Video-H26xPacketBuffer")) { + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + } rtp_video_stream_receiver_->OnReceivedPayloadData(data, rtp_packet, video_header, 0); } @@ -613,13 +635,14 @@ TEST_P(RtpVideoStreamReceiver2TestH264, OutOfBandFmtpSpsPps) { TEST_P(RtpVideoStreamReceiver2TestH264, ForceSpsPpsIdrIsKeyframe) { constexpr int kPayloadType = 99; webrtc::CodecParameterMap codec_params; - if (GetParam() == - "") { // Forcing can be done either with field trial or codec_params. + // Forcing can be done either with field trial or codec_params. + if (!field_trials_.IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe")) { codec_params.insert({cricket::kH264FmtpSpsPpsIdrInKeyframe, ""}); } rtp_video_stream_receiver_->AddReceiveCodec(kPayloadType, kVideoCodecH264, codec_params, /*raw_payload=*/false); + rtp_video_stream_receiver_->StartReceive(); rtc::CopyOnWriteBuffer sps_data; RtpPacketReceived rtp_packet; RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader(); @@ -652,6 +675,7 @@ TEST_P(RtpVideoStreamReceiver2TestH264, ForceSpsPpsIdrIsKeyframe) { RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader(); AddIdr(&idr_video_header, 1); rtp_packet.SetSequenceNumber(2); + rtp_packet.SetMarker(true); idr_video_header.is_first_packet_in_frame = true; idr_video_header.is_last_packet_in_frame = true; idr_video_header.frame_type = VideoFrameType::kVideoFrameKey; @@ -672,9 +696,16 @@ TEST_P(RtpVideoStreamReceiver2TestH264, ForceSpsPpsIdrIsKeyframe) { mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), idr_data.size()); rtp_packet.SetSequenceNumber(3); - EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) - .WillOnce( - [&](EncodedFrame* frame) { EXPECT_FALSE(frame->is_keyframe()); }); + // IDR frames without SPS/PPS are not returned by + // |H26xPacketBuffer.InsertPacket| until SPS and PPS are received, while + // |PacketBuffer| returns it as a delta frame. + if (field_trials_.IsEnabled("WebRTC-Video-H26xPacketBuffer")) { + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame).Times(0); + } else { + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame) + .WillOnce( + [&](EncodedFrame* frame) { EXPECT_FALSE(frame->is_keyframe()); }); + } rtp_video_stream_receiver_->OnReceivedPayloadData(idr_data, rtp_packet, idr_video_header, 0); } @@ -1333,4 +1364,73 @@ TEST_P(RtpVideoStreamReceiver2TestPlayoutDelay, PlayoutDelay) { received_packet.PayloadBuffer(), received_packet, video_header, 0); } +#ifdef RTC_ENABLE_H265 +RTPVideoHeader GetDefaultH265VideoHeader() { + RTPVideoHeader video_header; + video_header.codec = kVideoCodecH265; + return video_header; +} + +class RtpVideoStreamReceiver2TestH265 : public RtpVideoStreamReceiver2Test { + protected: + RtpVideoStreamReceiver2TestH265() + : RtpVideoStreamReceiver2Test("WebRTC-Video-H26xPacketBuffer/Enabled/") {} +}; + +TEST_F(RtpVideoStreamReceiver2TestH265, H265Bitstream) { + constexpr int kH265PayloadType = 98; + webrtc::CodecParameterMap codec_params; + rtp_video_stream_receiver_->AddReceiveCodec(kH265PayloadType, kVideoCodecH265, + codec_params, + /*raw_payload=*/false); + rtp_video_stream_receiver_->StartReceive(); + + // Data is generated by WebCodecs H265 encoder, with 720p fake media capturer. + // IDR is not complete. + constexpr uint8_t vps[] = {0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, + 0xff, 0xff, 0x21, 0x40, 0x00, 0x00, 0x03, 0x00, + 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, + 0x7b, 0x3c, 0x0c, 0x00, 0x00, 0x03, 0x00, 0x04, + 0x00, 0x00, 0x03, 0x00, 0x79, 0x40}; + constexpr uint8_t sps[] = { + 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x21, 0x40, 0x00, 0x00, 0x03, + 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x7b, 0xa0, 0x02, + 0x80, 0x80, 0x2d, 0x16, 0x8f, 0x92, 0x46, 0xd9, 0x3f, 0xf6, 0x02, 0x80, + 0x10, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, 0x01, 0xe6, 0x45, + 0xde, 0xf7, 0xe0, 0x04, 0x65, 0x00, 0x23, 0x28, 0x80}; + constexpr uint8_t pps[] = {0x00, 0x00, 0x00, 0x01, 0x44, 0x01, + 0xc0, 0x25, 0x64, 0xc0, 0xed, 0x90}; + constexpr uint8_t idr[] = {0x00, 0x00, 0x00, 0x01, 0x26, 0x01, 0xaf, + 0xb0, 0x87, 0x11, 0x7a, 0xc1, 0x45, 0x57, + 0x3f, 0xff, 0x57, 0x14, 0x5f, 0xf7, 0x7a, + 0x37, 0xfd, 0xe3, 0xd9}; + + RtpPacketReceived rtp_packet; + rtp_packet.SetPayloadType(kPayloadType); + rtp_packet.SetSequenceNumber(0); + rtp_packet.SetPayloadType(kH265PayloadType); + RTPVideoHeader video_header = GetDefaultH265VideoHeader(); + mock_on_complete_frame_callback_.AppendExpectedBitstream(vps, sizeof(vps)); + rtp_video_stream_receiver_->OnReceivedPayloadData( + rtc::CopyOnWriteBuffer(vps, sizeof(vps)), rtp_packet, video_header, 0); + + rtp_packet.SetSequenceNumber(1); + mock_on_complete_frame_callback_.AppendExpectedBitstream(sps, sizeof(sps)); + rtp_video_stream_receiver_->OnReceivedPayloadData( + rtc::CopyOnWriteBuffer(sps, sizeof(sps)), rtp_packet, video_header, 0); + + rtp_packet.SetSequenceNumber(2); + mock_on_complete_frame_callback_.AppendExpectedBitstream(pps, sizeof(pps)); + rtp_video_stream_receiver_->OnReceivedPayloadData( + rtc::CopyOnWriteBuffer(pps, sizeof(pps)), rtp_packet, video_header, 0); + + rtp_packet.SetSequenceNumber(3); + rtp_packet.SetMarker(true); + mock_on_complete_frame_callback_.AppendExpectedBitstream(idr, sizeof(idr)); + EXPECT_CALL(mock_on_complete_frame_callback_, DoOnCompleteFrame(_)); + rtp_video_stream_receiver_->OnReceivedPayloadData( + rtc::CopyOnWriteBuffer(idr, sizeof(idr)), rtp_packet, video_header, 0); +} +#endif // RTC_ENABLE_H265 + } // namespace webrtc