From cebdbf650dbd3a718b4b77389a183ebf8928ed99 Mon Sep 17 00:00:00 2001 From: Danil Chapovalov Date: Fri, 13 Dec 2019 16:08:18 +0100 Subject: [PATCH] switch RtpVideoStreamReceiver to use VideoRtpDepacketizer interface instead of creating each time an object with RtpDepacketizer interface this moves packet payload memcpy from RtpVideoStreamReceiver into the depacketizers with possibility to remove it from there in follow ups. Bug: webrtc:11152 Change-Id: If474207eb84d7e9d0207075bd395e60895f0d842 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/162185 Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#30095} --- video/rtp_video_stream_receiver.cc | 41 ++++++++++----------- video/rtp_video_stream_receiver.h | 7 ++-- video/rtp_video_stream_receiver_unittest.cc | 38 +++++++++---------- 3 files changed, 41 insertions(+), 45 deletions(-) diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 3373024986..ae3475b7d0 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -17,6 +17,7 @@ #include "absl/algorithm/container.h" #include "absl/memory/memory.h" +#include "absl/types/optional.h" #include "media/base/media_constants.h" #include "modules/pacing/packet_router.h" #include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" @@ -24,11 +25,14 @@ #include "modules/rtp_rtcp/include/rtp_cvo.h" #include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/ulpfec_receiver.h" +#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h" #include "modules/rtp_rtcp/source/rtp_format.h" #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h" #include "modules/utility/include/process_thread.h" #include "modules/video_coding/frame_object.h" #include "modules/video_coding/h264_sprop_parameter_sets.h" @@ -295,11 +299,10 @@ void RtpVideoStreamReceiver::AddReceiveCodec( const VideoCodec& video_codec, const std::map& codec_params, bool raw_payload) { - absl::optional video_type; - if (!raw_payload) { - video_type = video_codec.codecType; - } - payload_type_map_.emplace(video_codec.plType, video_type); + payload_type_map_.emplace( + video_codec.plType, + raw_payload ? std::make_unique() + : CreateVideoRtpDepacketizer(video_codec.codecType)); pt_codec_params_.emplace(video_codec.plType, codec_params); } @@ -324,7 +327,7 @@ absl::optional RtpVideoStreamReceiver::GetSyncInfo() const { } void RtpVideoStreamReceiver::OnReceivedPayloadData( - rtc::ArrayView codec_payload, + rtc::CopyOnWriteBuffer codec_payload, const RtpPacketReceived& rtp_packet, const RTPVideoHeader& video) { RTC_DCHECK_RUN_ON(&worker_task_checker_); @@ -440,7 +443,7 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData( packet.times_nacked = -1; } - if (codec_payload.empty()) { + if (codec_payload.size() == 0) { NotifyReceiverOfEmptyPacket(packet.seq_num); rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); return; @@ -456,7 +459,9 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData( } video_coding::H264SpsPpsTracker::FixedBitstream fixed = - tracker_.CopyAndFixBitstream(codec_payload, &packet.video_header); + tracker_.CopyAndFixBitstream( + rtc::MakeArrayView(codec_payload.cdata(), codec_payload.size()), + &packet.video_header); switch (fixed.action) { case video_coding::H264SpsPpsTracker::kRequestKeyframe: @@ -471,7 +476,7 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData( } } else { - packet.video_payload.SetData(codec_payload.data(), codec_payload.size()); + packet.video_payload = std::move(codec_payload); } rtcp_feedback_buffer_.SendBufferedRtcpFeedback(); @@ -755,23 +760,15 @@ void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) { if (type_it == payload_type_map_.end()) { return; } - auto depacketizer = - absl::WrapUnique(RtpDepacketizer::Create(type_it->second)); - - if (!depacketizer) { - RTC_LOG(LS_ERROR) << "Failed to create depacketizer."; - return; - } - RtpDepacketizer::ParsedPayload parsed_payload; - if (!depacketizer->Parse(&parsed_payload, packet.payload().data(), - packet.payload().size())) { + absl::optional parsed_payload = + type_it->second->Parse(packet.PayloadBuffer()); + if (parsed_payload == absl::nullopt) { RTC_LOG(LS_WARNING) << "Failed parsing payload."; return; } - OnReceivedPayloadData( - rtc::MakeArrayView(parsed_payload.payload, parsed_payload.payload_length), - packet, parsed_payload.video); + OnReceivedPayloadData(std::move(parsed_payload->video_payload), packet, + parsed_payload->video_header); } void RtpVideoStreamReceiver::ParseAndHandleEncapsulatingHeader( diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h index 5bd5061de8..7fa3e0b7e4 100644 --- a/video/rtp_video_stream_receiver.h +++ b/video/rtp_video_stream_receiver.h @@ -34,6 +34,7 @@ #include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" #include "modules/video_coding/h264_sps_pps_tracker.h" #include "modules/video_coding/loss_notification_controller.h" #include "modules/video_coding/packet_buffer.h" @@ -116,7 +117,7 @@ class RtpVideoStreamReceiver : public LossNotificationSender, // TODO(philipel): Stop using VCMPacket in the new jitter buffer and then // remove this function. Public only for tests. - void OnReceivedPayloadData(rtc::ArrayView codec_payload, + void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, const RtpPacketReceived& rtp_packet, const RTPVideoHeader& video); @@ -288,8 +289,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender, RTC_GUARDED_BY(last_seq_num_cs_); video_coding::H264SpsPpsTracker tracker_; - // Maps payload type to codec type, for packetization. - std::map> payload_type_map_; + // Maps payload id to the depacketizer. + std::map> payload_type_map_; // TODO(johan): Remove pt_codec_params_ once // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. diff --git a/video/rtp_video_stream_receiver_unittest.cc b/video/rtp_video_stream_receiver_unittest.cc index f7e6269c06..569c5150f8 100644 --- a/video/rtp_video_stream_receiver_unittest.cc +++ b/video/rtp_video_stream_receiver_unittest.cc @@ -11,6 +11,7 @@ #include "video/rtp_video_stream_receiver.h" #include +#include #include "api/video/video_codec_type.h" #include "api/video/video_frame_type.h" @@ -170,13 +171,12 @@ class RtpVideoStreamReceiverTest : public ::testing::Test { // code. void AddSps(RTPVideoHeader* video_header, uint8_t sps_id, - std::vector* data) { + rtc::CopyOnWriteBuffer* data) { NaluInfo info; info.type = H264::NaluType::kSps; info.sps_id = sps_id; info.pps_id = -1; - data->push_back(H264::NaluType::kSps); - data->push_back(sps_id); + data->AppendData({H264::NaluType::kSps, sps_id}); auto& h264 = absl::get(video_header->video_type_header); h264.nalus[h264.nalus_length++] = info; } @@ -184,13 +184,12 @@ class RtpVideoStreamReceiverTest : public ::testing::Test { void AddPps(RTPVideoHeader* video_header, uint8_t sps_id, uint8_t pps_id, - std::vector* data) { + rtc::CopyOnWriteBuffer* data) { NaluInfo info; info.type = H264::NaluType::kPps; info.sps_id = sps_id; info.pps_id = pps_id; - data->push_back(H264::NaluType::kPps); - data->push_back(pps_id); + data->AppendData({H264::NaluType::kPps, pps_id}); auto& h264 = absl::get(video_header->video_type_header); h264.nalus[h264.nalus_length++] = info; } @@ -342,7 +341,7 @@ TEST_F(RtpVideoStreamReceiverTest, CacheColorSpaceFromLastPacketOfKeyframe) { TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrame) { RtpPacketReceived rtp_packet; RTPVideoHeader video_header; - const std::vector data({1, 2, 3, 4}); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); rtp_packet.SetSequenceNumber(1); video_header.is_first_packet_in_frame = true; video_header.is_last_packet_in_frame = true; @@ -363,7 +362,7 @@ TEST_F(RtpVideoStreamReceiverTest, PacketInfoIsPropagatedIntoVideoFrames) { extension_map.Register(kId0); RtpPacketReceived rtp_packet(&extension_map); RTPVideoHeader video_header; - const std::vector data({1, 2, 3, 4}); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); rtp_packet.SetSequenceNumber(1); rtp_packet.SetTimestamp(1); rtp_packet.SetSsrc(kSsrc); @@ -397,7 +396,7 @@ TEST_F(RtpVideoStreamReceiverTest, RtpPacketReceived rtp_packet(&extension_map); RTPVideoHeader video_header; - const std::vector data({1, 2, 3, 4}); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); uint16_t sequence_number = 1; uint32_t rtp_timestamp = 1; rtp_packet.SetSequenceNumber(sequence_number); @@ -479,7 +478,7 @@ TEST_F(RtpVideoStreamReceiverTest, TEST_F(RtpVideoStreamReceiverTest, GenericKeyFrameBitstreamError) { RtpPacketReceived rtp_packet; RTPVideoHeader video_header; - const std::vector data({1, 2, 3, 4}); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); rtp_packet.SetSequenceNumber(1); video_header.is_first_packet_in_frame = true; video_header.is_last_packet_in_frame = true; @@ -506,7 +505,7 @@ INSTANTIATE_TEST_SUITE_P(SpsPpsIdrIsKeyframe, Values("", "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/")); TEST_P(RtpVideoStreamReceiverTestH264, InBandSpsPps) { - std::vector sps_data; + rtc::CopyOnWriteBuffer sps_data; RtpPacketReceived rtp_packet; RTPVideoHeader sps_video_header = GetDefaultH264VideoHeader(); AddSps(&sps_video_header, 0, &sps_data); @@ -520,7 +519,7 @@ TEST_P(RtpVideoStreamReceiverTestH264, InBandSpsPps) { rtp_video_stream_receiver_->OnReceivedPayloadData(sps_data, rtp_packet, sps_video_header); - std::vector pps_data; + rtc::CopyOnWriteBuffer pps_data; RTPVideoHeader pps_video_header = GetDefaultH264VideoHeader(); AddPps(&pps_video_header, 0, 1, &pps_data); rtp_packet.SetSequenceNumber(1); @@ -533,14 +532,15 @@ TEST_P(RtpVideoStreamReceiverTestH264, InBandSpsPps) { rtp_video_stream_receiver_->OnReceivedPayloadData(pps_data, rtp_packet, pps_video_header); - std::vector idr_data; + rtc::CopyOnWriteBuffer idr_data; RTPVideoHeader idr_video_header = GetDefaultH264VideoHeader(); AddIdr(&idr_video_header, 1); rtp_packet.SetSequenceNumber(2); idr_video_header.is_first_packet_in_frame = true; idr_video_header.is_last_packet_in_frame = true; idr_video_header.frame_type = VideoFrameType::kVideoFrameKey; - idr_data.insert(idr_data.end(), {0x65, 1, 2, 3}); + const uint8_t idr[] = {0x65, 1, 2, 3}; + idr_data.AppendData(idr); mock_on_complete_frame_callback_.AppendExpectedBitstream( kH264StartCode, sizeof(kH264StartCode)); mock_on_complete_frame_callback_.AppendExpectedBitstream(idr_data.data(), @@ -573,7 +573,6 @@ TEST_P(RtpVideoStreamReceiverTestH264, OutOfBandFmtpSpsPps) { mock_on_complete_frame_callback_.AppendExpectedBitstream(binary_pps, sizeof(binary_pps)); - std::vector data; RtpPacketReceived rtp_packet; RTPVideoHeader video_header = GetDefaultH264VideoHeader(); AddIdr(&video_header, 0); @@ -583,7 +582,7 @@ TEST_P(RtpVideoStreamReceiverTestH264, OutOfBandFmtpSpsPps) { video_header.is_last_packet_in_frame = true; video_header.codec = kVideoCodecH264; video_header.frame_type = VideoFrameType::kVideoFrameKey; - data.insert(data.end(), {1, 2, 3}); + rtc::CopyOnWriteBuffer data({1, 2, 3}); mock_on_complete_frame_callback_.AppendExpectedBitstream( kH264StartCode, sizeof(kH264StartCode)); mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(), @@ -596,8 +595,7 @@ TEST_P(RtpVideoStreamReceiverTestH264, OutOfBandFmtpSpsPps) { TEST_F(RtpVideoStreamReceiverTest, PaddingInMediaStream) { RtpPacketReceived rtp_packet; RTPVideoHeader video_header = GetDefaultH264VideoHeader(); - std::vector data; - data.insert(data.end(), {1, 2, 3}); + rtc::CopyOnWriteBuffer data({1, 2, 3}); rtp_packet.SetPayloadType(99); rtp_packet.SetSequenceNumber(2); video_header.is_first_packet_in_frame = true; @@ -634,7 +632,7 @@ TEST_F(RtpVideoStreamReceiverTest, PaddingInMediaStream) { TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeIfFirstFrameIsDelta) { RtpPacketReceived rtp_packet; RTPVideoHeader video_header; - const std::vector data({1, 2, 3, 4}); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); rtp_packet.SetSequenceNumber(1); video_header.is_first_packet_in_frame = true; video_header.is_last_packet_in_frame = true; @@ -650,7 +648,7 @@ TEST_F(RtpVideoStreamReceiverTest, RequestKeyframeWhenPacketBufferGetsFull) { RtpPacketReceived rtp_packet; RTPVideoHeader video_header; - const std::vector data({1, 2, 3, 4}); + rtc::CopyOnWriteBuffer data({1, 2, 3, 4}); video_header.is_first_packet_in_frame = true; // Incomplete frames so that the packet buffer is filling up. video_header.is_last_packet_in_frame = false;