From 022b54e86a538236b6a3cb97b80dfa1b3e3d5c56 Mon Sep 17 00:00:00 2001 From: philipel Date: Tue, 20 Dec 2016 04:15:59 -0800 Subject: [PATCH] Wire up H264 fmtp sprop-parameter-sets with H264SpsPpsTracker. To avoid making this CL large unittests will be added in a followup CL. BUG=webrtc:5948 patch from issue 2570073002 at patchset 20001 (http://crrev.com/2570073002#ps20001) Review-Url: https://codereview.webrtc.org/2565173009 Cr-Commit-Position: refs/heads/master@{#15710} --- .../video_coding/h264_sps_pps_tracker.cc | 31 +++++++++++++++ .../video_coding/h264_sps_pps_tracker.h | 3 ++ webrtc/video/BUILD.gn | 1 + webrtc/video/rtp_stream_receiver.cc | 39 ++++++++++++++++++- webrtc/video/rtp_stream_receiver.h | 9 +++++ webrtc/video/video_receive_stream.cc | 3 +- 6 files changed, 84 insertions(+), 2 deletions(-) diff --git a/webrtc/modules/video_coding/h264_sps_pps_tracker.cc b/webrtc/modules/video_coding/h264_sps_pps_tracker.cc index 7bcc2b7c5e..d8eb20fb1a 100644 --- a/webrtc/modules/video_coding/h264_sps_pps_tracker.cc +++ b/webrtc/modules/video_coding/h264_sps_pps_tracker.cc @@ -11,10 +11,13 @@ #include "webrtc/modules/video_coding/h264_sps_pps_tracker.h" #include +#include #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/common_video/h264/h264_common.h" +#include "webrtc/common_video/h264/pps_parser.h" +#include "webrtc/common_video/h264/sps_parser.h" #include "webrtc/modules/video_coding/frame_object.h" #include "webrtc/modules/video_coding/packet_buffer.h" @@ -179,5 +182,33 @@ H264SpsPpsTracker::PacketAction H264SpsPpsTracker::CopyAndFixBitstream( return kInsert; } +void H264SpsPpsTracker::InsertSpsPps(const std::vector& sps, + const std::vector& pps) { + rtc::Optional parsed_sps = + SpsParser::ParseSps(sps.data(), sps.size()); + rtc::Optional parsed_pps = + PpsParser::ParsePps(pps.data(), pps.size()); + + if (!parsed_pps || !parsed_sps) { + LOG(LS_WARNING) << "Failed to parse SPS or PPS parameters."; + return; + } + + SpsInfo sps_info; + sps_info.size = sps.size(); + uint8_t* sps_data = new uint8_t[sps_info.size]; + memcpy(sps_data, sps.data(), sps_info.size); + sps_info.data.reset(sps_data); + sps_data_[parsed_sps->id] = std::move(sps_info); + + PpsInfo pps_info; + pps_info.size = pps.size(); + pps_info.sps_id = parsed_pps->sps_id; + uint8_t* pps_data = new uint8_t[pps_info.size]; + memcpy(pps_data, pps.data(), pps_info.size); + pps_info.data.reset(pps_data); + pps_data_[parsed_pps->id] = std::move(pps_info); +} + } // namespace video_coding } // namespace webrtc diff --git a/webrtc/modules/video_coding/h264_sps_pps_tracker.h b/webrtc/modules/video_coding/h264_sps_pps_tracker.h index b54afbb260..db44ea5f41 100644 --- a/webrtc/modules/video_coding/h264_sps_pps_tracker.h +++ b/webrtc/modules/video_coding/h264_sps_pps_tracker.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "webrtc/modules/include/module_common_types.h" @@ -28,6 +29,8 @@ class H264SpsPpsTracker { enum PacketAction { kInsert, kDrop, kRequestKeyframe }; PacketAction CopyAndFixBitstream(VCMPacket* packet); + void InsertSpsPps(const std::vector& sps, + const std::vector& pps); private: struct PpsInfo { diff --git a/webrtc/video/BUILD.gn b/webrtc/video/BUILD.gn index 69756096de..312f3f450c 100644 --- a/webrtc/video/BUILD.gn +++ b/webrtc/video/BUILD.gn @@ -63,6 +63,7 @@ rtc_static_library("video") { "../base:rtc_task_queue", "../common_video", "../logging:rtc_event_log_api", + "../media:rtc_media_base", "../modules/bitrate_controller", "../modules/congestion_controller", "../modules/pacing", diff --git a/webrtc/video/rtp_stream_receiver.cc b/webrtc/video/rtp_stream_receiver.cc index 73ca531b12..d236085872 100644 --- a/webrtc/video/rtp_stream_receiver.cc +++ b/webrtc/video/rtp_stream_receiver.cc @@ -17,6 +17,7 @@ #include "webrtc/base/logging.h" #include "webrtc/common_types.h" #include "webrtc/config.h" +#include "webrtc/media/base/mediaconstants.h" #include "webrtc/modules/pacing/packet_router.h" #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "webrtc/modules/rtp_rtcp/include/receive_statistics.h" @@ -26,6 +27,7 @@ #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" #include "webrtc/modules/rtp_rtcp/include/ulpfec_receiver.h" #include "webrtc/modules/video_coding/frame_object.h" +#include "webrtc/modules/video_coding/h264_sprop_parameter_sets.h" #include "webrtc/modules/video_coding/h264_sps_pps_tracker.h" #include "webrtc/modules/video_coding/packet_buffer.h" #include "webrtc/modules/video_coding/video_coding_impl.h" @@ -223,13 +225,19 @@ RtpStreamReceiver::~RtpStreamReceiver() { UpdateHistograms(); } +bool RtpStreamReceiver::AddReceiveCodec( + const VideoCodec& video_codec, + const std::map& codec_params) { + pt_codec_params_.insert(make_pair(video_codec.plType, codec_params)); + return AddReceiveCodec(video_codec); +} + bool RtpStreamReceiver::AddReceiveCodec(const VideoCodec& video_codec) { int8_t old_pltype = -1; if (rtp_payload_registry_.ReceivePayloadType(video_codec, &old_pltype) != -1) { rtp_payload_registry_.DeRegisterReceivePayload(old_pltype); } - return rtp_payload_registry_.RegisterReceivePayload(video_codec) == 0; } @@ -259,6 +267,14 @@ int32_t RtpStreamReceiver::OnReceivedPayloadData( packet.timesNacked = nack_module_->OnReceivedPacket(packet); if (packet.codec == kVideoCodecH264) { + // Only when we start to receive packets will we know what payload type + // that will be used. When we know the payload type insert the correct + // sps/pps into the tracker. + if (packet.payloadType != last_payload_type_) { + last_payload_type_ = packet.payloadType; + InsertSpsPpsIntoTracker(packet.payloadType); + } + switch (tracker_.CopyAndFixBitstream(&packet)) { case video_coding::H264SpsPpsTracker::kRequestKeyframe: keyframe_request_sender_->RequestKeyFrame(); @@ -650,4 +666,25 @@ void RtpStreamReceiver::EnableReceiveRtpHeaderExtension( StringToRtpExtensionType(extension), id)); } +void RtpStreamReceiver::InsertSpsPpsIntoTracker(uint8_t payload_type) { + auto codec_params_it = pt_codec_params_.find(payload_type); + if (codec_params_it == pt_codec_params_.end()) + return; + + LOG(LS_INFO) << "Found out of band supplied codec parameters for" + << " payload type: " << payload_type; + + H264SpropParameterSets sprop_decoder; + auto sprop_base64_it = + codec_params_it->second.find(cricket::kH264FmtpSpropParameterSets); + + if (sprop_base64_it == codec_params_it->second.end()) + return; + + if (!sprop_decoder.DecodeSprop(sprop_base64_it->second)) + return; + + tracker_.InsertSpsPps(sprop_decoder.sps_nalu(), sprop_decoder.pps_nalu()); +} + } // namespace webrtc diff --git a/webrtc/video/rtp_stream_receiver.h b/webrtc/video/rtp_stream_receiver.h index 6968cc6428..b1e1db4b14 100644 --- a/webrtc/video/rtp_stream_receiver.h +++ b/webrtc/video/rtp_stream_receiver.h @@ -82,6 +82,9 @@ class RtpStreamReceiver : public RtpData, VCMTiming* timing); ~RtpStreamReceiver(); + bool AddReceiveCodec(const VideoCodec& video_codec, + const std::map& codec_params); + bool AddReceiveCodec(const VideoCodec& video_codec); uint32_t GetRemoteSsrc() const; @@ -158,6 +161,7 @@ class RtpStreamReceiver : public RtpData, void UpdateHistograms(); void EnableReceiveRtpHeaderExtension(const std::string& extension, int id); bool IsRedEnabled() const; + void InsertSpsPpsIntoTracker(uint8_t payload_type); Clock* const clock_; // Ownership of this object lies with VideoReceiveStream, which owns |this|. @@ -196,6 +200,11 @@ class RtpStreamReceiver : public RtpData, std::map> last_seq_num_for_pic_id_ GUARDED_BY(last_seq_num_cs_); video_coding::H264SpsPpsTracker tracker_; + // TODO(johan): Remove pt_codec_params_ once + // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. + // Maps a payload type to a map of out-of-band supplied codec parameters. + std::map> pt_codec_params_; + int16_t last_payload_type_ = -1; }; } // namespace webrtc diff --git a/webrtc/video/video_receive_stream.cc b/webrtc/video/video_receive_stream.cc index 0d3365810c..183f72b537 100644 --- a/webrtc/video/video_receive_stream.cc +++ b/webrtc/video/video_receive_stream.cc @@ -315,7 +315,8 @@ void VideoReceiveStream::Start() { for (const Decoder& decoder : config_.decoders) { video_receiver_.RegisterExternalDecoder(decoder.decoder, decoder.payload_type); - + // TODO(johan): make Decoder.codec_params accessible for RtpStreamReceiver + // which holds H264SpsPpsTracker VideoCodec codec = CreateDecoderVideoCodec(decoder); RTC_CHECK(rtp_stream_receiver_.AddReceiveCodec(codec)); RTC_CHECK_EQ(VCM_OK, video_receiver_.RegisterReceiveCodec(