diff --git a/webrtc/media/base/codec.cc b/webrtc/media/base/codec.cc index 835cf77b3e..75e5a7bf71 100644 --- a/webrtc/media/base/codec.cc +++ b/webrtc/media/base/codec.cc @@ -210,10 +210,13 @@ std::string VideoCodec::ToString() const { } VideoCodec::VideoCodec(int id, const std::string& name) - : Codec(id, name, kVideoCodecClockrate) {} + : Codec(id, name, kVideoCodecClockrate) { + SetDefaultParameters(); +} -VideoCodec::VideoCodec(const std::string& name) - : VideoCodec(0 /* id */, name) {} +VideoCodec::VideoCodec(const std::string& name) : VideoCodec(0 /* id */, name) { + SetDefaultParameters(); +} VideoCodec::VideoCodec() : Codec() { clockrate = kVideoCodecClockrate; @@ -224,6 +227,16 @@ VideoCodec::VideoCodec(VideoCodec&& c) = default; VideoCodec& VideoCodec::operator=(const VideoCodec& c) = default; VideoCodec& VideoCodec::operator=(VideoCodec&& c) = default; +void VideoCodec::SetDefaultParameters() { + if (_stricmp(kH264CodecName, name.c_str()) == 0) { + // This default is set for all H.264 codecs created because + // that was the default before packetization mode support was added. + // TODO(hta): Move this to the places that create VideoCodecs from + // SDP or from knowledge of implementation capabilities. + SetParam(kH264FmtpPacketizationMode, "1"); + } +} + bool VideoCodec::operator==(const VideoCodec& c) const { return Codec::operator==(c); } diff --git a/webrtc/media/base/codec.h b/webrtc/media/base/codec.h index 85b4327bb5..2280082c22 100644 --- a/webrtc/media/base/codec.h +++ b/webrtc/media/base/codec.h @@ -184,6 +184,9 @@ struct VideoCodec : public Codec { // don't make sense (such as max < min bitrate), and error is logged and // ValidateCodecFormat returns false. bool ValidateCodecFormat() const; + + private: + void SetDefaultParameters(); }; struct DataCodec : public Codec { diff --git a/webrtc/media/engine/internalencoderfactory.cc b/webrtc/media/engine/internalencoderfactory.cc index e714e5a5d8..ec9837579b 100644 --- a/webrtc/media/engine/internalencoderfactory.cc +++ b/webrtc/media/engine/internalencoderfactory.cc @@ -37,11 +37,9 @@ InternalEncoderFactory::InternalEncoderFactory() { cricket::VideoCodec codec(kH264CodecName); // TODO(magjed): Move setting these parameters into webrtc::H264Encoder // instead. - // TODO(hta): Set FMTP parameters for all codecs of type H264. codec.SetParam(kH264FmtpProfileLevelId, kH264ProfileLevelConstrainedBaseline); codec.SetParam(kH264FmtpLevelAsymmetryAllowed, "1"); - codec.SetParam(kH264FmtpPacketizationMode, "1"); supported_codecs_.push_back(std::move(codec)); } diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 6471ac4040..bc2a0f0691 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -544,6 +544,10 @@ if (rtc_include_tests) { [ "video_coding/codecs/vp9/vp9_screenshare_layers_unittest.cc" ] } + if (rtc_use_h264) { + sources += [ "video_coding/codecs/h264/h264_encoder_impl_unittest.cc" ] + } + if (rtc_desktop_capture_supported || is_android) { deps += [ "desktop_capture" ] sources += [ diff --git a/webrtc/modules/include/module_common_types.h b/webrtc/modules/include/module_common_types.h index 5de5eb78f6..3df93b6c89 100644 --- a/webrtc/modules/include/module_common_types.h +++ b/webrtc/modules/include/module_common_types.h @@ -262,6 +262,15 @@ enum H264PacketizationTypes { // that was too large to fit into a single packet. }; +// Packetization modes are defined in RFC 6184 section 6 +// Due to the structure containing this being initialized with zeroes +// in some places, and mode 1 being default, mode 1 needs to have the value +// zero. https://crbug.com/webrtc/6803 +enum class H264PacketizationMode { + NonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed + SingleNalUnit // Mode 0 - only single NALU allowed +}; + struct NaluInfo { uint8_t type; int sps_id; @@ -275,14 +284,19 @@ struct NaluInfo { const size_t kMaxNalusPerPacket = 10; struct RTPVideoHeaderH264 { - uint8_t nalu_type; // The NAL unit type. If this is a header for a - // fragmented packet, it's the NAL unit type of - // the original data. If this is the header for an - // aggregated packet, it's the NAL unit type of - // the first NAL unit in the packet. + // The NAL unit type. If this is a header for a + // fragmented packet, it's the NAL unit type of + // the original data. If this is the header for an + // aggregated packet, it's the NAL unit type of + // the first NAL unit in the packet. + uint8_t nalu_type; + // The packetization type of this buffer - single, aggregated or fragmented. H264PacketizationTypes packetization_type; NaluInfo nalus[kMaxNalusPerPacket]; size_t nalus_length; + // The packetization mode of this transport. Packetization mode + // determines which packetization types are allowed when packetizing. + H264PacketizationMode packetization_mode; }; union RTPVideoTypeHeader { diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format.cc b/webrtc/modules/rtp_rtcp/source/rtp_format.cc index cdb9c4920e..753fc2ec41 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format.cc @@ -10,6 +10,8 @@ #include "webrtc/modules/rtp_rtcp/source/rtp_format.h" +#include + #include "webrtc/modules/rtp_rtcp/source/rtp_format_h264.h" #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h" #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h" @@ -22,17 +24,19 @@ RtpPacketizer* RtpPacketizer::Create(RtpVideoCodecTypes type, FrameType frame_type) { switch (type) { case kRtpVideoH264: - return new RtpPacketizerH264(frame_type, max_payload_len); + RTC_CHECK(rtp_type_header); + return new RtpPacketizerH264(max_payload_len, + rtp_type_header->H264.packetization_mode); case kRtpVideoVp8: - assert(rtp_type_header != NULL); + RTC_CHECK(rtp_type_header); return new RtpPacketizerVp8(rtp_type_header->VP8, max_payload_len); case kRtpVideoVp9: - assert(rtp_type_header != NULL); + RTC_CHECK(rtp_type_header); return new RtpPacketizerVp9(rtp_type_header->VP9, max_payload_len); case kRtpVideoGeneric: return new RtpPacketizerGeneric(frame_type, max_payload_len); case kRtpVideoNone: - assert(false); + RTC_NOTREACHED(); } return NULL; } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc index b82b66f5fe..9d71803f3b 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -78,9 +78,10 @@ bool ParseStapAStartOffsets(const uint8_t* nalu_ptr, } // namespace -RtpPacketizerH264::RtpPacketizerH264(FrameType frame_type, - size_t max_payload_len) - : max_payload_len_(max_payload_len) {} +RtpPacketizerH264::RtpPacketizerH264(size_t max_payload_len, + H264PacketizationMode packetization_mode) + : max_payload_len_(max_payload_len), + packetization_mode_(packetization_mode) {} RtpPacketizerH264::~RtpPacketizerH264() { } @@ -163,11 +164,19 @@ void RtpPacketizerH264::SetPayloadData( void RtpPacketizerH264::GeneratePackets() { for (size_t i = 0; i < input_fragments_.size();) { - if (input_fragments_[i].length > max_payload_len_) { - PacketizeFuA(i); - ++i; - } else { - i = PacketizeStapA(i); + switch (packetization_mode_) { + case H264PacketizationMode::SingleNalUnit: + PacketizeSingleNalu(i); + ++i; + break; + case H264PacketizationMode::NonInterleaved: + if (input_fragments_[i].length > max_payload_len_) { + PacketizeFuA(i); + ++i; + } else { + i = PacketizeStapA(i); + } + break; } } } @@ -230,6 +239,21 @@ size_t RtpPacketizerH264::PacketizeStapA(size_t fragment_index) { return fragment_index; } +void RtpPacketizerH264::PacketizeSingleNalu(size_t fragment_index) { + // Add a single NALU to the queue, no aggregation. + size_t payload_size_left = max_payload_len_; + const Fragment* fragment = &input_fragments_[fragment_index]; + RTC_CHECK_GE(payload_size_left, fragment->length) + << "Payload size left " << payload_size_left << ", fragment length " + << fragment->length << ", packetization mode " + << (packetization_mode_ == H264PacketizationMode::SingleNalUnit + ? "SingleNalUnit" + : "NonInterleaved"); + RTC_CHECK_GT(fragment->length, 0u); + packets_.push(PacketUnit(*fragment, true /* first */, true /* last */, + false /* aggregated */, fragment->buffer[0])); +} + bool RtpPacketizerH264::NextPacket(RtpPacketToSend* rtp_packet, bool* last_packet) { RTC_DCHECK(rtp_packet); @@ -248,8 +272,10 @@ bool RtpPacketizerH264::NextPacket(RtpPacketToSend* rtp_packet, packets_.pop(); input_fragments_.pop_front(); } else if (packet.aggregated) { + RTC_CHECK(packetization_mode_ == H264PacketizationMode::NonInterleaved); NextAggregatePacket(rtp_packet); } else { + RTC_CHECK(packetization_mode_ == H264PacketizationMode::NonInterleaved); NextFragmentPacket(rtp_packet); } RTC_DCHECK_LE(rtp_packet->payload_size(), max_payload_len_); diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h index bc3240106d..fe7b37832a 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264.h @@ -26,7 +26,8 @@ class RtpPacketizerH264 : public RtpPacketizer { public: // Initialize with payload from encoder. // The payload_data must be exactly one encoded H264 frame. - RtpPacketizerH264(FrameType frame_type, size_t max_payload_len); + RtpPacketizerH264(size_t max_payload_len, + H264PacketizationMode packetization_mode); virtual ~RtpPacketizerH264(); @@ -86,10 +87,12 @@ class RtpPacketizerH264 : public RtpPacketizer { void GeneratePackets(); void PacketizeFuA(size_t fragment_index); size_t PacketizeStapA(size_t fragment_index); + void PacketizeSingleNalu(size_t fragment_index); void NextAggregatePacket(RtpPacketToSend* rtp_packet); void NextFragmentPacket(RtpPacketToSend* rtp_packet); const size_t max_payload_len_; + const H264PacketizationMode packetization_mode_; std::deque input_fragments_; std::queue packets_; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc index 03082a420d..ccaef1a9f4 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc @@ -49,6 +49,27 @@ enum NalDefs { kFBit = 0x80, kNriMask = 0x60, kTypeMask = 0x1F }; // Bit masks for FU (A and B) headers. enum FuDefs { kSBit = 0x80, kEBit = 0x40, kRBit = 0x20 }; +void CreateThreeFragments(RTPFragmentationHeader* fragmentation, + size_t frameSize, + size_t payloadOffset) { + fragmentation->VerifyAndAllocateFragmentationHeader(3); + fragmentation->fragmentationOffset[0] = 0; + fragmentation->fragmentationLength[0] = 2; + fragmentation->fragmentationOffset[1] = 2; + fragmentation->fragmentationLength[1] = 2; + fragmentation->fragmentationOffset[2] = 4; + fragmentation->fragmentationLength[2] = + kNalHeaderSize + frameSize - payloadOffset; +} + +RtpPacketizer* CreateH264Packetizer(H264PacketizationMode mode, + size_t max_payload_size) { + RTPVideoTypeHeader type_header; + type_header.H264.packetization_mode = mode; + return RtpPacketizer::Create(kRtpVideoH264, max_payload_size, &type_header, + kEmptyFrame); +} + void VerifyFua(size_t fua_index, const uint8_t* expected_payload, int offset, @@ -88,8 +109,8 @@ void TestFua(size_t frame_size, fragmentation.VerifyAndAllocateFragmentationHeader(1); fragmentation.fragmentationOffset[0] = 0; fragmentation.fragmentationLength[0] = frame_size; - std::unique_ptr packetizer(RtpPacketizer::Create( - kRtpVideoH264, max_payload_size, NULL, kEmptyFrame)); + std::unique_ptr packetizer(CreateH264Packetizer( + H264PacketizationMode::NonInterleaved, max_payload_size)); packetizer->SetPayloadData(frame.get(), frame_size, &fragmentation); RtpPacketToSend packet(kNoExtensions); @@ -149,14 +170,19 @@ void VerifySingleNaluPayload(const RTPFragmentationHeader& fragmentation, } } // namespace -TEST(RtpPacketizerH264Test, TestSingleNalu) { +// Tests that should work with both packetization mode 0 and +// packetization mode 1. +class RtpPacketizerH264ModeTest + : public ::testing::TestWithParam {}; + +TEST_P(RtpPacketizerH264ModeTest, TestSingleNalu) { const uint8_t frame[2] = {0x05, 0xFF}; // F=0, NRI=0, Type=5. RTPFragmentationHeader fragmentation; fragmentation.VerifyAndAllocateFragmentationHeader(1); fragmentation.fragmentationOffset[0] = 0; fragmentation.fragmentationLength[0] = sizeof(frame); std::unique_ptr packetizer( - RtpPacketizer::Create(kRtpVideoH264, kMaxPayloadSize, NULL, kEmptyFrame)); + CreateH264Packetizer(GetParam(), kMaxPayloadSize)); packetizer->SetPayloadData(frame, sizeof(frame), &fragmentation); RtpPacketToSend packet(kNoExtensions); ASSERT_LE(kMaxPayloadSize, packet.FreeCapacity()); @@ -168,7 +194,7 @@ TEST(RtpPacketizerH264Test, TestSingleNalu) { EXPECT_FALSE(packetizer->NextPacket(&packet, &last)); } -TEST(RtpPacketizerH264Test, TestSingleNaluTwoPackets) { +TEST_P(RtpPacketizerH264ModeTest, TestSingleNaluTwoPackets) { const size_t kFrameSize = kMaxPayloadSize + 100; uint8_t frame[kFrameSize] = {0}; for (size_t i = 0; i < kFrameSize; ++i) @@ -184,7 +210,7 @@ TEST(RtpPacketizerH264Test, TestSingleNaluTwoPackets) { frame[fragmentation.fragmentationOffset[1]] = 0x01; std::unique_ptr packetizer( - RtpPacketizer::Create(kRtpVideoH264, kMaxPayloadSize, NULL, kEmptyFrame)); + CreateH264Packetizer(GetParam(), kMaxPayloadSize)); packetizer->SetPayloadData(frame, kFrameSize, &fragmentation); RtpPacketToSend packet(kNoExtensions); @@ -201,6 +227,12 @@ TEST(RtpPacketizerH264Test, TestSingleNaluTwoPackets) { EXPECT_FALSE(packetizer->NextPacket(&packet, &last)); } +INSTANTIATE_TEST_CASE_P( + PacketMode, + RtpPacketizerH264ModeTest, + ::testing::Values(H264PacketizationMode::SingleNalUnit, + H264PacketizationMode::NonInterleaved)); + TEST(RtpPacketizerH264Test, TestStapA) { const size_t kFrameSize = kMaxPayloadSize - 3 * kLengthFieldLength - kNalHeaderSize; @@ -211,16 +243,9 @@ TEST(RtpPacketizerH264Test, TestStapA) { for (size_t i = 0; i < kFrameSize - kPayloadOffset; ++i) frame[i + kPayloadOffset] = i; RTPFragmentationHeader fragmentation; - fragmentation.VerifyAndAllocateFragmentationHeader(3); - fragmentation.fragmentationOffset[0] = 0; - fragmentation.fragmentationLength[0] = 2; - fragmentation.fragmentationOffset[1] = 2; - fragmentation.fragmentationLength[1] = 2; - fragmentation.fragmentationOffset[2] = 4; - fragmentation.fragmentationLength[2] = - kNalHeaderSize + kFrameSize - kPayloadOffset; - std::unique_ptr packetizer( - RtpPacketizer::Create(kRtpVideoH264, kMaxPayloadSize, NULL, kEmptyFrame)); + CreateThreeFragments(&fragmentation, kFrameSize, kPayloadOffset); + std::unique_ptr packetizer(CreateH264Packetizer( + H264PacketizationMode::NonInterleaved, kMaxPayloadSize)); packetizer->SetPayloadData(frame, kFrameSize, &fragmentation); RtpPacketToSend packet(kNoExtensions); @@ -237,6 +262,31 @@ TEST(RtpPacketizerH264Test, TestStapA) { EXPECT_FALSE(packetizer->NextPacket(&packet, &last)); } +TEST(RtpPacketizerH264Test, TestSingleNalUnitModeHasNoStapA) { + // This is the same setup as for the TestStapA test. + const size_t kFrameSize = + kMaxPayloadSize - 3 * kLengthFieldLength - kNalHeaderSize; + uint8_t frame[kFrameSize] = {0x07, 0xFF, // F=0, NRI=0, Type=7 (SPS). + 0x08, 0xFF, // F=0, NRI=0, Type=8 (PPS). + 0x05}; // F=0, NRI=0, Type=5 (IDR). + const size_t kPayloadOffset = 5; + for (size_t i = 0; i < kFrameSize - kPayloadOffset; ++i) + frame[i + kPayloadOffset] = i; + RTPFragmentationHeader fragmentation; + CreateThreeFragments(&fragmentation, kFrameSize, kPayloadOffset); + std::unique_ptr packetizer(CreateH264Packetizer( + H264PacketizationMode::SingleNalUnit, kMaxPayloadSize)); + packetizer->SetPayloadData(frame, kFrameSize, &fragmentation); + + RtpPacketToSend packet(kNoExtensions); + bool last = false; + // The three fragments should be returned as three packets. + ASSERT_TRUE(packetizer->NextPacket(&packet, &last)); + ASSERT_TRUE(packetizer->NextPacket(&packet, &last)); + ASSERT_TRUE(packetizer->NextPacket(&packet, &last)); + EXPECT_FALSE(packetizer->NextPacket(&packet, &last)); +} + TEST(RtpPacketizerH264Test, TestTooSmallForStapAHeaders) { const size_t kFrameSize = kMaxPayloadSize - 1; uint8_t frame[kFrameSize] = {0x07, 0xFF, // F=0, NRI=0, Type=7. @@ -254,8 +304,8 @@ TEST(RtpPacketizerH264Test, TestTooSmallForStapAHeaders) { fragmentation.fragmentationOffset[2] = 4; fragmentation.fragmentationLength[2] = kNalHeaderSize + kFrameSize - kPayloadOffset; - std::unique_ptr packetizer( - RtpPacketizer::Create(kRtpVideoH264, kMaxPayloadSize, NULL, kEmptyFrame)); + std::unique_ptr packetizer(CreateH264Packetizer( + H264PacketizationMode::NonInterleaved, kMaxPayloadSize)); packetizer->SetPayloadData(frame, kFrameSize, &fragmentation); RtpPacketToSend packet(kNoExtensions); @@ -302,8 +352,8 @@ TEST(RtpPacketizerH264Test, TestMixedStapA_FUA) { frame[nalu_offset + j] = i + j; } } - std::unique_ptr packetizer( - RtpPacketizer::Create(kRtpVideoH264, kMaxPayloadSize, NULL, kEmptyFrame)); + std::unique_ptr packetizer(CreateH264Packetizer( + H264PacketizationMode::NonInterleaved, kMaxPayloadSize)); packetizer->SetPayloadData(frame, kFrameSize, &fragmentation); // First expecting two FU-A packets. @@ -376,6 +426,28 @@ TEST(RtpPacketizerH264Test, TestFUABig) { sizeof(kExpectedPayloadSizes) / sizeof(size_t))); } +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +TEST(RtpPacketizerH264DeathTest, SendOverlongDataInPacketizationMode0) { + const size_t kFrameSize = kMaxPayloadSize + 1; + uint8_t frame[kFrameSize] = {0}; + for (size_t i = 0; i < kFrameSize; ++i) + frame[i] = i; + RTPFragmentationHeader fragmentation; + fragmentation.VerifyAndAllocateFragmentationHeader(1); + fragmentation.fragmentationOffset[0] = 0; + fragmentation.fragmentationLength[0] = kFrameSize; + // Set NAL headers. + frame[fragmentation.fragmentationOffset[0]] = 0x01; + + std::unique_ptr packetizer(CreateH264Packetizer( + H264PacketizationMode::SingleNalUnit, kMaxPayloadSize)); + EXPECT_DEATH(packetizer->SetPayloadData(frame, kFrameSize, &fragmentation), + "payload_size"); +} + +#endif // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + namespace { const uint8_t kStartSequence[] = {0x00, 0x00, 0x00, 0x01}; const uint8_t kOriginalSps[] = {kSps, 0x00, 0x00, 0x03, 0x03, @@ -416,9 +488,9 @@ TEST_F(RtpPacketizerH264TestSpsRewriting, FuASps) { const size_t kHeaderOverhead = kFuAHeaderSize + 1; // Set size to fragment SPS into two FU-A packets. - packetizer_.reset(RtpPacketizer::Create( - kRtpVideoH264, sizeof(kOriginalSps) - 2 + kHeaderOverhead, nullptr, - kEmptyFrame)); + packetizer_.reset( + CreateH264Packetizer(H264PacketizationMode::NonInterleaved, + sizeof(kOriginalSps) - 2 + kHeaderOverhead)); packetizer_->SetPayloadData(in_buffer_.data(), in_buffer_.size(), &fragmentation_header_); @@ -450,9 +522,8 @@ TEST_F(RtpPacketizerH264TestSpsRewriting, StapASps) { sizeof(kIdrTwo) + (kLengthFieldLength * 3); // Set size to include SPS and the rest of the packets in a Stap-A package. - packetizer_.reset(RtpPacketizer::Create(kRtpVideoH264, - kExpectedTotalSize + kHeaderOverhead, - nullptr, kEmptyFrame)); + packetizer_.reset(CreateH264Packetizer(H264PacketizationMode::NonInterleaved, + kExpectedTotalSize + kHeaderOverhead)); packetizer_->SetPayloadData(in_buffer_.data(), in_buffer_.size(), &fragmentation_header_); diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index c8f640490f..cd43b83217 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -158,6 +158,7 @@ rtc_static_library("webrtc_h264") { ] deps += [ "../../common_video", + "../../media:rtc_media_base", "//third_party/ffmpeg:ffmpeg", "//third_party/openh264:encoder", ] diff --git a/webrtc/modules/video_coding/codecs/h264/h264.cc b/webrtc/modules/video_coding/codecs/h264/h264.cc index f03a83d781..1e3a5809eb 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264.cc @@ -49,7 +49,7 @@ H264Encoder* H264Encoder::Create(const cricket::VideoCodec& codec) { #if defined(WEBRTC_USE_H264) RTC_CHECK(g_rtc_use_h264); LOG(LS_INFO) << "Creating H264EncoderImpl."; - return new H264EncoderImpl(); + return new H264EncoderImpl(codec); #else RTC_NOTREACHED(); return nullptr; diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc index d7df12209d..6150aa89c0 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -12,6 +12,7 @@ #include "webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h" #include +#include #include "third_party/openh264/src/codec/api/svc/codec_api.h" #include "third_party/openh264/src/codec/api/svc/codec_app_def.h" @@ -21,6 +22,7 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/media/base/mediaconstants.h" #include "webrtc/system_wrappers/include/metrics.h" namespace webrtc { @@ -150,7 +152,7 @@ static void RtpFragmentize(EncodedImage* encoded_image, } } -H264EncoderImpl::H264EncoderImpl() +H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec) : openh264_encoder_(nullptr), width_(0), height_(0), @@ -160,10 +162,20 @@ H264EncoderImpl::H264EncoderImpl() mode_(kRealtimeVideo), frame_dropping_on_(false), key_frame_interval_(0), + packetization_mode_(H264PacketizationMode::SingleNalUnit), + max_payload_size_(0), number_of_cores_(0), encoded_image_callback_(nullptr), has_reported_init_(false), - has_reported_error_(false) {} + has_reported_error_(false) { + RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName)); + std::string packetization_mode_string; + if (codec.GetParam(cricket::kH264FmtpPacketizationMode, + &packetization_mode_string) && + packetization_mode_string == "1") { + packetization_mode_ = H264PacketizationMode::NonInterleaved; + } +} H264EncoderImpl::~H264EncoderImpl() { Release(); @@ -171,7 +183,7 @@ H264EncoderImpl::~H264EncoderImpl() { int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings, int32_t number_of_cores, - size_t /*max_payload_size*/) { + size_t max_payload_size) { ReportInit(); if (!codec_settings || codec_settings->codecType != kVideoCodecH264) { @@ -218,6 +230,7 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings, mode_ = codec_settings->mode; frame_dropping_on_ = codec_settings->H264().frameDroppingOn; key_frame_interval_ = codec_settings->H264().keyFrameInterval; + max_payload_size_ = max_payload_size; // Codec_settings uses kbits/second; encoder uses bits/second. max_bps_ = codec_settings->maxBitrate * 1000; @@ -227,6 +240,7 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings, target_bps_ = codec_settings->targetBitrate * 1000; SEncParamExt encoder_params = CreateEncoderParams(); + // Initialize. if (openh264_encoder_->InitializeExt(&encoder_params) != 0) { LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder"; @@ -370,6 +384,7 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame, // Deliver encoded image. CodecSpecificInfo codec_specific; codec_specific.codecType = kVideoCodecH264; + codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_; encoded_image_callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_header); @@ -434,19 +449,46 @@ SEncParamExt H264EncoderImpl::CreateEncoderParams() const { encoder_params.iTargetBitrate; encoder_params.sSpatialLayers[0].iMaxSpatialBitrate = encoder_params.iMaxBitrate; + LOG(INFO) << "OpenH264 version is " << OPENH264_MAJOR << "." + << OPENH264_MINOR; + switch (packetization_mode_) { + case H264PacketizationMode::SingleNalUnit: +// Limit the size of the packets produced. #if (OPENH264_MAJOR == 1) && (OPENH264_MINOR <= 5) - // Slice num according to number of threads. - encoder_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = SM_AUTO_SLICE; + encoder_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = SM_DYN_SLICE; + // The slice size is max payload size - room for a NAL header. + // The constant 50 is NAL_HEADER_ADD_0X30BYTES in openh264 source, + // but is not exported. + const kNalHeaderSizeAllocation = 50; + encoder_params.sSpatialLayers[0] + .sSliceCfg.sSliceArgument.uiSliceSizeConstraint = + static_cast(max_payload_size_ - + kNalHeaderSizeAllocation); + encoder_params.uiMaxNalSize = + static_cast(max_payload_size_); #else - // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto design - // it with cpu core number. - // TODO(sprang): Set to 0 when we understand why the rate controller borks - // when uiSliceNum > 1. - encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; - encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = - SM_FIXEDSLCNUM_SLICE; + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = + SM_SIZELIMITED_SLICE; + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceSizeConstraint = + static_cast(max_payload_size_); #endif - + break; + case H264PacketizationMode::NonInterleaved: +#if (OPENH264_MAJOR == 1) && (OPENH264_MINOR <= 5) + // Slice num according to number of threads. + encoder_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = SM_AUTO_SLICE; +#else + // When uiSliceMode = SM_FIXEDSLCNUM_SLICE, uiSliceNum = 0 means auto + // design it with cpu core number. + // TODO(sprang): Set to 0 when we understand why the rate controller borks + // when uiSliceNum > 1. + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; + encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceMode = + SM_FIXEDSLCNUM_SLICE; +#endif + break; + } return encoder_params; } diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h index aab16acf36..a455259bf2 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h @@ -27,7 +27,7 @@ namespace webrtc { class H264EncoderImpl : public H264Encoder { public: - H264EncoderImpl(); + explicit H264EncoderImpl(const cricket::VideoCodec& codec); ~H264EncoderImpl() override; // |max_payload_size| is ignored. @@ -39,7 +39,7 @@ class H264EncoderImpl : public H264Encoder { // - height int32_t InitEncode(const VideoCodec* codec_settings, int32_t number_of_cores, - size_t /*max_payload_size*/) override; + size_t max_payload_size) override; int32_t Release() override; int32_t RegisterEncodeCompleteCallback( @@ -61,6 +61,11 @@ class H264EncoderImpl : public H264Encoder { int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; int32_t SetPeriodicKeyFrames(bool enable) override; + // Exposed for testing. + H264PacketizationMode PacketizationModeForTesting() const { + return packetization_mode_; + } + private: bool IsInitialized() const; SEncParamExt CreateEncoderParams() const; @@ -81,7 +86,9 @@ class H264EncoderImpl : public H264Encoder { // H.264 specifc parameters bool frame_dropping_on_; int key_frame_interval_; + H264PacketizationMode packetization_mode_; + size_t max_payload_size_; int32_t number_of_cores_; EncodedImage encoded_image_; diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc new file mode 100644 index 0000000000..2d236cf1f9 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl_unittest.cc @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h" + +#include "webrtc/test/gtest.h" + +namespace webrtc { + +namespace { + +const int kMaxPayloadSize = 1024; +const int kNumCores = 1; + +void SetDefaultSettings(VideoCodec* codec_settings) { + codec_settings->codecType = kVideoCodecH264; + codec_settings->maxFramerate = 60; + codec_settings->width = 640; + codec_settings->height = 480; + // If frame dropping is false, we get a warning that bitrate can't + // be controlled for RC_QUALITY_MODE; RC_BITRATE_MODE and RC_TIMESTAMP_MODE + codec_settings->H264()->frameDroppingOn = true; + codec_settings->targetBitrate = 2000; + codec_settings->maxBitrate = 4000; +} + +TEST(H264EncoderImplTest, CanInitializeWithDefaultParameters) { + H264EncoderImpl encoder(cricket::VideoCodec("H264")); + VideoCodec codec_settings; + SetDefaultSettings(&codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(H264PacketizationMode::NonInterleaved, + encoder.PacketizationModeForTesting()); +} + +TEST(H264EncoderImplTest, CanInitializeWithNonInterleavedModeExplicitly) { + cricket::VideoCodec codec("H264"); + codec.SetParam(cricket::kH264FmtpPacketizationMode, "1"); + H264EncoderImpl encoder(codec); + VideoCodec codec_settings; + SetDefaultSettings(&codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(H264PacketizationMode::NonInterleaved, + encoder.PacketizationModeForTesting()); +} + +TEST(H264EncoderImplTest, CanInitializeWithSingleNalUnitModeExplicitly) { + cricket::VideoCodec codec("H264"); + codec.SetParam(cricket::kH264FmtpPacketizationMode, "0"); + H264EncoderImpl encoder(codec); + VideoCodec codec_settings; + SetDefaultSettings(&codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(H264PacketizationMode::SingleNalUnit, + encoder.PacketizationModeForTesting()); +} + +TEST(H264EncoderImplTest, CanInitializeWithRemovedParameter) { + cricket::VideoCodec codec("H264"); + codec.RemoveParam(cricket::kH264FmtpPacketizationMode); + H264EncoderImpl encoder(codec); + VideoCodec codec_settings; + SetDefaultSettings(&codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(H264PacketizationMode::SingleNalUnit, + encoder.PacketizationModeForTesting()); +} + +} // anonymous namespace + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/include/video_codec_interface.h b/webrtc/modules/video_coding/include/video_codec_interface.h index f0a11e7a53..5dcfaffadb 100644 --- a/webrtc/modules/video_coding/include/video_codec_interface.h +++ b/webrtc/modules/video_coding/include/video_codec_interface.h @@ -77,7 +77,9 @@ struct CodecSpecificInfoGeneric { uint8_t simulcast_idx; }; -struct CodecSpecificInfoH264 {}; +struct CodecSpecificInfoH264 { + H264PacketizationMode packetization_mode; +}; union CodecSpecificInfoUnion { CodecSpecificInfoGeneric generic; diff --git a/webrtc/test/fake_encoder.cc b/webrtc/test/fake_encoder.cc index 26ce0c6608..ae1788bf6c 100644 --- a/webrtc/test/fake_encoder.cc +++ b/webrtc/test/fake_encoder.cc @@ -200,6 +200,8 @@ EncodedImageCallback::Result FakeH264Encoder::OnEncodedImage( CodecSpecificInfo specifics; memset(&specifics, 0, sizeof(specifics)); specifics.codecType = kVideoCodecH264; + specifics.codecSpecific.H264.packetization_mode = + H264PacketizationMode::NonInterleaved; return callback_->OnEncodedImage(encoded_image, &specifics, &fragmentation); } diff --git a/webrtc/video/BUILD.gn b/webrtc/video/BUILD.gn index 66191efb8a..9709e45c2a 100644 --- a/webrtc/video/BUILD.gn +++ b/webrtc/video/BUILD.gn @@ -144,6 +144,7 @@ if (rtc_include_tests) { # TODO(pbos): Rename test suite. rtc_source_set("video_tests") { testonly = true + defines = [] sources = [ "call_stats_unittest.cc", "encoder_rtcp_feedback_unittest.cc", @@ -171,7 +172,7 @@ if (rtc_include_tests) { suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] } if (rtc_use_h264) { - defines = [ "WEBRTC_USE_H264" ] + defines += [ "WEBRTC_USE_H264" ] } } } diff --git a/webrtc/video/end_to_end_tests.cc b/webrtc/video/end_to_end_tests.cc index 4de6db49cd..9d37e28f87 100644 --- a/webrtc/video/end_to_end_tests.cc +++ b/webrtc/video/end_to_end_tests.cc @@ -314,7 +314,9 @@ class CodecObserver : public test::EndToEndTest, const std::string& payload_name, webrtc::VideoEncoder* encoder, webrtc::VideoDecoder* decoder) - : EndToEndTest(2 * webrtc::EndToEndTest::kDefaultTimeoutMs), + : EndToEndTest(4 * webrtc::EndToEndTest::kDefaultTimeoutMs), + // TODO(hta): This timeout (120 seconds) is excessive. + // https://bugs.webrtc.org/6830 no_frames_to_wait_for_(no_frames_to_wait_for), expected_rotation_(rotation_to_test), payload_name_(payload_name), @@ -404,6 +406,23 @@ TEST_P(EndToEndTest, SendsAndReceivesH264VideoRotation90) { H264Decoder::Create()); RunBaseTest(&test); } + +TEST_P(EndToEndTest, SendsAndReceivesH264PacketizationMode0) { + cricket::VideoCodec codec = cricket::VideoCodec("H264"); + codec.SetParam(cricket::kH264FmtpPacketizationMode, "0"); + CodecObserver test(500, kVideoRotation_0, "H264", H264Encoder::Create(codec), + H264Decoder::Create()); + RunBaseTest(&test); +} + +TEST_P(EndToEndTest, SendsAndReceivesH264PacketizationMode1) { + cricket::VideoCodec codec = cricket::VideoCodec("H264"); + codec.SetParam(cricket::kH264FmtpPacketizationMode, "1"); + CodecObserver test(500, kVideoRotation_0, "H264", H264Encoder::Create(codec), + H264Decoder::Create()); + RunBaseTest(&test); +} + #endif // defined(WEBRTC_USE_H264) TEST_P(EndToEndTest, ReceiverUsesLocalSsrc) { diff --git a/webrtc/video/payload_router.cc b/webrtc/video/payload_router.cc index 6c8e36f72f..8b29e818f0 100644 --- a/webrtc/video/payload_router.cc +++ b/webrtc/video/payload_router.cc @@ -75,6 +75,8 @@ void CopyCodecSpecific(const CodecSpecificInfo* info, RTPVideoHeader* rtp) { } case kVideoCodecH264: rtp->codec = kRtpVideoH264; + rtp->codecHeader.H264.packetization_mode = + info->codecSpecific.H264.packetization_mode; return; case kVideoCodecGeneric: rtp->codec = kRtpVideoGeneric;