From 9345dcfeea73f9dd68dc83dee60255e86dac3f97 Mon Sep 17 00:00:00 2001 From: "Hua, Chunbo" Date: Tue, 22 Sep 2020 10:34:56 +0800 Subject: [PATCH] Strip AUD NALUs from H.264 bitstream Bug: webrtc:11919 Change-Id: I33085faa3378f6a4059af32bab9988b640c968a3 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/181180 Commit-Queue: Stefan Holmer Reviewed-by: Stefan Holmer Reviewed-by: Sergey Silkin Cr-Commit-Position: refs/heads/master@{#32160} --- common_video/h264/sps_vui_rewriter.cc | 11 +++--- common_video/h264/sps_vui_rewriter.h | 5 +-- .../h264/sps_vui_rewriter_unittest.cc | 35 +++++++++++++++---- video/frame_encode_metadata_writer.cc | 2 +- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/common_video/h264/sps_vui_rewriter.cc b/common_video/h264/sps_vui_rewriter.cc index 3fc794fb2b..0d16be8254 100644 --- a/common_video/h264/sps_vui_rewriter.cc +++ b/common_video/h264/sps_vui_rewriter.cc @@ -210,7 +210,7 @@ SpsVuiRewriter::ParseResult SpsVuiRewriter::ParseAndRewriteSps( return result; } -rtc::Buffer SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( +rtc::Buffer SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite( rtc::ArrayView buffer, const webrtc::ColorSpace* color_space) { std::vector nalus = @@ -225,8 +225,6 @@ rtc::Buffer SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( const uint8_t* start_code_ptr = buffer.data() + nalu.start_offset; const size_t start_code_length = nalu.payload_start_offset - nalu.start_offset; - output_buffer.AppendData(start_code_ptr, start_code_length); - const uint8_t* nalu_ptr = buffer.data() + nalu.payload_start_offset; const size_t nalu_length = nalu.payload_size; @@ -253,12 +251,17 @@ rtc::Buffer SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( nalu_ptr + H264::kNaluTypeSize, nalu_length - H264::kNaluTypeSize, &sps, color_space, &output_nalu, Direction::kOutgoing); if (result == ParseResult::kVuiRewritten) { + output_buffer.AppendData(start_code_ptr, start_code_length); output_buffer.AppendData(output_nalu.data(), output_nalu.size()); continue; } + } else if (H264::ParseNaluType(nalu_ptr[0]) == H264::NaluType::kAud) { + // Skip the access unit delimiter copy. + continue; } - // vui wasn't rewritten, copy the nal unit as is. + // vui wasn't rewritten and it is not aud, copy the nal unit as is. + output_buffer.AppendData(start_code_ptr, start_code_length); output_buffer.AppendData(nalu_ptr, nalu_length); } return output_buffer; diff --git a/common_video/h264/sps_vui_rewriter.h b/common_video/h264/sps_vui_rewriter.h index 9e79c3f316..311db30d50 100644 --- a/common_video/h264/sps_vui_rewriter.h +++ b/common_video/h264/sps_vui_rewriter.h @@ -50,8 +50,9 @@ class SpsVuiRewriter : private SpsParser { rtc::Buffer* destination, Direction Direction); - // Parses NAL units from |buffer| and rewrites VUI in SPS blocks if necessary. - static rtc::Buffer ParseOutgoingBitstreamAndRewriteSps( + // Parses NAL units from |buffer|, strips AUD blocks and rewrites VUI in SPS + // blocks if necessary. + static rtc::Buffer ParseOutgoingBitstreamAndRewrite( rtc::ArrayView buffer, const ColorSpace* color_space); diff --git a/common_video/h264/sps_vui_rewriter_unittest.cc b/common_video/h264/sps_vui_rewriter_unittest.cc index 9a9cce3845..2907949e6c 100644 --- a/common_video/h264/sps_vui_rewriter_unittest.cc +++ b/common_video/h264/sps_vui_rewriter_unittest.cc @@ -36,6 +36,7 @@ static const size_t kWidth = 640; static const size_t kHeight = 480; static const uint8_t kStartSequence[] = {0x00, 0x00, 0x00, 0x01}; +static const uint8_t kAud[] = {H264::NaluType::kAud, 0x09, 0x10}; static const uint8_t kSpsNaluType[] = {H264::NaluType::kSps}; static const uint8_t kIdr1[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x00, 0x04}; static const uint8_t kIdr2[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x11}; @@ -401,9 +402,8 @@ TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamOptimalVui) { buffer.AppendData(kStartSequence); buffer.AppendData(kIdr1); - EXPECT_THAT( - SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(buffer, nullptr), - ::testing::ElementsAreArray(buffer)); + EXPECT_THAT(SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite(buffer, nullptr), + ::testing::ElementsAreArray(buffer)); } TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamNoVui) { @@ -433,8 +433,31 @@ TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamNoVui) { expected_buffer.AppendData(kStartSequence); expected_buffer.AppendData(kIdr2); - EXPECT_THAT( - SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(buffer, nullptr), - ::testing::ElementsAreArray(expected_buffer)); + EXPECT_THAT(SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite(buffer, nullptr), + ::testing::ElementsAreArray(expected_buffer)); +} + +TEST(SpsVuiRewriterOutgoingAudTest, ParseOutgoingBitstreamWithAud) { + rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE); + + rtc::Buffer optimal_sps; + GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps); + + rtc::Buffer buffer; + buffer.AppendData(kStartSequence); + buffer.AppendData(kAud); + buffer.AppendData(kStartSequence); + buffer.AppendData(optimal_sps); + buffer.AppendData(kStartSequence); + buffer.AppendData(kIdr1); + + rtc::Buffer expected_buffer; + expected_buffer.AppendData(kStartSequence); + expected_buffer.AppendData(optimal_sps); + expected_buffer.AppendData(kStartSequence); + expected_buffer.AppendData(kIdr1); + + EXPECT_THAT(SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite(buffer, nullptr), + ::testing::ElementsAreArray(expected_buffer)); } } // namespace webrtc diff --git a/video/frame_encode_metadata_writer.cc b/video/frame_encode_metadata_writer.cc index b30eeeace8..0e604cd765 100644 --- a/video/frame_encode_metadata_writer.cc +++ b/video/frame_encode_metadata_writer.cc @@ -213,7 +213,7 @@ void FrameEncodeMetadataWriter::UpdateBitstream( // Make sure that the data is not copied if owned by EncodedImage. const EncodedImage& buffer = *encoded_image; rtc::Buffer modified_buffer = - SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( + SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite( buffer, encoded_image->ColorSpace()); encoded_image->SetEncodedData(