diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 2371d08224..785533ace7 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -377,6 +377,7 @@ if (rtc_include_tests) { } deps = [ + ":codec_globals_headers", ":video_coding", ":video_coding_utility", ":webrtc_vp8", diff --git a/modules/video_coding/codecs/test/stats.h b/modules/video_coding/codecs/test/stats.h index 8022f277c8..fd6fa874b2 100644 --- a/modules/video_coding/codecs/test/stats.h +++ b/modules/video_coding/codecs/test/stats.h @@ -32,6 +32,9 @@ struct FrameStatistic { size_t encoded_frame_size_bytes = 0; webrtc::FrameType frame_type = kVideoFrameDelta; + // H264 specific. + rtc::Optional max_nalu_length; + // Decoding. int64_t decode_start_ns = 0; int decode_return_code = 0; diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc index af5b4b35b5..cc453fab17 100644 --- a/modules/video_coding/codecs/test/videoprocessor.cc +++ b/modules/video_coding/codecs/test/videoprocessor.cc @@ -12,6 +12,7 @@ #include +#include #include #include #include @@ -19,6 +20,7 @@ #include "api/video/i420_buffer.h" #include "common_types.h" // NOLINT(build/include) +#include "common_video/h264/h264_common.h" #include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h" #include "modules/video_coding/include/video_codec_initializer.h" #include "modules/video_coding/utility/default_video_bitrate_allocator.h" @@ -108,6 +110,24 @@ void VerifyQpParser(const EncodedImage& encoded_frame, EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP."; } +rtc::Optional GetMaxNaluLength(const EncodedImage& encoded_frame, + const TestConfig& config) { + if (config.codec_settings.codecType != kVideoCodecH264) + return rtc::Optional(); + + std::vector nalu_indices = + webrtc::H264::FindNaluIndices(encoded_frame._buffer, + encoded_frame._length); + + RTC_CHECK(!nalu_indices.empty()); + + size_t max_length = 0; + for (const webrtc::H264::NaluIndex& index : nalu_indices) + max_length = std::max(max_length, index.payload_size); + + return rtc::Optional(max_length); +} + int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) { int64_t diff_us = (stop_ns - start_ns) / rtc::kNumNanosecsPerMicrosec; RTC_DCHECK_GE(diff_us, std::numeric_limits::min()); @@ -351,6 +371,8 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec, encoded_image._length / config_.networking_config.packet_size_in_bytes + 1; + frame_stat->max_nalu_length = GetMaxNaluLength(encoded_image, config_); + // Simulate packet loss. bool exclude_this_frame = false; if (encoded_image._frameType == kVideoFrameKey) { diff --git a/modules/video_coding/codecs/test/videoprocessor.h b/modules/video_coding/codecs/test/videoprocessor.h index 98b9140940..cf9692319e 100644 --- a/modules/video_coding/codecs/test/videoprocessor.h +++ b/modules/video_coding/codecs/test/videoprocessor.h @@ -18,6 +18,7 @@ #include "api/video/video_frame.h" #include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/test/packet_manipulator.h" #include "modules/video_coding/codecs/test/stats.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -100,6 +101,10 @@ struct TestConfig { // Should the hardware codecs be wrapped in software fallbacks? bool sw_fallback_encoder = false; bool sw_fallback_decoder = false; + + // RTP H264 packetization mode. + H264PacketizationMode packetization_mode = + H264PacketizationMode::NonInterleaved; }; // Handles encoding/decoding of video using the VideoEncoder/VideoDecoder diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 99ecb10a8c..7d9a81aa36 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -176,6 +176,7 @@ void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( const RateProfile& rate_profile, const std::vector* rc_thresholds, const QualityThresholds* quality_thresholds, + const BitstreamThresholds* bs_thresholds, const VisualizationParams* visualization_params) { // The Android HW codec needs to be run on a task queue, so we simply always // run the test on a task queue. @@ -246,6 +247,10 @@ void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( while (frame_number < num_frames) { UpdateRateControlMetrics(frame_number); + if (bs_thresholds) { + VerifyBitstream(frame_number, *bs_thresholds); + } + ++frame_number; if (frame_number == @@ -336,6 +341,13 @@ void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { case kVideoCodecH264: // TODO(brandtr): Generalize so that we support multiple profiles here. codec = cricket::VideoCodec(cricket::kH264CodecName); + if (config_.packetization_mode == H264PacketizationMode::NonInterleaved) { + codec.SetParam(cricket::kH264FmtpPacketizationMode, "1"); + } else { + RTC_CHECK_EQ(config_.packetization_mode, + H264PacketizationMode::SingleNalUnit); + codec.SetParam(cricket::kH264FmtpPacketizationMode, "0"); + } encoder_.reset(encoder_factory->CreateVideoEncoder(codec)); decoder_.reset( decoder_factory->CreateVideoDecoderWithParams(codec, decoder_params)); @@ -562,6 +574,14 @@ void VideoProcessorIntegrationTest::PrintRateControlMetrics( printf("\n"); } +void VideoProcessorIntegrationTest::VerifyBitstream( + int frame_number, + const BitstreamThresholds& bs_thresholds) { + RTC_CHECK_GE(frame_number, 0); + const FrameStatistic* frame_stat = stats_.GetFrame(frame_number); + EXPECT_LE(*(frame_stat->max_nalu_length), bs_thresholds.max_nalu_length); +} + // Temporal layer index corresponding to frame number, for up to 3 layers. int VideoProcessorIntegrationTest::TemporalLayerIndexForFrame( int frame_number) const { diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/modules/video_coding/codecs/test/videoprocessor_integrationtest.h index 9831e483f2..d38391540b 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.h @@ -71,6 +71,12 @@ struct QualityThresholds { double min_min_ssim; }; +struct BitstreamThresholds { + explicit BitstreamThresholds(size_t max_nalu_length) + : max_nalu_length(max_nalu_length) {} + size_t max_nalu_length; +}; + // Should video files be saved persistently to disk for post-run visualization? struct VisualizationParams { bool save_encoded_ivf; @@ -122,6 +128,7 @@ class VideoProcessorIntegrationTest : public testing::Test { const RateProfile& rate_profile, const std::vector* rc_thresholds, const QualityThresholds* quality_thresholds, + const BitstreamThresholds* bs_thresholds, const VisualizationParams* visualization_params); // Config. @@ -192,6 +199,9 @@ class VideoProcessorIntegrationTest : public testing::Test { const std::vector& num_dropped_frames, const std::vector& num_spatial_resizes) const; + void VerifyBitstream(int frame_number, + const BitstreamThresholds& bs_thresholds); + // Codecs. std::unique_ptr encoder_; std::unique_ptr decoder_; diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc index 9f3c1d3149..3e43a7693e 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc @@ -70,7 +70,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, Process0PercentPacketLossVP9) { QualityThresholds quality_thresholds(37.0, 36.0, 0.93, 0.92); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // VP9: Run with 5% packet loss and fixed bitrate. Quality should be a bit @@ -91,7 +91,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, Process5PercentPacketLossVP9) { QualityThresholds quality_thresholds(17.0, 14.0, 0.45, 0.36); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // VP9: Run with no packet loss, with varying bitrate (3 rate updates): @@ -117,7 +117,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessNoLossChangeBitRateVP9) { QualityThresholds quality_thresholds(35.5, 30.0, 0.90, 0.85); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // VP9: Run with no packet loss, with an update (decrease) in frame rate. @@ -147,7 +147,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, QualityThresholds quality_thresholds(31.5, 18.0, 0.80, 0.43); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // VP9: Run with no packet loss and denoiser on. One key frame (first frame). @@ -166,7 +166,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessNoLossDenoiserOnVP9) { QualityThresholds quality_thresholds(36.8, 35.8, 0.92, 0.91); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // Run with no packet loss, at low bitrate. @@ -188,7 +188,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, QualityThresholds quality_thresholds(24.0, 13.0, 0.65, 0.37); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // TODO(marpan): Add temporal layer test for VP9, once changes are in @@ -214,7 +214,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessZeroPacketLoss) { QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // VP8: Run with 5% packet loss and fixed bitrate. Quality should be a bit @@ -235,7 +235,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, Process5PercentPacketLoss) { QualityThresholds quality_thresholds(20.0, 16.0, 0.60, 0.40); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // VP8: Run with 10% packet loss and fixed bitrate. Quality should be lower. @@ -256,7 +256,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, Process10PercentPacketLoss) { QualityThresholds quality_thresholds(19.0, 16.0, 0.50, 0.35); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } #endif // !defined(WEBRTC_IOS) @@ -301,7 +301,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, QualityThresholds quality_thresholds(34.0, 32.0, 0.85, 0.80); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // VP8: Run with no packet loss, with an update (decrease) in frame rate. @@ -339,7 +339,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, QualityThresholds quality_thresholds(31.0, 22.0, 0.80, 0.65); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } // VP8: Run with no packet loss, with 3 temporal layers, with a rate update in @@ -372,7 +372,7 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, QualityThresholds quality_thresholds(32.5, 30.0, 0.85, 0.80); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } } // namespace test diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc index 293ac54e29..6e20244a26 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc @@ -57,7 +57,7 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } TEST_F(VideoProcessorIntegrationTestMediaCodec, @@ -89,7 +89,7 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, QualityThresholds quality_thresholds(33.0, 30.0, 0.90, 0.85); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); } #endif // defined(WEBRTC_ANDROID) diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc index e6db027e1d..df3e4d0bef 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc @@ -53,7 +53,7 @@ class VideoProcessorIntegrationTestOpenH264 // these unittests appears to drop "packets" in a way that is not compatible // with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have // not been added. -TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLossH264) { +TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLoss) { SetCodecSettings(&config_, kVideoCodecH264, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); @@ -68,7 +68,32 @@ TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLossH264) { QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70); ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, - kNoVisualizationParams); + nullptr, kNoVisualizationParams); +} + +// H264: Enable SingleNalUnit packetization mode. Encoder should split +// large frames into multiple slices and limit length of NAL units. +TEST_F(VideoProcessorIntegrationTestOpenH264, ProcessNoLossSingleNalUnit) { + config_.packetization_mode = H264PacketizationMode::SingleNalUnit; + config_.networking_config.max_payload_size_in_bytes = 500; + SetCodecSettings(&config_, kVideoCodecH264, 1, false, false, true, false, + kResilienceOn, kCifWidth, kCifHeight); + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFrames + 1; + rate_profile.num_frames = kNumFrames; + + std::vector rc_thresholds; + AddRateControlThresholds(2, 60, 30, 10, 20, 0, 1, &rc_thresholds); + + QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70); + + BitstreamThresholds bs_thresholds( + config_.networking_config.max_payload_size_in_bytes); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + &bs_thresholds, kNoVisualizationParams); } #endif // defined(WEBRTC_USE_H264) diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc index 4bd5fec78f..8a182f9b41 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc @@ -77,7 +77,7 @@ class VideoProcessorIntegrationTestParameterized rate_profile.frame_index_rate_update[1] = kNumFrames + 1; rate_profile.num_frames = kNumFrames; - ProcessFramesAndMaybeVerify(rate_profile, nullptr, nullptr, + ProcessFramesAndMaybeVerify(rate_profile, nullptr, nullptr, nullptr, &kVisualizationParams); }