diff --git a/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc index 4aeed569a4..198bb8a682 100644 --- a/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc @@ -58,7 +58,12 @@ class PlotVideoProcessorIntegrationTest int height, int framerate, const std::string& filename) { - // Bitrate and frame rate profile. + SetTestConfig(&config_, hw_codec_, kUseSingleCore, kPacketLoss, filename, + kVerboseLogging, kBatchMode); + SetCodecSettings(&config_, codec_type_, kNumTemporalLayers, + kErrorConcealmentOn, kDenoisingOn, kFrameDropperOn, + kSpatialResizeOn, kResilienceOn, width, height); + RateProfile rate_profile; SetRateProfile(&rate_profile, 0, // update_index @@ -67,34 +72,8 @@ class PlotVideoProcessorIntegrationTest rate_profile.frame_index_rate_update[1] = kNumFrames + 1; rate_profile.num_frames = kNumFrames; - // Codec/network settings. - SetTestConfig(&config_, hw_codec_, kUseSingleCore, kPacketLoss, filename, - kVerboseLogging, kBatchMode); - SetCodecSettings(&config_, codec_type_, kNumTemporalLayers, - kErrorConcealmentOn, kDenoisingOn, kFrameDropperOn, - kSpatialResizeOn, kResilienceOn, width, height); - - // Use default thresholds for quality (PSNR and SSIM). - QualityThresholds quality_thresholds; - - // Use very loose thresholds for rate control, so even poor HW codecs will - // pass the requirements. - RateControlThresholds rc_thresholds[1]; - // clang-format off - SetRateControlThresholds( - rc_thresholds, - 0, // update_index - kNumFrames + 1, // max_num_dropped_frames - 10000, // max_key_frame_size_mismatch - 10000, // max_delta_frame_size_mismatch - 10000, // max_encoding_rate_mismatch - kNumFrames + 1, // max_time_hit_target - 0, // num_spatial_resizes - 1); // num_key_frames - // clang-format on - - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - &kVisualizationParams); + ProcessFramesAndMaybeVerify(rate_profile, nullptr, nullptr, + &kVisualizationParams); } const int bitrate_; diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 2dab008822..d188d3466b 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -10,6 +10,8 @@ #include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h" +#include + namespace webrtc { namespace test { @@ -34,6 +36,8 @@ const int kNumFramesShort = 100; #endif const int kNumFramesLong = 299; +const std::nullptr_t kNoVisualizationParams = nullptr; + } // namespace #if defined(WEBRTC_USE_H264) @@ -44,23 +48,23 @@ const int kNumFramesLong = 299; // with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have // not been added. TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossH264) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 500, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; - rate_profile.num_frames = kNumFramesShort; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, kVerboseLogging, kBatchMode); SetCodecSettings(&config_, kVideoCodecH264, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; + rate_profile.num_frames = kNumFramesShort; + + std::vector rc_thresholds; + AddRateControlThresholds(2, 60, 20, 10, 20, 0, 1, &rc_thresholds); + QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 2, 60, 20, 10, 20, 0, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } #endif // defined(WEBRTC_USE_H264) @@ -72,45 +76,45 @@ TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossH264) { // VP9: Run with no packet loss and fixed bitrate. Quality should be very high. // One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossVP9) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 500, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; - rate_profile.num_frames = kNumFramesShort; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, kVerboseLogging, kBatchMode); SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; + rate_profile.num_frames = kNumFramesShort; + + std::vector rc_thresholds; + AddRateControlThresholds(0, 40, 20, 10, 20, 0, 1, &rc_thresholds); + QualityThresholds quality_thresholds(37.0, 36.0, 0.93, 0.92); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 20, 0, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // VP9: Run with 5% packet loss and fixed bitrate. Quality should be a bit // lower. One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLossVP9) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 500, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; - rate_profile.num_frames = kNumFramesShort; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.05f, kForemanCif, kVerboseLogging, kBatchMode); SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; + rate_profile.num_frames = kNumFramesShort; + + std::vector rc_thresholds; + AddRateControlThresholds(0, 40, 20, 10, 20, 0, 1, &rc_thresholds); + QualityThresholds quality_thresholds(17.0, 14.0, 0.45, 0.36); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 20, 0, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // VP9: Run with no packet loss, with varying bitrate (3 rate updates): @@ -118,27 +122,27 @@ TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLossVP9) { // target rate/per-frame bandwidth (for each rate update) is within limits. // One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRateVP9) { - // Bit rate and frame rate profile. + SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, + kVerboseLogging, kBatchMode); + SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false, + kResilienceOn, kCifWidth, kCifHeight); + RateProfile rate_profile; SetRateProfile(&rate_profile, 0, 200, 30, 0); SetRateProfile(&rate_profile, 1, 700, 30, 100); SetRateProfile(&rate_profile, 2, 500, 30, 200); rate_profile.frame_index_rate_update[3] = kNumFramesLong + 1; rate_profile.num_frames = kNumFramesLong; - // Codec/network settings. - SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, - kVerboseLogging, kBatchMode); - SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false, - kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + std::vector rc_thresholds; + AddRateControlThresholds(0, 30, 20, 20, 35, 0, 1, &rc_thresholds); + AddRateControlThresholds(2, 0, 20, 20, 60, 0, 0, &rc_thresholds); + AddRateControlThresholds(0, 0, 25, 20, 40, 0, 0, &rc_thresholds); + QualityThresholds quality_thresholds(35.5, 30.0, 0.90, 0.85); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[3]; - SetRateControlThresholds(rc_thresholds, 0, 0, 30, 20, 20, 35, 0, 1); - SetRateControlThresholds(rc_thresholds, 1, 2, 0, 20, 20, 60, 0, 0); - SetRateControlThresholds(rc_thresholds, 2, 0, 0, 25, 20, 40, 0, 0); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // VP9: Run with no packet loss, with an update (decrease) in frame rate. @@ -150,48 +154,48 @@ TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRateVP9) { // metrics averaged over whole sequence run. TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeFrameRateFrameDropVP9) { - // Bit rate and frame rate profile. + SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, + kVerboseLogging, kBatchMode); + SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false, + kResilienceOn, kCifWidth, kCifHeight); + RateProfile rate_profile; SetRateProfile(&rate_profile, 0, 100, 24, 0); SetRateProfile(&rate_profile, 1, 100, 15, 100); SetRateProfile(&rate_profile, 2, 100, 10, 200); rate_profile.frame_index_rate_update[3] = kNumFramesLong + 1; rate_profile.num_frames = kNumFramesLong; - // Codec/network settings. - SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, - kVerboseLogging, kBatchMode); - SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false, - kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + std::vector rc_thresholds; + AddRateControlThresholds(45, 50, 95, 15, 45, 0, 1, &rc_thresholds); + AddRateControlThresholds(20, 0, 50, 10, 30, 0, 0, &rc_thresholds); + AddRateControlThresholds(5, 0, 30, 5, 25, 0, 0, &rc_thresholds); + QualityThresholds quality_thresholds(31.5, 18.0, 0.80, 0.43); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[3]; - SetRateControlThresholds(rc_thresholds, 0, 45, 50, 95, 15, 45, 0, 1); - SetRateControlThresholds(rc_thresholds, 1, 20, 0, 50, 10, 30, 0, 0); - SetRateControlThresholds(rc_thresholds, 2, 5, 0, 30, 5, 25, 0, 0); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // VP9: Run with no packet loss and denoiser on. One key frame (first frame). TEST_F(VideoProcessorIntegrationTest, ProcessNoLossDenoiserOnVP9) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 500, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; - rate_profile.num_frames = kNumFramesShort; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, kVerboseLogging, kBatchMode); SetCodecSettings(&config_, kVideoCodecVP9, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; + rate_profile.num_frames = kNumFramesShort; + + std::vector rc_thresholds; + AddRateControlThresholds(0, 40, 20, 10, 20, 0, 1, &rc_thresholds); + QualityThresholds quality_thresholds(36.8, 35.8, 0.92, 0.91); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 20, 0, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // Run with no packet loss, at low bitrate. @@ -199,23 +203,23 @@ TEST_F(VideoProcessorIntegrationTest, ProcessNoLossDenoiserOnVP9) { // Resize happens on delta frame. Expect only one key frame (first frame). TEST_F(VideoProcessorIntegrationTest, DISABLED_ProcessNoLossSpatialResizeFrameDropVP9) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 50, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesLong + 1; - rate_profile.num_frames = kNumFramesLong; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, kVerboseLogging, kBatchMode); SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, true, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 50, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesLong + 1; + rate_profile.num_frames = kNumFramesLong; + + std::vector rc_thresholds; + AddRateControlThresholds(228, 70, 160, 15, 80, 1, 1, &rc_thresholds); + QualityThresholds quality_thresholds(24.0, 13.0, 0.65, 0.37); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 228, 70, 160, 15, 80, 1, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // TODO(marpan): Add temporal layer test for VP9, once changes are in @@ -227,67 +231,67 @@ TEST_F(VideoProcessorIntegrationTest, // One key frame (first frame only) in sequence. Setting |key_frame_interval| // to -1 below means no periodic key frames in test. TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 500, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; - rate_profile.num_frames = kNumFramesShort; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, kVerboseLogging, kBatchMode); SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; + rate_profile.num_frames = kNumFramesShort; + + std::vector rc_thresholds; + AddRateControlThresholds(0, 40, 20, 10, 15, 0, 1, &rc_thresholds); + QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 15, 0, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // VP8: Run with 5% packet loss and fixed bitrate. Quality should be a bit // lower. One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 500, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; - rate_profile.num_frames = kNumFramesShort; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.05f, kForemanCif, kVerboseLogging, kBatchMode); SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; + rate_profile.num_frames = kNumFramesShort; + + std::vector rc_thresholds; + AddRateControlThresholds(0, 40, 20, 10, 15, 0, 1, &rc_thresholds); + QualityThresholds quality_thresholds(20.0, 16.0, 0.60, 0.40); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 15, 0, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // VP8: Run with 10% packet loss and fixed bitrate. Quality should be lower. // One key frame (first frame only) in sequence. TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 500, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; - rate_profile.num_frames = kNumFramesShort; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.1f, kForemanCif, kVerboseLogging, kBatchMode); SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; + rate_profile.num_frames = kNumFramesShort; + + std::vector rc_thresholds; + AddRateControlThresholds(0, 40, 20, 10, 15, 0, 1, &rc_thresholds); + QualityThresholds quality_thresholds(19.0, 16.0, 0.50, 0.35); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 15, 0, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // This test is identical to VideoProcessorIntegrationTest.ProcessZeroPacketLoss @@ -295,23 +299,23 @@ TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) { // that the reported stats are not wildly varying between batch mode and the // regular online mode. TEST_F(VideoProcessorIntegrationTest, ProcessInBatchMode) { - // Bit rate and frame rate profile. - RateProfile rate_profile; - SetRateProfile(&rate_profile, 0, 500, 30, 0); - rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; - rate_profile.num_frames = kNumFramesShort; - // Codec/network settings. SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, kVerboseLogging, true /* batch_mode */); SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + RateProfile rate_profile; + SetRateProfile(&rate_profile, 0, 500, 30, 0); + rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1; + rate_profile.num_frames = kNumFramesShort; + + std::vector rc_thresholds; + AddRateControlThresholds(0, 40, 20, 10, 15, 0, 1, &rc_thresholds); + QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[1]; - SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 15, 0, 1); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } #endif // !defined(WEBRTC_IOS) @@ -337,27 +341,27 @@ TEST_F(VideoProcessorIntegrationTest, ProcessInBatchMode) { #define MAYBE_ProcessNoLossChangeBitRateVP8 ProcessNoLossChangeBitRateVP8 #endif TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossChangeBitRateVP8) { - // Bit rate and frame rate profile. + SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, + kVerboseLogging, kBatchMode); + SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false, + kResilienceOn, kCifWidth, kCifHeight); + RateProfile rate_profile; SetRateProfile(&rate_profile, 0, 200, 30, 0); SetRateProfile(&rate_profile, 1, 800, 30, 100); SetRateProfile(&rate_profile, 2, 500, 30, 200); rate_profile.frame_index_rate_update[3] = kNumFramesLong + 1; rate_profile.num_frames = kNumFramesLong; - // Codec/network settings. - SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, - kVerboseLogging, kBatchMode); - SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false, - kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + std::vector rc_thresholds; + AddRateControlThresholds(0, 45, 20, 10, 15, 0, 1, &rc_thresholds); + AddRateControlThresholds(0, 0, 25, 20, 10, 0, 0, &rc_thresholds); + AddRateControlThresholds(0, 0, 25, 15, 10, 0, 0, &rc_thresholds); + QualityThresholds quality_thresholds(34.0, 32.0, 0.85, 0.80); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[3]; - SetRateControlThresholds(rc_thresholds, 0, 0, 45, 20, 10, 15, 0, 1); - SetRateControlThresholds(rc_thresholds, 1, 0, 0, 25, 20, 10, 0, 0); - SetRateControlThresholds(rc_thresholds, 2, 0, 0, 25, 15, 10, 0, 0); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // VP8: Run with no packet loss, with an update (decrease) in frame rate. @@ -377,27 +381,27 @@ TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossChangeBitRateVP8) { #endif TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8) { - // Bit rate and frame rate profile. + SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, + kVerboseLogging, kBatchMode); + SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false, + kResilienceOn, kCifWidth, kCifHeight); + RateProfile rate_profile; SetRateProfile(&rate_profile, 0, 80, 24, 0); SetRateProfile(&rate_profile, 1, 80, 15, 100); SetRateProfile(&rate_profile, 2, 80, 10, 200); rate_profile.frame_index_rate_update[3] = kNumFramesLong + 1; rate_profile.num_frames = kNumFramesLong; - // Codec/network settings. - SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, - kVerboseLogging, kBatchMode); - SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false, - kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + std::vector rc_thresholds; + AddRateControlThresholds(40, 20, 75, 15, 60, 0, 1, &rc_thresholds); + AddRateControlThresholds(10, 0, 25, 10, 35, 0, 0, &rc_thresholds); + AddRateControlThresholds(0, 0, 20, 10, 15, 0, 0, &rc_thresholds); + QualityThresholds quality_thresholds(31.0, 22.0, 0.80, 0.65); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[3]; - SetRateControlThresholds(rc_thresholds, 0, 40, 20, 75, 15, 60, 0, 1); - SetRateControlThresholds(rc_thresholds, 1, 10, 0, 25, 10, 35, 0, 0); - SetRateControlThresholds(rc_thresholds, 2, 0, 0, 20, 10, 15, 0, 0); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } // VP8: Run with no packet loss, with 3 temporal layers, with a rate update in @@ -413,25 +417,25 @@ TEST_F(VideoProcessorIntegrationTest, #define MAYBE_ProcessNoLossTemporalLayersVP8 ProcessNoLossTemporalLayersVP8 #endif TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossTemporalLayersVP8) { - // Bit rate and frame rate profile. + SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, + kVerboseLogging, kBatchMode); + SetCodecSettings(&config_, kVideoCodecVP8, 3, false, true, true, false, + kResilienceOn, kCifWidth, kCifHeight); + RateProfile rate_profile; SetRateProfile(&rate_profile, 0, 200, 30, 0); SetRateProfile(&rate_profile, 1, 400, 30, 150); rate_profile.frame_index_rate_update[2] = kNumFramesLong + 1; rate_profile.num_frames = kNumFramesLong; - // Codec/network settings. - SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif, - kVerboseLogging, kBatchMode); - SetCodecSettings(&config_, kVideoCodecVP8, 3, false, true, true, false, - kResilienceOn, kCifWidth, kCifHeight); - // Thresholds for expected quality. + + std::vector rc_thresholds; + AddRateControlThresholds(0, 20, 30, 10, 10, 0, 1, &rc_thresholds); + AddRateControlThresholds(0, 0, 30, 15, 10, 0, 0, &rc_thresholds); + QualityThresholds quality_thresholds(32.5, 30.0, 0.85, 0.80); - // Thresholds for rate control. - RateControlThresholds rc_thresholds[2]; - SetRateControlThresholds(rc_thresholds, 0, 0, 20, 30, 10, 10, 0, 1); - SetRateControlThresholds(rc_thresholds, 1, 0, 0, 30, 15, 10, 0, 0); - ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds, - nullptr /* visualization_params */); + + ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds, + kNoVisualizationParams); } } // namespace test } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h index e47df371f6..3d43d64380 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h @@ -17,6 +17,7 @@ #include #include #include +#include #if defined(WEBRTC_ANDROID) #include "webrtc/modules/video_coding/codecs/test/android_test_initializer.h" @@ -65,7 +66,6 @@ const float kScaleKeyFrameSize = 0.5f; // Thresholds for the quality metrics. Defaults are maximally minimal. struct QualityThresholds { - QualityThresholds() {} QualityThresholds(double min_avg_psnr, double min_min_psnr, double min_avg_ssim, @@ -74,10 +74,10 @@ struct QualityThresholds { min_min_psnr(min_min_psnr), min_avg_ssim(min_avg_ssim), min_min_ssim(min_min_ssim) {} - double min_avg_psnr = std::numeric_limits::min(); - double min_min_psnr = std::numeric_limits::min(); - double min_avg_ssim = 0.0; - double min_min_ssim = 0.0; + double min_avg_psnr; + double min_min_psnr; + double min_avg_ssim; + double min_min_ssim; }; // The sequence of bit rate and frame rate changes for the encoder, the frame @@ -101,8 +101,8 @@ struct RateControlThresholds { int max_delta_frame_size_mismatch; int max_encoding_rate_mismatch; int max_time_hit_target; - int num_spatial_resizes; // Set to -1 to disable check. - int num_key_frames; // Set to -1 to disable check. + int num_spatial_resizes; + int num_key_frames; }; // Should video files be saved persistently to disk for post-run visualization? @@ -306,8 +306,9 @@ class VideoProcessorIntegrationTest : public testing::Test { } // Verify expected behavior of rate control and print out data. - void VerifyRateControlMetrics(int rate_update_index, - const RateControlThresholds& rc_expected) { + void PrintAndMaybeVerifyRateControlMetrics( + int rate_update_index, + const std::vector* rc_thresholds) { int num_dropped_frames = processor_->NumberDroppedFrames(); int num_resize_actions = processor_->NumberSpatialResizes(); printf( @@ -324,8 +325,13 @@ class VideoProcessorIntegrationTest : public testing::Test { num_frames_total_, num_frames_to_hit_target_, num_dropped_frames, num_resize_actions); - EXPECT_LE(perc_encoding_rate_mismatch_, - rc_expected.max_encoding_rate_mismatch); + const RateControlThresholds* rc_threshold = nullptr; + if (rc_thresholds) { + rc_threshold = &(*rc_thresholds)[rate_update_index]; + + EXPECT_LE(perc_encoding_rate_mismatch_, + rc_threshold->max_encoding_rate_mismatch); + } if (num_key_frames_ > 0) { int perc_key_frame_size_mismatch = 100 * sum_key_frame_size_mismatch_ / num_key_frames_; @@ -333,8 +339,10 @@ class VideoProcessorIntegrationTest : public testing::Test { " # key frames : %d\n" " Key frame rate mismatch: %d\n", num_key_frames_, perc_key_frame_size_mismatch); - EXPECT_LE(perc_key_frame_size_mismatch, - rc_expected.max_key_frame_size_mismatch); + if (rc_threshold) { + EXPECT_LE(perc_key_frame_size_mismatch, + rc_threshold->max_key_frame_size_mismatch); + } } const int num_temporal_layers = @@ -357,20 +365,20 @@ class VideoProcessorIntegrationTest : public testing::Test { i, bitrate_layer_[i], framerate_layer_[i], per_frame_bandwidth_[i], encoding_bitrate_[i], perc_frame_size_mismatch, perc_encoding_rate_mismatch, num_frames_per_update_[i]); - EXPECT_LE(perc_frame_size_mismatch, - rc_expected.max_delta_frame_size_mismatch); - EXPECT_LE(perc_encoding_rate_mismatch, - rc_expected.max_encoding_rate_mismatch); + if (rc_threshold) { + EXPECT_LE(perc_frame_size_mismatch, + rc_threshold->max_delta_frame_size_mismatch); + EXPECT_LE(perc_encoding_rate_mismatch, + rc_threshold->max_encoding_rate_mismatch); + } } printf("\n"); - EXPECT_LE(num_frames_to_hit_target_, rc_expected.max_time_hit_target); - EXPECT_LE(num_dropped_frames, rc_expected.max_num_dropped_frames); - if (rc_expected.num_spatial_resizes >= 0) { - EXPECT_EQ(rc_expected.num_spatial_resizes, num_resize_actions); - } - if (rc_expected.num_key_frames >= 0) { - EXPECT_EQ(rc_expected.num_key_frames, num_key_frames_); + if (rc_threshold) { + EXPECT_LE(num_frames_to_hit_target_, rc_threshold->max_time_hit_target); + EXPECT_LE(num_dropped_frames, rc_threshold->max_num_dropped_frames); + EXPECT_EQ(rc_threshold->num_spatial_resizes, num_resize_actions); + EXPECT_EQ(rc_threshold->num_key_frames, num_key_frames_); } } @@ -459,14 +467,11 @@ class VideoProcessorIntegrationTest : public testing::Test { } // Processes all frames in the clip and verifies the result. - // TODO(brandtr): Change the second last argument to be a - // const std::vector&, so we can ensure that the user - // does not expect us to do mid-clip rate updates when we are not able to, - // e.g., when we are operating in batch mode. - void ProcessFramesAndVerify(const QualityThresholds& quality_thresholds, - const RateProfile& rate_profile, - RateControlThresholds* rc_thresholds, - const VisualizationParams* visualization_params) { + void ProcessFramesAndMaybeVerify( + const RateProfile& rate_profile, + const std::vector* rc_thresholds, + const QualityThresholds* quality_thresholds, + const VisualizationParams* visualization_params) { config_.codec_settings.startBitrate = rate_profile.target_bit_rate[0]; SetUpObjects(visualization_params, rate_profile.target_bit_rate[0], rate_profile.input_frame_rate[0]); @@ -526,7 +531,7 @@ class VideoProcessorIntegrationTest : public testing::Test { // update layers and codec with new rates. if (frame_number == rate_profile.frame_index_rate_update[update_index + 1]) { - VerifyRateControlMetrics(update_index, rc_thresholds[update_index]); + PrintAndMaybeVerifyRateControlMetrics(update_index, rc_thresholds); // Update layer rates and the codec with new rates. ++update_index; @@ -545,7 +550,7 @@ class VideoProcessorIntegrationTest : public testing::Test { // Verify rate control metrics for all frames (if in batch mode), or for all // frames since the last rate update (if not in batch mode). - VerifyRateControlMetrics(update_index, rc_thresholds[update_index]); + PrintAndMaybeVerifyRateControlMetrics(update_index, rc_thresholds); EXPECT_EQ(num_frames, frame_number); EXPECT_EQ(num_frames + 1, static_cast(stats_.stats_.size())); @@ -572,7 +577,9 @@ class VideoProcessorIntegrationTest : public testing::Test { config_.codec_settings.width, config_.codec_settings.height, &psnr_result, &ssim_result)); - VerifyQuality(psnr_result, ssim_result, quality_thresholds); + if (quality_thresholds) { + VerifyQuality(psnr_result, ssim_result, *quality_thresholds); + } stats_.PrintSummary(); printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n", psnr_result.average, psnr_result.min, ssim_result.average, @@ -662,25 +669,26 @@ class VideoProcessorIntegrationTest : public testing::Test { frame_index_rate_update; } - static void SetRateControlThresholds(RateControlThresholds* rc_thresholds, - int update_index, - int max_num_dropped_frames, - int max_key_frame_size_mismatch, - int max_delta_frame_size_mismatch, - int max_encoding_rate_mismatch, - int max_time_hit_target, - int num_spatial_resizes, - int num_key_frames) { - rc_thresholds[update_index].max_num_dropped_frames = max_num_dropped_frames; - rc_thresholds[update_index].max_key_frame_size_mismatch = - max_key_frame_size_mismatch; - rc_thresholds[update_index].max_delta_frame_size_mismatch = - max_delta_frame_size_mismatch; - rc_thresholds[update_index].max_encoding_rate_mismatch = - max_encoding_rate_mismatch; - rc_thresholds[update_index].max_time_hit_target = max_time_hit_target; - rc_thresholds[update_index].num_spatial_resizes = num_spatial_resizes; - rc_thresholds[update_index].num_key_frames = num_key_frames; + static void AddRateControlThresholds( + int max_num_dropped_frames, + int max_key_frame_size_mismatch, + int max_delta_frame_size_mismatch, + int max_encoding_rate_mismatch, + int max_time_hit_target, + int num_spatial_resizes, + int num_key_frames, + std::vector* rc_thresholds) { + RTC_DCHECK(rc_thresholds); + + rc_thresholds->emplace_back(); + RateControlThresholds* rc_threshold = &rc_thresholds->back(); + rc_threshold->max_num_dropped_frames = max_num_dropped_frames; + rc_threshold->max_key_frame_size_mismatch = max_key_frame_size_mismatch; + rc_threshold->max_delta_frame_size_mismatch = max_delta_frame_size_mismatch; + rc_threshold->max_encoding_rate_mismatch = max_encoding_rate_mismatch; + rc_threshold->max_time_hit_target = max_time_hit_target; + rc_threshold->num_spatial_resizes = num_spatial_resizes; + rc_threshold->num_key_frames = num_key_frames; } // Config.