admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make rate control and quality thresholds optional in VideoProcessorIntegrationTest.

Browse Source 
This CL is split out from https://codereview.webrtc.org/2997283002/.

BUG=webrtc:6634

Review-Url: https://codereview.webrtc.org/3008543002
Cr-Commit-Position: refs/heads/master@{#19513}
This commit is contained in:
brandtr 2017-08-25 04:33:49 -07:00 committed by Commit Bot
parent e5eb7247ff
commit 87f5c5a4d3
3 changed files with 251 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc
View File

@ -58,7 +58,12 @@ class PlotVideoProcessorIntegrationTest
int height,
int framerate,
const std::string& filename) {
// Bitrate and frame rate profile.
SetTestConfig(&config_, hw_codec_, kUseSingleCore, kPacketLoss, filename,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, codec_type_, kNumTemporalLayers,
kErrorConcealmentOn, kDenoisingOn, kFrameDropperOn,
kSpatialResizeOn, kResilienceOn, width, height);
RateProfile rate_profile;
SetRateProfile(&rate_profile,
0, // update_index
@ -67,34 +72,8 @@ class PlotVideoProcessorIntegrationTest
rate_profile.frame_index_rate_update[1] = kNumFrames + 1;
rate_profile.num_frames = kNumFrames;
// Codec/network settings.
SetTestConfig(&config_, hw_codec_, kUseSingleCore, kPacketLoss, filename,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, codec_type_, kNumTemporalLayers,
kErrorConcealmentOn, kDenoisingOn, kFrameDropperOn,
kSpatialResizeOn, kResilienceOn, width, height);
// Use default thresholds for quality (PSNR and SSIM).
QualityThresholds quality_thresholds;
// Use very loose thresholds for rate control, so even poor HW codecs will
// pass the requirements.
RateControlThresholds rc_thresholds[1];
// clang-format off
SetRateControlThresholds(
rc_thresholds,
0, // update_index
kNumFrames + 1, // max_num_dropped_frames
10000, // max_key_frame_size_mismatch
10000, // max_delta_frame_size_mismatch
10000, // max_encoding_rate_mismatch
kNumFrames + 1, // max_time_hit_target
0, // num_spatial_resizes
1); // num_key_frames
// clang-format on
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
&kVisualizationParams);
ProcessFramesAndMaybeVerify(rate_profile, nullptr, nullptr,
&kVisualizationParams);
}
const int bitrate_;

358
webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
View File

@ -10,6 +10,8 @@
#include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h"
#include <vector>
namespace webrtc {
namespace test {
@ -34,6 +36,8 @@ const int kNumFramesShort = 100;
#endif
const int kNumFramesLong = 299;
const std::nullptr_t kNoVisualizationParams = nullptr;
} // namespace
#if defined(WEBRTC_USE_H264)
@ -44,23 +48,23 @@ const int kNumFramesLong = 299;
// with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have
// not been added.
TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossH264) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecH264, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(2, 60, 20, 10, 20, 0, 1, &rc_thresholds);
QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 2, 60, 20, 10, 20, 0, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
#endif // defined(WEBRTC_USE_H264)
@ -72,45 +76,45 @@ TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossH264) {
// VP9: Run with no packet loss and fixed bitrate. Quality should be very high.
// One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossVP9) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 40, 20, 10, 20, 0, 1, &rc_thresholds);
QualityThresholds quality_thresholds(37.0, 36.0, 0.93, 0.92);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 20, 0, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// VP9: Run with 5% packet loss and fixed bitrate. Quality should be a bit
// lower. One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLossVP9) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.05f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 40, 20, 10, 20, 0, 1, &rc_thresholds);
QualityThresholds quality_thresholds(17.0, 14.0, 0.45, 0.36);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 20, 0, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// VP9: Run with no packet loss, with varying bitrate (3 rate updates):
@ -118,27 +122,27 @@ TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLossVP9) {
// target rate/per-frame bandwidth (for each rate update) is within limits.
// One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRateVP9) {
// Bit rate and frame rate profile.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 200, 30, 0);
SetRateProfile(&rate_profile, 1, 700, 30, 100);
SetRateProfile(&rate_profile, 2, 500, 30, 200);
rate_profile.frame_index_rate_update[3] = kNumFramesLong + 1;
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 30, 20, 20, 35, 0, 1, &rc_thresholds);
AddRateControlThresholds(2, 0, 20, 20, 60, 0, 0, &rc_thresholds);
AddRateControlThresholds(0, 0, 25, 20, 40, 0, 0, &rc_thresholds);
QualityThresholds quality_thresholds(35.5, 30.0, 0.90, 0.85);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[3];
SetRateControlThresholds(rc_thresholds, 0, 0, 30, 20, 20, 35, 0, 1);
SetRateControlThresholds(rc_thresholds, 1, 2, 0, 20, 20, 60, 0, 0);
SetRateControlThresholds(rc_thresholds, 2, 0, 0, 25, 20, 40, 0, 0);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// VP9: Run with no packet loss, with an update (decrease) in frame rate.
@ -150,48 +154,48 @@ TEST_F(VideoProcessorIntegrationTest, ProcessNoLossChangeBitRateVP9) {
// metrics averaged over whole sequence run.
TEST_F(VideoProcessorIntegrationTest,
ProcessNoLossChangeFrameRateFrameDropVP9) {
// Bit rate and frame rate profile.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 100, 24, 0);
SetRateProfile(&rate_profile, 1, 100, 15, 100);
SetRateProfile(&rate_profile, 2, 100, 10, 200);
rate_profile.frame_index_rate_update[3] = kNumFramesLong + 1;
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(45, 50, 95, 15, 45, 0, 1, &rc_thresholds);
AddRateControlThresholds(20, 0, 50, 10, 30, 0, 0, &rc_thresholds);
AddRateControlThresholds(5, 0, 30, 5, 25, 0, 0, &rc_thresholds);
QualityThresholds quality_thresholds(31.5, 18.0, 0.80, 0.43);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[3];
SetRateControlThresholds(rc_thresholds, 0, 45, 50, 95, 15, 45, 0, 1);
SetRateControlThresholds(rc_thresholds, 1, 20, 0, 50, 10, 30, 0, 0);
SetRateControlThresholds(rc_thresholds, 2, 5, 0, 30, 5, 25, 0, 0);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// VP9: Run with no packet loss and denoiser on. One key frame (first frame).
TEST_F(VideoProcessorIntegrationTest, ProcessNoLossDenoiserOnVP9) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP9, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 40, 20, 10, 20, 0, 1, &rc_thresholds);
QualityThresholds quality_thresholds(36.8, 35.8, 0.92, 0.91);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 20, 0, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// Run with no packet loss, at low bitrate.
@ -199,23 +203,23 @@ TEST_F(VideoProcessorIntegrationTest, ProcessNoLossDenoiserOnVP9) {
// Resize happens on delta frame. Expect only one key frame (first frame).
TEST_F(VideoProcessorIntegrationTest,
DISABLED_ProcessNoLossSpatialResizeFrameDropVP9) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 50, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesLong + 1;
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP9, 1, false, false, true, true,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 50, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesLong + 1;
rate_profile.num_frames = kNumFramesLong;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(228, 70, 160, 15, 80, 1, 1, &rc_thresholds);
QualityThresholds quality_thresholds(24.0, 13.0, 0.65, 0.37);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 228, 70, 160, 15, 80, 1, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// TODO(marpan): Add temporal layer test for VP9, once changes are in
@ -227,67 +231,67 @@ TEST_F(VideoProcessorIntegrationTest,
// One key frame (first frame only) in sequence. Setting |key_frame_interval|
// to -1 below means no periodic key frames in test.
TEST_F(VideoProcessorIntegrationTest, ProcessZeroPacketLoss) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 40, 20, 10, 15, 0, 1, &rc_thresholds);
QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 15, 0, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// VP8: Run with 5% packet loss and fixed bitrate. Quality should be a bit
// lower. One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTest, Process5PercentPacketLoss) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.05f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 40, 20, 10, 15, 0, 1, &rc_thresholds);
QualityThresholds quality_thresholds(20.0, 16.0, 0.60, 0.40);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 15, 0, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// VP8: Run with 10% packet loss and fixed bitrate. Quality should be lower.
// One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.1f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 40, 20, 10, 15, 0, 1, &rc_thresholds);
QualityThresholds quality_thresholds(19.0, 16.0, 0.50, 0.35);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 15, 0, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// This test is identical to VideoProcessorIntegrationTest.ProcessZeroPacketLoss
@ -295,23 +299,23 @@ TEST_F(VideoProcessorIntegrationTest, Process10PercentPacketLoss) {
// that the reported stats are not wildly varying between batch mode and the
// regular online mode.
TEST_F(VideoProcessorIntegrationTest, ProcessInBatchMode) {
// Bit rate and frame rate profile.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, true /* batch_mode */);
SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 500, 30, 0);
rate_profile.frame_index_rate_update[1] = kNumFramesShort + 1;
rate_profile.num_frames = kNumFramesShort;
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 40, 20, 10, 15, 0, 1, &rc_thresholds);
QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[1];
SetRateControlThresholds(rc_thresholds, 0, 0, 40, 20, 10, 15, 0, 1);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
#endif // !defined(WEBRTC_IOS)
@ -337,27 +341,27 @@ TEST_F(VideoProcessorIntegrationTest, ProcessInBatchMode) {
#define MAYBE_ProcessNoLossChangeBitRateVP8 ProcessNoLossChangeBitRateVP8
#endif
TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossChangeBitRateVP8) {
// Bit rate and frame rate profile.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 200, 30, 0);
SetRateProfile(&rate_profile, 1, 800, 30, 100);
SetRateProfile(&rate_profile, 2, 500, 30, 200);
rate_profile.frame_index_rate_update[3] = kNumFramesLong + 1;
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 45, 20, 10, 15, 0, 1, &rc_thresholds);
AddRateControlThresholds(0, 0, 25, 20, 10, 0, 0, &rc_thresholds);
AddRateControlThresholds(0, 0, 25, 15, 10, 0, 0, &rc_thresholds);
QualityThresholds quality_thresholds(34.0, 32.0, 0.85, 0.80);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[3];
SetRateControlThresholds(rc_thresholds, 0, 0, 45, 20, 10, 15, 0, 1);
SetRateControlThresholds(rc_thresholds, 1, 0, 0, 25, 20, 10, 0, 0);
SetRateControlThresholds(rc_thresholds, 2, 0, 0, 25, 15, 10, 0, 0);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// VP8: Run with no packet loss, with an update (decrease) in frame rate.
@ -377,27 +381,27 @@ TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossChangeBitRateVP8) {
#endif
TEST_F(VideoProcessorIntegrationTest,
MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8) {
// Bit rate and frame rate profile.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 80, 24, 0);
SetRateProfile(&rate_profile, 1, 80, 15, 100);
SetRateProfile(&rate_profile, 2, 80, 10, 200);
rate_profile.frame_index_rate_update[3] = kNumFramesLong + 1;
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(40, 20, 75, 15, 60, 0, 1, &rc_thresholds);
AddRateControlThresholds(10, 0, 25, 10, 35, 0, 0, &rc_thresholds);
AddRateControlThresholds(0, 0, 20, 10, 15, 0, 0, &rc_thresholds);
QualityThresholds quality_thresholds(31.0, 22.0, 0.80, 0.65);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[3];
SetRateControlThresholds(rc_thresholds, 0, 40, 20, 75, 15, 60, 0, 1);
SetRateControlThresholds(rc_thresholds, 1, 10, 0, 25, 10, 35, 0, 0);
SetRateControlThresholds(rc_thresholds, 2, 0, 0, 20, 10, 15, 0, 0);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
// VP8: Run with no packet loss, with 3 temporal layers, with a rate update in
@ -413,25 +417,25 @@ TEST_F(VideoProcessorIntegrationTest,
#define MAYBE_ProcessNoLossTemporalLayersVP8 ProcessNoLossTemporalLayersVP8
#endif
TEST_F(VideoProcessorIntegrationTest, MAYBE_ProcessNoLossTemporalLayersVP8) {
// Bit rate and frame rate profile.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 3, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
RateProfile rate_profile;
SetRateProfile(&rate_profile, 0, 200, 30, 0);
SetRateProfile(&rate_profile, 1, 400, 30, 150);
rate_profile.frame_index_rate_update[2] = kNumFramesLong + 1;
rate_profile.num_frames = kNumFramesLong;
// Codec/network settings.
SetTestConfig(&config_, kHwCodec, kUseSingleCore, 0.0f, kForemanCif,
kVerboseLogging, kBatchMode);
SetCodecSettings(&config_, kVideoCodecVP8, 3, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
// Thresholds for expected quality.
std::vector<RateControlThresholds> rc_thresholds;
AddRateControlThresholds(0, 20, 30, 10, 10, 0, 1, &rc_thresholds);
AddRateControlThresholds(0, 0, 30, 15, 10, 0, 0, &rc_thresholds);
QualityThresholds quality_thresholds(32.5, 30.0, 0.85, 0.80);
// Thresholds for rate control.
RateControlThresholds rc_thresholds[2];
SetRateControlThresholds(rc_thresholds, 0, 0, 20, 30, 10, 10, 0, 1);
SetRateControlThresholds(rc_thresholds, 1, 0, 0, 30, 15, 10, 0, 0);
ProcessFramesAndVerify(quality_thresholds, rate_profile, rc_thresholds,
nullptr /* visualization_params */);
ProcessFramesAndMaybeVerify(rate_profile, &rc_thresholds, &quality_thresholds,
kNoVisualizationParams);
}
} // namespace test
} // namespace webrtc

116
webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
View File

@ -17,6 +17,7 @@
#include <memory>
#include <string>
#include <utility>
#include <vector>
#if defined(WEBRTC_ANDROID)
#include "webrtc/modules/video_coding/codecs/test/android_test_initializer.h"
@ -65,7 +66,6 @@ const float kScaleKeyFrameSize = 0.5f;
// Thresholds for the quality metrics. Defaults are maximally minimal.
struct QualityThresholds {
QualityThresholds() {}
QualityThresholds(double min_avg_psnr,
double min_min_psnr,
double min_avg_ssim,
@ -74,10 +74,10 @@ struct QualityThresholds {
min_min_psnr(min_min_psnr),
min_avg_ssim(min_avg_ssim),
min_min_ssim(min_min_ssim) {}
double min_avg_psnr = std::numeric_limits<double>::min();
double min_min_psnr = std::numeric_limits<double>::min();
double min_avg_ssim = 0.0;
double min_min_ssim = 0.0;
double min_avg_psnr;
double min_min_psnr;
double min_avg_ssim;
double min_min_ssim;
};
// The sequence of bit rate and frame rate changes for the encoder, the frame
@ -101,8 +101,8 @@ struct RateControlThresholds {
int max_delta_frame_size_mismatch;
int max_encoding_rate_mismatch;
int max_time_hit_target;
int num_spatial_resizes; // Set to -1 to disable check.
int num_key_frames; // Set to -1 to disable check.
int num_spatial_resizes;
int num_key_frames;
};
// Should video files be saved persistently to disk for post-run visualization?
@ -306,8 +306,9 @@ class VideoProcessorIntegrationTest : public testing::Test {
}
// Verify expected behavior of rate control and print out data.
void VerifyRateControlMetrics(int rate_update_index,
const RateControlThresholds& rc_expected) {
void PrintAndMaybeVerifyRateControlMetrics(
int rate_update_index,
const std::vector<RateControlThresholds>* rc_thresholds) {
int num_dropped_frames = processor_->NumberDroppedFrames();
int num_resize_actions = processor_->NumberSpatialResizes();
printf(
@ -324,8 +325,13 @@ class VideoProcessorIntegrationTest : public testing::Test {
num_frames_total_, num_frames_to_hit_target_, num_dropped_frames,
num_resize_actions);
EXPECT_LE(perc_encoding_rate_mismatch_,
rc_expected.max_encoding_rate_mismatch);
const RateControlThresholds* rc_threshold = nullptr;
if (rc_thresholds) {
rc_threshold = &(*rc_thresholds)[rate_update_index];
EXPECT_LE(perc_encoding_rate_mismatch_,
rc_threshold->max_encoding_rate_mismatch);
}
if (num_key_frames_ > 0) {
int perc_key_frame_size_mismatch =
100 * sum_key_frame_size_mismatch_ / num_key_frames_;
@ -333,8 +339,10 @@ class VideoProcessorIntegrationTest : public testing::Test {
" # key frames : %d\n"
" Key frame rate mismatch: %d\n",
num_key_frames_, perc_key_frame_size_mismatch);
EXPECT_LE(perc_key_frame_size_mismatch,
rc_expected.max_key_frame_size_mismatch);
if (rc_threshold) {
EXPECT_LE(perc_key_frame_size_mismatch,
rc_threshold->max_key_frame_size_mismatch);
}
}
const int num_temporal_layers =
@ -357,20 +365,20 @@ class VideoProcessorIntegrationTest : public testing::Test {
i, bitrate_layer_[i], framerate_layer_[i], per_frame_bandwidth_[i],
encoding_bitrate_[i], perc_frame_size_mismatch,
perc_encoding_rate_mismatch, num_frames_per_update_[i]);
EXPECT_LE(perc_frame_size_mismatch,
rc_expected.max_delta_frame_size_mismatch);
EXPECT_LE(perc_encoding_rate_mismatch,
rc_expected.max_encoding_rate_mismatch);
if (rc_threshold) {
EXPECT_LE(perc_frame_size_mismatch,
rc_threshold->max_delta_frame_size_mismatch);
EXPECT_LE(perc_encoding_rate_mismatch,
rc_threshold->max_encoding_rate_mismatch);
}
}
printf("\n");
EXPECT_LE(num_frames_to_hit_target_, rc_expected.max_time_hit_target);
EXPECT_LE(num_dropped_frames, rc_expected.max_num_dropped_frames);
if (rc_expected.num_spatial_resizes >= 0) {
EXPECT_EQ(rc_expected.num_spatial_resizes, num_resize_actions);
}
if (rc_expected.num_key_frames >= 0) {
EXPECT_EQ(rc_expected.num_key_frames, num_key_frames_);
if (rc_threshold) {
EXPECT_LE(num_frames_to_hit_target_, rc_threshold->max_time_hit_target);
EXPECT_LE(num_dropped_frames, rc_threshold->max_num_dropped_frames);
EXPECT_EQ(rc_threshold->num_spatial_resizes, num_resize_actions);
EXPECT_EQ(rc_threshold->num_key_frames, num_key_frames_);
}
}
@ -459,14 +467,11 @@ class VideoProcessorIntegrationTest : public testing::Test {
}
// Processes all frames in the clip and verifies the result.
// TODO(brandtr): Change the second last argument to be a
// const std::vector<RateControlThresholds>&, so we can ensure that the user
// does not expect us to do mid-clip rate updates when we are not able to,
// e.g., when we are operating in batch mode.
void ProcessFramesAndVerify(const QualityThresholds& quality_thresholds,
const RateProfile& rate_profile,
RateControlThresholds* rc_thresholds,
const VisualizationParams* visualization_params) {
void ProcessFramesAndMaybeVerify(
const RateProfile& rate_profile,
const std::vector<RateControlThresholds>* rc_thresholds,
const QualityThresholds* quality_thresholds,
const VisualizationParams* visualization_params) {
config_.codec_settings.startBitrate = rate_profile.target_bit_rate[0];
SetUpObjects(visualization_params, rate_profile.target_bit_rate[0],
rate_profile.input_frame_rate[0]);
@ -526,7 +531,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
// update layers and codec with new rates.
if (frame_number ==
rate_profile.frame_index_rate_update[update_index + 1]) {
VerifyRateControlMetrics(update_index, rc_thresholds[update_index]);
PrintAndMaybeVerifyRateControlMetrics(update_index, rc_thresholds);
// Update layer rates and the codec with new rates.
++update_index;
@ -545,7 +550,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
// Verify rate control metrics for all frames (if in batch mode), or for all
// frames since the last rate update (if not in batch mode).
VerifyRateControlMetrics(update_index, rc_thresholds[update_index]);
PrintAndMaybeVerifyRateControlMetrics(update_index, rc_thresholds);
EXPECT_EQ(num_frames, frame_number);
EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
@ -572,7 +577,9 @@ class VideoProcessorIntegrationTest : public testing::Test {
config_.codec_settings.width,
config_.codec_settings.height,
&psnr_result, &ssim_result));
VerifyQuality(psnr_result, ssim_result, quality_thresholds);
if (quality_thresholds) {
VerifyQuality(psnr_result, ssim_result, *quality_thresholds);
}
stats_.PrintSummary();
printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n",
psnr_result.average, psnr_result.min, ssim_result.average,
@ -662,25 +669,26 @@ class VideoProcessorIntegrationTest : public testing::Test {
frame_index_rate_update;
}
static void SetRateControlThresholds(RateControlThresholds* rc_thresholds,
int update_index,
int max_num_dropped_frames,
int max_key_frame_size_mismatch,
int max_delta_frame_size_mismatch,
int max_encoding_rate_mismatch,
int max_time_hit_target,
int num_spatial_resizes,
int num_key_frames) {
rc_thresholds[update_index].max_num_dropped_frames = max_num_dropped_frames;
rc_thresholds[update_index].max_key_frame_size_mismatch =
max_key_frame_size_mismatch;
rc_thresholds[update_index].max_delta_frame_size_mismatch =
max_delta_frame_size_mismatch;
rc_thresholds[update_index].max_encoding_rate_mismatch =
max_encoding_rate_mismatch;
rc_thresholds[update_index].max_time_hit_target = max_time_hit_target;
rc_thresholds[update_index].num_spatial_resizes = num_spatial_resizes;
rc_thresholds[update_index].num_key_frames = num_key_frames;
static void AddRateControlThresholds(
int max_num_dropped_frames,
int max_key_frame_size_mismatch,
int max_delta_frame_size_mismatch,
int max_encoding_rate_mismatch,
int max_time_hit_target,
int num_spatial_resizes,
int num_key_frames,
std::vector<RateControlThresholds>* rc_thresholds) {
RTC_DCHECK(rc_thresholds);
rc_thresholds->emplace_back();
RateControlThresholds* rc_threshold = &rc_thresholds->back();
rc_threshold->max_num_dropped_frames = max_num_dropped_frames;
rc_threshold->max_key_frame_size_mismatch = max_key_frame_size_mismatch;
rc_threshold->max_delta_frame_size_mismatch = max_delta_frame_size_mismatch;
rc_threshold->max_encoding_rate_mismatch = max_encoding_rate_mismatch;
rc_threshold->max_time_hit_target = max_time_hit_target;
rc_threshold->num_spatial_resizes = num_spatial_resizes;
rc_threshold->num_key_frames = num_key_frames;
}
// Config.