diff --git a/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc index 1ed1272255..861af8a502 100644 --- a/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/src/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -26,6 +26,7 @@ namespace { // wouldn't like scaling, so this will work when we compare with the original. const int kInputWidth = 176; const int kInputHeight = 144; +const int kVerifyingTestMaxNumAttempts = 3; class ViEVideoVerificationTest : public testing::Test { protected: @@ -75,24 +76,19 @@ class ViEVideoVerificationTest : public testing::Test { void CompareFiles(const std::string& reference_file, const std::string& test_file, - double minimum_psnr, double minimum_ssim) { - static const char* kPsnrSsimExplanation = - "Don't worry too much about this error if it only happens once. " - "It may be because mundane things like unfortunate OS scheduling. " - "If it keeps happening over and over though it's a cause of concern."; - + double* psnr_result, double *ssim_result) { webrtc::test::QualityMetricsResult psnr; int error = I420PSNRFromFiles(reference_file.c_str(), test_file.c_str(), kInputWidth, kInputHeight, &psnr); EXPECT_EQ(0, error) << "PSNR routine failed - output files missing?"; - EXPECT_GT(psnr.average, minimum_psnr) << kPsnrSsimExplanation; + *psnr_result = psnr.average; webrtc::test::QualityMetricsResult ssim; error = I420SSIMFromFiles(reference_file.c_str(), test_file.c_str(), kInputWidth, kInputHeight, &ssim); EXPECT_EQ(0, error) << "SSIM routine failed - output files missing?"; - EXPECT_GT(ssim.average, minimum_ssim) << kPsnrSsimExplanation; + *ssim_result = ssim.average; ViETest::Log("Results: PSNR is %f (dB), SSIM is %f (1 is perfect)", psnr.average, ssim.average); @@ -120,31 +116,38 @@ class ViEVideoVerificationTest : public testing::Test { }; TEST_F(ViEVideoVerificationTest, RunsBaseStandardTestWithoutErrors) { - InitializeFileRenderers(); - ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight, - local_file_renderer_, - remote_file_renderer_)); - std::string output_file = remote_file_renderer_->GetFullOutputPath(); - StopRenderers(); - // The I420 test should give pretty good values since it's a lossless codec // running on the default bitrate. It should average about 30 dB but there - // may be cases where it dips as low as 26 under adverse conditions. - const double kExpectedMinimumPSNR = 28; + // may be cases where it dips as low as 26 under adverse conditions. That's + // why we have a retrying mechanism in place for this test. + const double kExpectedMinimumPSNR = 30; const double kExpectedMinimumSSIM = 0.95; - CompareFiles(input_file_, output_file, kExpectedMinimumPSNR, - kExpectedMinimumSSIM); + + for (int attempt = 0; attempt < kVerifyingTestMaxNumAttempts; attempt++) { + InitializeFileRenderers(); + ASSERT_TRUE(tests_.TestCallSetup(input_file_, kInputWidth, kInputHeight, + local_file_renderer_, + remote_file_renderer_)); + std::string output_file = remote_file_renderer_->GetFullOutputPath(); + StopRenderers(); + + double actual_psnr = 0; + double actual_ssim = 0; + CompareFiles(input_file_, output_file, &actual_psnr, &actual_ssim); + + if (actual_psnr >= kExpectedMinimumPSNR && + actual_ssim >= kExpectedMinimumSSIM) { + // Test succeeded! + return; + } + } + + ADD_FAILURE() << "Failed to achieve PSNR " << kExpectedMinimumPSNR << + " and SSIM " << kExpectedMinimumSSIM << " after " << + kVerifyingTestMaxNumAttempts << " attempts."; } TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) { - InitializeFileRenderers(); - ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight, - local_file_renderer_, - remote_file_renderer_)); - std::string reference_file = local_file_renderer_->GetFullOutputPath(); - std::string output_file = remote_file_renderer_->GetFullOutputPath(); - StopRenderers(); - // We compare the local and remote here instead of with the original. // The reason is that it is hard to say when the three consecutive tests // switch over into each other, at which point we would have to restart the @@ -155,8 +158,26 @@ TEST_F(ViEVideoVerificationTest, RunsCodecTestWithoutErrors) { // acceptable, but it probably shouldn't get worse than this. const double kExpectedMinimumPSNR = 20; const double kExpectedMinimumSSIM = 0.7; - CompareFiles(reference_file, output_file, kExpectedMinimumPSNR, - kExpectedMinimumSSIM); + + for (int attempt = 0; attempt < kVerifyingTestMaxNumAttempts; attempt++) { + InitializeFileRenderers(); + ASSERT_TRUE(tests_.TestCodecs(input_file_, kInputWidth, kInputHeight, + local_file_renderer_, + remote_file_renderer_)); + std::string reference_file = local_file_renderer_->GetFullOutputPath(); + std::string output_file = remote_file_renderer_->GetFullOutputPath(); + StopRenderers(); + + double actual_psnr = 0; + double actual_ssim = 0; + CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim); + + if (actual_psnr >= kExpectedMinimumPSNR && + actual_ssim >= kExpectedMinimumSSIM) { + // Test succeeded! + return; + } + } } // Runs a whole stack processing with tracking of which frames are dropped @@ -218,10 +239,15 @@ TEST_F(ViEVideoVerificationTest, RunsFullStackWithoutErrors) { // We are running on a lower bitrate here so we need to settle for somewhat // lower PSNR and SSIM values. + double actual_psnr = 0; + double actual_ssim = 0; + CompareFiles(reference_file, output_file, &actual_psnr, &actual_ssim); + const double kExpectedMinimumPSNR = 24; const double kExpectedMinimumSSIM = 0.7; - CompareFiles(reference_file, output_file, kExpectedMinimumPSNR, - kExpectedMinimumSSIM); + + EXPECT_GE(actual_psnr, kExpectedMinimumPSNR); + EXPECT_GE(actual_ssim, kExpectedMinimumSSIM); } } // namespace