From e7dc7f8553150feefa785a4f65548a0d3bc13b67 Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Fri, 11 Jan 2013 12:55:19 +0000 Subject: [PATCH] Disable full stack PSNR/SSIM triggers on Mac and Win for now due to flakiness. Adding plots of PSNR and SSIM. TBR=mflodman BUG=1271 Review URL: https://webrtc-codereview.appspot.com/1032005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@3360 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../automated/vie_video_verification_test.cc | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc index 339c9625b7..d09c7f4d7e 100644 --- a/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc +++ b/webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc @@ -12,6 +12,7 @@ #include "gtest/gtest.h" #include "webrtc/test/testsupport/fileutils.h" +#include "webrtc/test/testsupport/perf_test.h" #include "webrtc/test/testsupport/metrics/video_metrics.h" #include "webrtc/video_engine/test/auto_test/interface/vie_autotest.h" #include "webrtc/video_engine/test/auto_test/interface/vie_file_based_comparison_tests.h" @@ -146,8 +147,15 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest, parameter_table_[i].network.mean_one_way_delay = 0; parameter_table_[i].network.std_dev_one_way_delay = 0; parameter_table_[i].bitrate = 300; + // TODO(holmer): Enable for Win and Mac when the file rendering has been + // moved to a separate thread. +#ifdef WEBRTC_LINUX parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_ssim_threshold = 0.96; +#else + parameter_table_[i].avg_psnr_threshold = 0; + parameter_table_[i].avg_ssim_threshold = 0.0; +#endif parameter_table_[i].test_label = "net_delay_0_0_plr_0"; ++i; parameter_table_[i].protection_method = kNack; @@ -157,8 +165,15 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest, parameter_table_[i].network.mean_one_way_delay = 50; parameter_table_[i].network.std_dev_one_way_delay = 5; parameter_table_[i].bitrate = 300; + // TODO(holmer): Enable for Win and Mac when the file rendering has been + // moved to a separate thread. +#ifdef WEBRTC_LINUX parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_ssim_threshold = 0.96; +#else + parameter_table_[i].avg_psnr_threshold = 0; + parameter_table_[i].avg_ssim_threshold = 0.0; +#endif parameter_table_[i].test_label = "net_delay_50_5_plr_5"; ++i; parameter_table_[i].protection_method = kNack; @@ -168,8 +183,15 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest, parameter_table_[i].network.mean_one_way_delay = 100; parameter_table_[i].network.std_dev_one_way_delay = 10; parameter_table_[i].bitrate = 300; + // TODO(holmer): Enable for Win and Mac when the file rendering has been + // moved to a separate thread. +#ifdef WEBRTC_LINUX parameter_table_[i].avg_psnr_threshold = 35; parameter_table_[i].avg_ssim_threshold = 0.96; +#else + parameter_table_[i].avg_psnr_threshold = 0; + parameter_table_[i].avg_ssim_threshold = 0.0; +#endif parameter_table_[i].test_label = "net_delay_100_10_plr_0"; ++i; parameter_table_[i].protection_method = kNack; @@ -305,6 +327,12 @@ TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors) { EXPECT_GE(actual_psnr, kExpectedMinimumPSNR); EXPECT_GE(actual_ssim, kExpectedMinimumSSIM); + webrtc::test::PrintResult( + "psnr", "", parameter_table_[GetParam()].test_label, + actual_psnr, "dB", false); + webrtc::test::PrintResult( + "ssim", "", parameter_table_[GetParam()].test_label, + actual_ssim, "", false); } INSTANTIATE_TEST_CASE_P(FullStackTests, ParameterizedFullStackTest,