diff --git a/call/BUILD.gn b/call/BUILD.gn index ac9445750f..80a74d4a4c 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -255,6 +255,7 @@ if (rtc_include_tests) { "../test:direct_transport", "../test:fake_audio_device", "../test:field_trial", + "../test:perf_test", "../test:test_common", "../test:test_support", "../test:video_test_common", diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn index 557247ffd3..dbc821fe60 100644 --- a/modules/audio_coding/BUILD.gn +++ b/modules/audio_coding/BUILD.gn @@ -1318,6 +1318,7 @@ if (rtc_include_tests) { "../../rtc_base:rtc_base_approved", "../../system_wrappers", "../../system_wrappers:field_trial_api", + "../../test:perf_test", "../../test:test_support", ] @@ -1806,6 +1807,7 @@ if (rtc_include_tests) { deps = [ ":isac_fix", + "../../test:perf_test", "../../test:test_support", ] diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn index 9ea74b84fc..639424bd36 100644 --- a/modules/audio_processing/BUILD.gn +++ b/modules/audio_processing/BUILD.gn @@ -713,6 +713,7 @@ if (rtc_include_tests) { "../../rtc_base:protobuf_utils", "../../rtc_base:rtc_base_approved", "../../system_wrappers", + "../../test:perf_test", "../../test:test_support", ] diff --git a/modules/remote_bitrate_estimator/BUILD.gn b/modules/remote_bitrate_estimator/BUILD.gn index 2dc4b220ca..7c32ce13e0 100644 --- a/modules/remote_bitrate_estimator/BUILD.gn +++ b/modules/remote_bitrate_estimator/BUILD.gn @@ -146,6 +146,7 @@ if (rtc_include_tests) { "../../rtc_base:rtc_numerics", "../../system_wrappers", "../../system_wrappers:field_trial_api", + "../../test:perf_test", "../../test:test_support", "../../voice_engine", "../bitrate_controller", diff --git a/rtc_base/unittest_main.cc b/rtc_base/unittest_main.cc index 80b988d256..330bd3f813 100644 --- a/rtc_base/unittest_main.cc +++ b/rtc_base/unittest_main.cc @@ -109,7 +109,7 @@ int main(int argc, char* argv[]) { rtc::SSLStreamAdapter::enable_time_callback_for_testing(); #if defined(WEBRTC_IOS) - rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv); + rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv, false); rtc::test::RunTestsFromIOSApp(); #endif const int res = RUN_ALL_TESTS(); diff --git a/test/BUILD.gn b/test/BUILD.gn index 936d3750ff..a74683315c 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -113,6 +113,19 @@ rtc_source_set("field_trial") { ] } +rtc_source_set("perf_test") { + testonly = true + sources = [ + "testsupport/perf_test.cc", + "testsupport/perf_test.h", + ] + deps = [ + "..:webrtc_common", + "../api:array_view", + "../rtc_base:rtc_base_approved", + ] +} + if (is_ios) { rtc_source_set("test_support_objc") { testonly = true @@ -122,6 +135,7 @@ if (is_ios) { "ios/test_support.mm", ] deps = [ + ":perf_test", "../sdk:common_objc", ] } @@ -135,8 +149,6 @@ rtc_source_set("test_support") { "gtest.h", "testsupport/packet_reader.cc", "testsupport/packet_reader.h", - "testsupport/perf_test.cc", - "testsupport/perf_test.h", "testsupport/unittest_utils.h", ] @@ -188,6 +200,7 @@ if (rtc_include_tests) { ] deps = [ ":field_trial", + ":perf_test", "../rtc_base:rtc_base_approved", "../system_wrappers:metrics_default", "//testing/gmock", @@ -287,6 +300,7 @@ if (rtc_include_tests) { rtc_test("test_support_unittests") { deps = [ ":fake_audio_device", + ":perf_test", ":rtp_test_utils", "../api:video_frame_api", "../api:video_frame_api_i420", diff --git a/test/ios/test_support.h b/test/ios/test_support.h index 7bc2993b9e..0178fca76b 100644 --- a/test/ios/test_support.h +++ b/test/ios/test_support.h @@ -17,7 +17,8 @@ namespace test { // This is necessary as iOS doesn't like processes without a gui // running for longer than a few seconds. void RunTestsFromIOSApp(); -void InitTestSuite(int (*test_suite)(void), int argc, char* argv[]); +void InitTestSuite(int (*test_suite)(void), int argc, char* argv[], + bool save_chartjson_result); } // namespace test } // namespace rtc diff --git a/test/ios/test_support.mm b/test/ios/test_support.mm index 603bc8c19c..5b7f2f9350 100644 --- a/test/ios/test_support.mm +++ b/test/ios/test_support.mm @@ -11,8 +11,10 @@ #import #include "test/ios/test_support.h" +#include "test/testsupport/perf_test.h" #import "sdk/objc/Framework/Classes/Common/RTCUIApplicationStatusObserver.h" +#include "sdk/objc/Framework/Classes/Common/helpers.h" // Springboard will kill any iOS app that fails to check in after launch within // a given time. Starting a UIApplication before invoking TestSuite::Run @@ -31,6 +33,7 @@ static int (*g_test_suite)(void) = NULL; static int g_argc; static char **g_argv; +static bool g_save_chartjson_result; @interface UIApplication (Testing) - (void)_terminateWithStatus:(int)status; @@ -75,6 +78,20 @@ static char **g_argv; - (void)runTests { int exitStatus = g_test_suite(); + if (g_save_chartjson_result) { + // Stores data into a json file under the app's document directory. + NSString* fileName = @"perf_result.json"; + NSArray* outputDirectories = NSSearchPathForDirectoriesInDomains( + NSDocumentDirectory, NSUserDomainMask, YES); + if ([outputDirectories count] != 0) { + NSString* outputPath = + [outputDirectories[0] stringByAppendingPathComponent:fileName]; + + webrtc::test::WritePerfResults( + webrtc::ios::StdStringFromNSString(outputPath)); + } + } + // If a test app is too fast, it will exit before Instruments has has a // a chance to initialize and no test results will be seen. // TODO(crbug.com/137010): Figure out how much time is actually needed, and @@ -93,10 +110,14 @@ static char **g_argv; namespace rtc { namespace test { -void InitTestSuite(int (*test_suite)(void), int argc, char *argv[]) { +// Note: This is not thread safe, and must be called from the same thread as +// runTests above. +void InitTestSuite(int (*test_suite)(void), int argc, char *argv[], + bool save_chartjson_result) { g_test_suite = test_suite; g_argc = argc; g_argv = argv; + g_save_chartjson_result = save_chartjson_result; } void RunTestsFromIOSApp() { diff --git a/test/test_main.cc b/test/test_main.cc index c6c00c599d..4b36271333 100644 --- a/test/test_main.cc +++ b/test/test_main.cc @@ -8,8 +8,6 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include - #include "rtc_base/flags.h" #include "rtc_base/logging.h" #include "system_wrappers/include/metrics_default.h" @@ -26,6 +24,24 @@ DEFINE_string(NSTreatUnknownArgumentsAsOpen, "", "Intentionally ignored flag intended for iOS simulator."); DEFINE_string(ApplePersistenceIgnoreState, "", "Intentionally ignored flag intended for iOS simulator."); +DEFINE_bool( + save_chartjson_result, + false, + "Store the perf results in Documents/perf_result.json in the format " + "described by " + "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/" + "data-format.md."); + +#else + +DEFINE_string( + chartjson_result_file, + "", + "Path where the perf results should be stored in the JSON format described " + "by " + "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/" + "data-format.md."); + #endif DEFINE_bool(logs, false, "print logs to stderr"); @@ -35,13 +51,6 @@ DEFINE_string(force_fieldtrials, "", "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/" " will assign the group Enable to field trial WebRTC-FooFeature."); -DEFINE_string( - chartjson_result_file, - "", - "Path where the perf results should be stored it the JSON format described " - "by " - "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/" - "data-format.md."); DEFINE_bool(help, false, "Print this message."); @@ -66,21 +75,25 @@ int main(int argc, char* argv[]) { webrtc::test::InitFieldTrialsFromString(fieldtrials); webrtc::metrics::Enable(); + rtc::LogMessage::SetLogToStderr(FLAG_logs); + #if defined(WEBRTC_IOS) - rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv); + + rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv, + FLAG_save_chartjson_result); rtc::test::RunTestsFromIOSApp(); -#endif + +#else int exit_code = RUN_ALL_TESTS(); std::string chartjson_result_file = FLAG_chartjson_result_file; if (chartjson_result_file != "") { - std::string json_results = webrtc::test::GetPerfResultsJSON(); - std::fstream json_file(chartjson_result_file, std::fstream::out); - json_file << json_results; - json_file.close(); + webrtc::test::WritePerfResults(chartjson_result_file); } return exit_code; + +#endif } diff --git a/test/testsupport/perf_test.cc b/test/testsupport/perf_test.cc index c2a2a502f9..3719916423 100644 --- a/test/testsupport/perf_test.cc +++ b/test/testsupport/perf_test.cc @@ -12,6 +12,7 @@ #include "rtc_base/criticalsection.h" #include +#include #include #include #include @@ -154,6 +155,13 @@ std::string GetPerfResultsJSON() { return GetPerfResultsLogger().ToJSON(); } +void WritePerfResults(const std::string& output_path) { + std::string json_results = GetPerfResultsJSON(); + std::fstream json_file(output_path, std::fstream::out); + json_file << json_results; + json_file.close(); +} + void PrintResult(const std::string& measurement, const std::string& modifier, const std::string& trace, diff --git a/test/testsupport/perf_test.h b/test/testsupport/perf_test.h index 1a72c042fe..f28dd2b2ac 100644 --- a/test/testsupport/perf_test.h +++ b/test/testsupport/perf_test.h @@ -61,8 +61,10 @@ void PrintResultList(const std::string& measurement, const std::string& units, bool important); -// Get all perf results to date in a JSON format as described in +// Write all perf results to date to a JSON file formatted as described in // https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md +void WritePerfResults(const std::string& output_path); + std::string GetPerfResultsJSON(); // You shouldn't use this function. It's only used to test the functions above. diff --git a/tools_webrtc/ios/tests/perf_tests.json b/tools_webrtc/ios/tests/perf_tests.json index f87322153a..b871173c96 100644 --- a/tools_webrtc/ios/tests/perf_tests.json +++ b/tools_webrtc/ios/tests/perf_tests.json @@ -3,7 +3,7 @@ { "app": "webrtc_perf_tests", "test args": [ - "--perf_results_json_path=Documents/perf_result.json" + "--save_chartjson_result" ], "bot_id": "build15-a7", "pool": "WebRTC" diff --git a/tools_webrtc/ios/tests/perf_trybot_tests.json b/tools_webrtc/ios/tests/perf_trybot_tests.json index 3b53e94480..17301ba148 100644 --- a/tools_webrtc/ios/tests/perf_trybot_tests.json +++ b/tools_webrtc/ios/tests/perf_trybot_tests.json @@ -2,6 +2,10 @@ "tests": [ { "app": "webrtc_perf_tests", + "test args": [ + "--gtest_filter=AudioEncoderOpusComplexityAdaptationTest.AdaptationOn", + "--save_chartjson_result" + ], "bot_id": "build16-a7", "pool": "WebRTC" } diff --git a/video/BUILD.gn b/video/BUILD.gn index ea31bf54ad..16e97090c6 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -298,6 +298,7 @@ if (rtc_include_tests) { "../system_wrappers:metrics_default", "../test:direct_transport", "../test:field_trial", + "../test:perf_test", "../test:rtp_test_utils", "../test:test_common", "../test:test_support",