diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 5c7db2e9d6..f01338f2ef 100755 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -820,9 +820,10 @@ def RunPythonTests(input_api, output_api): return input_api.os_path.join(input_api.PresubmitLocalPath(), *args) excluded_files = [ - # This test should be run manually after webrtc_dashboard_upload target + # These tests should be run manually after webrtc_dashboard_upload target # has been built. - 'catapult_uploader_test.py' + 'catapult_uploader_test.py', + 'process_perf_results_test.py', ] test_directories = [ diff --git a/tools_webrtc/perf/catapult_uploader.py b/tools_webrtc/perf/catapult_uploader.py index c33ac89d0c..d07c287f28 100644 --- a/tools_webrtc/perf/catapult_uploader.py +++ b/tools_webrtc/perf/catapult_uploader.py @@ -14,6 +14,7 @@ import subprocess import time import zlib +from typing import Optional import dataclasses import httplib2 @@ -53,7 +54,7 @@ class UploaderOptions(): build_page_url: str dashboard_url: str input_results_file: str - output_json_file: str + output_json_file: Optional[str] = None wait_timeout_sec: datetime.timedelta = datetime.timedelta(seconds=1200) wait_polling_period_sec: datetime.timedelta = datetime.timedelta(seconds=120) @@ -305,5 +306,5 @@ def UploadToDashboard(options): exit_code = UploadToDashboardImpl(options) except RuntimeError as e: print(e) - return 2 + return 1 return exit_code diff --git a/tools_webrtc/perf/process_perf_results.py b/tools_webrtc/perf/process_perf_results.py new file mode 100644 index 0000000000..e91b1f66e9 --- /dev/null +++ b/tools_webrtc/perf/process_perf_results.py @@ -0,0 +1,123 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Adds build info to perf results and uploads them. + +The tests don't know which bot executed the tests or at what revision, so we +need to take their output and enrich it with this information. We load the proto +from the tests, add the build information as shared diagnostics and then +upload it to the dashboard. + +This script can't be in recipes, because we can't access the catapult APIs from +there. It needs to be here source-side. +""" + +import argparse +import json +import os +import sys + +from pathlib import Path + +# Even if protobuf is not used directly, this allows transitive imports +# of the protobuf library to use the vpython wheel specified in the root +# level .vpython (see bugs.webrtc.org/12211 for context). +import google.protobuf # pylint: disable=unused-import + + +def _ConfigurePythonPath(outdir): + # We just yank the python scripts we require into the PYTHONPATH. You could + # also imagine a solution where we use for instance + # protobuf:py_proto_runtime to copy catapult and protobuf code to out/. + # This is the convention in Chromium and WebRTC python scripts. We do need + # to build histogram_pb2 however, so that's why we add out/ to sys.path + # below. + # + # It would be better if there was an equivalent to py_binary in GN, but + # there's not. + script_dir = os.path.dirname(os.path.realpath(__file__)) + checkout_root = os.path.abspath(os.path.join(script_dir, os.pardir, + os.pardir)) + + sys.path.insert( + 0, os.path.join(checkout_root, 'third_party', 'catapult', 'tracing')) + sys.path.insert( + 0, os.path.join(checkout_root, 'third_party', 'protobuf', 'python')) + + # The webrtc_dashboard_upload gn rule will build the protobuf stub for + # python, so put it in the path for this script before we attempt to import + # it. + histogram_proto_path = os.path.join(outdir, 'pyproto', 'tracing', 'tracing', + 'proto') + sys.path.insert(0, histogram_proto_path) + + # Fail early in case the proto hasn't been built. + from tracing.proto import histogram_proto + if not histogram_proto.HAS_PROTO: + print('Could not find histogram_pb2. You need to build the ' + 'webrtc_dashboard_upload target before invoking this ' + 'script. Expected to find ' + 'histogram_pb2.py in %s.' % histogram_proto_path) + return 1 + return 0 + + +def _UploadToDasboard(args): + build_properties = json.loads(args.build_properties) + exit_code = _ConfigurePythonPath(build_properties['outdir']) + if exit_code != 0: + return exit_code + + import catapult_uploader + + perftest_outputs = [ + f.absolute() for f in Path(args.task_output_dir).rglob('perftest-output*') + if f.is_file() + ] + for perftest_output in perftest_outputs: + uploader_options = catapult_uploader.UploaderOptions( + perf_dashboard_machine_group=( + build_properties['perf_dashboard_machine_group']), + bot=build_properties['bot'], + webrtc_git_hash=build_properties['webrtc_git_hash'], + commit_position=build_properties['commit_position'], + build_page_url=build_properties['build_page_url'], + dashboard_url=build_properties['dashboard_url'], + test_suite=args.test_suite, + input_results_file=perftest_output, + ) + exit_code = catapult_uploader.UploadToDashboard(uploader_options) + if exit_code != 0: + return exit_code + return 0 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--build-properties', help=argparse.SUPPRESS) + parser.add_argument('--summary-json', help=argparse.SUPPRESS) + parser.add_argument('--task-output-dir', help=argparse.SUPPRESS) + parser.add_argument('--test-suite', help=argparse.SUPPRESS) + parser.add_argument('-o', '--output-json', help=argparse.SUPPRESS) + parser.add_argument('json_files', nargs='*', help=argparse.SUPPRESS) + args = parser.parse_args() + + exit_code = _UploadToDasboard(args) + if exit_code != 0: + with open(args.output_json, 'w') as f: + json.dump({ + "global_tags": ["UNRELIABLE_RESULTS"], + "missing_shards": [0] + }, f) + return exit_code + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools_webrtc/perf/process_perf_results_py2.py b/tools_webrtc/perf/process_perf_results_py2.py new file mode 100644 index 0000000000..14b6858093 --- /dev/null +++ b/tools_webrtc/perf/process_perf_results_py2.py @@ -0,0 +1,25 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Calls process_perf_results.py with a python 3 interpreter.""" + +import sys +import subprocess + + +# TODO(crbug.com/webrtc/13835): Delete this file and use +# process_perf_results.py instead. +def main(): + cmd = sys.argv[0].replace('_py2', '') + print('Calling "%s" with py3 in case this script was called with py2.' % cmd) + return subprocess.call(['vpython3', cmd] + sys.argv[1:]) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools_webrtc/perf/process_perf_results_test.py b/tools_webrtc/perf/process_perf_results_test.py new file mode 100644 index 0000000000..3aa5afd75c --- /dev/null +++ b/tools_webrtc/perf/process_perf_results_test.py @@ -0,0 +1,70 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import os +import sys + +import unittest +from unittest import mock + +_SCRIPT_DIR = os.path.dirname(__file__) +_SRC_DIR = os.path.normpath(os.path.join(_SCRIPT_DIR, '..', '..')) + +sys.path.insert(0, os.path.join(_SRC_DIR, 'third_party', 'protobuf', 'python')) +import process_perf_results + + +class ProcessPerfResultsTest(unittest.TestCase): + def testConfigurePythonPath(self): + # pylint: disable=protected-access + self.assertEqual( + 0, + process_perf_results._ConfigurePythonPath( + os.path.join(_SRC_DIR, 'out/Default'))) + + def testUploadToDasboard(self): + outdir = os.path.join(_SRC_DIR, 'out/Default') + args = mock.Mock( + build_properties='{' + '"outdir":"' + outdir + '", ' + + '"perf_dashboard_machine_group":"mock_machine_group", ' + + '"bot":"mock_bot", ' + '"webrtc_git_hash":"mock_webrtc_git_hash", ' + + '"commit_position":"123456", ' + + '"build_page_url":"mock_build_page_url", ' + + '"dashboard_url":"mock_dashboard_url"' + '}', + summary_json='mock_sumary_json', + task_output_dir='mock_task_output_dir', + test_suite='mock_test_suite', + ) + perftest_output = mock.Mock( + absolute=lambda: 'dummy_path/perftest-output.pb', + is_file=lambda: True, + ) + with mock.patch('pathlib.Path.rglob') as mocked_rglob: + with mock.patch('catapult_uploader.UploadToDashboard') as mocked_upload: + mocked_rglob.return_value = [perftest_output] + mocked_upload.return_value = 0 + # pylint: disable=protected-access + self.assertEqual(0, process_perf_results._UploadToDasboard(args)) + + import catapult_uploader + mocked_upload.assert_called_once_with( + catapult_uploader.UploaderOptions( + perf_dashboard_machine_group='mock_machine_group', + bot='mock_bot', + test_suite='mock_test_suite', + webrtc_git_hash='mock_webrtc_git_hash', + commit_position='123456', + build_page_url='mock_build_page_url', + dashboard_url='mock_dashboard_url', + input_results_file=perftest_output.absolute())) + + +if (__name__) == '__main__': + unittest.main()