From 137173c29659ca86139f8a891b427a330b00119e Mon Sep 17 00:00:00 2001 From: Jeremy Leconte Date: Tue, 29 Mar 2022 07:12:54 +0000 Subject: [PATCH] Revert "Reland "Make tools_webrtc/mb inherit from tools/mb."" This reverts commit aa0d4061ba1d65bfbd562c6a173440911c0357d8. Reason for revert: All green except for the android bots compiling low_bandwidth_audio_perf_test. https://ci.chromium.org/ui/p/webrtc/builders/ci/Android32%20Builder%20arm/12133/overview Original change's description: > Reland "Make tools_webrtc/mb inherit from tools/mb." > > This is a reland of commit 7a324b977c5ab6f9b88bcce3353feade943ccefe > > Original change's description: > > Make tools_webrtc/mb inherit from tools/mb. > > > > Bug: webrtc:13867 > > Change-Id: I33e998d260454d16120b09fedecf0c25d2654611 > > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/256809 > > Reviewed-by: Mirko Bonadei > > Commit-Queue: Jeremy Leconte > > Cr-Commit-Position: refs/heads/main@{#36347} > > Bug: webrtc:13867 > Change-Id: I8fe4424771876ea725f5c9a3c5d13b2f6ad83917 > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/256970 > Reviewed-by: Mirko Bonadei > Commit-Queue: Jeremy Leconte > Cr-Commit-Position: refs/heads/main@{#36361} Bug: webrtc:13867 Change-Id: I69948eb028a57b915feba1037e71e82e2d8bc7c7 No-Presubmit: true No-Tree-Checks: true No-Try: true Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/256971 Auto-Submit: Jeremy Leconte Bot-Commit: rubber-stamper@appspot.gserviceaccount.com Reviewed-by: Mirko Bonadei Commit-Queue: Mirko Bonadei Cr-Commit-Position: refs/heads/main@{#36362} --- tools_webrtc/mb/mb.py | 1206 +++++++++++++++++++++++++++++++- tools_webrtc/mb/mb_config.pyl | 3 - tools_webrtc/mb/mb_unittest.py | 91 +-- 3 files changed, 1214 insertions(+), 86 deletions(-) diff --git a/tools_webrtc/mb/mb.py b/tools_webrtc/mb/mb.py index a9b560d8e6..01f4914f3c 100755 --- a/tools_webrtc/mb/mb.py +++ b/tools_webrtc/mb/mb.py @@ -14,14 +14,26 @@ MB is a wrapper script for GN that can be used to generate build files for sets of canned configurations and analyze them. """ +import argparse +import ast +import errno +import json import os +import pipes +import pprint +import re +import shutil import sys +import subprocess +import tempfile +import traceback +from urllib.request import urlopen -_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -_SRC_DIR = os.path.dirname(os.path.dirname(_SCRIPT_DIR)) -sys.path.insert(0, _SRC_DIR) +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +SRC_DIR = os.path.dirname(os.path.dirname(SCRIPT_DIR)) +sys.path = [os.path.join(SRC_DIR, 'build')] + sys.path -from tools.mb import mb +import gn_helpers def _get_executable(target, platform): @@ -31,20 +43,862 @@ def _get_executable(target, platform): def main(args): - mbw = WebRTCMetaBuildWrapper() + mbw = MetaBuildWrapper() return mbw.Main(args) -class WebRTCMetaBuildWrapper(mb.MetaBuildWrapper): +class MetaBuildWrapper: def __init__(self): - super(WebRTCMetaBuildWrapper, self).__init__() - # Make sure default_config and default_isolate_map are attributes of the - # parent class before changing their values. - # pylint: disable=access-member-before-definition - assert self.default_config - assert self.default_isolate_map - self.default_config = os.path.join(_SCRIPT_DIR, 'mb_config.pyl') - self.default_isolate_map = os.path.join(_SCRIPT_DIR, 'gn_isolate_map.pyl') + self.src_dir = SRC_DIR + self.default_config = os.path.join(SCRIPT_DIR, 'mb_config.pyl') + self.default_isolate_map = os.path.join(SCRIPT_DIR, 'gn_isolate_map.pyl') + self.executable = sys.executable + self.platform = sys.platform + self.sep = os.sep + self.args = argparse.Namespace() + self.configs = {} + self.builder_groups = {} + self.mixins = {} + self.isolate_exe = 'isolate.exe' if self.platform.startswith( + 'win') else 'isolate' + + def Main(self, args): + self.ParseArgs(args) + try: + ret = self.args.func() + if ret: + self.DumpInputFiles() + return ret + except KeyboardInterrupt: + self.Print('interrupted, exiting') + return 130 + except Exception: + self.DumpInputFiles() + s = traceback.format_exc() + for l in s.splitlines(): + self.Print(l) + return 1 + + def ParseArgs(self, argv): + def AddCommonOptions(subp): + subp.add_argument('-b', + '--builder', + help='builder name to look up config from') + subp.add_argument('-m', + '--builder-group', + help='builder group name to look up config from') + subp.add_argument('-c', '--config', help='configuration to analyze') + subp.add_argument('--phase', + help='optional phase name (used when builders ' + 'do multiple compiles with different ' + 'arguments in a single build)') + subp.add_argument('-f', + '--config-file', + metavar='PATH', + default=self.default_config, + help='path to config file ' + '(default is %(default)s)') + subp.add_argument('-i', + '--isolate-map-file', + metavar='PATH', + default=self.default_isolate_map, + help='path to isolate map file ' + '(default is %(default)s)') + subp.add_argument('-r', + '--realm', + default='webrtc:try', + help='optional LUCI realm to use (for example ' + 'when triggering tasks on Swarming)') + subp.add_argument('-g', '--goma-dir', help='path to goma directory') + subp.add_argument('--android-version-code', + help='Sets GN arg android_default_version_code') + subp.add_argument('--android-version-name', + help='Sets GN arg android_default_version_name') + subp.add_argument('-n', + '--dryrun', + action='store_true', + help='Do a dry run (i.e., do nothing, just ' + 'print the commands that will run)') + subp.add_argument('-v', + '--verbose', + action='store_true', + help='verbose logging') + + parser = argparse.ArgumentParser(prog='mb') + subps = parser.add_subparsers() + + subp = subps.add_parser('analyze', + help='analyze whether changes to a set of ' + 'files will cause a set of binaries ' + 'to be rebuilt.') + AddCommonOptions(subp) + subp.add_argument('path', nargs=1, help='path build was generated into.') + subp.add_argument('input_path', + nargs=1, + help='path to a file containing the input ' + 'arguments as a JSON object.') + subp.add_argument('output_path', + nargs=1, + help='path to a file containing the output ' + 'arguments as a JSON object.') + subp.add_argument('--json-output', help='Write errors to json.output') + subp.set_defaults(func=self.CmdAnalyze) + + subp = subps.add_parser('export', + help='print out the expanded configuration for' + 'each builder as a JSON object') + subp.add_argument('-f', + '--config-file', + metavar='PATH', + default=self.default_config, + help='path to config file (default is %(default)s)') + subp.add_argument('-g', '--goma-dir', help='path to goma directory') + subp.set_defaults(func=self.CmdExport) + + subp = subps.add_parser('gen', help='generate a new set of build files') + AddCommonOptions(subp) + subp.add_argument('--swarming-targets-file', + help='save runtime dependencies for targets listed ' + 'in file.') + subp.add_argument('--json-output', help='Write errors to json.output') + subp.add_argument('path', nargs=1, help='path to generate build into') + subp.set_defaults(func=self.CmdGen) + + subp = subps.add_parser('isolate', + help='generate the .isolate files for a given' + 'binary') + AddCommonOptions(subp) + subp.add_argument('path', nargs=1, help='path build was generated into') + subp.add_argument('target', + nargs=1, + help='ninja target to generate the isolate for') + subp.set_defaults(func=self.CmdIsolate) + + subp = subps.add_parser('lookup', + help='look up the command for a given config ' + 'or builder') + AddCommonOptions(subp) + subp.add_argument('--quiet', + default=False, + action='store_true', + help='Print out just the arguments, do ' + 'not emulate the output of the gen subcommand.') + subp.set_defaults(func=self.CmdLookup) + + subp = subps.add_parser( + 'run', + help='build and run the isolated version of a ' + 'binary', + formatter_class=argparse.RawDescriptionHelpFormatter) + subp.description = ( + 'Build, isolate, and run the given binary with the command line\n' + 'listed in the isolate. You may pass extra arguments after the\n' + 'target; use "--" if the extra arguments need to include switches.' + '\n\n' + 'Examples:\n' + '\n' + ' % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n' + ' //out/Default content_browsertests\n' + '\n' + ' % tools/mb/mb.py run out/Default content_browsertests\n' + '\n' + ' % tools/mb/mb.py run out/Default content_browsertests -- \\\n' + ' --test-launcher-retry-limit=0' + '\n') + AddCommonOptions(subp) + subp.add_argument('-j', + '--jobs', + dest='jobs', + type=int, + help='Number of jobs to pass to ninja') + subp.add_argument('--no-build', + dest='build', + default=True, + action='store_false', + help='Do not build, just isolate and run') + subp.add_argument('path', + nargs=1, + help=('path to generate build into (or use).' + ' This can be either a regular path or a ' + 'GN-style source-relative path like ' + '//out/Default.')) + subp.add_argument('-s', + '--swarmed', + action='store_true', + help='Run under swarming') + subp.add_argument('-d', + '--dimension', + default=[], + action='append', + nargs=2, + dest='dimensions', + metavar='FOO bar', + help='dimension to filter on') + subp.add_argument('target', nargs=1, help='ninja target to build and run') + subp.add_argument('extra_args', + nargs='*', + help=('extra args to pass to the isolate to run. ' + 'Use "--" as the first arg if you need to ' + 'pass switches')) + subp.set_defaults(func=self.CmdRun) + + subp = subps.add_parser('validate', help='validate the config file') + subp.add_argument('-f', + '--config-file', + metavar='PATH', + default=self.default_config, + help='path to config file (default is %(default)s)') + subp.set_defaults(func=self.CmdValidate) + + subp = subps.add_parser('help', help='Get help on a subcommand.') + subp.add_argument(nargs='?', + action='store', + dest='subcommand', + help='The command to get help for.') + subp.set_defaults(func=self.CmdHelp) + + self.args = parser.parse_args(argv) + + def DumpInputFiles(self): + def DumpContentsOfFilePassedTo(arg_name, path): + if path and self.Exists(path): + self.Print("\n# To recreate the file passed to %s:" % arg_name) + self.Print("%% cat > %s < returned %d' % ret) + if out: + self.Print(out, end='') + if err: + self.Print(err, end='', file=sys.stderr) + + return ret + + try: + archive_hashes = json.loads(self.ReadFile(archive_json_path)) + except Exception: + self.Print('Failed to read JSON file "%s"' % archive_json_path, + file=sys.stderr) + return 1 + try: + cas_digest = archive_hashes[target] + except Exception: + self.Print('Cannot find hash for "%s" in "%s", file content: %s' % + (target, archive_json_path, archive_hashes), + file=sys.stderr) + return 1 + + try: + json_dir = self.TempDir() + json_file = self.PathJoin(json_dir, 'task.json') + + cmd = [ + self.PathJoin('tools', 'luci-go', 'swarming'), + 'trigger', + '-realm', + self.args.realm, + '-digest', + cas_digest, + '-server', + swarming_server, + '-tag=purpose:user-debug-mb', + '-relative-cwd', + self.ToSrcRelPath(build_dir), + '-dump-json', + json_file, + ] + dimensions + ['--'] + list(isolate_cmd) + + if self.args.extra_args: + cmd += ['--'] + self.args.extra_args + self.Print('') + ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) + if ret: + return ret + task_json = self.ReadFile(json_file) + task_id = json.loads(task_json)["tasks"][0]['task_id'] + finally: + if json_dir: + self.RemoveDirectory(json_dir) + + cmd = [ + self.PathJoin('tools', 'luci-go', 'swarming'), + 'collect', + '-server', + swarming_server, + '-task-output-stdout=console', + task_id, + ] + ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) + return ret + + def _RunLocallyIsolated(self, build_dir, target): + cmd = [ + self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), + 'run', + '-i', + self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), + ] + if self.args.extra_args: + cmd += ['--'] + self.args.extra_args + ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False) + return ret + + def CmdValidate(self, print_ok=True): + errs = [] + + # Read the file to make sure it parses. + self.ReadConfigFile() + + # Build a list of all of the configs referenced by builders. + all_configs = {} + for builder_group in self.builder_groups: + for config in list(self.builder_groups[builder_group].values()): + if isinstance(config, dict): + for c in list(config.values()): + all_configs[c] = builder_group + else: + all_configs[config] = builder_group + + # Check that every referenced args file or config actually exists. + for config, loc in list(all_configs.items()): + if config.startswith('//'): + if not self.Exists(self.ToAbsPath(config)): + errs.append('Unknown args file "%s" referenced from "%s".' % + (config, loc)) + elif not config in self.configs: + errs.append('Unknown config "%s" referenced from "%s".' % (config, loc)) + + # Check that every actual config is actually referenced. + for config in self.configs: + if not config in all_configs: + errs.append('Unused config "%s".' % config) + + # Figure out the whole list of mixins, and check that every mixin + # listed by a config or another mixin actually exists. + referenced_mixins = set() + for config, mixins in list(self.configs.items()): + for mixin in mixins: + if not mixin in self.mixins: + errs.append('Unknown mixin "%s" referenced by config "%s".' % + (mixin, config)) + referenced_mixins.add(mixin) + + for mixin in self.mixins: + for sub_mixin in self.mixins[mixin].get('mixins', []): + if not sub_mixin in self.mixins: + errs.append('Unknown mixin "%s" referenced by mixin "%s".' % + (sub_mixin, mixin)) + referenced_mixins.add(sub_mixin) + + # Check that every mixin defined is actually referenced somewhere. + for mixin in self.mixins: + if not mixin in referenced_mixins: + errs.append('Unreferenced mixin "%s".' % mixin) + + if errs: + raise MBErr(('mb config file %s has problems:' % self.args.config_file) + + '\n ' + '\n '.join(errs)) + + if print_ok: + self.Print('mb config file %s looks ok.' % self.args.config_file) + return 0 + + def GetConfig(self): + build_dir = self.args.path[0] + + vals = self.DefaultVals() + if self.args.builder or self.args.builder_group or self.args.config: + vals = self.Lookup() + # Re-run gn gen in order to ensure the config is consistent with + # the build dir. + self.RunGNGen(vals) + return vals + + toolchain_path = self.PathJoin(self.ToAbsPath(build_dir), 'toolchain.ninja') + if not self.Exists(toolchain_path): + self.Print('Must either specify a path to an existing GN build ' + 'dir or pass in a -m/-b pair or a -c flag to specify ' + 'the configuration') + return {} + + vals['gn_args'] = self.GNArgsFromDir(build_dir) + return vals + + def GNArgsFromDir(self, build_dir): + args_contents = "" + gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn') + if self.Exists(gn_args_path): + args_contents = self.ReadFile(gn_args_path) + gn_args = [] + for l in args_contents.splitlines(): + fields = l.split(' ') + name = fields[0] + val = ' '.join(fields[2:]) + gn_args.append('%s=%s' % (name, val)) + + return ' '.join(gn_args) + + def Lookup(self): + self.ReadConfigFile() + config = self.ConfigFromArgs() + if config.startswith('//'): + if not self.Exists(self.ToAbsPath(config)): + raise MBErr('args file "%s" not found' % config) + vals = self.DefaultVals() + vals['args_file'] = config + else: + if not config in self.configs: + raise MBErr('Config "%s" not found in %s' % + (config, self.args.config_file)) + vals = self.FlattenConfig(config) + return vals + + def ReadConfigFile(self): + if not self.Exists(self.args.config_file): + raise MBErr('config file not found at %s' % self.args.config_file) + + try: + contents = ast.literal_eval(self.ReadFile(self.args.config_file)) + except SyntaxError as e: + raise MBErr('Failed to parse config file "%s"' % + self.args.config_file) from e + + self.configs = contents['configs'] + self.builder_groups = contents['builder_groups'] + self.mixins = contents['mixins'] + + def ReadIsolateMap(self): + isolate_map = self.args.isolate_map_file + if not self.Exists(isolate_map): + raise MBErr('isolate map file not found at %s' % isolate_map) + try: + return ast.literal_eval(self.ReadFile(isolate_map)) + except SyntaxError as e: + raise MBErr('Failed to parse isolate map file "%s"' % isolate_map) from e + + def ConfigFromArgs(self): + if self.args.config: + if self.args.builder_group or self.args.builder: + raise MBErr('Can not specific both -c/--config and ' + '-m/--builder-group or -b/--builder') + + return self.args.config + + if not self.args.builder_group or not self.args.builder: + raise MBErr('Must specify either -c/--config or ' + '(-m/--builder-group and -b/--builder)') + + if not self.args.builder_group in self.builder_groups: + raise MBErr('Master name "%s" not found in "%s"' % + (self.args.builder_group, self.args.config_file)) + + if not self.args.builder in self.builder_groups[self.args.builder_group]: + raise MBErr( + 'Builder name "%s" not found under builder_groups[%s] in "%s"' % + (self.args.builder, self.args.builder_group, self.args.config_file)) + + config = (self.builder_groups[self.args.builder_group][self.args.builder]) + if isinstance(config, dict): + if self.args.phase is None: + raise MBErr('Must specify a build --phase for %s on %s' % + (self.args.builder, self.args.builder_group)) + phase = str(self.args.phase) + if phase not in config: + raise MBErr('Phase %s doesn\'t exist for %s on %s' % + (phase, self.args.builder, self.args.builder_group)) + return config[phase] + + if self.args.phase is not None: + raise MBErr('Must not specify a build --phase for %s on %s' % + (self.args.builder, self.args.builder_group)) + return config + + def FlattenConfig(self, config): + mixins = self.configs[config] + vals = self.DefaultVals() + + visited = [] + self.FlattenMixins(mixins, vals, visited) + return vals + + @staticmethod + def DefaultVals(): + return { + 'args_file': '', + 'cros_passthrough': False, + 'gn_args': '', + } + + def FlattenMixins(self, mixins, vals, visited): + for m in mixins: + if m not in self.mixins: + raise MBErr('Unknown mixin "%s"' % m) + + visited.append(m) + + mixin_vals = self.mixins[m] + + if 'cros_passthrough' in mixin_vals: + vals['cros_passthrough'] = mixin_vals['cros_passthrough'] + if 'gn_args' in mixin_vals: + if vals['gn_args']: + vals['gn_args'] += ' ' + mixin_vals['gn_args'] + else: + vals['gn_args'] = mixin_vals['gn_args'] + + if 'mixins' in mixin_vals: + self.FlattenMixins(mixin_vals['mixins'], vals, visited) + return vals + + def RunGNGen(self, vals): + build_dir = self.args.path[0] + + cmd = self.GNCmd('gen', build_dir, '--check') + gn_args = self.GNArgs(vals) + + # Since GN hasn't run yet, the build directory may not even exist. + self.MaybeMakeDirectory(self.ToAbsPath(build_dir)) + + gn_args_path = self.ToAbsPath(build_dir, 'args.gn') + self.WriteFile(gn_args_path, gn_args, force_verbose=True) + + swarming_targets = set() + if getattr(self.args, 'swarming_targets_file', None): + # We need GN to generate the list of runtime dependencies for + # the compile targets listed (one per line) in the file so + # we can run them via swarming. We use gn_isolate_map.pyl to + # convert the compile targets to the matching GN labels. + path = self.args.swarming_targets_file + if not self.Exists(path): + self.WriteFailureAndRaise('"%s" does not exist' % path, + output_path=None) + contents = self.ReadFile(path) + swarming_targets = set(contents.splitlines()) + + isolate_map = self.ReadIsolateMap() + err, labels = self.MapTargetsToLabels(isolate_map, swarming_targets) + if err: + raise MBErr(err) + + gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps') + self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n') + cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path) + + ret, output, _ = self.Run(cmd) + if ret: + if self.args.json_output: + # write errors to json.output + self.WriteJSON({'output': output}, self.args.json_output) + # If `gn gen` failed, we should exit early rather than trying to + # generate isolates. Run() will have already logged any error + # output. + self.Print('GN gen failed: %d' % ret) + return ret + + android = 'target_os="android"' in vals['gn_args'] + for target in swarming_targets: + if android: + # Android targets may be either android_apk or executable. The + # former will result in runtime_deps associated with the stamp + # file, while the latter will result in runtime_deps associated + # with the executable. + label = isolate_map[target]['label'] + runtime_deps_targets = [ + target + '.runtime_deps', + 'obj/%s.stamp.runtime_deps' % label.replace(':', '/') + ] + elif isolate_map[target]['type'] == 'gpu_browser_test': + if self.platform == 'win32': + runtime_deps_targets = ['browser_tests.exe.runtime_deps'] + else: + runtime_deps_targets = ['browser_tests.runtime_deps'] + elif isolate_map[target]['type'] == 'script': + label = isolate_map[target]['label'].split(':')[1] + runtime_deps_targets = ['%s.runtime_deps' % label] + if self.platform == 'win32': + runtime_deps_targets += [label + '.exe.runtime_deps'] + else: + runtime_deps_targets += [label + '.runtime_deps'] + elif self.platform == 'win32': + runtime_deps_targets = [target + '.exe.runtime_deps'] + else: + runtime_deps_targets = [target + '.runtime_deps'] + + for r in runtime_deps_targets: + runtime_deps_path = self.ToAbsPath(build_dir, r) + if self.Exists(runtime_deps_path): + break + else: + raise MBErr('did not generate any of %s' % + ', '.join(runtime_deps_targets)) + + command, extra_files = self.GetSwarmingCommand(target, vals) + + runtime_deps = self.ReadFile(runtime_deps_path).splitlines() + + self.WriteIsolateFiles(build_dir, command, target, runtime_deps, + extra_files) + + return 0 + + def RunGNIsolate(self, vals): + target = self.args.target[0] + isolate_map = self.ReadIsolateMap() + err, labels = self.MapTargetsToLabels(isolate_map, [target]) + if err: + raise MBErr(err) + label = labels[0] + + build_dir = self.args.path[0] + command, extra_files = self.GetSwarmingCommand(target, vals) + + cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps') + ret, out, _ = self.Call(cmd) + if ret: + if out: + self.Print(out) + return ret + + runtime_deps = out.splitlines() + + self.WriteIsolateFiles(build_dir, command, target, runtime_deps, + extra_files) + + ret, _, _ = self.Run([ + self.PathJoin(self.src_dir, 'tools', 'luci-go', self.isolate_exe), + 'check', '-i', + self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)) + ], + buffer_output=False) + + return ret + + def WriteIsolateFiles(self, build_dir, command, target, runtime_deps, + extra_files): + isolate_path = self.ToAbsPath(build_dir, target + '.isolate') + self.WriteFile( + isolate_path, + pprint.pformat({ + 'variables': { + 'command': command, + 'files': sorted(runtime_deps + extra_files), + } + }) + '\n') + + self.WriteJSON( + { + 'args': [ + '--isolate', + self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), + ], + 'dir': + self.src_dir, + 'version': + 1, + }, + isolate_path + 'd.gen.json', + ) + + @staticmethod + def MapTargetsToLabels(isolate_map, targets): + labels = [] + err = '' + + def StripTestSuffixes(target): + for suffix in ('_apk_run', '_apk', '_run'): + if target.endswith(suffix): + return target[:-len(suffix)], suffix + return None, None + + for target in targets: + if target == 'all': + labels.append(target) + elif target.startswith('//'): + labels.append(target) + else: + if target in isolate_map: + stripped_target, suffix = target, '' + else: + stripped_target, suffix = StripTestSuffixes(target) + if stripped_target in isolate_map: + if isolate_map[stripped_target]['type'] == 'unknown': + err += ('test target "%s" type is unknown\n' % target) + else: + labels.append(isolate_map[stripped_target]['label'] + suffix) + else: + err += ('target "%s" not found in ' + '//testing/buildbot/gn_isolate_map.pyl\n' % target) + + return err, labels + + def GNCmd(self, subcommand, path, *args): + if self.platform.startswith('linux'): + subdir, exe = 'linux64', 'gn' + elif self.platform == 'darwin': + subdir, exe = 'mac', 'gn' + else: + subdir, exe = 'win', 'gn.exe' + + gn_path = self.PathJoin(self.src_dir, 'buildtools', subdir, exe) + return [gn_path, subcommand, path] + list(args) + + def GNArgs(self, vals): + if vals['cros_passthrough']: + if not 'GN_ARGS' in os.environ: + raise MBErr('MB is expecting GN_ARGS to be in the environment') + gn_args = os.environ['GN_ARGS'] + if not re.search('target_os.*=.*"chromeos"', gn_args): + raise MBErr('GN_ARGS is missing target_os = "chromeos": ' + '(GN_ARGS=%s)' % gn_args) + else: + gn_args = vals['gn_args'] + + if self.args.goma_dir: + gn_args += ' goma_dir="%s"' % self.args.goma_dir + + android_version_code = self.args.android_version_code + if android_version_code: + gn_args += (' android_default_version_code="%s"' % android_version_code) + + android_version_name = self.args.android_version_name + if android_version_name: + gn_args += (' android_default_version_name="%s"' % android_version_name) + + # Canonicalize the arg string into a sorted, newline-separated list + # of key-value pairs, and de-dup the keys if need be so that only + # the last instance of each arg is listed. + gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args)) + + args_file = vals.get('args_file', None) + if args_file: + gn_args = ('import("%s")\n' % vals['args_file']) + gn_args + return gn_args def GetSwarmingCommand(self, target, vals): isolate_map = self.ReadIsolateMap() @@ -157,5 +1011,329 @@ class WebRTCMetaBuildWrapper(mb.MetaBuildWrapper): return cmdline, extra_files + def ToAbsPath(self, build_path, *comps): + return self.PathJoin(self.src_dir, self.ToSrcRelPath(build_path), *comps) + + def ToSrcRelPath(self, path): + """Returns a relative path from the top of the repo.""" + if path.startswith('//'): + return path[2:].replace('/', self.sep) + return self.RelPath(path, self.src_dir) + + def RunGNAnalyze(self, vals): + # Analyze runs before 'gn gen' now, so we need to run gn gen + # in order to ensure that we have a build directory. + ret = self.RunGNGen(vals) + if ret: + return ret + + build_path = self.args.path[0] + input_path = self.args.input_path[0] + gn_input_path = input_path + '.gn' + output_path = self.args.output_path[0] + gn_output_path = output_path + '.gn' + + inp = self.ReadInputJSON( + ['files', 'test_targets', 'additional_compile_targets']) + if self.args.verbose: + self.Print() + self.Print('analyze input:') + self.PrintJSON(inp) + self.Print() + + # This shouldn't normally happen, but could due to unusual race + # conditions, like a try job that gets scheduled before a patch + # lands but runs after the patch has landed. + if not inp['files']: + self.Print('Warning: No files modified in patch, bailing out early.') + self.WriteJSON( + { + 'status': 'No dependency', + 'compile_targets': [], + 'test_targets': [], + }, output_path) + return 0 + + gn_inp = {} + gn_inp['files'] = ['//' + f for f in inp['files'] if not f.startswith('//')] + + isolate_map = self.ReadIsolateMap() + err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels( + isolate_map, inp['additional_compile_targets']) + if err: + raise MBErr(err) + + err, gn_inp['test_targets'] = self.MapTargetsToLabels( + isolate_map, inp['test_targets']) + if err: + raise MBErr(err) + labels_to_targets = {} + for i, label in enumerate(gn_inp['test_targets']): + labels_to_targets[label] = inp['test_targets'][i] + + try: + self.WriteJSON(gn_inp, gn_input_path) + cmd = self.GNCmd('analyze', build_path, gn_input_path, gn_output_path) + ret, output, _ = self.Run(cmd, force_verbose=True) + if ret: + if self.args.json_output: + # write errors to json.output + self.WriteJSON({'output': output}, self.args.json_output) + return ret + + gn_outp_str = self.ReadFile(gn_output_path) + try: + gn_outp = json.loads(gn_outp_str) + except Exception as e: + self.Print("Failed to parse the JSON string GN " + "returned: %s\n%s" % (repr(gn_outp_str), str(e))) + raise + + outp = {} + if 'status' in gn_outp: + outp['status'] = gn_outp['status'] + if 'error' in gn_outp: + outp['error'] = gn_outp['error'] + if 'invalid_targets' in gn_outp: + outp['invalid_targets'] = gn_outp['invalid_targets'] + if 'compile_targets' in gn_outp: + if 'all' in gn_outp['compile_targets']: + outp['compile_targets'] = ['all'] + else: + outp['compile_targets'] = [ + label.replace('//', '') for label in gn_outp['compile_targets'] + ] + if 'test_targets' in gn_outp: + outp['test_targets'] = [ + labels_to_targets[label] for label in gn_outp['test_targets'] + ] + + if self.args.verbose: + self.Print() + self.Print('analyze output:') + self.PrintJSON(outp) + self.Print() + + self.WriteJSON(outp, output_path) + + finally: + if self.Exists(gn_input_path): + self.RemoveFile(gn_input_path) + if self.Exists(gn_output_path): + self.RemoveFile(gn_output_path) + + return 0 + + def ReadInputJSON(self, required_keys): + path = self.args.input_path[0] + output_path = self.args.output_path[0] + if not self.Exists(path): + self.WriteFailureAndRaise('"%s" does not exist' % path, output_path) + + try: + inp = json.loads(self.ReadFile(path)) + except Exception as e: + self.WriteFailureAndRaise( + 'Failed to read JSON input from "%s": %s' % (path, e), output_path) + + for k in required_keys: + if not k in inp: + self.WriteFailureAndRaise('input file is missing a "%s" key' % k, + output_path) + + return inp + + def WriteFailureAndRaise(self, msg, output_path): + if output_path: + self.WriteJSON({'error': msg}, output_path, force_verbose=True) + raise MBErr(msg) + + def WriteJSON(self, obj, path, force_verbose=False): + try: + self.WriteFile(path, + json.dumps(obj, indent=2, sort_keys=True) + '\n', + force_verbose=force_verbose) + except Exception as e: + raise MBErr('Error writing to the output path "%s"' % path) from e + + def PrintCmd(self, cmd, env): + if self.platform == 'win32': + env_prefix = 'set ' + env_quoter = QuoteForSet + shell_quoter = QuoteForCmd + else: + env_prefix = '' + env_quoter = pipes.quote + shell_quoter = pipes.quote + + var = 'LLVM_FORCE_HEAD_REVISION' + if env and var in env: + self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var]))) + + if cmd[0] == self.executable: + cmd = ['vpython3'] + cmd[1:] + self.Print(*[shell_quoter(arg) for arg in cmd]) + + def PrintJSON(self, obj): + self.Print(json.dumps(obj, indent=2, sort_keys=True)) + + def Build(self, target): + build_dir = self.ToSrcRelPath(self.args.path[0]) + ninja_cmd = ['ninja', '-C', build_dir] + if self.args.jobs: + ninja_cmd.extend(['-j', '%d' % self.args.jobs]) + ninja_cmd.append(target) + ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False) + return ret + + def Run(self, cmd, env=None, force_verbose=True, buffer_output=True): + # This function largely exists so it can be overridden for testing. + if self.args.dryrun or self.args.verbose or force_verbose: + self.PrintCmd(cmd, env) + if self.args.dryrun: + return 0, '', '' + + ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output) + if self.args.verbose or force_verbose: + if ret: + self.Print(' -> returned %d' % ret) + if out: + self.Print(out, end='') + if err: + self.Print(err, end='', file=sys.stderr) + return ret, out, err + + def Call(self, cmd, env=None, buffer_output=True): + if buffer_output: + p = subprocess.Popen(cmd, + shell=False, + cwd=self.src_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + out, err = p.communicate() + out = out.decode('utf-8') + err = err.decode('utf-8') + else: + p = subprocess.Popen(cmd, shell=False, cwd=self.src_dir, env=env) + p.wait() + out = err = '' + return p.returncode, out, err + + @staticmethod + def ExpandUser(path): + # This function largely exists so it can be overridden for testing. + return os.path.expanduser(path) + + @staticmethod + def Exists(path): + # This function largely exists so it can be overridden for testing. + return os.path.exists(path) + + @staticmethod + def Fetch(url): + # This function largely exists so it can be overridden for testing. + f = urlopen(url) + contents = f.read() + f.close() + return contents + + @staticmethod + def MaybeMakeDirectory(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + @staticmethod + def PathJoin(*comps): + # This function largely exists so it can be overriden for testing. + return os.path.join(*comps) + + @staticmethod + def Print(*args, **kwargs): + # This function largely exists so it can be overridden for testing. + print(*args, **kwargs) + if kwargs.get('stream', sys.stdout) == sys.stdout: + sys.stdout.flush() + + @staticmethod + def ReadFile(path): + # This function largely exists so it can be overriden for testing. + with open(path) as fp: + return fp.read() + + @staticmethod + def RelPath(path, start='.'): + # This function largely exists so it can be overriden for testing. + return os.path.relpath(path, start) + + @staticmethod + def RemoveFile(path): + # This function largely exists so it can be overriden for testing. + os.remove(path) + + def RemoveDirectory(self, abs_path): + if self.platform == 'win32': + # In other places in chromium, we often have to retry this command + # because we're worried about other processes still holding on to + # file handles, but when MB is invoked, it will be early enough in + # the build that their should be no other processes to interfere. + # We can change this if need be. + self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path]) + else: + shutil.rmtree(abs_path, ignore_errors=True) + + @staticmethod + def TempDir(): + # This function largely exists so it can be overriden for testing. + return tempfile.mkdtemp(prefix='mb_') + + @staticmethod + def TempFile(mode='w'): + # This function largely exists so it can be overriden for testing. + return tempfile.NamedTemporaryFile(mode=mode, delete=False) + + def WriteFile(self, path, contents, force_verbose=False): + # This function largely exists so it can be overriden for testing. + if self.args.dryrun or self.args.verbose or force_verbose: + self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) + with open(path, 'w') as fp: + return fp.write(contents) + + +class MBErr(Exception): + pass + + +# See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful +# details of this next section, which handles escaping command lines +# so that they can be copied and pasted into a cmd window. +UNSAFE_FOR_SET = set('^<>&|') +UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%')) +ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"')) + + +def QuoteForSet(arg): + if any(a in UNSAFE_FOR_SET for a in arg): + arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg) + return arg + + +def QuoteForCmd(arg): + # First, escape the arg so that CommandLineToArgvW will parse it properly. + if arg == '' or ' ' in arg or '"' in arg: + quote_re = re.compile(r'(\\*)"') + arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) + + # Then check to see if the arg contains any metacharacters other than + # double quotes; if it does, quote everything (including the double + # quotes) for safety. + if any(a in UNSAFE_FOR_CMD for a in arg): + arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg) + return arg + + if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/tools_webrtc/mb/mb_config.pyl b/tools_webrtc/mb/mb_config.pyl index 0080ae23c5..89dd294f87 100644 --- a/tools_webrtc/mb/mb_config.pyl +++ b/tools_webrtc/mb/mb_config.pyl @@ -18,9 +18,6 @@ # The builders should be sorted by the order they appear in the /builders # page on the buildbots, *not* alphabetically. 'builder_groups': { - # This is required because WebRTC mb.py overwrites the default configs - # and Chromium's mb.py checks the default config contains 'chromium'. - 'chromium': {}, 'client.webrtc': { # iOS 'iOS64 Debug': 'ios_debug_bot_arm64', diff --git a/tools_webrtc/mb/mb_unittest.py b/tools_webrtc/mb/mb_unittest.py index 713d16992a..a79186a0d0 100755 --- a/tools_webrtc/mb/mb_unittest.py +++ b/tools_webrtc/mb/mb_unittest.py @@ -19,20 +19,16 @@ import sys import tempfile import unittest -_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -_SRC_DIR = os.path.dirname(os.path.dirname(_SCRIPT_DIR)) -sys.path.insert(0, _SRC_DIR) - -from tools_webrtc.mb import mb +import mb -class FakeMBW(mb.WebRTCMetaBuildWrapper): +class FakeMBW(mb.MetaBuildWrapper): def __init__(self, win32=False): super(FakeMBW, self).__init__() # Override vars for test portability. if win32: - self.chromium_src_dir = 'c:\\fake_src' + self.src_dir = 'c:\\fake_src' self.default_config = 'c:\\fake_src\\tools_webrtc\\mb\\mb_config.pyl' self.default_isolate_map = ('c:\\fake_src\\testing\\buildbot\\' 'gn_isolate_map.pyl') @@ -41,7 +37,7 @@ class FakeMBW(mb.WebRTCMetaBuildWrapper): self.sep = '\\' self.cwd = 'c:\\fake_src\\out\\Default' else: - self.chromium_src_dir = '/fake_src' + self.src_dir = '/fake_src' self.default_config = '/fake_src/tools_webrtc/mb/mb_config.pyl' self.default_isolate_map = '/fake_src/testing/buildbot/gn_isolate_map.pyl' self.executable = '/usr/bin/vpython3' @@ -63,15 +59,7 @@ class FakeMBW(mb.WebRTCMetaBuildWrapper): def Exists(self, path): abs_path = self._AbsPath(path) - return (self.files.get(abs_path) is not None or abs_path in self.dirs) - - def ListDir(self, path): - dir_contents = [] - for f in list(self.files.keys()) + list(self.dirs): - head, _ = os.path.split(f) - if head == path: - dir_contents.append(f) - return dir_contents + return self.files.get(abs_path) is not None or abs_path in self.dirs def MaybeMakeDirectory(self, path): abpath = self._AbsPath(path) @@ -81,10 +69,7 @@ class FakeMBW(mb.WebRTCMetaBuildWrapper): return self.sep.join(comps) def ReadFile(self, path): - try: - return self.files[self._AbsPath(path)] - except KeyError: - raise IOError('%s not found' % path) + return self.files[self._AbsPath(path)] def WriteFile(self, path, contents, force_verbose=False): if self.args.dryrun or self.args.verbose or force_verbose: @@ -92,10 +77,7 @@ class FakeMBW(mb.WebRTCMetaBuildWrapper): abpath = self._AbsPath(path) self.files[abpath] = contents - def Call(self, cmd, env=None, buffer_output=True, stdin=None): - del env - del buffer_output - del stdin + def Call(self, cmd, env=None, buffer_output=True): self.calls.append(cmd) if self.cmds: return self.cmds.pop(0) @@ -115,20 +97,17 @@ class FakeMBW(mb.WebRTCMetaBuildWrapper): self.dirs.add(tmp_dir) return tmp_dir - def TempFile(self, mode='w'): - del mode + def TempFile(self): return FakeFile(self.files) def RemoveFile(self, path): abpath = self._AbsPath(path) self.files[abpath] = None - def RemoveDirectory(self, abs_path): - # Normalize the passed-in path to handle different working directories - # used during unit testing. - abs_path = self._AbsPath(abs_path) - self.rmdirs.append(abs_path) - files_to_delete = [f for f in self.files if f.startswith(abs_path)] + def RemoveDirectory(self, path): + abpath = self._AbsPath(path) + self.rmdirs.append(abpath) + files_to_delete = [f for f in self.files if f.startswith(abpath)] for f in files_to_delete: self.files[f] = None @@ -161,30 +140,19 @@ TEST_CONFIG = """\ 'fake_group': { 'fake_builder': 'rel_bot', 'fake_debug_builder': 'debug_goma', - 'fake_args_bot': 'fake_args_bot', + 'fake_args_bot': '//build/args/bots/fake_group/fake_args_bot.gn', 'fake_multi_phase': { 'phase_1': 'phase_1', 'phase_2': 'phase_2'}, 'fake_android_bot': 'android_bot', - 'fake_args_file': 'args_file_goma', - 'fake_ios_error': 'ios_error', }, }, 'configs': { - 'args_file_goma': ['fake_args_bot', 'goma'], - 'fake_args_bot': ['fake_args_bot'], 'rel_bot': ['rel', 'goma', 'fake_feature1'], 'debug_goma': ['debug', 'goma'], - 'phase_1': ['rel', 'phase_1'], - 'phase_2': ['rel', 'phase_2'], + 'phase_1': ['phase_1'], + 'phase_2': ['phase_2'], 'android_bot': ['android'], - 'ios_error': ['error'], }, 'mixins': { - 'error': { - 'gn_args': 'error', - }, - 'fake_args_bot': { - 'args_file': '//build/args/bots/fake_group/fake_args_bot.gn', - }, 'fake_feature1': { 'gn_args': 'enable_doom_melon=true', }, @@ -198,13 +166,13 @@ TEST_CONFIG = """\ 'gn_args': 'phase=2', }, 'rel': { - 'gn_args': 'is_debug=false dcheck_always_on=false', + 'gn_args': 'is_debug=false', }, 'debug': { 'gn_args': 'is_debug=true', }, 'android': { - 'gn_args': 'target_os="android" dcheck_always_on=false', + 'gn_args': 'target_os="android"', } }, } @@ -225,23 +193,10 @@ class UnitTest(unittest.TestCase): }''') mbw.files.setdefault( mbw.ToAbsPath('//build/args/bots/fake_group/fake_args_bot.gn'), - 'is_debug = false\ndcheck_always_on=false\n') - mbw.files.setdefault(mbw.ToAbsPath('//tools/mb/rts_banned_suites.json'), - '{}') + 'is_debug = false\n') if files: for path, contents in list(files.items()): mbw.files[path] = contents - if path.endswith('.runtime_deps'): - - def fake_call(cmd, env=None, buffer_output=True, stdin=None): - del cmd - del env - del buffer_output - del stdin - mbw.files[path] = contents - return 0, '', '' - - mbw.Call = fake_call return mbw def check(self, @@ -375,7 +330,7 @@ class UnitTest(unittest.TestCase): '/fake_src/testing/buildbot/gn_isolate_map.pyl': ("{'base_unittests': {" " 'label': '//base:base_unittests'," - " 'type': 'console_test_launcher'," + " 'type': 'additional_compile_target'," "}}\n"), '/fake_src/out/Default/base_unittests.runtime_deps': ("base_unittests\n"), @@ -503,10 +458,10 @@ class UnitTest(unittest.TestCase): '/fake_src/testing/buildbot/gn_isolate_map.pyl': ("{'base_unittests_script': {" " 'label': '//base:base_unittests'," - " 'type': 'console_test_launcher'," + " 'type': 'script'," " 'script': '//base/base_unittests_script.py'," "}}\n"), - '/fake_src/out/Default/base_unittests_script.runtime_deps': + '/fake_src/out/Default/base_unittests.runtime_deps': ("base_unittests\n" "base_unittests_script.py\n"), } @@ -842,7 +797,7 @@ class UnitTest(unittest.TestCase): ret=0) # test running isolate on an existing build_dir - files['/fake_src/out/Default/args.gn'] = 'is_debug = true\n' + files['/fake_src/out/Default/args.gn'] = 'is_debug = True\n' self.check(['isolate', '//out/Default', 'base_unittests'], files=files, ret=0) @@ -866,7 +821,6 @@ class UnitTest(unittest.TestCase): ret=0) def test_run_swarmed(self): - # pylint: disable=attribute-defined-outside-init files = { '/fake_src/testing/buildbot/gn_isolate_map.pyl': ("{'base_unittests': {" @@ -925,7 +879,6 @@ class UnitTest(unittest.TestCase): ret=0, out=('\n' 'Writing """\\\n' - 'dcheck_always_on = false\n' 'enable_doom_melon = true\n' 'goma_dir = "/foo"\n' 'is_debug = false\n'