diff --git a/tools_webrtc/presubmit_checks_lib/build_helpers.py b/tools_webrtc/presubmit_checks_lib/build_helpers.py index d64c2f457b..849fc771ae 100644 --- a/tools_webrtc/presubmit_checks_lib/build_helpers.py +++ b/tools_webrtc/presubmit_checks_lib/build_helpers.py @@ -7,8 +7,8 @@ # tree. An additional intellectual property rights grant can be found # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. -"""This script helps to invoke gn and ninja -which lie in depot_tools repository.""" +"""This script helps to invoke gn and ninja which lie in depot_tools +repository.""" import json import os @@ -19,30 +19,30 @@ import sys import tempfile -def FindSrcDirPath(): - """Returns the abs path to the src/ dir of the project.""" - src_dir = os.path.dirname(os.path.abspath(__file__)) - while os.path.basename(src_dir) != 'src': - src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) - return src_dir +def find_src_dir_path(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir -SRC_DIR = FindSrcDirPath() +SRC_DIR = find_src_dir_path() sys.path.append(os.path.join(SRC_DIR, 'build')) import find_depot_tools -def RunGnCommand(args, root_dir=None): - """Runs `gn` with provided args and return error if any.""" - try: - command = [ - sys.executable, - os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py') - ] + args - subprocess.check_output(command, cwd=root_dir) - except subprocess.CalledProcessError as err: - return err.output - return None +def run_gn_command(args, root_dir=None): + """Runs `gn` with provided args and return error if any.""" + try: + command = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py') + ] + args + subprocess.check_output(command, cwd=root_dir) + except subprocess.CalledProcessError as err: + return err.output + return None # GN_ERROR_RE matches the summary of an error output by `gn check`. @@ -51,50 +51,50 @@ def RunGnCommand(args, root_dir=None): GN_ERROR_RE = re.compile(r'^ERROR .+(?:\n.*[^_\n].*$)+', re.MULTILINE) -def RunGnCheck(root_dir=None): - """Runs `gn gen --check` with default args to detect mismatches between +def run_gn_check(root_dir=None): + """Runs `gn gen --check` with default args to detect mismatches between #includes and dependencies in the BUILD.gn files, as well as general build errors. Returns a list of error summary strings. """ - out_dir = tempfile.mkdtemp('gn') - try: - error = RunGnCommand(['gen', '--check', out_dir], root_dir) - finally: - shutil.rmtree(out_dir, ignore_errors=True) - return GN_ERROR_RE.findall(error.decode('utf-8')) if error else [] + out_dir = tempfile.mkdtemp('gn') + try: + error = run_gn_command(['gen', '--check', out_dir], root_dir) + finally: + shutil.rmtree(out_dir, ignore_errors=True) + return GN_ERROR_RE.findall(error.decode('utf-8')) if error else [] -def RunNinjaCommand(args, root_dir=None): - """Runs ninja quietly. Any failure (e.g. clang not found) is +def run_ninja_command(args, root_dir=None): + """Runs ninja quietly. Any failure (e.g. clang not found) is silently discarded, since this is unlikely an error in submitted CL.""" - command = [os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja')] + args - p = subprocess.Popen(command, - cwd=root_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, _ = p.communicate() - return out + command = [os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja')] + args + proc = subprocess.Popen(command, + cwd=root_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, _ = proc.communicate() + return out -def GetClangTidyPath(): - """POC/WIP! Use the one we have, even it doesn't match clang's version.""" - tidy = ('third_party/android_toolchain/toolchains/' - 'llvm/prebuilt/linux-x86_64/bin/clang-tidy') - return os.path.join(SRC_DIR, tidy) +def get_clang_tidy_path(): + """POC/WIP! Use the one we have, even it doesn't match clang's version.""" + tidy = ('third_party/android_toolchain/toolchains/' + 'llvm/prebuilt/linux-x86_64/bin/clang-tidy') + return os.path.join(SRC_DIR, tidy) -def GetCompilationDb(root_dir=None): - """Run ninja compdb tool to get proper flags, defines and include paths.""" - # The compdb tool expect a rule. - commands = json.loads(RunNinjaCommand(['-t', 'compdb', 'cxx'], root_dir)) - # Turns 'file' field into a key. - return {v['file']: v for v in commands} +def get_compilation_db(root_dir=None): + """Run ninja compdb tool to get proper flags, defines and include paths.""" + # The compdb tool expect a rule. + commands = json.loads(run_ninja_command(['-t', 'compdb', 'cxx'], root_dir)) + # Turns 'file' field into a key. + return {v['file']: v for v in commands} -def GetCompilationCommand(filepath, gn_args, work_dir): - """Get the whole command used to compile one cc file. +def get_compilation_command(filepath, gn_args, work_dir): + """Get the whole command used to compile one cc file. Typically, clang++ with flags, defines and include paths. Args: @@ -105,30 +105,30 @@ def GetCompilationCommand(filepath, gn_args, work_dir): Returns: Command as a list, ready to be consumed by subprocess.Popen. """ - gn_errors = RunGnCommand(['gen'] + gn_args + [work_dir]) - if gn_errors: - raise RuntimeError('FYI, cannot complete check due to gn error:\n%s\n' - 'Please open a bug.' % gn_errors) + gn_errors = run_gn_command(['gen'] + gn_args + [work_dir]) + if gn_errors: + raise RuntimeError('FYI, cannot complete check due to gn error:\n%s\n' + 'Please open a bug.' % gn_errors) - # Needed for single file compilation. - commands = GetCompilationDb(work_dir) + # Needed for single file compilation. + commands = get_compilation_db(work_dir) - # Path as referenced by ninja. - rel_path = os.path.relpath(os.path.abspath(filepath), work_dir) + # Path as referenced by ninja. + rel_path = os.path.relpath(os.path.abspath(filepath), work_dir) - # Gather defines, include path and flags (such as -std=c++11). - try: - compilation_entry = commands[rel_path] - except KeyError as not_found: - raise ValueError('%s: Not found in compilation database.\n' - 'Please check the path.' % filepath) from not_found - command = compilation_entry['command'].split() + # Gather defines, include path and flags (such as -std=c++11). + try: + compilation_entry = commands[rel_path] + except KeyError as not_found: + raise ValueError('%s: Not found in compilation database.\n' + 'Please check the path.' % filepath) from not_found + command = compilation_entry['command'].split() - # Remove troublesome flags. May trigger an error otherwise. - if '-MMD' in command: - command.remove('-MMD') - if '-MF' in command: - index = command.index('-MF') - del command[index:index + 2] # Remove filename as well. + # Remove troublesome flags. May trigger an error otherwise. + if '-MMD' in command: + command.remove('-MMD') + if '-MF' in command: + index = command.index('-MF') + del command[index:index + 2] # Remove filename as well. - return command + return command diff --git a/tools_webrtc/presubmit_checks_lib/build_helpers_test.py b/tools_webrtc/presubmit_checks_lib/build_helpers_test.py index 42b94d6c29..431f72cc0f 100755 --- a/tools_webrtc/presubmit_checks_lib/build_helpers_test.py +++ b/tools_webrtc/presubmit_checks_lib/build_helpers_test.py @@ -20,13 +20,13 @@ TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), class GnCheckTest(unittest.TestCase): - def testCircularDependencyError(self): - test_dir = os.path.join(TESTDATA_DIR, 'circular_dependency') - expected_error = re.compile('ERROR Dependency cycle') - gn_output = build_helpers.RunGnCheck(test_dir) - self.assertEqual(1, len(gn_output)) - self.assertRegex(gn_output[0], expected_error) + def test_circular_dependency_error(self): + test_dir = os.path.join(TESTDATA_DIR, 'circular_dependency') + expected_error = re.compile('ERROR Dependency cycle') + gn_output = build_helpers.run_gn_check(test_dir) + self.assertEqual(1, len(gn_output)) + self.assertRegex(gn_output[0], expected_error) if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/tools_webrtc/version_updater/update_version.py b/tools_webrtc/version_updater/update_version.py index 2a693cd630..bc1bc9d364 100644 --- a/tools_webrtc/version_updater/update_version.py +++ b/tools_webrtc/version_updater/update_version.py @@ -19,150 +19,153 @@ import subprocess import sys -def FindSrcDirPath(): - """Returns the abs path to the src/ dir of the project.""" - src_dir = os.path.dirname(os.path.abspath(__file__)) - while os.path.basename(src_dir) != 'src': - src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) - return src_dir +def find_src_dir_path(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir UPDATE_BRANCH_NAME = 'webrtc_version_update' -CHECKOUT_SRC_DIR = FindSrcDirPath() +CHECKOUT_SRC_DIR = find_src_dir_path() NOTIFY_EMAIL = 'webrtc-trooper@webrtc.org' -def _RemovePreviousUpdateBranch(): - active_branch, branches = _GetBranches() - if active_branch == UPDATE_BRANCH_NAME: - active_branch = 'main' - if UPDATE_BRANCH_NAME in branches: - logging.info('Removing previous update branch (%s)', UPDATE_BRANCH_NAME) - subprocess.check_call(['git', 'checkout', active_branch]) - subprocess.check_call(['git', 'branch', '-D', UPDATE_BRANCH_NAME]) - logging.info('No branch to remove') +def _remove_previous_update_branch(): + active_branch, branches = _get_branches() + if active_branch == UPDATE_BRANCH_NAME: + active_branch = 'main' + if UPDATE_BRANCH_NAME in branches: + logging.info('Removing previous update branch (%s)', + UPDATE_BRANCH_NAME) + subprocess.check_call(['git', 'checkout', active_branch]) + subprocess.check_call(['git', 'branch', '-D', UPDATE_BRANCH_NAME]) + logging.info('No branch to remove') -def _GetLastAuthor(): - """Returns a string with the author of the last commit.""" - author = subprocess.check_output( - ['git', 'log', '-1', '--pretty=format:"%an"'], - universal_newlines=True).splitlines() - return author +def _get_last_author(): + """Returns a string with the author of the last commit.""" + author = subprocess.check_output( + ['git', 'log', '-1', '--pretty=format:"%an"'], + universal_newlines=True).splitlines() + return author -def _GetBranches(): - """Returns a tuple (active, branches). +def _get_branches(): + """Returns a tuple (active, branches). 'active' is a string with name of the currently active branch, while 'branches' is the list of all branches. """ - lines = subprocess.check_output(['git', 'branch'], - universal_newlines=True).splitlines() - branches = [] - active = '' - for line in lines: - if '*' in line: - # The assumption is that the first char will always be the '*'. - active = line[1:].strip() - branches.append(active) - else: - branch = line.strip() - if branch: - branches.append(branch) - return active, branches + lines = subprocess.check_output(['git', 'branch'], + universal_newlines=True).splitlines() + branches = [] + active = '' + for line in lines: + if '*' in line: + # The assumption is that the first char will always be the '*'. + active = line[1:].strip() + branches.append(active) + else: + branch = line.strip() + if branch: + branches.append(branch) + return active, branches -def _CreateUpdateBranch(): - logging.info('Creating update branch: %s', UPDATE_BRANCH_NAME) - subprocess.check_call(['git', 'checkout', '-b', UPDATE_BRANCH_NAME]) +def _create_update_branch(): + logging.info('Creating update branch: %s', UPDATE_BRANCH_NAME) + subprocess.check_call(['git', 'checkout', '-b', UPDATE_BRANCH_NAME]) -def _UpdateWebRTCVersion(filename): - with open(filename, 'rb') as f: - content = f.read().decode('utf-8') - d = datetime.datetime.utcnow() - # pylint: disable=line-too-long - new_content = re.sub( - r'WebRTC source stamp [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}', - r'WebRTC source stamp %02d-%02d-%02dT%02d:%02d:%02d' % - (d.year, d.month, d.day, d.hour, d.minute, d.second), - content, - flags=re.MULTILINE) - # pylint: enable=line-too-long - with open(filename, 'wb') as f: - f.write(new_content.encode('utf-8')) +def _update_webrtc_version(filename): + with open(filename, 'rb') as file: + content = file.read().decode('utf-8') + date = datetime.datetime.utcnow() + # pylint: disable=line-too-long + new_content = re.sub( + r'WebRTC source stamp [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}', + r'WebRTC source stamp %02d-%02d-%02dT%02d:%02d:%02d' % + (date.year, date.month, date.day, date.hour, date.minute, date.second), + content, + flags=re.MULTILINE) + # pylint: enable=line-too-long + with open(filename, 'wb') as file: + file.write(new_content.encode('utf-8')) -def _IsTreeClean(): - stdout = subprocess.check_output(['git', 'status', '--porcelain'], - universal_newlines=True) - if len(stdout) == 0: - return True - return False +def _is_tree_clean(): + stdout = subprocess.check_output(['git', 'status', '--porcelain'], + universal_newlines=True) + if len(stdout) == 0: + return True + return False -def _LocalCommit(): - logging.info('Committing changes locally.') - d = datetime.datetime.utcnow() +def _local_commit(): + logging.info('Committing changes locally.') + date = datetime.datetime.utcnow() - commit_msg = ('Update WebRTC code version (%02d-%02d-%02dT%02d:%02d:%02d).' - '\n\nBug: None') - commit_msg = commit_msg % (d.year, d.month, d.day, d.hour, d.minute, d.second) - subprocess.check_call(['git', 'add', '--update', '.']) - subprocess.check_call(['git', 'commit', '-m', commit_msg]) + msg = ('Update WebRTC code version (%02d-%02d-%02dT%02d:%02d:%02d).' + '\n\nBug: None') + msg = msg % (date.year, date.month, date.day, date.hour, date.minute, + date.second) + subprocess.check_call(['git', 'add', '--update', '.']) + subprocess.check_call(['git', 'commit', '-m', msg]) -def _UploadCL(commit_queue_mode): - """Upload the committed changes as a changelist to Gerrit. +def _upload_cl(commit_queue_mode): + """Upload the committed changes as a changelist to Gerrit. commit_queue_mode: - 2: Submit to commit queue. - 1: Run trybots but do not submit to CQ. - 0: Skip CQ, upload only. """ - cmd = [ - 'git', 'cl', 'upload', '--force', '--bypass-hooks', '--bypass-watchlist' - ] - if commit_queue_mode >= 2: - logging.info('Sending the CL to the CQ...') - cmd.extend(['-o', 'label=Bot-Commit+1']) - cmd.extend(['-o', 'label=Commit-Queue+2']) - cmd.extend(['--send-mail', '--cc', NOTIFY_EMAIL]) - elif commit_queue_mode >= 1: - logging.info('Starting CQ dry run...') - cmd.extend(['-o', 'label=Commit-Queue+1']) - subprocess.check_call(cmd) + cmd = [ + 'git', 'cl', 'upload', '--force', '--bypass-hooks', + '--bypass-watchlist' + ] + if commit_queue_mode >= 2: + logging.info('Sending the CL to the CQ...') + cmd.extend(['-o', 'label=Bot-Commit+1']) + cmd.extend(['-o', 'label=Commit-Queue+2']) + cmd.extend(['--send-mail', '--cc', NOTIFY_EMAIL]) + elif commit_queue_mode >= 1: + logging.info('Starting CQ dry run...') + cmd.extend(['-o', 'label=Commit-Queue+1']) + subprocess.check_call(cmd) def main(): - logging.basicConfig(level=logging.INFO) - p = argparse.ArgumentParser() - p.add_argument('--clean', - action='store_true', - default=False, - help='Removes any previous local update branch.') - opts = p.parse_args() + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() + parser.add_argument('--clean', + action='store_true', + default=False, + help='Removes any previous local update branch.') + opts = parser.parse_args() - if opts.clean: - _RemovePreviousUpdateBranch() + if opts.clean: + _remove_previous_update_branch() - if _GetLastAuthor() == 'webrtc-version-updater': - logging.info('Last commit is a version change, skipping CL.') + if _get_last_author() == 'webrtc-version-updater': + logging.info('Last commit is a version change, skipping CL.') + return 0 + + version_filename = os.path.join(CHECKOUT_SRC_DIR, 'call', 'version.cc') + _create_update_branch() + _update_webrtc_version(version_filename) + if _is_tree_clean(): + logging.info('No WebRTC version change detected, skipping CL.') + else: + _local_commit() + logging.info('Uploading CL...') + _upload_cl(2) return 0 - version_filename = os.path.join(CHECKOUT_SRC_DIR, 'call', 'version.cc') - _CreateUpdateBranch() - _UpdateWebRTCVersion(version_filename) - if _IsTreeClean(): - logging.info('No WebRTC version change detected, skipping CL.') - else: - _LocalCommit() - logging.info('Uploading CL...') - _UploadCL(2) - return 0 - if __name__ == '__main__': - sys.exit(main()) + sys.exit(main())