admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make update_version.py and build_helpers(_test).py pylint compliant

Browse Source 
Trying to change any one of these files will make presubmits
complain that the file isn't properly formatted. Format and rename
variables to be PEP-8 and pylint compliant.

Bug: b/333744051
Change-Id: I8dd4f7f05e52777a62b49659a3c264fe28926539
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/358160
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42700}
This commit is contained in:
Gavin Mak 2024-07-30 20:14:40 +00:00 committed by WebRTC LUCI CQ
parent 4ddd931023
commit bde30d393b
3 changed files with 188 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
tools_webrtc/presubmit_checks_lib/build_helpers.py
View File

@ -7,8 +7,8 @@
# tree. An additional intellectual property rights grant can be found # tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may # in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree. # be found in the AUTHORS file in the root of the source tree.
"""This script helps to invoke gn and ninja """This script helps to invoke gn and ninja which lie in depot_tools
which lie in depot_tools repository.""" repository."""
import json import json
import os import os
@ -19,30 +19,30 @@ import sys
import tempfile import tempfile
def FindSrcDirPath(): def find_src_dir_path():
"""Returns the abs path to the src/ dir of the project.""" """Returns the abs path to the src/ dir of the project."""
src_dir = os.path.dirname(os.path.abspath(__file__)) src_dir = os.path.dirname(os.path.abspath(__file__))
while os.path.basename(src_dir) != 'src': while os.path.basename(src_dir) != 'src':
src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
return src_dir return src_dir
SRC_DIR = FindSrcDirPath() SRC_DIR = find_src_dir_path()
sys.path.append(os.path.join(SRC_DIR, 'build')) sys.path.append(os.path.join(SRC_DIR, 'build'))
import find_depot_tools import find_depot_tools
def RunGnCommand(args, root_dir=None): def run_gn_command(args, root_dir=None):
"""Runs `gn` with provided args and return error if any.""" """Runs `gn` with provided args and return error if any."""
try: try:
command = [ command = [
sys.executable, sys.executable,
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py') os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py')
] + args ] + args
subprocess.check_output(command, cwd=root_dir) subprocess.check_output(command, cwd=root_dir)
except subprocess.CalledProcessError as err: except subprocess.CalledProcessError as err:
return err.output return err.output
return None return None
# GN_ERROR_RE matches the summary of an error output by `gn check`. # GN_ERROR_RE matches the summary of an error output by `gn check`.
@ -51,50 +51,50 @@ def RunGnCommand(args, root_dir=None):
GN_ERROR_RE = re.compile(r'^ERROR .+(?:\n.*[^_\n].*$)+', re.MULTILINE) GN_ERROR_RE = re.compile(r'^ERROR .+(?:\n.*[^_\n].*$)+', re.MULTILINE)
def RunGnCheck(root_dir=None): def run_gn_check(root_dir=None):
"""Runs `gn gen --check` with default args to detect mismatches between """Runs `gn gen --check` with default args to detect mismatches between
#includes and dependencies in the BUILD.gn files, as well as general build #includes and dependencies in the BUILD.gn files, as well as general build
errors. errors.
Returns a list of error summary strings. Returns a list of error summary strings.
""" """
out_dir = tempfile.mkdtemp('gn') out_dir = tempfile.mkdtemp('gn')
try: try:
error = RunGnCommand(['gen', '--check', out_dir], root_dir) error = run_gn_command(['gen', '--check', out_dir], root_dir)
finally: finally:
shutil.rmtree(out_dir, ignore_errors=True) shutil.rmtree(out_dir, ignore_errors=True)
return GN_ERROR_RE.findall(error.decode('utf-8')) if error else [] return GN_ERROR_RE.findall(error.decode('utf-8')) if error else []
def RunNinjaCommand(args, root_dir=None): def run_ninja_command(args, root_dir=None):
"""Runs ninja quietly. Any failure (e.g. clang not found) is """Runs ninja quietly. Any failure (e.g. clang not found) is
silently discarded, since this is unlikely an error in submitted CL.""" silently discarded, since this is unlikely an error in submitted CL."""
command = [os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja')] + args command = [os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja')] + args
p = subprocess.Popen(command, proc = subprocess.Popen(command,
cwd=root_dir, cwd=root_dir,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE)
out, _ = p.communicate() out, _ = proc.communicate()
return out return out
def GetClangTidyPath(): def get_clang_tidy_path():
"""POC/WIP! Use the one we have, even it doesn't match clang's version.""" """POC/WIP! Use the one we have, even it doesn't match clang's version."""
tidy = ('third_party/android_toolchain/toolchains/' tidy = ('third_party/android_toolchain/toolchains/'
'llvm/prebuilt/linux-x86_64/bin/clang-tidy') 'llvm/prebuilt/linux-x86_64/bin/clang-tidy')
return os.path.join(SRC_DIR, tidy) return os.path.join(SRC_DIR, tidy)
def GetCompilationDb(root_dir=None): def get_compilation_db(root_dir=None):
"""Run ninja compdb tool to get proper flags, defines and include paths.""" """Run ninja compdb tool to get proper flags, defines and include paths."""
# The compdb tool expect a rule. # The compdb tool expect a rule.
commands = json.loads(RunNinjaCommand(['-t', 'compdb', 'cxx'], root_dir)) commands = json.loads(run_ninja_command(['-t', 'compdb', 'cxx'], root_dir))
# Turns 'file' field into a key. # Turns 'file' field into a key.
return {v['file']: v for v in commands} return {v['file']: v for v in commands}
def GetCompilationCommand(filepath, gn_args, work_dir): def get_compilation_command(filepath, gn_args, work_dir):
"""Get the whole command used to compile one cc file. """Get the whole command used to compile one cc file.
Typically, clang++ with flags, defines and include paths. Typically, clang++ with flags, defines and include paths.
Args: Args:
@ -105,30 +105,30 @@ def GetCompilationCommand(filepath, gn_args, work_dir):
Returns: Returns:
Command as a list, ready to be consumed by subprocess.Popen. Command as a list, ready to be consumed by subprocess.Popen.
""" """
gn_errors = RunGnCommand(['gen'] + gn_args + [work_dir]) gn_errors = run_gn_command(['gen'] + gn_args + [work_dir])
if gn_errors: if gn_errors:
raise RuntimeError('FYI, cannot complete check due to gn error:\n%s\n' raise RuntimeError('FYI, cannot complete check due to gn error:\n%s\n'
'Please open a bug.' % gn_errors) 'Please open a bug.' % gn_errors)
# Needed for single file compilation. # Needed for single file compilation.
commands = GetCompilationDb(work_dir) commands = get_compilation_db(work_dir)
# Path as referenced by ninja. # Path as referenced by ninja.
rel_path = os.path.relpath(os.path.abspath(filepath), work_dir) rel_path = os.path.relpath(os.path.abspath(filepath), work_dir)
# Gather defines, include path and flags (such as -std=c++11). # Gather defines, include path and flags (such as -std=c++11).
try: try:
compilation_entry = commands[rel_path] compilation_entry = commands[rel_path]
except KeyError as not_found: except KeyError as not_found:
raise ValueError('%s: Not found in compilation database.\n' raise ValueError('%s: Not found in compilation database.\n'
'Please check the path.' % filepath) from not_found 'Please check the path.' % filepath) from not_found
command = compilation_entry['command'].split() command = compilation_entry['command'].split()
# Remove troublesome flags. May trigger an error otherwise. # Remove troublesome flags. May trigger an error otherwise.
if '-MMD' in command: if '-MMD' in command:
command.remove('-MMD') command.remove('-MMD')
if '-MF' in command: if '-MF' in command:
index = command.index('-MF') index = command.index('-MF')
del command[index:index + 2] # Remove filename as well. del command[index:index + 2] # Remove filename as well.
return command return command

14
tools_webrtc/presubmit_checks_lib/build_helpers_test.py
View File

@ -20,13 +20,13 @@ TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
class GnCheckTest(unittest.TestCase): class GnCheckTest(unittest.TestCase):
def testCircularDependencyError(self): def test_circular_dependency_error(self):
test_dir = os.path.join(TESTDATA_DIR, 'circular_dependency') test_dir = os.path.join(TESTDATA_DIR, 'circular_dependency')
expected_error = re.compile('ERROR Dependency cycle') expected_error = re.compile('ERROR Dependency cycle')
gn_output = build_helpers.RunGnCheck(test_dir) gn_output = build_helpers.run_gn_check(test_dir)
self.assertEqual(1, len(gn_output)) self.assertEqual(1, len(gn_output))
self.assertRegex(gn_output[0], expected_error) self.assertRegex(gn_output[0], expected_error)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

215
tools_webrtc/version_updater/update_version.py
View File

@ -19,150 +19,153 @@ import subprocess
import sys import sys
def FindSrcDirPath(): def find_src_dir_path():
"""Returns the abs path to the src/ dir of the project.""" """Returns the abs path to the src/ dir of the project."""
src_dir = os.path.dirname(os.path.abspath(__file__)) src_dir = os.path.dirname(os.path.abspath(__file__))
while os.path.basename(src_dir) != 'src': while os.path.basename(src_dir) != 'src':
src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
return src_dir return src_dir
UPDATE_BRANCH_NAME = 'webrtc_version_update' UPDATE_BRANCH_NAME = 'webrtc_version_update'
CHECKOUT_SRC_DIR = FindSrcDirPath() CHECKOUT_SRC_DIR = find_src_dir_path()
NOTIFY_EMAIL = 'webrtc-trooper@webrtc.org' NOTIFY_EMAIL = 'webrtc-trooper@webrtc.org'
def _RemovePreviousUpdateBranch(): def _remove_previous_update_branch():
active_branch, branches = _GetBranches() active_branch, branches = _get_branches()
if active_branch == UPDATE_BRANCH_NAME: if active_branch == UPDATE_BRANCH_NAME:
active_branch = 'main' active_branch = 'main'
if UPDATE_BRANCH_NAME in branches: if UPDATE_BRANCH_NAME in branches:
logging.info('Removing previous update branch (%s)', UPDATE_BRANCH_NAME) logging.info('Removing previous update branch (%s)',
subprocess.check_call(['git', 'checkout', active_branch]) UPDATE_BRANCH_NAME)
subprocess.check_call(['git', 'branch', '-D', UPDATE_BRANCH_NAME]) subprocess.check_call(['git', 'checkout', active_branch])
logging.info('No branch to remove') subprocess.check_call(['git', 'branch', '-D', UPDATE_BRANCH_NAME])
logging.info('No branch to remove')
def _GetLastAuthor(): def _get_last_author():
"""Returns a string with the author of the last commit.""" """Returns a string with the author of the last commit."""
author = subprocess.check_output( author = subprocess.check_output(
['git', 'log', '-1', '--pretty=format:"%an"'], ['git', 'log', '-1', '--pretty=format:"%an"'],
universal_newlines=True).splitlines() universal_newlines=True).splitlines()
return author return author
def _GetBranches(): def _get_branches():
"""Returns a tuple (active, branches). """Returns a tuple (active, branches).
'active' is a string with name of the currently active branch, while 'active' is a string with name of the currently active branch, while
'branches' is the list of all branches. 'branches' is the list of all branches.
""" """
lines = subprocess.check_output(['git', 'branch'], lines = subprocess.check_output(['git', 'branch'],
universal_newlines=True).splitlines() universal_newlines=True).splitlines()
branches = [] branches = []
active = '' active = ''
for line in lines: for line in lines:
if '*' in line: if '*' in line:
# The assumption is that the first char will always be the '*'. # The assumption is that the first char will always be the '*'.
active = line[1:].strip() active = line[1:].strip()
branches.append(active) branches.append(active)
else: else:
branch = line.strip() branch = line.strip()
if branch: if branch:
branches.append(branch) branches.append(branch)
return active, branches return active, branches
def _CreateUpdateBranch(): def _create_update_branch():
logging.info('Creating update branch: %s', UPDATE_BRANCH_NAME) logging.info('Creating update branch: %s', UPDATE_BRANCH_NAME)
subprocess.check_call(['git', 'checkout', '-b', UPDATE_BRANCH_NAME]) subprocess.check_call(['git', 'checkout', '-b', UPDATE_BRANCH_NAME])
def _UpdateWebRTCVersion(filename): def _update_webrtc_version(filename):
with open(filename, 'rb') as f: with open(filename, 'rb') as file:
content = f.read().decode('utf-8') content = file.read().decode('utf-8')
d = datetime.datetime.utcnow() date = datetime.datetime.utcnow()
# pylint: disable=line-too-long # pylint: disable=line-too-long
new_content = re.sub( new_content = re.sub(
r'WebRTC source stamp [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}', r'WebRTC source stamp [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}',
r'WebRTC source stamp %02d-%02d-%02dT%02d:%02d:%02d' % r'WebRTC source stamp %02d-%02d-%02dT%02d:%02d:%02d' %
(d.year, d.month, d.day, d.hour, d.minute, d.second), (date.year, date.month, date.day, date.hour, date.minute, date.second),
content, content,
flags=re.MULTILINE) flags=re.MULTILINE)
# pylint: enable=line-too-long # pylint: enable=line-too-long
with open(filename, 'wb') as f: with open(filename, 'wb') as file:
f.write(new_content.encode('utf-8')) file.write(new_content.encode('utf-8'))
def _IsTreeClean(): def _is_tree_clean():
stdout = subprocess.check_output(['git', 'status', '--porcelain'], stdout = subprocess.check_output(['git', 'status', '--porcelain'],
universal_newlines=True) universal_newlines=True)
if len(stdout) == 0: if len(stdout) == 0:
return True return True
return False return False
def _LocalCommit(): def _local_commit():
logging.info('Committing changes locally.') logging.info('Committing changes locally.')
d = datetime.datetime.utcnow() date = datetime.datetime.utcnow()
commit_msg = ('Update WebRTC code version (%02d-%02d-%02dT%02d:%02d:%02d).' msg = ('Update WebRTC code version (%02d-%02d-%02dT%02d:%02d:%02d).'
'\n\nBug: None') '\n\nBug: None')
commit_msg = commit_msg % (d.year, d.month, d.day, d.hour, d.minute, d.second) msg = msg % (date.year, date.month, date.day, date.hour, date.minute,
subprocess.check_call(['git', 'add', '--update', '.']) date.second)
subprocess.check_call(['git', 'commit', '-m', commit_msg]) subprocess.check_call(['git', 'add', '--update', '.'])
subprocess.check_call(['git', 'commit', '-m', msg])
def _UploadCL(commit_queue_mode): def _upload_cl(commit_queue_mode):
"""Upload the committed changes as a changelist to Gerrit. """Upload the committed changes as a changelist to Gerrit.
commit_queue_mode: commit_queue_mode:
- 2: Submit to commit queue. - 2: Submit to commit queue.
- 1: Run trybots but do not submit to CQ. - 1: Run trybots but do not submit to CQ.
- 0: Skip CQ, upload only. - 0: Skip CQ, upload only.
""" """
cmd = [ cmd = [
'git', 'cl', 'upload', '--force', '--bypass-hooks', '--bypass-watchlist' 'git', 'cl', 'upload', '--force', '--bypass-hooks',
] '--bypass-watchlist'
if commit_queue_mode >= 2: ]
logging.info('Sending the CL to the CQ...') if commit_queue_mode >= 2:
cmd.extend(['-o', 'label=Bot-Commit+1']) logging.info('Sending the CL to the CQ...')
cmd.extend(['-o', 'label=Commit-Queue+2']) cmd.extend(['-o', 'label=Bot-Commit+1'])
cmd.extend(['--send-mail', '--cc', NOTIFY_EMAIL]) cmd.extend(['-o', 'label=Commit-Queue+2'])
elif commit_queue_mode >= 1: cmd.extend(['--send-mail', '--cc', NOTIFY_EMAIL])
logging.info('Starting CQ dry run...') elif commit_queue_mode >= 1:
cmd.extend(['-o', 'label=Commit-Queue+1']) logging.info('Starting CQ dry run...')
subprocess.check_call(cmd) cmd.extend(['-o', 'label=Commit-Queue+1'])
subprocess.check_call(cmd)
def main(): def main():
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
p = argparse.ArgumentParser() parser = argparse.ArgumentParser()
p.add_argument('--clean', parser.add_argument('--clean',
action='store_true', action='store_true',
default=False, default=False,
help='Removes any previous local update branch.') help='Removes any previous local update branch.')
opts = p.parse_args() opts = parser.parse_args()
if opts.clean: if opts.clean:
_RemovePreviousUpdateBranch() _remove_previous_update_branch()
if _GetLastAuthor() == 'webrtc-version-updater': if _get_last_author() == 'webrtc-version-updater':
logging.info('Last commit is a version change, skipping CL.') logging.info('Last commit is a version change, skipping CL.')
return 0
version_filename = os.path.join(CHECKOUT_SRC_DIR, 'call', 'version.cc')
_create_update_branch()
_update_webrtc_version(version_filename)
if _is_tree_clean():
logging.info('No WebRTC version change detected, skipping CL.')
else:
_local_commit()
logging.info('Uploading CL...')
_upload_cl(2)
return 0 return 0
version_filename = os.path.join(CHECKOUT_SRC_DIR, 'call', 'version.cc')
_CreateUpdateBranch()
_UpdateWebRTCVersion(version_filename)
if _IsTreeClean():
logging.info('No WebRTC version change detected, skipping CL.')
else:
_LocalCommit()
logging.info('Uploading CL...')
_UploadCL(2)
return 0
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main()) sys.exit(main())