Make update_version.py and build_helpers(_test).py pylint compliant
Trying to change any one of these files will make presubmits complain that the file isn't properly formatted. Format and rename variables to be PEP-8 and pylint compliant. Bug: b/333744051 Change-Id: I8dd4f7f05e52777a62b49659a3c264fe28926539 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/358160 Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org> Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org> Cr-Commit-Position: refs/heads/main@{#42700}
This commit is contained in:
Gavin Mak
WebRTC LUCI CQ
parent
4ddd931023
commit
bde30d393b
@ -7,8 +7,8 @@
|
||||
# tree. An additional intellectual property rights grant can be found
|
||||
# in the file PATENTS. All contributing project authors may
|
||||
# be found in the AUTHORS file in the root of the source tree.
|
||||
"""This script helps to invoke gn and ninja
|
||||
which lie in depot_tools repository."""
|
||||
"""This script helps to invoke gn and ninja which lie in depot_tools
|
||||
repository."""
|
||||
|
||||
import json
|
||||
import os
|
||||
@ -19,30 +19,30 @@ import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
def FindSrcDirPath():
|
||||
"""Returns the abs path to the src/ dir of the project."""
|
||||
src_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
while os.path.basename(src_dir) != 'src':
|
||||
src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
|
||||
return src_dir
|
||||
def find_src_dir_path():
|
||||
"""Returns the abs path to the src/ dir of the project."""
|
||||
src_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
while os.path.basename(src_dir) != 'src':
|
||||
src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
|
||||
return src_dir
|
||||
|
||||
|
||||
SRC_DIR = FindSrcDirPath()
|
||||
SRC_DIR = find_src_dir_path()
|
||||
sys.path.append(os.path.join(SRC_DIR, 'build'))
|
||||
import find_depot_tools
|
||||
|
||||
|
||||
def RunGnCommand(args, root_dir=None):
|
||||
"""Runs `gn` with provided args and return error if any."""
|
||||
try:
|
||||
command = [
|
||||
sys.executable,
|
||||
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py')
|
||||
] + args
|
||||
subprocess.check_output(command, cwd=root_dir)
|
||||
except subprocess.CalledProcessError as err:
|
||||
return err.output
|
||||
return None
|
||||
def run_gn_command(args, root_dir=None):
|
||||
"""Runs `gn` with provided args and return error if any."""
|
||||
try:
|
||||
command = [
|
||||
sys.executable,
|
||||
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py')
|
||||
] + args
|
||||
subprocess.check_output(command, cwd=root_dir)
|
||||
except subprocess.CalledProcessError as err:
|
||||
return err.output
|
||||
return None
|
||||
|
||||
|
||||
# GN_ERROR_RE matches the summary of an error output by `gn check`.
|
||||
@ -51,50 +51,50 @@ def RunGnCommand(args, root_dir=None):
|
||||
GN_ERROR_RE = re.compile(r'^ERROR .+(?:\n.*[^_\n].*$)+', re.MULTILINE)
|
||||
|
||||
|
||||
def RunGnCheck(root_dir=None):
|
||||
"""Runs `gn gen --check` with default args to detect mismatches between
|
||||
def run_gn_check(root_dir=None):
|
||||
"""Runs `gn gen --check` with default args to detect mismatches between
|
||||
#includes and dependencies in the BUILD.gn files, as well as general build
|
||||
errors.
|
||||
|
||||
Returns a list of error summary strings.
|
||||
"""
|
||||
out_dir = tempfile.mkdtemp('gn')
|
||||
try:
|
||||
error = RunGnCommand(['gen', '--check', out_dir], root_dir)
|
||||
finally:
|
||||
shutil.rmtree(out_dir, ignore_errors=True)
|
||||
return GN_ERROR_RE.findall(error.decode('utf-8')) if error else []
|
||||
out_dir = tempfile.mkdtemp('gn')
|
||||
try:
|
||||
error = run_gn_command(['gen', '--check', out_dir], root_dir)
|
||||
finally:
|
||||
shutil.rmtree(out_dir, ignore_errors=True)
|
||||
return GN_ERROR_RE.findall(error.decode('utf-8')) if error else []
|
||||
|
||||
|
||||
def RunNinjaCommand(args, root_dir=None):
|
||||
"""Runs ninja quietly. Any failure (e.g. clang not found) is
|
||||
def run_ninja_command(args, root_dir=None):
|
||||
"""Runs ninja quietly. Any failure (e.g. clang not found) is
|
||||
silently discarded, since this is unlikely an error in submitted CL."""
|
||||
command = [os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja')] + args
|
||||
p = subprocess.Popen(command,
|
||||
cwd=root_dir,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
out, _ = p.communicate()
|
||||
return out
|
||||
command = [os.path.join(SRC_DIR, 'third_party', 'ninja', 'ninja')] + args
|
||||
proc = subprocess.Popen(command,
|
||||
cwd=root_dir,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
out, _ = proc.communicate()
|
||||
return out
|
||||
|
||||
|
||||
def GetClangTidyPath():
|
||||
"""POC/WIP! Use the one we have, even it doesn't match clang's version."""
|
||||
tidy = ('third_party/android_toolchain/toolchains/'
|
||||
'llvm/prebuilt/linux-x86_64/bin/clang-tidy')
|
||||
return os.path.join(SRC_DIR, tidy)
|
||||
def get_clang_tidy_path():
|
||||
"""POC/WIP! Use the one we have, even it doesn't match clang's version."""
|
||||
tidy = ('third_party/android_toolchain/toolchains/'
|
||||
'llvm/prebuilt/linux-x86_64/bin/clang-tidy')
|
||||
return os.path.join(SRC_DIR, tidy)
|
||||
|
||||
|
||||
def GetCompilationDb(root_dir=None):
|
||||
"""Run ninja compdb tool to get proper flags, defines and include paths."""
|
||||
# The compdb tool expect a rule.
|
||||
commands = json.loads(RunNinjaCommand(['-t', 'compdb', 'cxx'], root_dir))
|
||||
# Turns 'file' field into a key.
|
||||
return {v['file']: v for v in commands}
|
||||
def get_compilation_db(root_dir=None):
|
||||
"""Run ninja compdb tool to get proper flags, defines and include paths."""
|
||||
# The compdb tool expect a rule.
|
||||
commands = json.loads(run_ninja_command(['-t', 'compdb', 'cxx'], root_dir))
|
||||
# Turns 'file' field into a key.
|
||||
return {v['file']: v for v in commands}
|
||||
|
||||
|
||||
def GetCompilationCommand(filepath, gn_args, work_dir):
|
||||
"""Get the whole command used to compile one cc file.
|
||||
def get_compilation_command(filepath, gn_args, work_dir):
|
||||
"""Get the whole command used to compile one cc file.
|
||||
Typically, clang++ with flags, defines and include paths.
|
||||
|
||||
Args:
|
||||
@ -105,30 +105,30 @@ def GetCompilationCommand(filepath, gn_args, work_dir):
|
||||
Returns:
|
||||
Command as a list, ready to be consumed by subprocess.Popen.
|
||||
"""
|
||||
gn_errors = RunGnCommand(['gen'] + gn_args + [work_dir])
|
||||
if gn_errors:
|
||||
raise RuntimeError('FYI, cannot complete check due to gn error:\n%s\n'
|
||||
'Please open a bug.' % gn_errors)
|
||||
gn_errors = run_gn_command(['gen'] + gn_args + [work_dir])
|
||||
if gn_errors:
|
||||
raise RuntimeError('FYI, cannot complete check due to gn error:\n%s\n'
|
||||
'Please open a bug.' % gn_errors)
|
||||
|
||||
# Needed for single file compilation.
|
||||
commands = GetCompilationDb(work_dir)
|
||||
# Needed for single file compilation.
|
||||
commands = get_compilation_db(work_dir)
|
||||
|
||||
# Path as referenced by ninja.
|
||||
rel_path = os.path.relpath(os.path.abspath(filepath), work_dir)
|
||||
# Path as referenced by ninja.
|
||||
rel_path = os.path.relpath(os.path.abspath(filepath), work_dir)
|
||||
|
||||
# Gather defines, include path and flags (such as -std=c++11).
|
||||
try:
|
||||
compilation_entry = commands[rel_path]
|
||||
except KeyError as not_found:
|
||||
raise ValueError('%s: Not found in compilation database.\n'
|
||||
'Please check the path.' % filepath) from not_found
|
||||
command = compilation_entry['command'].split()
|
||||
# Gather defines, include path and flags (such as -std=c++11).
|
||||
try:
|
||||
compilation_entry = commands[rel_path]
|
||||
except KeyError as not_found:
|
||||
raise ValueError('%s: Not found in compilation database.\n'
|
||||
'Please check the path.' % filepath) from not_found
|
||||
command = compilation_entry['command'].split()
|
||||
|
||||
# Remove troublesome flags. May trigger an error otherwise.
|
||||
if '-MMD' in command:
|
||||
command.remove('-MMD')
|
||||
if '-MF' in command:
|
||||
index = command.index('-MF')
|
||||
del command[index:index + 2] # Remove filename as well.
|
||||
# Remove troublesome flags. May trigger an error otherwise.
|
||||
if '-MMD' in command:
|
||||
command.remove('-MMD')
|
||||
if '-MF' in command:
|
||||
index = command.index('-MF')
|
||||
del command[index:index + 2] # Remove filename as well.
|
||||
|
||||
return command
|
||||
return command
|
||||
|
||||
@ -20,13 +20,13 @@ TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
|
||||
class GnCheckTest(unittest.TestCase):
|
||||
|
||||
def testCircularDependencyError(self):
|
||||
test_dir = os.path.join(TESTDATA_DIR, 'circular_dependency')
|
||||
expected_error = re.compile('ERROR Dependency cycle')
|
||||
gn_output = build_helpers.RunGnCheck(test_dir)
|
||||
self.assertEqual(1, len(gn_output))
|
||||
self.assertRegex(gn_output[0], expected_error)
|
||||
def test_circular_dependency_error(self):
|
||||
test_dir = os.path.join(TESTDATA_DIR, 'circular_dependency')
|
||||
expected_error = re.compile('ERROR Dependency cycle')
|
||||
gn_output = build_helpers.run_gn_check(test_dir)
|
||||
self.assertEqual(1, len(gn_output))
|
||||
self.assertRegex(gn_output[0], expected_error)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
unittest.main()
|
||||
|
||||
@ -19,150 +19,153 @@ import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def FindSrcDirPath():
|
||||
"""Returns the abs path to the src/ dir of the project."""
|
||||
src_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
while os.path.basename(src_dir) != 'src':
|
||||
src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
|
||||
return src_dir
|
||||
def find_src_dir_path():
|
||||
"""Returns the abs path to the src/ dir of the project."""
|
||||
src_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
while os.path.basename(src_dir) != 'src':
|
||||
src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
|
||||
return src_dir
|
||||
|
||||
|
||||
UPDATE_BRANCH_NAME = 'webrtc_version_update'
|
||||
CHECKOUT_SRC_DIR = FindSrcDirPath()
|
||||
CHECKOUT_SRC_DIR = find_src_dir_path()
|
||||
|
||||
NOTIFY_EMAIL = 'webrtc-trooper@webrtc.org'
|
||||
|
||||
|
||||
def _RemovePreviousUpdateBranch():
|
||||
active_branch, branches = _GetBranches()
|
||||
if active_branch == UPDATE_BRANCH_NAME:
|
||||
active_branch = 'main'
|
||||
if UPDATE_BRANCH_NAME in branches:
|
||||
logging.info('Removing previous update branch (%s)', UPDATE_BRANCH_NAME)
|
||||
subprocess.check_call(['git', 'checkout', active_branch])
|
||||
subprocess.check_call(['git', 'branch', '-D', UPDATE_BRANCH_NAME])
|
||||
logging.info('No branch to remove')
|
||||
def _remove_previous_update_branch():
|
||||
active_branch, branches = _get_branches()
|
||||
if active_branch == UPDATE_BRANCH_NAME:
|
||||
active_branch = 'main'
|
||||
if UPDATE_BRANCH_NAME in branches:
|
||||
logging.info('Removing previous update branch (%s)',
|
||||
UPDATE_BRANCH_NAME)
|
||||
subprocess.check_call(['git', 'checkout', active_branch])
|
||||
subprocess.check_call(['git', 'branch', '-D', UPDATE_BRANCH_NAME])
|
||||
logging.info('No branch to remove')
|
||||
|
||||
|
||||
def _GetLastAuthor():
|
||||
"""Returns a string with the author of the last commit."""
|
||||
author = subprocess.check_output(
|
||||
['git', 'log', '-1', '--pretty=format:"%an"'],
|
||||
universal_newlines=True).splitlines()
|
||||
return author
|
||||
def _get_last_author():
|
||||
"""Returns a string with the author of the last commit."""
|
||||
author = subprocess.check_output(
|
||||
['git', 'log', '-1', '--pretty=format:"%an"'],
|
||||
universal_newlines=True).splitlines()
|
||||
return author
|
||||
|
||||
|
||||
def _GetBranches():
|
||||
"""Returns a tuple (active, branches).
|
||||
def _get_branches():
|
||||
"""Returns a tuple (active, branches).
|
||||
|
||||
'active' is a string with name of the currently active branch, while
|
||||
'branches' is the list of all branches.
|
||||
"""
|
||||
lines = subprocess.check_output(['git', 'branch'],
|
||||
universal_newlines=True).splitlines()
|
||||
branches = []
|
||||
active = ''
|
||||
for line in lines:
|
||||
if '*' in line:
|
||||
# The assumption is that the first char will always be the '*'.
|
||||
active = line[1:].strip()
|
||||
branches.append(active)
|
||||
else:
|
||||
branch = line.strip()
|
||||
if branch:
|
||||
branches.append(branch)
|
||||
return active, branches
|
||||
lines = subprocess.check_output(['git', 'branch'],
|
||||
universal_newlines=True).splitlines()
|
||||
branches = []
|
||||
active = ''
|
||||
for line in lines:
|
||||
if '*' in line:
|
||||
# The assumption is that the first char will always be the '*'.
|
||||
active = line[1:].strip()
|
||||
branches.append(active)
|
||||
else:
|
||||
branch = line.strip()
|
||||
if branch:
|
||||
branches.append(branch)
|
||||
return active, branches
|
||||
|
||||
|
||||
def _CreateUpdateBranch():
|
||||
logging.info('Creating update branch: %s', UPDATE_BRANCH_NAME)
|
||||
subprocess.check_call(['git', 'checkout', '-b', UPDATE_BRANCH_NAME])
|
||||
def _create_update_branch():
|
||||
logging.info('Creating update branch: %s', UPDATE_BRANCH_NAME)
|
||||
subprocess.check_call(['git', 'checkout', '-b', UPDATE_BRANCH_NAME])
|
||||
|
||||
|
||||
def _UpdateWebRTCVersion(filename):
|
||||
with open(filename, 'rb') as f:
|
||||
content = f.read().decode('utf-8')
|
||||
d = datetime.datetime.utcnow()
|
||||
# pylint: disable=line-too-long
|
||||
new_content = re.sub(
|
||||
r'WebRTC source stamp [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}',
|
||||
r'WebRTC source stamp %02d-%02d-%02dT%02d:%02d:%02d' %
|
||||
(d.year, d.month, d.day, d.hour, d.minute, d.second),
|
||||
content,
|
||||
flags=re.MULTILINE)
|
||||
# pylint: enable=line-too-long
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(new_content.encode('utf-8'))
|
||||
def _update_webrtc_version(filename):
|
||||
with open(filename, 'rb') as file:
|
||||
content = file.read().decode('utf-8')
|
||||
date = datetime.datetime.utcnow()
|
||||
# pylint: disable=line-too-long
|
||||
new_content = re.sub(
|
||||
r'WebRTC source stamp [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}',
|
||||
r'WebRTC source stamp %02d-%02d-%02dT%02d:%02d:%02d' %
|
||||
(date.year, date.month, date.day, date.hour, date.minute, date.second),
|
||||
content,
|
||||
flags=re.MULTILINE)
|
||||
# pylint: enable=line-too-long
|
||||
with open(filename, 'wb') as file:
|
||||
file.write(new_content.encode('utf-8'))
|
||||
|
||||
|
||||
def _IsTreeClean():
|
||||
stdout = subprocess.check_output(['git', 'status', '--porcelain'],
|
||||
universal_newlines=True)
|
||||
if len(stdout) == 0:
|
||||
return True
|
||||
return False
|
||||
def _is_tree_clean():
|
||||
stdout = subprocess.check_output(['git', 'status', '--porcelain'],
|
||||
universal_newlines=True)
|
||||
if len(stdout) == 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _LocalCommit():
|
||||
logging.info('Committing changes locally.')
|
||||
d = datetime.datetime.utcnow()
|
||||
def _local_commit():
|
||||
logging.info('Committing changes locally.')
|
||||
date = datetime.datetime.utcnow()
|
||||
|
||||
commit_msg = ('Update WebRTC code version (%02d-%02d-%02dT%02d:%02d:%02d).'
|
||||
'\n\nBug: None')
|
||||
commit_msg = commit_msg % (d.year, d.month, d.day, d.hour, d.minute, d.second)
|
||||
subprocess.check_call(['git', 'add', '--update', '.'])
|
||||
subprocess.check_call(['git', 'commit', '-m', commit_msg])
|
||||
msg = ('Update WebRTC code version (%02d-%02d-%02dT%02d:%02d:%02d).'
|
||||
'\n\nBug: None')
|
||||
msg = msg % (date.year, date.month, date.day, date.hour, date.minute,
|
||||
date.second)
|
||||
subprocess.check_call(['git', 'add', '--update', '.'])
|
||||
subprocess.check_call(['git', 'commit', '-m', msg])
|
||||
|
||||
|
||||
def _UploadCL(commit_queue_mode):
|
||||
"""Upload the committed changes as a changelist to Gerrit.
|
||||
def _upload_cl(commit_queue_mode):
|
||||
"""Upload the committed changes as a changelist to Gerrit.
|
||||
|
||||
commit_queue_mode:
|
||||
- 2: Submit to commit queue.
|
||||
- 1: Run trybots but do not submit to CQ.
|
||||
- 0: Skip CQ, upload only.
|
||||
"""
|
||||
cmd = [
|
||||
'git', 'cl', 'upload', '--force', '--bypass-hooks', '--bypass-watchlist'
|
||||
]
|
||||
if commit_queue_mode >= 2:
|
||||
logging.info('Sending the CL to the CQ...')
|
||||
cmd.extend(['-o', 'label=Bot-Commit+1'])
|
||||
cmd.extend(['-o', 'label=Commit-Queue+2'])
|
||||
cmd.extend(['--send-mail', '--cc', NOTIFY_EMAIL])
|
||||
elif commit_queue_mode >= 1:
|
||||
logging.info('Starting CQ dry run...')
|
||||
cmd.extend(['-o', 'label=Commit-Queue+1'])
|
||||
subprocess.check_call(cmd)
|
||||
cmd = [
|
||||
'git', 'cl', 'upload', '--force', '--bypass-hooks',
|
||||
'--bypass-watchlist'
|
||||
]
|
||||
if commit_queue_mode >= 2:
|
||||
logging.info('Sending the CL to the CQ...')
|
||||
cmd.extend(['-o', 'label=Bot-Commit+1'])
|
||||
cmd.extend(['-o', 'label=Commit-Queue+2'])
|
||||
cmd.extend(['--send-mail', '--cc', NOTIFY_EMAIL])
|
||||
elif commit_queue_mode >= 1:
|
||||
logging.info('Starting CQ dry run...')
|
||||
cmd.extend(['-o', 'label=Commit-Queue+1'])
|
||||
subprocess.check_call(cmd)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--clean',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Removes any previous local update branch.')
|
||||
opts = p.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--clean',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Removes any previous local update branch.')
|
||||
opts = parser.parse_args()
|
||||
|
||||
if opts.clean:
|
||||
_RemovePreviousUpdateBranch()
|
||||
if opts.clean:
|
||||
_remove_previous_update_branch()
|
||||
|
||||
if _GetLastAuthor() == 'webrtc-version-updater':
|
||||
logging.info('Last commit is a version change, skipping CL.')
|
||||
if _get_last_author() == 'webrtc-version-updater':
|
||||
logging.info('Last commit is a version change, skipping CL.')
|
||||
return 0
|
||||
|
||||
version_filename = os.path.join(CHECKOUT_SRC_DIR, 'call', 'version.cc')
|
||||
_create_update_branch()
|
||||
_update_webrtc_version(version_filename)
|
||||
if _is_tree_clean():
|
||||
logging.info('No WebRTC version change detected, skipping CL.')
|
||||
else:
|
||||
_local_commit()
|
||||
logging.info('Uploading CL...')
|
||||
_upload_cl(2)
|
||||
return 0
|
||||
|
||||
version_filename = os.path.join(CHECKOUT_SRC_DIR, 'call', 'version.cc')
|
||||
_CreateUpdateBranch()
|
||||
_UpdateWebRTCVersion(version_filename)
|
||||
if _IsTreeClean():
|
||||
logging.info('No WebRTC version change detected, skipping CL.')
|
||||
else:
|
||||
_LocalCommit()
|
||||
logging.info('Uploading CL...')
|
||||
_UploadCL(2)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
sys.exit(main())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user