diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 8c75942ccf..bc4eaf2721 100755 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -580,7 +580,7 @@ def CheckGnGen(input_api, output_api): """ with _AddToPath(input_api.os_path.join( input_api.PresubmitLocalPath(), 'tools_webrtc', 'presubmit_checks_lib')): - from gn_check import RunGnCheck + from build_helpers import RunGnCheck errors = RunGnCheck(FindSrcDirPath(input_api.PresubmitLocalPath()))[:5] if errors: return [output_api.PresubmitPromptWarning( diff --git a/tools_webrtc/__init__.py b/tools_webrtc/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools_webrtc/clang_tidy.py b/tools_webrtc/clang_tidy.py new file mode 100755 index 0000000000..49a9427f09 --- /dev/null +++ b/tools_webrtc/clang_tidy.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Invoke clang-tidy tool. + +Usage: clang_tidy.py file.cc [clang-tidy-args...] + +Just a proof of concept! +We use an embedded clang-tidy whose version doesn't match clang++. +""" + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +#pylint: disable=relative-import +from presubmit_checks_lib.build_helpers import GetClangTidyPath, \ + GetCompilationCommand + + +# We enable all checkers by default for investigation purpose. +# This includes clang-analyzer-* checks. +# Individual checkers can be disabled via command line options. +# TODO(bugs.webrtc.com/10258): Select checkers relevant to webrtc guidelines. +CHECKER_OPTION = '-checks=*' + + +def Process(filepath, args): + # Build directory is needed to gather compilation flags. + # Create a temporary one (instead of reusing an existing one) + # to keep the CLI simple and unencumbered. + out_dir = tempfile.mkdtemp('clang_tidy') + + try: + gn_args = [] # Use default build. + command = GetCompilationCommand(filepath, gn_args, out_dir) + + # Remove warning flags. They aren't needed and they cause trouble + # when clang-tidy doesn't match most recent clang. + # Same battle for -f (e.g. -fcomplete-member-pointers). + command = [arg for arg in command if not (arg.startswith('-W') or + arg.startswith('-f'))] + + # Path from build dir. + rel_path = os.path.relpath(os.path.abspath(filepath), out_dir) + + # Replace clang++ by clang-tidy + command[0:1] = [GetClangTidyPath(), + CHECKER_OPTION, + rel_path] + args + ['--'] # Separator for clang flags. + print "Running: %s" % ' '.join(command) + # Run from build dir so that relative paths are correct. + p = subprocess.Popen(command, cwd=out_dir, + stdout=sys.stdout, stderr=sys.stderr) + p.communicate() + return p.returncode + finally: + shutil.rmtree(out_dir, ignore_errors=True) + + +def ValidateCC(filepath): + """We can only analyze .cc files. Provide explicit message about that.""" + if filepath.endswith('.cc'): + return filepath + msg = ('%s not supported.\n' + 'For now, we can only analyze translation units (.cc files).' % + filepath) + raise argparse.ArgumentTypeError(msg) + + +def Main(): + description = ( + "Run clang-tidy on single cc file.\n" + "Use flags, defines and include paths as in default debug build.\n" + "WARNING, this is a POC version with rough edges.") + parser = argparse.ArgumentParser(description=description) + parser.add_argument('filepath', + help='Specifies the path of the .cc file to analyze.', + type=ValidateCC) + parser.add_argument('args', + nargs=argparse.REMAINDER, + help='Arguments passed to clang-tidy') + parsed_args = parser.parse_args() + return Process(parsed_args.filepath, parsed_args.args) + + +if __name__ == '__main__': + sys.exit(Main()) diff --git a/tools_webrtc/presubmit_checks_lib/__init__.py b/tools_webrtc/presubmit_checks_lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools_webrtc/presubmit_checks_lib/build_helpers.py b/tools_webrtc/presubmit_checks_lib/build_helpers.py new file mode 100644 index 0000000000..1ad59bfd49 --- /dev/null +++ b/tools_webrtc/presubmit_checks_lib/build_helpers.py @@ -0,0 +1,134 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""This script helps to invoke gn and ninja +which lie in depot_tools repository.""" + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + + +def FindSrcDirPath(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir + + +SRC_DIR = FindSrcDirPath() +sys.path.append(os.path.join(SRC_DIR, 'build')) +import find_depot_tools + + +def RunGnCommand(args, root_dir=None): + """Runs `gn` with provided args and return error if any.""" + try: + command = [ + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py') + ] + args + subprocess.check_output(command, cwd=root_dir) + except subprocess.CalledProcessError as err: + return err.output + return None + + +# GN_ERROR_RE matches the summary of an error output by `gn check`. +# Matches "ERROR" and following lines until it sees an empty line or a line +# containing just underscores. +GN_ERROR_RE = re.compile(r'^ERROR .+(?:\n.*[^_\n].*$)+', re.MULTILINE) + + +def RunGnCheck(root_dir=None): + """Runs `gn gen --check` with default args to detect mismatches between + #includes and dependencies in the BUILD.gn files, as well as general build + errors. + + Returns a list of error summary strings. + """ + out_dir = tempfile.mkdtemp('gn') + try: + error = RunGnCommand(['gen', '--check', out_dir], root_dir) + finally: + shutil.rmtree(out_dir, ignore_errors=True) + return GN_ERROR_RE.findall(error) if error else [] + + +def RunNinjaCommand(args, root_dir=None): + """Runs ninja quietly. Any failure (e.g. clang not found) is + silently discarded, since this is unlikely an error in submitted CL.""" + command = [ + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'ninja') + ] + args + p = subprocess.Popen(command, cwd=root_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, _ = p.communicate() + return out + + +def GetClangTidyPath(): + """POC/WIP! Use the one we have, even it doesn't match clang's version.""" + tidy = ('third_party/android_ndk/toolchains/' + 'llvm/prebuilt/linux-x86_64/bin/clang-tidy') + return os.path.join(SRC_DIR, tidy) + + +def GetCompilationDb(root_dir=None): + """Run ninja compdb tool to get proper flags, defines and include paths.""" + # The compdb tool expect a rule. + commands = json.loads(RunNinjaCommand(['-t', 'compdb', 'cxx'], root_dir)) + # Turns 'file' field into a key. + return {v['file']: v for v in commands} + + +def GetCompilationCommand(filepath, gn_args, work_dir): + """Get the whole command used to compile one cc file. + Typically, clang++ with flags, defines and include paths. + + Args: + filepath: path to .cc file. + gen_args: build configuration for gn. + work_dir: build dir. + + Returns: + Command as a list, ready to be consumed by subprocess.Popen. + """ + gn_errors = RunGnCommand(['gen'] + gn_args + [work_dir]) + if gn_errors: + raise(RuntimeError( + 'FYI, cannot complete check due to gn error:\n%s\n' + 'Please open a bug.' % gn_errors)) + + # Needed for single file compilation. + commands = GetCompilationDb(work_dir) + + # Path as referenced by ninja. + rel_path = os.path.relpath(os.path.abspath(filepath), work_dir) + + # Gather defines, include path and flags (such as -std=c++11). + try: + compilation_entry = commands[rel_path] + except KeyError: + raise ValueError('%s: Not found in compilation database.\n' + 'Please check the path.' % filepath) + command = compilation_entry['command'].split() + + # Remove troublesome flags. May trigger an error otherwise. + if '-MMD' in command: + command.remove('-MMD') + if '-MF' in command: + index = command.index('-MF') + del command[index:index+2] # Remove filename as well. + + return command diff --git a/tools_webrtc/presubmit_checks_lib/gn_check_test.py b/tools_webrtc/presubmit_checks_lib/build_helpers_test.py similarity index 84% rename from tools_webrtc/presubmit_checks_lib/gn_check_test.py rename to tools_webrtc/presubmit_checks_lib/build_helpers_test.py index f7e158cd9b..78973282f9 100755 --- a/tools_webrtc/presubmit_checks_lib/gn_check_test.py +++ b/tools_webrtc/presubmit_checks_lib/build_helpers_test.py @@ -11,7 +11,8 @@ import os import unittest -from gn_check import RunGnCheck +#pylint: disable=relative-import +import build_helpers TESTDATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), @@ -23,7 +24,8 @@ class GnCheckTest(unittest.TestCase): test_dir = os.path.join(TESTDATA_DIR, 'circular_dependency') expected_errors = ['ERROR Dependency cycle:\n' ' //:bar ->\n //:foo ->\n //:bar'] - self.assertListEqual(expected_errors, RunGnCheck(test_dir)) + self.assertListEqual(expected_errors, + build_helpers.RunGnCheck(test_dir)) if __name__ == '__main__': diff --git a/tools_webrtc/presubmit_checks_lib/check_orphan_headers_test.py b/tools_webrtc/presubmit_checks_lib/check_orphan_headers_test.py index 9acf25a517..2dfc18999d 100755 --- a/tools_webrtc/presubmit_checks_lib/check_orphan_headers_test.py +++ b/tools_webrtc/presubmit_checks_lib/check_orphan_headers_test.py @@ -11,6 +11,7 @@ import os import sys import unittest +#pylint: disable=relative-import import check_orphan_headers diff --git a/tools_webrtc/presubmit_checks_lib/check_package_boundaries_test.py b/tools_webrtc/presubmit_checks_lib/check_package_boundaries_test.py index 7a5874cf26..abf232e678 100755 --- a/tools_webrtc/presubmit_checks_lib/check_package_boundaries_test.py +++ b/tools_webrtc/presubmit_checks_lib/check_package_boundaries_test.py @@ -12,6 +12,7 @@ import ast import os import unittest +#pylint: disable=relative-import from check_package_boundaries import CheckPackageBoundaries diff --git a/tools_webrtc/presubmit_checks_lib/gn_check.py b/tools_webrtc/presubmit_checks_lib/gn_check.py deleted file mode 100644 index 459dcd88ee..0000000000 --- a/tools_webrtc/presubmit_checks_lib/gn_check.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -import os -import re -import shutil -import subprocess -import sys -import tempfile - - -def FindSrcDirPath(): - """Returns the abs path to the src/ dir of the project.""" - src_dir = os.path.dirname(os.path.abspath(__file__)) - while os.path.basename(src_dir) != 'src': - src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) - return src_dir - - -SRC_DIR = FindSrcDirPath() -sys.path.append(os.path.join(SRC_DIR, 'build')) -import find_depot_tools - - -# GN_ERROR_RE matches the summary of an error output by `gn check`. -# Matches "ERROR" and following lines until it sees an empty line or a line -# containing just underscores. -GN_ERROR_RE = re.compile(r'^ERROR .+(?:\n.*[^_\n].*$)+', re.MULTILINE) - - -def RunGnCheck(root_dir=None): - """Runs `gn gen --check` with default args to detect mismatches between - #includes and dependencies in the BUILD.gn files, as well as general build - errors. - - Returns a list of error summary strings. - """ - out_dir = tempfile.mkdtemp('gn') - try: - command = [ - sys.executable, - os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), - 'gen', - '--check', - out_dir, - ] - subprocess.check_output(command, cwd=root_dir) - except subprocess.CalledProcessError as err: - return GN_ERROR_RE.findall(err.output) - else: - return [] - finally: - shutil.rmtree(out_dir, ignore_errors=True)