diff --git a/tools_webrtc/libs/generate_licenses.py b/tools_webrtc/libs/generate_licenses.py index 7a76dbbe74..746694ca28 100755 --- a/tools_webrtc/libs/generate_licenses.py +++ b/tools_webrtc/libs/generate_licenses.py @@ -7,8 +7,18 @@ # tree. An additional intellectual property rights grant can be found # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. +"""Generates license markdown for a prebuilt version of WebRTC. -"""Generates license markdown for a prebuilt version of WebRTC.""" +Licenses are taken from dependent libraries which are determined by +GN desc command `gn desc` on all targets specified via `--target` argument. + +One can see all dependencies by invoking this command: +$ gn.py desc --all --format=json | python -m json.tool +(see "deps" subarray) + +Libraries are mapped to licenses via LIB_TO_LICENSES_DICT dictionary. + +""" import sys @@ -20,20 +30,11 @@ import os import re import subprocess - -def FindSrcDirPath(): - """Returns the abs path to the src/ dir of the project.""" - src_dir = os.path.dirname(os.path.abspath(__file__)) - while os.path.basename(src_dir) != 'src': - src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) - return src_dir - - +# Third_party library to licences mapping. Keys are names of the libraries +# (right after the `third_party/` prefix) LIB_TO_LICENSES_DICT = { 'abseil-cpp': ['third_party/abseil-cpp/LICENSE'], 'android_tools': ['third_party/android_tools/LICENSE'], - 'android_deps': ['third_party/android_deps/libs/' + - 'com_android_support_support_annotations/LICENSE'], 'auto': ['third_party/auto/src/LICENSE.txt'], 'bazel': ['third_party/bazel/LICENSE'], 'boringssl': ['third_party/boringssl/src/LICENSE'], @@ -69,59 +70,112 @@ LIB_TO_LICENSES_DICT = { 'ow2_asm': [], } +# Third_party library _regex_ to licences mapping. Keys are regular expression +# with names of the libraries (right after the `third_party/` prefix) +LIB_REGEX_TO_LICENSES_DICT = { + 'android_deps:android_support_annotations.*': [ + 'third_party/android_deps/libs/' + + 'com_android_support_support_annotations/LICENSE' + ], + + # Internal dependencies, licenses are already included by other dependencies + 'android_deps:com_android_support_support_annotations.*': [], +} + + +def FindSrcDirPath(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir + + SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0])) WEBRTC_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) SRC_DIR = FindSrcDirPath() sys.path.append(os.path.join(SRC_DIR, 'build')) import find_depot_tools -THIRD_PARTY_LIB_REGEX = r'^.*/third_party/([\w\-+]+).*$' +THIRD_PARTY_LIB_SIMPLE_NAME_REGEX = r'^.*/third_party/([\w\-+]+).*$' +THIRD_PARTY_LIB_REGEX_TEMPLATE = r'^.*/third_party/%s$' + class LicenseBuilder(object): - def __init__(self, buildfile_dirs, targets): + def __init__(self, + buildfile_dirs, + targets, + lib_to_licenses_dict=None, + lib_regex_to_licenses_dict=None): + if lib_to_licenses_dict is None: + lib_to_licenses_dict = LIB_TO_LICENSES_DICT + + if lib_regex_to_licenses_dict is None: + lib_regex_to_licenses_dict = LIB_REGEX_TO_LICENSES_DICT + self.buildfile_dirs = buildfile_dirs self.targets = targets + self.lib_to_licenses_dict = lib_to_licenses_dict + self.lib_regex_to_licenses_dict = lib_regex_to_licenses_dict + + self.common_licenses_dict = self.lib_to_licenses_dict.copy() + self.common_licenses_dict.update(self.lib_regex_to_licenses_dict) @staticmethod - def _ParseLibrary(dep): - """ - Returns a regex match containing library name after third_party + def _ParseLibraryName(dep): + """Returns library name after third_party Input one of: //a/b/third_party/libname:c //a/b/third_party/libname:c(//d/e/f:g) //a/b/third_party/libname/c:d(//e/f/g:h) - Outputs match with libname in group 1 or None if this is not a third_party - dependency. + Outputs libname or None if this is not a third_party dependency. """ - return re.match(THIRD_PARTY_LIB_REGEX, dep) + groups = re.match(THIRD_PARTY_LIB_SIMPLE_NAME_REGEX, dep) + return groups.group(1) if groups else None + + def _ParseLibrary(self, dep): + """Returns library simple or regex name that matches `dep` after third_party + + This method matches `dep` dependency against simple names in + LIB_TO_LICENSES_DICT and regular expression names in + LIB_REGEX_TO_LICENSES_DICT keys + + Outputs matched dict key or None if this is not a third_party dependency. + """ + libname = LicenseBuilder._ParseLibraryName(dep) + + for lib_regex in self.lib_regex_to_licenses_dict: + if re.match(THIRD_PARTY_LIB_REGEX_TEMPLATE % lib_regex, dep): + return lib_regex + + return libname @staticmethod def _RunGN(buildfile_dir, target): cmd = [ - sys.executable, - os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), - 'desc', - '--all', - '--format=json', - os.path.abspath(buildfile_dir), - target, + sys.executable, + os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gn.py'), + 'desc', + '--all', + '--format=json', + os.path.abspath(buildfile_dir), + target, ] - logging.debug("Running: %r", cmd) + logging.debug('Running: %r', cmd) output_json = subprocess.check_output(cmd, cwd=WEBRTC_ROOT) - logging.debug("Output: %s", output_json) + logging.debug('Output: %s', output_json) return output_json - @staticmethod - def _GetThirdPartyLibraries(buildfile_dir, target): + def _GetThirdPartyLibraries(self, buildfile_dir, target): output = json.loads(LicenseBuilder._RunGN(buildfile_dir, target)) libraries = set() - for target in output.values(): - third_party_matches = ( - LicenseBuilder._ParseLibrary(dep) for dep in target['deps']) - libraries |= set(match.group(1) for match in third_party_matches if match) + for described_target in output.values(): + third_party_libs = ( + self._ParseLibrary(dep) for dep in described_target['deps']) + libraries |= set(lib for lib in third_party_libs if lib) return libraries def GenerateLicenseText(self, output_dir): @@ -130,13 +184,13 @@ class LicenseBuilder(object): third_party_libs = set() for buildfile in self.buildfile_dirs: for target in self.targets: - third_party_libs |= LicenseBuilder._GetThirdPartyLibraries( - buildfile, target) + third_party_libs |= self._GetThirdPartyLibraries(buildfile, target) assert len(third_party_libs) > 0 - missing_licenses = third_party_libs - set(LIB_TO_LICENSES_DICT.keys()) + missing_licenses = third_party_libs - set(self.common_licenses_dict.keys()) if missing_licenses: - error_msg = 'Missing licenses: %s' % ', '.join(missing_licenses) + error_msg = 'Missing licenses for following third_party targets: %s' % \ + ', '.join(missing_licenses) logging.error(error_msg) raise Exception(error_msg) @@ -144,18 +198,19 @@ class LicenseBuilder(object): license_libs = sorted(third_party_libs) license_libs.insert(0, 'webrtc') - logging.info("List of licenses: %s", ', '.join(license_libs)) + logging.info('List of licenses: %s', ', '.join(license_libs)) # Generate markdown. output_license_file = open(os.path.join(output_dir, 'LICENSE.md'), 'w+') for license_lib in license_libs: - if len(LIB_TO_LICENSES_DICT[license_lib]) == 0: - logging.info("Skipping compile time dependency: %s", license_lib) - continue # Compile time dependency + if len(self.common_licenses_dict[license_lib]) == 0: + logging.info('Skipping compile time or internal dependency: %s', + license_lib) + continue # Compile time dependency output_license_file.write('# %s\n' % license_lib) output_license_file.write('```\n') - for path in LIB_TO_LICENSES_DICT[license_lib]: + for path in self.common_licenses_dict[license_lib]: license_path = os.path.join(WEBRTC_ROOT, path) with open(license_path, 'r') as license_file: license_text = cgi.escape(license_file.read(), quote=True) @@ -168,14 +223,19 @@ class LicenseBuilder(object): def main(): parser = argparse.ArgumentParser(description='Generate WebRTC LICENSE.md') - parser.add_argument('--verbose', action='store_true', default=False, - help='Debug logging.') - parser.add_argument('--target', required=True, action='append', default=[], - help='Name of the GN target to generate a license for') - parser.add_argument('output_dir', - help='Directory to output LICENSE.md to.') - parser.add_argument('buildfile_dirs', nargs="+", - help='Directories containing gn generated ninja files') + parser.add_argument( + '--verbose', action='store_true', default=False, help='Debug logging.') + parser.add_argument( + '--target', + required=True, + action='append', + default=[], + help='Name of the GN target to generate a license for') + parser.add_argument('output_dir', help='Directory to output LICENSE.md to.') + parser.add_argument( + 'buildfile_dirs', + nargs='+', + help='Directories containing gn generated ninja files') args = parser.parse_args() logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) diff --git a/tools_webrtc/libs/generate_licenses_test.py b/tools_webrtc/libs/generate_licenses_test.py index c28a226af2..03f14459c2 100755 --- a/tools_webrtc/libs/generate_licenses_test.py +++ b/tools_webrtc/libs/generate_licenses_test.py @@ -12,8 +12,8 @@ import os import sys -SRC = os.path.abspath(os.path.join( - os.path.dirname((__file__)), os.pardir, os.pardir)) +SRC = os.path.abspath( + os.path.join(os.path.dirname((__file__)), os.pardir, os.pardir)) sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) import unittest @@ -23,6 +23,7 @@ from generate_licenses import LicenseBuilder class TestLicenseBuilder(unittest.TestCase): + @staticmethod def _FakeRunGN(buildfile_dir, target): return """ @@ -38,25 +39,91 @@ class TestLicenseBuilder(unittest.TestCase): } """ - def testParseLibrary(self): - self.assertEquals(LicenseBuilder._ParseLibrary( - '//a/b/third_party/libname1:c').group(1), + def testParseLibraryName(self): + self.assertEquals( + LicenseBuilder._ParseLibraryName('//a/b/third_party/libname1:c'), 'libname1') - self.assertEquals(LicenseBuilder._ParseLibrary( - '//a/b/third_party/libname2:c(d)').group(1), + self.assertEquals( + LicenseBuilder._ParseLibraryName('//a/b/third_party/libname2:c(d)'), 'libname2') - self.assertEquals(LicenseBuilder._ParseLibrary( - '//a/b/third_party/libname3/c:d(e)').group(1), + self.assertEquals( + LicenseBuilder._ParseLibraryName('//a/b/third_party/libname3/c:d(e)'), 'libname3') - self.assertEquals(LicenseBuilder._ParseLibrary('//a/b/not_third_party/c'), - None) + self.assertEquals( + LicenseBuilder._ParseLibraryName('//a/b/not_third_party/c'), None) + + def testParseLibrarySimpleMatch(self): + builder = LicenseBuilder([], [], {}, {}) + self.assertEquals( + builder._ParseLibrary('//a/b/third_party/libname:c'), 'libname') + + def testParseLibraryRegExNoMatchFallbacksToDefaultLibname(self): + lib_dict = { + 'libname:foo.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], lib_dict, {}) + self.assertEquals( + builder._ParseLibrary('//a/b/third_party/libname:bar_java'), 'libname') + + def testParseLibraryRegExMatch(self): + lib_regex_dict = { + 'libname:foo.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEquals( + builder._ParseLibrary('//a/b/third_party/libname:foo_bar_java'), + 'libname:foo.*') + + def testParseLibraryRegExMatchWithSubDirectory(self): + lib_regex_dict = { + 'libname/foo:bar.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEquals( + builder._ParseLibrary('//a/b/third_party/libname/foo:bar_java'), + 'libname/foo:bar.*') + + def testParseLibraryRegExMatchWithStarInside(self): + lib_regex_dict = { + 'libname/foo.*bar.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEquals( + builder._ParseLibrary('//a/b/third_party/libname/fooHAHA:bar_java'), + 'libname/foo.*bar.*') @mock.patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) - def testGetThirdPartyLibraries(self): - self.assertEquals(LicenseBuilder._GetThirdPartyLibraries( - 'out/arm', 'target1'), + def testGetThirdPartyLibrariesWithoutRegex(self): + builder = LicenseBuilder([], [], {}, {}) + self.assertEquals( + builder._GetThirdPartyLibraries('out/arm', 'target1'), set(['libname1', 'libname2', 'libname3'])) + @mock.patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) + def testGetThirdPartyLibrariesWithRegex(self): + lib_regex_dict = { + 'libname2:c.*': ['path/to/LICENSE'], + } + builder = LicenseBuilder([], [], {}, lib_regex_dict) + self.assertEquals( + builder._GetThirdPartyLibraries('out/arm', 'target1'), + set(['libname1', 'libname2:c.*', 'libname3'])) + + @mock.patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) + def testGenerateLicenseTextFailIfUnknownLibrary(self): + lib_dict = { + 'simple_library': ['path/to/LICENSE'], + } + builder = LicenseBuilder(['dummy_dir'], ['dummy_target'], lib_dict, {}) + + with self.assertRaises(Exception) as context: + builder.GenerateLicenseText('dummy/dir') + + self.assertEquals( + context.exception.message, + 'Missing licenses for following third_party targets: ' + 'libname1, libname2, libname3') + if __name__ == '__main__': unittest.main()