diff --git a/tools/autoroller/roll_chromium_revision.py b/tools/autoroller/roll_chromium_revision.py index d6457d0fa3..787e3db7fb 100755 --- a/tools/autoroller/roll_chromium_revision.py +++ b/tools/autoroller/roll_chromium_revision.py @@ -20,6 +20,13 @@ import sys import urllib +# Skip these dependencies (list without solution name prefix). +DONT_AUTOROLL_THESE = [ + 'src/third_party/gflags/src', + 'src/third_party/winsdk_samples/src', +] + +WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc' CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' @@ -30,12 +37,15 @@ CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$') ROLL_BRANCH_NAME = 'roll_chromium_revision' SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, - os.pardir)) -sys.path.append(CHECKOUT_ROOT_DIR) +CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, + os.pardir)) +CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) +CHROMIUM_CHECKOUT_SRC_DIR = os.path.join(CHECKOUT_SRC_DIR, 'chromium', 'src') + +sys.path.append(CHECKOUT_SRC_DIR) import setup_links -sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'build')) +sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) import find_depot_tools find_depot_tools.add_depot_tools_to_path() from gclient import GClientKeywords @@ -94,7 +104,7 @@ def _RunCommand(command, working_dir=None, ignore_exit_code=False, Returns: A tuple containing the stdout and stderr outputs as strings. """ - working_dir = working_dir or CHECKOUT_ROOT_DIR + working_dir = working_dir or CHECKOUT_SRC_DIR logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) env = os.environ.copy() if extra_env: @@ -193,7 +203,7 @@ def GetMatchingDepsEntries(depsentry_dict, dir_path): def BuildDepsentryDict(deps_dict): - """Builds a dict of DepsEntry object from a raw parsed deps dict.""" + """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict.""" result = {} def AddDepsEntries(deps_subdict): for path, deps_url in deps_subdict.iteritems(): @@ -203,25 +213,74 @@ def BuildDepsentryDict(deps_dict): AddDepsEntries(deps_dict['deps']) for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: - AddDepsEntries(deps_dict['deps_os'].get(deps_os, {})) + AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) return result -def CalculateChangedDeps(current_deps, new_deps): - result = [] - current_entries = BuildDepsentryDict(current_deps) - new_entries = BuildDepsentryDict(new_deps) +def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps): + """ + Calculate changed deps entries based on: + 1. Entries defined in the WebRTC DEPS file: + - If a shared dependency with the Chromium DEPS file: roll it to the same + revision as Chromium (i.e. entry in the new_cr_deps dict) + - If it's a Chromium sub-directory, roll it to the HEAD revision (notice + this means it may be ahead of the chromium_revision, but generally these + should be close). + - If it's another DEPS entry (not shared with Chromium), roll it to HEAD + unless it's configured to be skipped. + 2. Entries present in the setup_links.py file. If the dir has changed between + old_cr_deps and new_cr_deps, it is considered changed and updated to the + revision for the entry in the new_cr_deps dict. + Returns: + A list of ChangedDep objects representing the changed deps. + """ + return sorted(CalculateChangedDepsProper(webrtc_deps, new_cr_deps) + + CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps)) + + +def CalculateChangedDepsProper(webrtc_deps, new_cr_deps): + result = [] + webrtc_entries = BuildDepsentryDict(webrtc_deps) + new_cr_entries = BuildDepsentryDict(new_cr_deps) + for path, webrtc_deps_entry in webrtc_entries.iteritems(): + if path in DONT_AUTOROLL_THESE: + continue + cr_deps_entry = new_cr_entries.get(path) + if cr_deps_entry: + # Use the revision from Chromium's DEPS file. + new_rev = cr_deps_entry.revision + assert webrtc_deps_entry.url == cr_deps_entry.url, ( + 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' % + (path, webrtc_deps_entry.url, cr_deps_entry.url)) + else: + # Use the HEAD of the deps repo. + stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url, + 'HEAD']) + new_rev = stdout.strip().split('\t')[0] + + # Check if an update is necessary. + if webrtc_deps_entry.revision != new_rev: + logging.debug('Roll dependency %s to %s', path, new_rev) + result.append(ChangedDep(path, webrtc_deps_entry.url, + webrtc_deps_entry.revision, new_rev)) + return result + + +def CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps): + result = [] + new_cr_entries = BuildDepsentryDict(new_cr_deps) + old_cr_entries = BuildDepsentryDict(old_cr_deps) all_deps_dirs = setup_links.DIRECTORIES for deps_dir in all_deps_dirs: # All deps have 'src' prepended to the path in the Chromium DEPS file. dir_path = 'src/%s' % deps_dir - for entry in GetMatchingDepsEntries(current_entries, dir_path): - new_matching_entries = GetMatchingDepsEntries(new_entries, entry.path) + for entry in GetMatchingDepsEntries(old_cr_entries, dir_path): + new_matching_entries = GetMatchingDepsEntries(new_cr_entries, entry.path) assert len(new_matching_entries) <= 1, ( 'Should never find more than one entry matching %s in %s, found %d' % - (entry.path, new_entries, len(new_matching_entries))) + (entry.path, new_cr_entries, len(new_matching_entries))) if not new_matching_entries: result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None')) elif entry != new_matching_entries[0]: @@ -238,7 +297,7 @@ def CalculateChangedClang(new_cr_rev): return match.group(1) raise RollError('Could not parse Clang revision!') - chromium_src_path = os.path.join(CHECKOUT_ROOT_DIR, 'chromium', 'src', + chromium_src_path = os.path.join(CHROMIUM_CHECKOUT_SRC_DIR, CLANG_UPDATE_SCRIPT_LOCAL_PATH) with open(chromium_src_path, 'rb') as f: current_lines = f.readlines() @@ -294,14 +353,26 @@ def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, return '\n'.join(commit_msg) -def UpdateDeps(deps_filename, old_cr_revision, new_cr_revision): +def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, + changed_deps): """Update the DEPS file with the new revision.""" + + # Update the chromium_revision variable. with open(deps_filename, 'rb') as deps_file: deps_content = deps_file.read() deps_content = deps_content.replace(old_cr_revision, new_cr_revision) with open(deps_filename, 'wb') as deps_file: deps_file.write(deps_content) + # Update each individual DEPS entry. + for dep in changed_deps: + _, stderr = _RunCommand( + ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev], + working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True) + if stderr: + logging.warning('roll-dep-svn: %s', stderr) + + def _IsTreeClean(): stdout, _ = _RunCommand(['git', 'status', '--porcelain']) if len(stdout) == 0: @@ -378,9 +449,6 @@ def main(): help=('Calculate changes and modify DEPS, but don\'t create ' 'any local branch, commit, upload CL or send any ' 'tryjobs.')) - p.add_argument('--allow-reverse', action='store_true', default=False, - help=('Allow rolling back in time (disabled by default but ' - 'may be useful to be able do to manually).')) p.add_argument('--skip-cq', action='store_true', default=False, help='Skip sending the CL to the CQ (default: %(default)s)') p.add_argument('-v', '--verbose', action='store_true', default=False, @@ -408,9 +476,9 @@ def main(): logging.info('No revision specified. Using HEAD: %s', head_rev) new_cr_rev = head_rev - deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS') - local_deps = ParseLocalDepsFile(deps_filename) - current_cr_rev = local_deps['vars']['chromium_revision'] + deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') + webrtc_deps = ParseLocalDepsFile(deps_filename) + current_cr_rev = webrtc_deps['vars']['chromium_revision'] current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) @@ -418,23 +486,15 @@ def main(): current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev) new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) - if new_commit_pos > current_commit_pos or opts.allow_reverse: - changed_deps = sorted(CalculateChangedDeps(current_cr_deps, new_cr_deps)) - clang_change = CalculateChangedClang(new_cr_rev) - commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, - current_commit_pos, new_commit_pos, - changed_deps, clang_change) - logging.debug('Commit message:\n%s', commit_msg) - else: - logging.info('Currently pinned chromium_revision: %s (#%s) is newer than ' - '%s (#%s). To roll to older revisions, you must pass the ' - '--allow-reverse flag.\n' - 'Aborting without action.', current_cr_rev, current_commit_pos, - new_cr_rev, new_commit_pos) - return 0 + changed_deps = CalculateChangedDeps(webrtc_deps, current_cr_deps, new_cr_deps) + clang_change = CalculateChangedClang(new_cr_rev) + commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, + current_commit_pos, new_commit_pos, + changed_deps, clang_change) + logging.debug('Commit message:\n%s', commit_msg) _CreateRollBranch(opts.dry_run) - UpdateDeps(deps_filename, current_cr_rev, new_cr_rev) + UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps) _LocalCommit(commit_msg, opts.dry_run) _UploadCL(opts.dry_run, opts.rietveld_email) _SendToCQ(opts.dry_run, opts.skip_cq) diff --git a/tools/autoroller/unittests/DEPS b/tools/autoroller/unittests/DEPS deleted file mode 100644 index 82638f8ee9..0000000000 --- a/tools/autoroller/unittests/DEPS +++ /dev/null @@ -1,6 +0,0 @@ -# Sample DEPS file for testing. - -vars = { - 'chromium_git': 'https://chromium.googlesource.com', - 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', -} diff --git a/tools/autoroller/unittests/roll_chromium_revision_test.py b/tools/autoroller/unittests/roll_chromium_revision_test.py index 7c899e92a9..d7b3e26bde 100755 --- a/tools/autoroller/unittests/roll_chromium_revision_test.py +++ b/tools/autoroller/unittests/roll_chromium_revision_test.py @@ -7,6 +7,7 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. +import glob import os import shutil import sys @@ -17,8 +18,11 @@ import unittest SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir) sys.path.append(PARENT_DIR) -from roll_chromium_revision import ParseDepsDict, UpdateDeps, \ - GetMatchingDepsEntries +import roll_chromium_revision +from roll_chromium_revision import CalculateChangedDepsProper, \ + GetMatchingDepsEntries, ParseDepsDict, \ + ParseLocalDepsFile, UpdateDepsFile + TEST_DATA_VARS = { 'chromium_git': 'https://chromium.googlesource.com', @@ -32,28 +36,65 @@ DEPS_ENTRIES = { 'src/testing/gmock': 'https://gmock.com', } +BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f' +BUILD_NEW_REV = 'HEAD' +BUILDTOOLS_OLD_REV = '64e38f0cebdde27aa0cfb405f330063582f9ac76' +BUILDTOOLS_NEW_REV = '55ad626b08ef971fd82a62b7abb325359542952b' + + +class TestError(Exception): + pass + + +class FakeCmd(object): + def __init__(self): + self.expectations = [] + + def add_expectation(self, *args, **kwargs): + returns = kwargs.pop('_returns', None) + self.expectations.append((args, kwargs, returns)) + + def __call__(self, *args, **kwargs): + if not self.expectations: + raise TestError('Got unexpected\n%s\n%s' % (args, kwargs)) + exp_args, exp_kwargs, exp_returns = self.expectations.pop(0) + if args != exp_args or kwargs != exp_kwargs: + message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs) + message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs) + raise TestError(message) + return exp_returns + class TestRollChromiumRevision(unittest.TestCase): def setUp(self): self._output_dir = tempfile.mkdtemp() - shutil.copy(os.path.join(SCRIPT_DIR, 'DEPS'), self._output_dir) - self._deps_filename = os.path.join(self._output_dir, 'DEPS') + for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')): + shutil.copy(test_file, self._output_dir) + self._webrtc_depsfile = os.path.join(self._output_dir, 'DEPS') + self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old') + self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new') + + self.fake = FakeCmd() + self.old_RunCommand = getattr(roll_chromium_revision, '_RunCommand') + setattr(roll_chromium_revision, '_RunCommand', self.fake) def tearDown(self): shutil.rmtree(self._output_dir, ignore_errors=True) + self.assertEqual(self.fake.expectations, []) + setattr(roll_chromium_revision, '_RunCommand', self.old_RunCommand) - def testUpdateDeps(self): + def testUpdateDepsFile(self): new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111' current_rev = TEST_DATA_VARS['chromium_revision'] - UpdateDeps(self._deps_filename, current_rev, new_rev) - with open(self._deps_filename) as deps_file: + UpdateDepsFile(self._webrtc_depsfile, current_rev, new_rev, []) + with open(self._webrtc_depsfile) as deps_file: deps_contents = deps_file.read() self.assertTrue(new_rev in deps_contents, 'Failed to find %s in\n%s' % (new_rev, deps_contents)) def testParseDepsDict(self): - with open(self._deps_filename) as deps_file: + with open(self._webrtc_depsfile) as deps_file: deps_contents = deps_file.read() local_scope = ParseDepsDict(deps_contents) vars_dict = local_scope['vars'] @@ -62,6 +103,8 @@ class TestRollChromiumRevision(unittest.TestCase): self.assertEquals(vars_dict[variable_name], TEST_DATA_VARS[variable_name]) assertVar('chromium_git') assertVar('chromium_revision') + self.assertEquals(len(local_scope['deps']), 3) + self.assertEquals(len(local_scope['deps_os']), 1) def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self): entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest') @@ -77,5 +120,35 @@ class TestRollChromiumRevision(unittest.TestCase): self.assertEquals(len(entries), 1) self.assertEquals(entries[0], DEPS_ENTRIES['src/build']) + def testCalculateChangedDepsProper(self): + _SetupGitLsRemoteCall(self.fake, + 'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV) + webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) + changed_deps = CalculateChangedDepsProper(webrtc_deps, new_cr_deps) + self.assertEquals(len(changed_deps), 2) + self.assertEquals(changed_deps[0].path, 'src/build') + self.assertEquals(changed_deps[0].current_rev, BUILD_OLD_REV) + self.assertEquals(changed_deps[0].new_rev, BUILD_NEW_REV) + + self.assertEquals(changed_deps[1].path, 'src/buildtools') + self.assertEquals(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV) + self.assertEquals(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV) + + def testCalculateChangedDepsLegacy(self): + old_cr_deps = ParseLocalDepsFile(self._old_cr_depsfile) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) + changed_deps = CalculateChangedDepsProper(old_cr_deps, new_cr_deps) + self.assertEquals(len(changed_deps), 1) + self.assertEquals(changed_deps[0].path, 'src/buildtools') + self.assertEquals(changed_deps[0].current_rev, BUILDTOOLS_OLD_REV) + self.assertEquals(changed_deps[0].new_rev, BUILDTOOLS_NEW_REV) + + +def _SetupGitLsRemoteCall(cmd_fake, url, revision): + cmd = ['git', 'ls-remote', url, revision] + cmd_fake.add_expectation(cmd, _returns=(revision, None)) + + if __name__ == '__main__': unittest.main() diff --git a/tools/autoroller/unittests/testdata/DEPS b/tools/autoroller/unittests/testdata/DEPS new file mode 100644 index 0000000000..3f76e62e17 --- /dev/null +++ b/tools/autoroller/unittests/testdata/DEPS @@ -0,0 +1,28 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', +} + +deps = { + # Entry that is a directory in Chromium, so we're using a Git subtree mirror for it. + 'src/build': + Var('chromium_git') + '/chromium/src/build' + '@' + '52f7afeca991d96d68cf0507e20dbdd5b845691f', + + # Entry that's also a DEPS entry in the Chromium DEPS file. + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + '64e38f0cebdde27aa0cfb405f330063582f9ac76', + + # Entry only present in WebRTC, not Chromium. + 'src/third_party/gflags/src': + Var('chromium_git') + '/external/github.com/gflags/gflags@03bebcb065c83beff83d50ae025a55a4bf94dfca', +} + +deps_os = { + # Entry only present in WebRTC, not Chromium. + 'win': { + 'src/third_party/winsdk_samples/src': + Var('chromium_git') + '/external/webrtc/deps/third_party/winsdk_samples_v71@e71b549167a665d7424d6f1dadfbff4b4aad1589', + }, +} diff --git a/tools/autoroller/unittests/testdata/DEPS.chromium.new b/tools/autoroller/unittests/testdata/DEPS.chromium.new new file mode 100644 index 0000000000..d53083ceec --- /dev/null +++ b/tools/autoroller/unittests/testdata/DEPS.chromium.new @@ -0,0 +1,13 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + + # This is updated compared to the DEPS.chromium.old file. + 'buildtools_revision': '55ad626b08ef971fd82a62b7abb325359542952b', +} + +deps = { + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'), +} diff --git a/tools/autoroller/unittests/testdata/DEPS.chromium.old b/tools/autoroller/unittests/testdata/DEPS.chromium.old new file mode 100644 index 0000000000..dd6ddaec81 --- /dev/null +++ b/tools/autoroller/unittests/testdata/DEPS.chromium.old @@ -0,0 +1,13 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + + # This is and older revision than DEPS.chromium.new file. + 'buildtools_revision': '64e38f0cebdde27aa0cfb405f330063582f9ac76', +} + +deps = { + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'), +}