admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Autoroller: Support for rolling individual DEPS entries.

Browse Source 
Add functionality for automatic rolling of individual DEPS
entries. This will make it possible to move away from the
links created by setup_links.py to real DEPS entries.

There are two kinds of such deps we intend to use:
1. Third party dependencies used by both WebRTC and Chromium:
  those are rolled to the same revision as the Chromium DEPS file of the revision passed
  to the script
2. Chromium sub-directories needed for WebRTC (mainly for //build and BUILD.gn
   files of third_party deps): those are rolled to the HEAD revision
Notice that the latter kind could be rolled ahead of the chromium_revision,
but generally these should be close (and if it passes the bots, we don't really mind).

The new functionality can coexist with the old one, to
enable a smooth transition (not everything needs to change at once).

Some of the updating logic was inspired by
https://cs.chromium.org/chromium/build/scripts/slave/recipes/v8/auto_roll_v8_deps.py

Add extensive tests for the logic of figuring out deps changes,
including mocking git ls-remote call to keep the tests hermetic.

BUG=webrtc:5006
NOTRY=True

Review-Url: https://codereview.webrtc.org/2570603003
Cr-Commit-Position: refs/heads/master@{#15570}
This commit is contained in:
kjellander 2016-12-13 03:35:21 -08:00 committed by Commit bot
parent ef16e9960f
commit 38b6dbc793
6 changed files with 232 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
tools/autoroller/roll_chromium_revision.py
View File

@ -20,6 +20,13 @@ import sys
import urllib
# Skip these dependencies (list without solution name prefix).
DONT_AUTOROLL_THESE = [
'src/third_party/gflags/src',
'src/third_party/winsdk_samples/src',
]
WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc'
CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
@ -30,12 +37,15 @@ CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$')
ROLL_BRANCH_NAME = 'roll_chromium_revision'
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
os.pardir))
sys.path.append(CHECKOUT_ROOT_DIR)
CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
os.pardir))
CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
CHROMIUM_CHECKOUT_SRC_DIR = os.path.join(CHECKOUT_SRC_DIR, 'chromium', 'src')
sys.path.append(CHECKOUT_SRC_DIR)
import setup_links
sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'build'))
sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
import find_depot_tools
find_depot_tools.add_depot_tools_to_path()
from gclient import GClientKeywords
@ -94,7 +104,7 @@ def _RunCommand(command, working_dir=None, ignore_exit_code=False,
Returns:
A tuple containing the stdout and stderr outputs as strings.
"""
working_dir = working_dir or CHECKOUT_ROOT_DIR
working_dir = working_dir or CHECKOUT_SRC_DIR
logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
env = os.environ.copy()
if extra_env:
@ -193,7 +203,7 @@ def GetMatchingDepsEntries(depsentry_dict, dir_path):
def BuildDepsentryDict(deps_dict):
"""Builds a dict of DepsEntry object from a raw parsed deps dict."""
"""Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
result = {}
def AddDepsEntries(deps_subdict):
for path, deps_url in deps_subdict.iteritems():
@ -203,25 +213,74 @@ def BuildDepsentryDict(deps_dict):
AddDepsEntries(deps_dict['deps'])
for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
AddDepsEntries(deps_dict['deps_os'].get(deps_os, {}))
AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
return result
def CalculateChangedDeps(current_deps, new_deps):
result = []
current_entries = BuildDepsentryDict(current_deps)
new_entries = BuildDepsentryDict(new_deps)
def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps):
"""
Calculate changed deps entries based on:
1. Entries defined in the WebRTC DEPS file:
- If a shared dependency with the Chromium DEPS file: roll it to the same
revision as Chromium (i.e. entry in the new_cr_deps dict)
- If it's a Chromium sub-directory, roll it to the HEAD revision (notice
this means it may be ahead of the chromium_revision, but generally these
should be close).
- If it's another DEPS entry (not shared with Chromium), roll it to HEAD
unless it's configured to be skipped.
2. Entries present in the setup_links.py file. If the dir has changed between
old_cr_deps and new_cr_deps, it is considered changed and updated to the
revision for the entry in the new_cr_deps dict.
Returns:
A list of ChangedDep objects representing the changed deps.
"""
return sorted(CalculateChangedDepsProper(webrtc_deps, new_cr_deps) +
CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps))
def CalculateChangedDepsProper(webrtc_deps, new_cr_deps):
result = []
webrtc_entries = BuildDepsentryDict(webrtc_deps)
new_cr_entries = BuildDepsentryDict(new_cr_deps)
for path, webrtc_deps_entry in webrtc_entries.iteritems():
if path in DONT_AUTOROLL_THESE:
continue
cr_deps_entry = new_cr_entries.get(path)
if cr_deps_entry:
# Use the revision from Chromium's DEPS file.
new_rev = cr_deps_entry.revision
assert webrtc_deps_entry.url == cr_deps_entry.url, (
'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' %
(path, webrtc_deps_entry.url, cr_deps_entry.url))
else:
# Use the HEAD of the deps repo.
stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url,
'HEAD'])
new_rev = stdout.strip().split('\t')[0]
# Check if an update is necessary.
if webrtc_deps_entry.revision != new_rev:
logging.debug('Roll dependency %s to %s', path, new_rev)
result.append(ChangedDep(path, webrtc_deps_entry.url,
webrtc_deps_entry.revision, new_rev))
return result
def CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps):
result = []
new_cr_entries = BuildDepsentryDict(new_cr_deps)
old_cr_entries = BuildDepsentryDict(old_cr_deps)
all_deps_dirs = setup_links.DIRECTORIES
for deps_dir in all_deps_dirs:
# All deps have 'src' prepended to the path in the Chromium DEPS file.
dir_path = 'src/%s' % deps_dir
for entry in GetMatchingDepsEntries(current_entries, dir_path):
new_matching_entries = GetMatchingDepsEntries(new_entries, entry.path)
for entry in GetMatchingDepsEntries(old_cr_entries, dir_path):
new_matching_entries = GetMatchingDepsEntries(new_cr_entries, entry.path)
assert len(new_matching_entries) <= 1, (
'Should never find more than one entry matching %s in %s, found %d' %
(entry.path, new_entries, len(new_matching_entries)))
(entry.path, new_cr_entries, len(new_matching_entries)))
if not new_matching_entries:
result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None'))
elif entry != new_matching_entries[0]:
@ -238,7 +297,7 @@ def CalculateChangedClang(new_cr_rev):
return match.group(1)
raise RollError('Could not parse Clang revision!')
chromium_src_path = os.path.join(CHECKOUT_ROOT_DIR, 'chromium', 'src',
chromium_src_path = os.path.join(CHROMIUM_CHECKOUT_SRC_DIR,
CLANG_UPDATE_SCRIPT_LOCAL_PATH)
with open(chromium_src_path, 'rb') as f:
current_lines = f.readlines()
@ -294,14 +353,26 @@ def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
return '\n'.join(commit_msg)
def UpdateDeps(deps_filename, old_cr_revision, new_cr_revision):
def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
changed_deps):
"""Update the DEPS file with the new revision."""
# Update the chromium_revision variable.
with open(deps_filename, 'rb') as deps_file:
deps_content = deps_file.read()
deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
with open(deps_filename, 'wb') as deps_file:
deps_file.write(deps_content)
# Update each individual DEPS entry.
for dep in changed_deps:
_, stderr = _RunCommand(
['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev],
working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True)
if stderr:
logging.warning('roll-dep-svn: %s', stderr)
def _IsTreeClean():
stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
if len(stdout) == 0:
@ -378,9 +449,6 @@ def main():
help=('Calculate changes and modify DEPS, but don\'t create '
'any local branch, commit, upload CL or send any '
'tryjobs.'))
p.add_argument('--allow-reverse', action='store_true', default=False,
help=('Allow rolling back in time (disabled by default but '
'may be useful to be able do to manually).'))
p.add_argument('--skip-cq', action='store_true', default=False,
help='Skip sending the CL to the CQ (default: %(default)s)')
p.add_argument('-v', '--verbose', action='store_true', default=False,
@ -408,9 +476,9 @@ def main():
logging.info('No revision specified. Using HEAD: %s', head_rev)
new_cr_rev = head_rev
deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS')
local_deps = ParseLocalDepsFile(deps_filename)
current_cr_rev = local_deps['vars']['chromium_revision']
deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
webrtc_deps = ParseLocalDepsFile(deps_filename)
current_cr_rev = webrtc_deps['vars']['chromium_revision']
current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
@ -418,23 +486,15 @@ def main():
current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
if new_commit_pos > current_commit_pos or opts.allow_reverse:
changed_deps = sorted(CalculateChangedDeps(current_cr_deps, new_cr_deps))
clang_change = CalculateChangedClang(new_cr_rev)
commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
current_commit_pos, new_commit_pos,
changed_deps, clang_change)
logging.debug('Commit message:\n%s', commit_msg)
else:
logging.info('Currently pinned chromium_revision: %s (#%s) is newer than '
'%s (#%s). To roll to older revisions, you must pass the '
'--allow-reverse flag.\n'
'Aborting without action.', current_cr_rev, current_commit_pos,
new_cr_rev, new_commit_pos)
return 0
changed_deps = CalculateChangedDeps(webrtc_deps, current_cr_deps, new_cr_deps)
clang_change = CalculateChangedClang(new_cr_rev)
commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
current_commit_pos, new_commit_pos,
changed_deps, clang_change)
logging.debug('Commit message:\n%s', commit_msg)
_CreateRollBranch(opts.dry_run)
UpdateDeps(deps_filename, current_cr_rev, new_cr_rev)
UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
_LocalCommit(commit_msg, opts.dry_run)
_UploadCL(opts.dry_run, opts.rietveld_email)
_SendToCQ(opts.dry_run, opts.skip_cq)

6
tools/autoroller/unittests/DEPS
View File

@ -1,6 +0,0 @@
# Sample DEPS file for testing.
vars = {
'chromium_git': 'https://chromium.googlesource.com',
'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
}

89
tools/autoroller/unittests/roll_chromium_revision_test.py
View File

@ -7,6 +7,7 @@
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
import glob
import os
import shutil
import sys
@ -17,8 +18,11 @@ import unittest
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
sys.path.append(PARENT_DIR)
from roll_chromium_revision import ParseDepsDict, UpdateDeps, \
GetMatchingDepsEntries
import roll_chromium_revision
from roll_chromium_revision import CalculateChangedDepsProper, \
GetMatchingDepsEntries, ParseDepsDict, \
ParseLocalDepsFile, UpdateDepsFile
TEST_DATA_VARS = {
'chromium_git': 'https://chromium.googlesource.com',
@ -32,28 +36,65 @@ DEPS_ENTRIES = {
'src/testing/gmock': 'https://gmock.com',
}
BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f'
BUILD_NEW_REV = 'HEAD'
BUILDTOOLS_OLD_REV = '64e38f0cebdde27aa0cfb405f330063582f9ac76'
BUILDTOOLS_NEW_REV = '55ad626b08ef971fd82a62b7abb325359542952b'
class TestError(Exception):
pass
class FakeCmd(object):
def __init__(self):
self.expectations = []
def add_expectation(self, *args, **kwargs):
returns = kwargs.pop('_returns', None)
self.expectations.append((args, kwargs, returns))
def __call__(self, *args, **kwargs):
if not self.expectations:
raise TestError('Got unexpected\n%s\n%s' % (args, kwargs))
exp_args, exp_kwargs, exp_returns = self.expectations.pop(0)
if args != exp_args or kwargs != exp_kwargs:
message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs)
message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs)
raise TestError(message)
return exp_returns
class TestRollChromiumRevision(unittest.TestCase):
def setUp(self):
self._output_dir = tempfile.mkdtemp()
shutil.copy(os.path.join(SCRIPT_DIR, 'DEPS'), self._output_dir)
self._deps_filename = os.path.join(self._output_dir, 'DEPS')
for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
shutil.copy(test_file, self._output_dir)
self._webrtc_depsfile = os.path.join(self._output_dir, 'DEPS')
self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old')
self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new')
self.fake = FakeCmd()
self.old_RunCommand = getattr(roll_chromium_revision, '_RunCommand')
setattr(roll_chromium_revision, '_RunCommand', self.fake)
def tearDown(self):
shutil.rmtree(self._output_dir, ignore_errors=True)
self.assertEqual(self.fake.expectations, [])
setattr(roll_chromium_revision, '_RunCommand', self.old_RunCommand)
def testUpdateDeps(self):
def testUpdateDepsFile(self):
new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111'
current_rev = TEST_DATA_VARS['chromium_revision']
UpdateDeps(self._deps_filename, current_rev, new_rev)
with open(self._deps_filename) as deps_file:
UpdateDepsFile(self._webrtc_depsfile, current_rev, new_rev, [])
with open(self._webrtc_depsfile) as deps_file:
deps_contents = deps_file.read()
self.assertTrue(new_rev in deps_contents,
'Failed to find %s in\n%s' % (new_rev, deps_contents))
def testParseDepsDict(self):
with open(self._deps_filename) as deps_file:
with open(self._webrtc_depsfile) as deps_file:
deps_contents = deps_file.read()
local_scope = ParseDepsDict(deps_contents)
vars_dict = local_scope['vars']
@ -62,6 +103,8 @@ class TestRollChromiumRevision(unittest.TestCase):
self.assertEquals(vars_dict[variable_name], TEST_DATA_VARS[variable_name])
assertVar('chromium_git')
assertVar('chromium_revision')
self.assertEquals(len(local_scope['deps']), 3)
self.assertEquals(len(local_scope['deps_os']), 1)
def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest')
@ -77,5 +120,35 @@ class TestRollChromiumRevision(unittest.TestCase):
self.assertEquals(len(entries), 1)
self.assertEquals(entries[0], DEPS_ENTRIES['src/build'])
def testCalculateChangedDepsProper(self):
_SetupGitLsRemoteCall(self.fake,
'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV)
webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
changed_deps = CalculateChangedDepsProper(webrtc_deps, new_cr_deps)
self.assertEquals(len(changed_deps), 2)
self.assertEquals(changed_deps[0].path, 'src/build')
self.assertEquals(changed_deps[0].current_rev, BUILD_OLD_REV)
self.assertEquals(changed_deps[0].new_rev, BUILD_NEW_REV)
self.assertEquals(changed_deps[1].path, 'src/buildtools')
self.assertEquals(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
self.assertEquals(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
def testCalculateChangedDepsLegacy(self):
old_cr_deps = ParseLocalDepsFile(self._old_cr_depsfile)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
changed_deps = CalculateChangedDepsProper(old_cr_deps, new_cr_deps)
self.assertEquals(len(changed_deps), 1)
self.assertEquals(changed_deps[0].path, 'src/buildtools')
self.assertEquals(changed_deps[0].current_rev, BUILDTOOLS_OLD_REV)
self.assertEquals(changed_deps[0].new_rev, BUILDTOOLS_NEW_REV)
def _SetupGitLsRemoteCall(cmd_fake, url, revision):
cmd = ['git', 'ls-remote', url, revision]
cmd_fake.add_expectation(cmd, _returns=(revision, None))
if __name__ == '__main__':
unittest.main()

28
tools/autoroller/unittests/testdata/DEPS vendored Normal file
View File

@ -0,0 +1,28 @@
# DEPS file for unit tests.
vars = {
'chromium_git': 'https://chromium.googlesource.com',
'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
}
deps = {
# Entry that is a directory in Chromium, so we're using a Git subtree mirror for it.
'src/build':
Var('chromium_git') + '/chromium/src/build' + '@' + '52f7afeca991d96d68cf0507e20dbdd5b845691f',
# Entry that's also a DEPS entry in the Chromium DEPS file.
'src/buildtools':
Var('chromium_git') + '/chromium/buildtools.git' + '@' + '64e38f0cebdde27aa0cfb405f330063582f9ac76',
# Entry only present in WebRTC, not Chromium.
'src/third_party/gflags/src':
Var('chromium_git') + '/external/github.com/gflags/gflags@03bebcb065c83beff83d50ae025a55a4bf94dfca',
}
deps_os = {
# Entry only present in WebRTC, not Chromium.
'win': {
'src/third_party/winsdk_samples/src':
Var('chromium_git') + '/external/webrtc/deps/third_party/winsdk_samples_v71@e71b549167a665d7424d6f1dadfbff4b4aad1589',
},
}

13
tools/autoroller/unittests/testdata/DEPS.chromium.new vendored Normal file
View File

@ -0,0 +1,13 @@
# DEPS file for unit tests.
vars = {
'chromium_git': 'https://chromium.googlesource.com',
# This is updated compared to the DEPS.chromium.old file.
'buildtools_revision': '55ad626b08ef971fd82a62b7abb325359542952b',
}
deps = {
'src/buildtools':
Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'),
}

13
tools/autoroller/unittests/testdata/DEPS.chromium.old vendored Normal file
View File

@ -0,0 +1,13 @@
# DEPS file for unit tests.
vars = {
'chromium_git': 'https://chromium.googlesource.com',
# This is and older revision than DEPS.chromium.new file.
'buildtools_revision': '64e38f0cebdde27aa0cfb405f330063582f9ac76',
}
deps = {
'src/buildtools':
Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'),
}