Autoroller: Support for rolling individual DEPS entries.
Add functionality for automatic rolling of individual DEPS
entries. This will make it possible to move away from the
links created by setup_links.py to real DEPS entries.
There are two kinds of such deps we intend to use:
1. Third party dependencies used by both WebRTC and Chromium:
those are rolled to the same revision as the Chromium DEPS file of the revision passed
to the script
2. Chromium sub-directories needed for WebRTC (mainly for //build and BUILD.gn
files of third_party deps): those are rolled to the HEAD revision
Notice that the latter kind could be rolled ahead of the chromium_revision,
but generally these should be close (and if it passes the bots, we don't really mind).
The new functionality can coexist with the old one, to
enable a smooth transition (not everything needs to change at once).
Some of the updating logic was inspired by
https://cs.chromium.org/chromium/build/scripts/slave/recipes/v8/auto_roll_v8_deps.py
Add extensive tests for the logic of figuring out deps changes,
including mocking git ls-remote call to keep the tests hermetic.
BUG=webrtc:5006
NOTRY=True
Review-Url: https://codereview.webrtc.org/2570603003
Cr-Commit-Position: refs/heads/master@{#15570}
diff --git a/tools/autoroller/roll_chromium_revision.py b/tools/autoroller/roll_chromium_revision.py
index d6457d0..787e3db 100755
--- a/tools/autoroller/roll_chromium_revision.py
+++ b/tools/autoroller/roll_chromium_revision.py
@@ -20,6 +20,13 @@
import urllib
+# Skip these dependencies (list without solution name prefix).
+DONT_AUTOROLL_THESE = [
+ 'src/third_party/gflags/src',
+ 'src/third_party/winsdk_samples/src',
+]
+
+WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc'
CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
@@ -30,12 +37,15 @@
ROLL_BRANCH_NAME = 'roll_chromium_revision'
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
- os.pardir))
-sys.path.append(CHECKOUT_ROOT_DIR)
+CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
+ os.pardir))
+CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
+CHROMIUM_CHECKOUT_SRC_DIR = os.path.join(CHECKOUT_SRC_DIR, 'chromium', 'src')
+
+sys.path.append(CHECKOUT_SRC_DIR)
import setup_links
-sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'build'))
+sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
import find_depot_tools
find_depot_tools.add_depot_tools_to_path()
from gclient import GClientKeywords
@@ -94,7 +104,7 @@
Returns:
A tuple containing the stdout and stderr outputs as strings.
"""
- working_dir = working_dir or CHECKOUT_ROOT_DIR
+ working_dir = working_dir or CHECKOUT_SRC_DIR
logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
env = os.environ.copy()
if extra_env:
@@ -193,7 +203,7 @@
def BuildDepsentryDict(deps_dict):
- """Builds a dict of DepsEntry object from a raw parsed deps dict."""
+ """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
result = {}
def AddDepsEntries(deps_subdict):
for path, deps_url in deps_subdict.iteritems():
@@ -203,25 +213,74 @@
AddDepsEntries(deps_dict['deps'])
for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
- AddDepsEntries(deps_dict['deps_os'].get(deps_os, {}))
+ AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
return result
-def CalculateChangedDeps(current_deps, new_deps):
- result = []
- current_entries = BuildDepsentryDict(current_deps)
- new_entries = BuildDepsentryDict(new_deps)
+def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps):
+ """
+ Calculate changed deps entries based on:
+ 1. Entries defined in the WebRTC DEPS file:
+ - If a shared dependency with the Chromium DEPS file: roll it to the same
+ revision as Chromium (i.e. entry in the new_cr_deps dict)
+ - If it's a Chromium sub-directory, roll it to the HEAD revision (notice
+ this means it may be ahead of the chromium_revision, but generally these
+ should be close).
+ - If it's another DEPS entry (not shared with Chromium), roll it to HEAD
+ unless it's configured to be skipped.
+ 2. Entries present in the setup_links.py file. If the dir has changed between
+ old_cr_deps and new_cr_deps, it is considered changed and updated to the
+ revision for the entry in the new_cr_deps dict.
+ Returns:
+ A list of ChangedDep objects representing the changed deps.
+ """
+ return sorted(CalculateChangedDepsProper(webrtc_deps, new_cr_deps) +
+ CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps))
+
+
+def CalculateChangedDepsProper(webrtc_deps, new_cr_deps):
+ result = []
+ webrtc_entries = BuildDepsentryDict(webrtc_deps)
+ new_cr_entries = BuildDepsentryDict(new_cr_deps)
+ for path, webrtc_deps_entry in webrtc_entries.iteritems():
+ if path in DONT_AUTOROLL_THESE:
+ continue
+ cr_deps_entry = new_cr_entries.get(path)
+ if cr_deps_entry:
+ # Use the revision from Chromium's DEPS file.
+ new_rev = cr_deps_entry.revision
+ assert webrtc_deps_entry.url == cr_deps_entry.url, (
+ 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' %
+ (path, webrtc_deps_entry.url, cr_deps_entry.url))
+ else:
+ # Use the HEAD of the deps repo.
+ stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url,
+ 'HEAD'])
+ new_rev = stdout.strip().split('\t')[0]
+
+ # Check if an update is necessary.
+ if webrtc_deps_entry.revision != new_rev:
+ logging.debug('Roll dependency %s to %s', path, new_rev)
+ result.append(ChangedDep(path, webrtc_deps_entry.url,
+ webrtc_deps_entry.revision, new_rev))
+ return result
+
+
+def CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps):
+ result = []
+ new_cr_entries = BuildDepsentryDict(new_cr_deps)
+ old_cr_entries = BuildDepsentryDict(old_cr_deps)
all_deps_dirs = setup_links.DIRECTORIES
for deps_dir in all_deps_dirs:
# All deps have 'src' prepended to the path in the Chromium DEPS file.
dir_path = 'src/%s' % deps_dir
- for entry in GetMatchingDepsEntries(current_entries, dir_path):
- new_matching_entries = GetMatchingDepsEntries(new_entries, entry.path)
+ for entry in GetMatchingDepsEntries(old_cr_entries, dir_path):
+ new_matching_entries = GetMatchingDepsEntries(new_cr_entries, entry.path)
assert len(new_matching_entries) <= 1, (
'Should never find more than one entry matching %s in %s, found %d' %
- (entry.path, new_entries, len(new_matching_entries)))
+ (entry.path, new_cr_entries, len(new_matching_entries)))
if not new_matching_entries:
result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None'))
elif entry != new_matching_entries[0]:
@@ -238,7 +297,7 @@
return match.group(1)
raise RollError('Could not parse Clang revision!')
- chromium_src_path = os.path.join(CHECKOUT_ROOT_DIR, 'chromium', 'src',
+ chromium_src_path = os.path.join(CHROMIUM_CHECKOUT_SRC_DIR,
CLANG_UPDATE_SCRIPT_LOCAL_PATH)
with open(chromium_src_path, 'rb') as f:
current_lines = f.readlines()
@@ -294,14 +353,26 @@
return '\n'.join(commit_msg)
-def UpdateDeps(deps_filename, old_cr_revision, new_cr_revision):
+def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
+ changed_deps):
"""Update the DEPS file with the new revision."""
+
+ # Update the chromium_revision variable.
with open(deps_filename, 'rb') as deps_file:
deps_content = deps_file.read()
deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
with open(deps_filename, 'wb') as deps_file:
deps_file.write(deps_content)
+ # Update each individual DEPS entry.
+ for dep in changed_deps:
+ _, stderr = _RunCommand(
+ ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev],
+ working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True)
+ if stderr:
+ logging.warning('roll-dep-svn: %s', stderr)
+
+
def _IsTreeClean():
stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
if len(stdout) == 0:
@@ -378,9 +449,6 @@
help=('Calculate changes and modify DEPS, but don\'t create '
'any local branch, commit, upload CL or send any '
'tryjobs.'))
- p.add_argument('--allow-reverse', action='store_true', default=False,
- help=('Allow rolling back in time (disabled by default but '
- 'may be useful to be able do to manually).'))
p.add_argument('--skip-cq', action='store_true', default=False,
help='Skip sending the CL to the CQ (default: %(default)s)')
p.add_argument('-v', '--verbose', action='store_true', default=False,
@@ -408,9 +476,9 @@
logging.info('No revision specified. Using HEAD: %s', head_rev)
new_cr_rev = head_rev
- deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS')
- local_deps = ParseLocalDepsFile(deps_filename)
- current_cr_rev = local_deps['vars']['chromium_revision']
+ deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
+ webrtc_deps = ParseLocalDepsFile(deps_filename)
+ current_cr_rev = webrtc_deps['vars']['chromium_revision']
current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
@@ -418,23 +486,15 @@
current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
- if new_commit_pos > current_commit_pos or opts.allow_reverse:
- changed_deps = sorted(CalculateChangedDeps(current_cr_deps, new_cr_deps))
- clang_change = CalculateChangedClang(new_cr_rev)
- commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
- current_commit_pos, new_commit_pos,
- changed_deps, clang_change)
- logging.debug('Commit message:\n%s', commit_msg)
- else:
- logging.info('Currently pinned chromium_revision: %s (#%s) is newer than '
- '%s (#%s). To roll to older revisions, you must pass the '
- '--allow-reverse flag.\n'
- 'Aborting without action.', current_cr_rev, current_commit_pos,
- new_cr_rev, new_commit_pos)
- return 0
+ changed_deps = CalculateChangedDeps(webrtc_deps, current_cr_deps, new_cr_deps)
+ clang_change = CalculateChangedClang(new_cr_rev)
+ commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
+ current_commit_pos, new_commit_pos,
+ changed_deps, clang_change)
+ logging.debug('Commit message:\n%s', commit_msg)
_CreateRollBranch(opts.dry_run)
- UpdateDeps(deps_filename, current_cr_rev, new_cr_rev)
+ UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
_LocalCommit(commit_msg, opts.dry_run)
_UploadCL(opts.dry_run, opts.rietveld_email)
_SendToCQ(opts.dry_run, opts.skip_cq)
diff --git a/tools/autoroller/unittests/DEPS b/tools/autoroller/unittests/DEPS
deleted file mode 100644
index 82638f8..0000000
--- a/tools/autoroller/unittests/DEPS
+++ /dev/null
@@ -1,6 +0,0 @@
-# Sample DEPS file for testing.
-
-vars = {
- 'chromium_git': 'https://chromium.googlesource.com',
- 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
-}
diff --git a/tools/autoroller/unittests/roll_chromium_revision_test.py b/tools/autoroller/unittests/roll_chromium_revision_test.py
index 7c899e9..d7b3e26 100755
--- a/tools/autoroller/unittests/roll_chromium_revision_test.py
+++ b/tools/autoroller/unittests/roll_chromium_revision_test.py
@@ -7,6 +7,7 @@
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
+import glob
import os
import shutil
import sys
@@ -17,8 +18,11 @@
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
sys.path.append(PARENT_DIR)
-from roll_chromium_revision import ParseDepsDict, UpdateDeps, \
- GetMatchingDepsEntries
+import roll_chromium_revision
+from roll_chromium_revision import CalculateChangedDepsProper, \
+ GetMatchingDepsEntries, ParseDepsDict, \
+ ParseLocalDepsFile, UpdateDepsFile
+
TEST_DATA_VARS = {
'chromium_git': 'https://chromium.googlesource.com',
@@ -32,28 +36,65 @@
'src/testing/gmock': 'https://gmock.com',
}
+BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f'
+BUILD_NEW_REV = 'HEAD'
+BUILDTOOLS_OLD_REV = '64e38f0cebdde27aa0cfb405f330063582f9ac76'
+BUILDTOOLS_NEW_REV = '55ad626b08ef971fd82a62b7abb325359542952b'
+
+
+class TestError(Exception):
+ pass
+
+
+class FakeCmd(object):
+ def __init__(self):
+ self.expectations = []
+
+ def add_expectation(self, *args, **kwargs):
+ returns = kwargs.pop('_returns', None)
+ self.expectations.append((args, kwargs, returns))
+
+ def __call__(self, *args, **kwargs):
+ if not self.expectations:
+ raise TestError('Got unexpected\n%s\n%s' % (args, kwargs))
+ exp_args, exp_kwargs, exp_returns = self.expectations.pop(0)
+ if args != exp_args or kwargs != exp_kwargs:
+ message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs)
+ message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs)
+ raise TestError(message)
+ return exp_returns
+
class TestRollChromiumRevision(unittest.TestCase):
def setUp(self):
self._output_dir = tempfile.mkdtemp()
- shutil.copy(os.path.join(SCRIPT_DIR, 'DEPS'), self._output_dir)
- self._deps_filename = os.path.join(self._output_dir, 'DEPS')
+ for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
+ shutil.copy(test_file, self._output_dir)
+ self._webrtc_depsfile = os.path.join(self._output_dir, 'DEPS')
+ self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old')
+ self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new')
+
+ self.fake = FakeCmd()
+ self.old_RunCommand = getattr(roll_chromium_revision, '_RunCommand')
+ setattr(roll_chromium_revision, '_RunCommand', self.fake)
def tearDown(self):
shutil.rmtree(self._output_dir, ignore_errors=True)
+ self.assertEqual(self.fake.expectations, [])
+ setattr(roll_chromium_revision, '_RunCommand', self.old_RunCommand)
- def testUpdateDeps(self):
+ def testUpdateDepsFile(self):
new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111'
current_rev = TEST_DATA_VARS['chromium_revision']
- UpdateDeps(self._deps_filename, current_rev, new_rev)
- with open(self._deps_filename) as deps_file:
+ UpdateDepsFile(self._webrtc_depsfile, current_rev, new_rev, [])
+ with open(self._webrtc_depsfile) as deps_file:
deps_contents = deps_file.read()
self.assertTrue(new_rev in deps_contents,
'Failed to find %s in\n%s' % (new_rev, deps_contents))
def testParseDepsDict(self):
- with open(self._deps_filename) as deps_file:
+ with open(self._webrtc_depsfile) as deps_file:
deps_contents = deps_file.read()
local_scope = ParseDepsDict(deps_contents)
vars_dict = local_scope['vars']
@@ -62,6 +103,8 @@
self.assertEquals(vars_dict[variable_name], TEST_DATA_VARS[variable_name])
assertVar('chromium_git')
assertVar('chromium_revision')
+ self.assertEquals(len(local_scope['deps']), 3)
+ self.assertEquals(len(local_scope['deps_os']), 1)
def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest')
@@ -77,5 +120,35 @@
self.assertEquals(len(entries), 1)
self.assertEquals(entries[0], DEPS_ENTRIES['src/build'])
+ def testCalculateChangedDepsProper(self):
+ _SetupGitLsRemoteCall(self.fake,
+ 'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV)
+ webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile)
+ new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
+ changed_deps = CalculateChangedDepsProper(webrtc_deps, new_cr_deps)
+ self.assertEquals(len(changed_deps), 2)
+ self.assertEquals(changed_deps[0].path, 'src/build')
+ self.assertEquals(changed_deps[0].current_rev, BUILD_OLD_REV)
+ self.assertEquals(changed_deps[0].new_rev, BUILD_NEW_REV)
+
+ self.assertEquals(changed_deps[1].path, 'src/buildtools')
+ self.assertEquals(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
+ self.assertEquals(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
+
+ def testCalculateChangedDepsLegacy(self):
+ old_cr_deps = ParseLocalDepsFile(self._old_cr_depsfile)
+ new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
+ changed_deps = CalculateChangedDepsProper(old_cr_deps, new_cr_deps)
+ self.assertEquals(len(changed_deps), 1)
+ self.assertEquals(changed_deps[0].path, 'src/buildtools')
+ self.assertEquals(changed_deps[0].current_rev, BUILDTOOLS_OLD_REV)
+ self.assertEquals(changed_deps[0].new_rev, BUILDTOOLS_NEW_REV)
+
+
+def _SetupGitLsRemoteCall(cmd_fake, url, revision):
+ cmd = ['git', 'ls-remote', url, revision]
+ cmd_fake.add_expectation(cmd, _returns=(revision, None))
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/tools/autoroller/unittests/testdata/DEPS b/tools/autoroller/unittests/testdata/DEPS
new file mode 100644
index 0000000..3f76e62
--- /dev/null
+++ b/tools/autoroller/unittests/testdata/DEPS
@@ -0,0 +1,28 @@
+# DEPS file for unit tests.
+
+vars = {
+ 'chromium_git': 'https://chromium.googlesource.com',
+ 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
+}
+
+deps = {
+ # Entry that is a directory in Chromium, so we're using a Git subtree mirror for it.
+ 'src/build':
+ Var('chromium_git') + '/chromium/src/build' + '@' + '52f7afeca991d96d68cf0507e20dbdd5b845691f',
+
+ # Entry that's also a DEPS entry in the Chromium DEPS file.
+ 'src/buildtools':
+ Var('chromium_git') + '/chromium/buildtools.git' + '@' + '64e38f0cebdde27aa0cfb405f330063582f9ac76',
+
+ # Entry only present in WebRTC, not Chromium.
+ 'src/third_party/gflags/src':
+ Var('chromium_git') + '/external/github.com/gflags/gflags@03bebcb065c83beff83d50ae025a55a4bf94dfca',
+}
+
+deps_os = {
+ # Entry only present in WebRTC, not Chromium.
+ 'win': {
+ 'src/third_party/winsdk_samples/src':
+ Var('chromium_git') + '/external/webrtc/deps/third_party/winsdk_samples_v71@e71b549167a665d7424d6f1dadfbff4b4aad1589',
+ },
+}
diff --git a/tools/autoroller/unittests/testdata/DEPS.chromium.new b/tools/autoroller/unittests/testdata/DEPS.chromium.new
new file mode 100644
index 0000000..d53083c
--- /dev/null
+++ b/tools/autoroller/unittests/testdata/DEPS.chromium.new
@@ -0,0 +1,13 @@
+# DEPS file for unit tests.
+
+vars = {
+ 'chromium_git': 'https://chromium.googlesource.com',
+
+ # This is updated compared to the DEPS.chromium.old file.
+ 'buildtools_revision': '55ad626b08ef971fd82a62b7abb325359542952b',
+}
+
+deps = {
+ 'src/buildtools':
+ Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'),
+}
diff --git a/tools/autoroller/unittests/testdata/DEPS.chromium.old b/tools/autoroller/unittests/testdata/DEPS.chromium.old
new file mode 100644
index 0000000..dd6ddae
--- /dev/null
+++ b/tools/autoroller/unittests/testdata/DEPS.chromium.old
@@ -0,0 +1,13 @@
+# DEPS file for unit tests.
+
+vars = {
+ 'chromium_git': 'https://chromium.googlesource.com',
+
+ # This is and older revision than DEPS.chromium.new file.
+ 'buildtools_revision': '64e38f0cebdde27aa0cfb405f330063582f9ac76',
+}
+
+deps = {
+ 'src/buildtools':
+ Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'),
+}
