Support rolling CIPD deps in WebRTC
Bug: chromium:755920
Change-Id: I84fc76976ad9acc077035913ef738f0a974abb80
Reviewed-on: https://webrtc-review.googlesource.com/70581
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Commit-Queue: Oleh Prypin <oprypin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23091}diff --git a/tools_webrtc/autoroller/roll_deps.py b/tools_webrtc/autoroller/roll_deps.py
index f2a412d..469668b 100755
--- a/tools_webrtc/autoroller/roll_deps.py
+++ b/tools_webrtc/autoroller/roll_deps.py
@@ -54,8 +54,11 @@
'clang', 'scripts', 'update.py')
DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
-ChangedDep = collections.namedtuple('ChangedDep',
- 'path url current_rev new_rev')
+ChangedDep = collections.namedtuple(
+ 'ChangedDep', 'path url current_rev new_rev')
+CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages')
+ChangedCipdPackage = collections.namedtuple(
+ 'ChangedCipdPackage', 'path package current_version new_version')
class RollError(Exception):
pass
@@ -208,14 +211,14 @@
result = {}
def AddDepsEntries(deps_subdict):
for path, dep in deps_subdict.iteritems():
- if isinstance(dep, dict):
- if dep.get('dep_type') == 'cipd':
- continue
- deps_url = dep['url']
+ if path in result:
+ continue
+ if not isinstance(dep, dict):
+ dep = {'url': dep}
+ if dep.get('dep_type') == 'cipd':
+ result[path] = CipdDepsEntry(path, dep['packages'])
else:
- deps_url = dep
- if not result.has_key(path):
- url, revision = deps_url.split('@') if deps_url else (None, None)
+ url, revision = dep['url'].split('@')
result[path] = DepsEntry(path, url, revision)
AddDepsEntries(deps_dict['deps'])
@@ -224,6 +227,25 @@
return result
+def _RemoveStringPrefix(string, substring):
+ assert string.startswith(substring)
+ return string[len(substring):]
+
+
+def _FindChangedCipdPackages(path, old_pkgs, new_pkgs):
+ assert ({p['package'] for p in old_pkgs} ==
+ {p['package'] for p in new_pkgs})
+ for old_pkg in old_pkgs:
+ for new_pkg in new_pkgs:
+ old_version = _RemoveStringPrefix(old_pkg['version'], 'version:')
+ new_version = _RemoveStringPrefix(new_pkg['version'], 'version:')
+ if (old_pkg['package'] == new_pkg['package'] and
+ old_version != new_version):
+ logging.debug('Roll dependency %s to %s', path, new_version)
+ yield ChangedCipdPackage(path, old_pkg['package'],
+ old_version, new_version)
+
+
def CalculateChangedDeps(webrtc_deps, new_cr_deps):
"""
Calculate changed deps entries based on entries defined in the WebRTC DEPS
@@ -247,6 +269,13 @@
continue
cr_deps_entry = new_cr_entries.get(path)
if cr_deps_entry:
+ assert type(cr_deps_entry) is type(webrtc_deps_entry)
+
+ if isinstance(cr_deps_entry, CipdDepsEntry):
+ result.extend(_FindChangedCipdPackages(path, webrtc_deps_entry.packages,
+ cr_deps_entry.packages))
+ continue
+
# Use the revision from Chromium's DEPS file.
new_rev = cr_deps_entry.revision
assert webrtc_deps_entry.url == cr_deps_entry.url, (
@@ -301,9 +330,13 @@
commit_msg.append('Changed dependencies:')
for c in changed_deps_list:
- commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url,
- c.current_rev[0:10],
- c.new_rev[0:10]))
+ if isinstance(c, ChangedCipdPackage):
+ commit_msg.append('* %s: %s..%s' % (c.path, c.current_version,
+ c.new_version))
+ else:
+ commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url,
+ c.current_rev[0:10],
+ c.new_rev[0:10]))
if 'libvpx' in c.path:
tbr_authors += 'marpan@webrtc.org, '
@@ -354,9 +387,12 @@
'platforms in the target_os list, i.e.\n'
'target_os = ["android", "unix", "mac", "ios", "win"];\n'
'Then run "gclient sync" again.' % local_dep_dir)
- _RunCommand(
- ['gclient', 'setdep', '--revision', '%s@%s' % (dep.path, dep.new_rev)],
- working_dir=CHECKOUT_SRC_DIR)
+ if isinstance(dep, ChangedCipdPackage):
+ update = '%s:%s@%s' % (dep.path, dep.package, dep.new_version)
+ else:
+ update = '%s@%s' % (dep.path, dep.new_rev)
+ _RunCommand(['gclient', 'setdep', '--revision', update],
+ working_dir=CHECKOUT_SRC_DIR)
def _IsTreeClean():
diff --git a/tools_webrtc/autoroller/unittests/roll_deps_test.py b/tools_webrtc/autoroller/unittests/roll_deps_test.py
index db7ab02..b6ac1be 100755
--- a/tools_webrtc/autoroller/unittests/roll_deps_test.py
+++ b/tools_webrtc/autoroller/unittests/roll_deps_test.py
@@ -70,7 +70,6 @@
for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
shutil.copy(test_file, self._output_dir)
self._webrtc_depsfile = os.path.join(self._output_dir, 'DEPS')
- self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old')
self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new')
self.fake = FakeCmd()
@@ -107,7 +106,7 @@
self.assertEquals(vars_dict[variable_name], TEST_DATA_VARS[variable_name])
AssertVar('chromium_git')
AssertVar('chromium_revision')
- self.assertEquals(len(local_scope['deps']), 2)
+ self.assertEquals(len(local_scope['deps']), 3)
self.assertEquals(len(local_scope['deps_os']), 1)
def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
@@ -130,7 +129,7 @@
webrtc_deps = ParseLocalDepsFile(self._webrtc_depsfile)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps)
- self.assertEquals(len(changed_deps), 2)
+ self.assertEquals(len(changed_deps), 3)
self.assertEquals(changed_deps[0].path, 'src/build')
self.assertEquals(changed_deps[0].current_rev, BUILD_OLD_REV)
self.assertEquals(changed_deps[0].new_rev, BUILD_NEW_REV)
@@ -139,6 +138,11 @@
self.assertEquals(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
self.assertEquals(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
+ self.assertEquals(changed_deps[2].path, 'src/third_party/xstream')
+ self.assertEquals(changed_deps[2].package, 'chromium/third_party/xstream')
+ self.assertEquals(changed_deps[2].current_version, '1.4.8-cr0')
+ self.assertEquals(changed_deps[2].new_version, '1.10.0-cr0')
+
class TestChooseCQMode(unittest.TestCase):
def testSkip(self):
diff --git a/tools_webrtc/autoroller/unittests/testdata/DEPS b/tools_webrtc/autoroller/unittests/testdata/DEPS
index 8d6c0d6..fcd65a5 100644
--- a/tools_webrtc/autoroller/unittests/testdata/DEPS
+++ b/tools_webrtc/autoroller/unittests/testdata/DEPS
@@ -13,6 +13,18 @@
# Entry that's also a DEPS entry in the Chromium DEPS file.
'src/buildtools':
Var('chromium_git') + '/chromium/buildtools.git' + '@' + '64e38f0cebdde27aa0cfb405f330063582f9ac76',
+
+ # Entry that's also a CIPD entry in the Chromium DEPS file.
+ 'src/third_party/xstream': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/xstream',
+ 'version': 'version:1.4.8-cr0',
+ },
+ ],
+ 'condition': 'checkout_android',
+ 'dep_type': 'cipd',
+ },
}
deps_os = {
diff --git a/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.new b/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.new
index d53083c..41526c7 100644
--- a/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.new
+++ b/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.new
@@ -3,11 +3,23 @@
vars = {
'chromium_git': 'https://chromium.googlesource.com',
- # This is updated compared to the DEPS.chromium.old file.
+ # This is updated compared to the DEPS file.
'buildtools_revision': '55ad626b08ef971fd82a62b7abb325359542952b',
}
deps = {
'src/buildtools':
Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'),
+
+ 'src/third_party/xstream': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/xstream',
+ # This is updated compared to the DEPS file.
+ 'version': 'version:1.10.0-cr0',
+ },
+ ],
+ 'condition': 'checkout_android',
+ 'dep_type': 'cipd',
+ },
}
diff --git a/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.old b/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.old
deleted file mode 100644
index dd6ddae..0000000
--- a/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.old
+++ /dev/null
@@ -1,13 +0,0 @@
-# DEPS file for unit tests.
-
-vars = {
- 'chromium_git': 'https://chromium.googlesource.com',
-
- # This is and older revision than DEPS.chromium.new file.
- 'buildtools_revision': '64e38f0cebdde27aa0cfb405f330063582f9ac76',
-}
-
-deps = {
- 'src/buildtools':
- Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'),
-}