autotest: add a flag to force to run all the update commands when deployment
Previously, deploy script will skip certain update commands when the
corresponding repos or script are not changed. For some cases, we may
need to force to run these commands. Add an option for this use case.
BUG=chromium:648425
TEST=Test on test server
Change-Id: I464afc7bfd945e0f4ccf6dced6dd468df4397b97
Reviewed-on: https://chromium-review.googlesource.com/387292
Reviewed-by: Shuqian Zhao <shuqianz@chromium.org>
Tested-by: Shuqian Zhao <shuqianz@chromium.org>
diff --git a/site_utils/deploy_server.py b/site_utils/deploy_server.py
index a77a58f..e5fe5ea 100755
--- a/site_utils/deploy_server.py
+++ b/site_utils/deploy_server.py
@@ -117,6 +117,8 @@
help='What is the main server for this installation? (cautotest).')
parser.add_argument('--update_push_servers', action='store_true',
help='Indicate to update test_push servers.')
+ parser.add_argument('--force_update', action='store_true',
+ help='Force to run update commands for afe, tko, build_externals')
parser.add_argument('--dryrun', action='store_true',
help='Don\'t actually run remote commands.')
parser.add_argument('args', nargs=argparse.REMAINDER,
diff --git a/site_utils/deploy_server_local.py b/site_utils/deploy_server_local.py
index bf69343..6c81b70 100755
--- a/site_utils/deploy_server_local.py
+++ b/site_utils/deploy_server_local.py
@@ -36,6 +36,11 @@
# are both running test_importer, there is a chance to fail as both try to
# update the same table.
PRIMARY_ONLY_COMMANDS = ['test_importer']
+# A dict to map update_commands defined in config file to repos or files that
+# decide whether need to update these commands. E.g. if no changes under
+# frontend repo, no need to update afe.
+COMMANDS_TO_REPOS_DICT = {'afe': 'frontend/', 'tko': 'tko/',
+ 'build_externals': 'utils/external_packages.py'}
AFE = frontend_wrappers.RetryingAFE(
server=server_utils.get_global_afe_hostname(), timeout_min=5,
@@ -116,6 +121,24 @@
return project_heads
+def repo_versions_to_decide_whether_run_cmd_update():
+ """Collect versions of repos/files defined in COMMANDS_TO_REPOS_DICT.
+
+ For the update_commands defined in config files, no need to run the command
+ every time. Only run it when the repos/files related to the commands have
+ been changed.
+
+ @returns A set of tuples: {(cmd, repo_version), ()...}
+ """
+ results = set()
+ for cmd, repo in COMMANDS_TO_REPOS_DICT.iteritems():
+ version = subprocess.check_output(
+ ['git', 'log', '-1', '--pretty=tformat:%h',
+ '%s/%s' % (common.autotest_dir, repo)])
+ results.add((cmd, version.strip()))
+ return results
+
+
def repo_sync(update_push_servers=False):
"""Perform a repo sync.
@@ -278,9 +301,12 @@
raise UnstableServices(unstable_services)
-def run_deploy_actions(dryrun=False, skip_service_status=False):
+def run_deploy_actions(cmds_skip=set(), dryrun=False,
+ skip_service_status=False):
"""Run arbitrary update commands specified in global.ini.
+ @param cmds_skip: cmds no need to run since the corresponding repo/file
+ does not change.
@param dryrun: Don't really restart the service, just print out the command.
@param skip_service_status: Set to True to skip service status check.
Default is False.
@@ -288,7 +314,8 @@
@raises subprocess.CalledProcessError on a command failure.
@raises UnstableServices if any services are unstable after restart.
"""
- cmds = set(discover_update_commands())
+ defined_cmds = set(discover_update_commands())
+ cmds = defined_cmds - cmds_skip
if cmds:
print('Running update commands:', ', '.join(cmds))
for cmd in cmds:
@@ -379,6 +406,9 @@
parser.add_argument('--update_push_servers', action='store_true',
help='Indicate to update test_push server. If not '
'specify, then update server to production.')
+ parser.add_argument('--force_update', action='store_true',
+ help='Force to run the update commands for afe, tko '
+ 'and build_externals')
results = parser.parse_args(args)
@@ -416,16 +446,22 @@
versions_before = repo_versions()
versions_after = {}
+ cmd_versions_before = repo_versions_to_decide_whether_run_cmd_update()
+ cmd_versions_after = {}
if behaviors.update:
print('Updating Repo.')
repo_sync(behaviors.update_push_servers)
versions_after = repo_versions()
+ cmd_versions_after = repo_versions_to_decide_whether_run_cmd_update()
if behaviors.actions:
+ # If the corresponding repo/file not change, no need to run the cmd.
+ cmds_skip = (set() if behaviors.force_update else
+ {t[0] for t in cmd_versions_before & cmd_versions_after})
try:
run_deploy_actions(
- behaviors.dryrun, behaviors.skip_service_status)
+ cmds_skip, behaviors.dryrun, behaviors.skip_service_status)
except UnstableServices as e:
print('The following services were not stable after '
'the update:')
diff --git a/site_utils/deploy_server_unittest.py b/site_utils/deploy_server_unittest.py
index 340a032..f3533ce 100755
--- a/site_utils/deploy_server_unittest.py
+++ b/site_utils/deploy_server_unittest.py
@@ -22,7 +22,7 @@
self.assertEqual(
{'afe': 'foo', 'servers': [], 'args': [],
'cont': False, 'dryrun': False, 'verbose': False,
- 'update_push_servers': False},
+ 'force_update': False, 'update_push_servers': False},
vars(results))
# Dryrun, continue
@@ -31,7 +31,7 @@
self.assertDictContainsSubset(
{'afe': 'foo', 'servers': [], 'args': [],
'cont': True, 'dryrun': True, 'verbose': False,
- 'update_push_servers': False},
+ 'force_update': False, 'update_push_servers': False},
vars(results))
# List some servers
@@ -40,7 +40,7 @@
self.assertDictContainsSubset(
{'afe': 'foo', 'servers': ['dummy', 'bar'], 'args': [],
'cont': False, 'dryrun': False, 'verbose': False,
- 'update_push_servers': False},
+ 'force_update': False, 'update_push_servers': False},
vars(results))
# List some local args
@@ -49,17 +49,18 @@
self.assertDictContainsSubset(
{'afe': 'foo', 'servers': [], 'args': ['dummy', 'bar'],
'cont': False, 'dryrun': False, 'verbose': False,
- 'update_push_servers': False},
+ 'force_update': False, 'update_push_servers': False},
vars(results))
# List everything.
results = deploy_server.parse_arguments(
['--continue', '--afe', 'foo', '--dryrun', 'dummy', 'bar',
- '--', '--actions-only', '--dryrun', '--update_push_servers'])
+ '--', '--actions-only', '--dryrun', '--update_push_servers',
+ '--force_update'])
self.assertDictContainsSubset(
{'afe': 'foo', 'servers': ['dummy', 'bar'],
'args': ['--actions-only', '--dryrun',
- '--update_push_servers'],
+ '--update_push_servers', '--force_update'],
'cont': True, 'dryrun': True, 'verbose': False},
vars(results))