[autotest] Change DB update order.
Originally, we update shard DBs first and then the master DB, and
use the shard DB as ground truth when inconsistency between the
master DB and a shard DB is detected. This incurs a tricky bug
as explained in crbug.com/574966.
We should change the order. Update the master DB first and then
shard DBs, and use the master DB as ground truth for fixing
DB inconsistency.
BUG=chromium:574966
TEST=Test updated RPCs in puppylab.
Change-Id: Ic785f9f3c677bea997d4637266c8c223618070d9
Reviewed-on: https://chromium-review.googlesource.com/320599
Commit-Ready: Mungyung Ryu <mkryu@google.com>
Tested-by: Mungyung Ryu <mkryu@google.com>
Reviewed-by: Dan Shi <dshi@chromium.org>
diff --git a/frontend/afe/rpc_interface.py b/frontend/afe/rpc_interface.py
index 107562d..afe8a82 100644
--- a/frontend/afe/rpc_interface.py
+++ b/frontend/afe/rpc_interface.py
@@ -72,14 +72,13 @@
@param data: New data for a label.
"""
label_model = models.Label.smart_get(id)
+ label_model.update_object(data)
# Master forwards the RPC to shards
if not utils.is_shard():
rpc_utils.fanout_rpc(label_model.host_set.all(), 'modify_label', False,
id=id, **data)
- label_model.update_object(data)
-
def delete_label(id):
"""Delete a label.
@@ -87,13 +86,16 @@
@param id: id or name of a label. More often a label name.
"""
label_model = models.Label.smart_get(id)
+ # Hosts that have the label to be deleted. Save this info before
+ # the label is deleted to use it later.
+ hosts = []
+ for h in label_model.host_set.all():
+ hosts.append(models.Host.smart_get(h.id))
+ label_model.delete()
# Master forwards the RPC to shards
if not utils.is_shard():
- rpc_utils.fanout_rpc(label_model.host_set.all(), 'delete_label', False,
- id=id)
-
- label_model.delete()
+ rpc_utils.fanout_rpc(hosts, 'delete_label', False, id=id)
def add_label(name, ignore_exception_if_exists=False, **kwargs):
@@ -167,6 +169,7 @@
raise ValueError('Label id (%s) does not exist. Please specify '
'the argument, id, as a string (label name).'
% id)
+ add_label_to_hosts(id, hosts)
host_objs = models.Host.smart_get_bulk(hosts)
# Make sure the label exists on the shard with the same id
@@ -186,8 +189,6 @@
id=label.id, platform=label.platform)
rpc_utils.fanout_rpc(host_objs, 'add_label_to_hosts', id=id)
- add_label_to_hosts(id, hosts)
-
def remove_label_from_hosts(id, hosts):
"""Removes a label of the given id from the given hosts only in local DB.
@@ -209,10 +210,10 @@
@param hosts: A list of hostnames or ids. More often hostnames.
"""
host_objs = models.Host.smart_get_bulk(hosts)
- rpc_utils.fanout_rpc(host_objs, 'remove_label_from_hosts', id=id)
-
remove_label_from_hosts(id, hosts)
+ rpc_utils.fanout_rpc(host_objs, 'remove_label_from_hosts', id=id)
+
def get_labels(exclude_filters=(), **filter_data):
"""\
@@ -291,10 +292,10 @@
# between the master and a shard.
if kwargs.get('locked', None) and 'lock_time' not in kwargs:
kwargs['lock_time'] = datetime.datetime.now()
+ host.update_object(kwargs)
rpc_utils.fanout_rpc([host], 'modify_host_local',
include_hostnames=False, id=id, **kwargs)
- host.update_object(kwargs)
def modify_host_local(id, **kwargs):
@@ -348,6 +349,8 @@
# between the master and a shard.
if update_data.get('locked', None) and 'lock_time' not in update_data:
update_data['lock_time'] = datetime.datetime.now()
+ for host in hosts:
+ host.update_object(update_data)
# Caution: Changing the filter from the original here. See docstring.
rpc_utils.run_rpc_on_multiple_hostnames(
@@ -355,9 +358,6 @@
host_filter_data={'id__in': affected_host_ids},
update_data=update_data)
- for host in hosts:
- host.update_object(update_data)
-
def modify_hosts_local(host_filter_data, update_data):
"""Modify attributes of hosts in local DB.
@@ -398,10 +398,10 @@
host_obj = models.Host.smart_get(id)
if len(platforms) == 1:
models.Host.check_no_platform([host_obj])
+ add_labels_to_host(id, labels)
rpc_utils.fanout_rpc([host_obj], 'add_labels_to_host', False,
id=id, labels=labels)
- add_labels_to_host(id, labels)
def remove_labels_from_host(id, labels):
@@ -421,10 +421,11 @@
@param id: id or hostname for a host.
@param labels: ids or names for labels.
"""
+ remove_labels_from_host(id, labels)
+
host_obj = models.Host.smart_get(id)
rpc_utils.fanout_rpc([host_obj], 'remove_labels_from_host', False,
id=id, labels=labels)
- remove_labels_from_host(id, labels)
def get_host_attribute(attribute, **host_filter_data):
@@ -455,15 +456,14 @@
assert host_filter_data # disallow accidental actions on all hosts
hosts = models.Host.query_objects(host_filter_data)
models.AclGroup.check_for_acl_violation_hosts(hosts)
+ for host in hosts:
+ host.set_or_delete_attribute(attribute, value)
# Master forwards this RPC to shards.
if not utils.is_shard():
rpc_utils.fanout_rpc(hosts, 'set_host_attribute', False,
attribute=attribute, value=value, **host_filter_data)
- for host in hosts:
- host.set_or_delete_attribute(attribute, value)
-
@rpc_utils.forward_single_host_rpc_to_shard
def delete_host(id):