David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 1 | /****************************************************************************** |
| 2 | ******************************************************************************* |
| 3 | ** |
Christine Caulfield | 391fbdc | 2009-05-07 10:54:16 -0500 | [diff] [blame] | 4 | ** Copyright (C) 2005-2009 Red Hat, Inc. All rights reserved. |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 5 | ** |
| 6 | ** This copyrighted material is made available to anyone wishing to use, |
| 7 | ** modify, copy, or redistribute it subject to the terms and conditions |
| 8 | ** of the GNU General Public License v.2. |
| 9 | ** |
| 10 | ******************************************************************************* |
| 11 | ******************************************************************************/ |
| 12 | |
| 13 | #include "dlm_internal.h" |
| 14 | #include "lockspace.h" |
| 15 | #include "member.h" |
| 16 | #include "recoverd.h" |
| 17 | #include "recover.h" |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 18 | #include "rcom.h" |
| 19 | #include "config.h" |
Christine Caulfield | 391fbdc | 2009-05-07 10:54:16 -0500 | [diff] [blame] | 20 | #include "lowcomms.h" |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 21 | |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 22 | static void add_ordered_member(struct dlm_ls *ls, struct dlm_member *new) |
| 23 | { |
| 24 | struct dlm_member *memb = NULL; |
| 25 | struct list_head *tmp; |
| 26 | struct list_head *newlist = &new->list; |
| 27 | struct list_head *head = &ls->ls_nodes; |
| 28 | |
| 29 | list_for_each(tmp, head) { |
| 30 | memb = list_entry(tmp, struct dlm_member, list); |
| 31 | if (new->nodeid < memb->nodeid) |
| 32 | break; |
| 33 | } |
| 34 | |
| 35 | if (!memb) |
| 36 | list_add_tail(newlist, head); |
| 37 | else { |
| 38 | /* FIXME: can use list macro here */ |
| 39 | newlist->prev = tmp->prev; |
| 40 | newlist->next = tmp; |
| 41 | tmp->prev->next = newlist; |
| 42 | tmp->prev = newlist; |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | static int dlm_add_member(struct dlm_ls *ls, int nodeid) |
| 47 | { |
| 48 | struct dlm_member *memb; |
Christine Caulfield | 391fbdc | 2009-05-07 10:54:16 -0500 | [diff] [blame] | 49 | int w, error; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 50 | |
David Teigland | 573c24c | 2009-11-30 16:34:43 -0600 | [diff] [blame] | 51 | memb = kzalloc(sizeof(struct dlm_member), GFP_NOFS); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 52 | if (!memb) |
| 53 | return -ENOMEM; |
| 54 | |
| 55 | w = dlm_node_weight(ls->ls_name, nodeid); |
Jesper Juhl | 1a2bf2e | 2007-07-19 00:27:43 +0200 | [diff] [blame] | 56 | if (w < 0) { |
| 57 | kfree(memb); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 58 | return w; |
Jesper Juhl | 1a2bf2e | 2007-07-19 00:27:43 +0200 | [diff] [blame] | 59 | } |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 60 | |
Christine Caulfield | 391fbdc | 2009-05-07 10:54:16 -0500 | [diff] [blame] | 61 | error = dlm_lowcomms_connect_node(nodeid); |
| 62 | if (error < 0) { |
| 63 | kfree(memb); |
| 64 | return error; |
| 65 | } |
| 66 | |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 67 | memb->nodeid = nodeid; |
| 68 | memb->weight = w; |
| 69 | add_ordered_member(ls, memb); |
| 70 | ls->ls_num_nodes++; |
| 71 | return 0; |
| 72 | } |
| 73 | |
| 74 | static void dlm_remove_member(struct dlm_ls *ls, struct dlm_member *memb) |
| 75 | { |
| 76 | list_move(&memb->list, &ls->ls_nodes_gone); |
| 77 | ls->ls_num_nodes--; |
| 78 | } |
| 79 | |
David Teigland | 46b43ee | 2008-01-08 16:24:00 -0600 | [diff] [blame] | 80 | int dlm_is_member(struct dlm_ls *ls, int nodeid) |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 81 | { |
| 82 | struct dlm_member *memb; |
| 83 | |
| 84 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 85 | if (memb->nodeid == nodeid) |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 86 | return 1; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 87 | } |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 88 | return 0; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 89 | } |
| 90 | |
| 91 | int dlm_is_removed(struct dlm_ls *ls, int nodeid) |
| 92 | { |
| 93 | struct dlm_member *memb; |
| 94 | |
| 95 | list_for_each_entry(memb, &ls->ls_nodes_gone, list) { |
| 96 | if (memb->nodeid == nodeid) |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 97 | return 1; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 98 | } |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 99 | return 0; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 100 | } |
| 101 | |
| 102 | static void clear_memb_list(struct list_head *head) |
| 103 | { |
| 104 | struct dlm_member *memb; |
| 105 | |
| 106 | while (!list_empty(head)) { |
| 107 | memb = list_entry(head->next, struct dlm_member, list); |
| 108 | list_del(&memb->list); |
| 109 | kfree(memb); |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | void dlm_clear_members(struct dlm_ls *ls) |
| 114 | { |
| 115 | clear_memb_list(&ls->ls_nodes); |
| 116 | ls->ls_num_nodes = 0; |
| 117 | } |
| 118 | |
| 119 | void dlm_clear_members_gone(struct dlm_ls *ls) |
| 120 | { |
| 121 | clear_memb_list(&ls->ls_nodes_gone); |
| 122 | } |
| 123 | |
| 124 | static void make_member_array(struct dlm_ls *ls) |
| 125 | { |
| 126 | struct dlm_member *memb; |
| 127 | int i, w, x = 0, total = 0, all_zero = 0, *array; |
| 128 | |
| 129 | kfree(ls->ls_node_array); |
| 130 | ls->ls_node_array = NULL; |
| 131 | |
| 132 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 133 | if (memb->weight) |
| 134 | total += memb->weight; |
| 135 | } |
| 136 | |
| 137 | /* all nodes revert to weight of 1 if all have weight 0 */ |
| 138 | |
| 139 | if (!total) { |
| 140 | total = ls->ls_num_nodes; |
| 141 | all_zero = 1; |
| 142 | } |
| 143 | |
| 144 | ls->ls_total_weight = total; |
| 145 | |
David Teigland | 573c24c | 2009-11-30 16:34:43 -0600 | [diff] [blame] | 146 | array = kmalloc(sizeof(int) * total, GFP_NOFS); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 147 | if (!array) |
| 148 | return; |
| 149 | |
| 150 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 151 | if (!all_zero && !memb->weight) |
| 152 | continue; |
| 153 | |
| 154 | if (all_zero) |
| 155 | w = 1; |
| 156 | else |
| 157 | w = memb->weight; |
| 158 | |
| 159 | DLM_ASSERT(x < total, printk("total %d x %d\n", total, x);); |
| 160 | |
| 161 | for (i = 0; i < w; i++) |
| 162 | array[x++] = memb->nodeid; |
| 163 | } |
| 164 | |
| 165 | ls->ls_node_array = array; |
| 166 | } |
| 167 | |
| 168 | /* send a status request to all members just to establish comms connections */ |
| 169 | |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 170 | static int ping_members(struct dlm_ls *ls) |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 171 | { |
| 172 | struct dlm_member *memb; |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 173 | int error = 0; |
| 174 | |
| 175 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 176 | error = dlm_recovery_stopped(ls); |
| 177 | if (error) |
| 178 | break; |
| 179 | error = dlm_rcom_status(ls, memb->nodeid); |
| 180 | if (error) |
| 181 | break; |
| 182 | } |
| 183 | if (error) |
David Teigland | faa0f26 | 2006-08-08 17:08:42 -0500 | [diff] [blame] | 184 | log_debug(ls, "ping_members aborted %d last nodeid %d", |
| 185 | error, ls->ls_recover_nodeid); |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 186 | return error; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 187 | } |
| 188 | |
| 189 | int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out) |
| 190 | { |
| 191 | struct dlm_member *memb, *safe; |
| 192 | int i, error, found, pos = 0, neg = 0, low = -1; |
| 193 | |
David Teigland | 91c0dc9 | 2006-10-31 11:56:01 -0600 | [diff] [blame] | 194 | /* previously removed members that we've not finished removing need to |
| 195 | count as a negative change so the "neg" recovery steps will happen */ |
| 196 | |
| 197 | list_for_each_entry(memb, &ls->ls_nodes_gone, list) { |
| 198 | log_debug(ls, "prev removed member %d", memb->nodeid); |
| 199 | neg++; |
| 200 | } |
| 201 | |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 202 | /* move departed members from ls_nodes to ls_nodes_gone */ |
| 203 | |
| 204 | list_for_each_entry_safe(memb, safe, &ls->ls_nodes, list) { |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 205 | found = 0; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 206 | for (i = 0; i < rv->node_count; i++) { |
| 207 | if (memb->nodeid == rv->nodeids[i]) { |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 208 | found = 1; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 209 | break; |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | if (!found) { |
| 214 | neg++; |
| 215 | dlm_remove_member(ls, memb); |
| 216 | log_debug(ls, "remove member %d", memb->nodeid); |
| 217 | } |
| 218 | } |
| 219 | |
David Teigland | d44e0fc | 2008-03-18 14:22:11 -0500 | [diff] [blame] | 220 | /* Add an entry to ls_nodes_gone for members that were removed and |
| 221 | then added again, so that previous state for these nodes will be |
| 222 | cleared during recovery. */ |
| 223 | |
| 224 | for (i = 0; i < rv->new_count; i++) { |
| 225 | if (!dlm_is_member(ls, rv->new[i])) |
| 226 | continue; |
| 227 | log_debug(ls, "new nodeid %d is a re-added member", rv->new[i]); |
| 228 | |
David Teigland | 573c24c | 2009-11-30 16:34:43 -0600 | [diff] [blame] | 229 | memb = kzalloc(sizeof(struct dlm_member), GFP_NOFS); |
David Teigland | d44e0fc | 2008-03-18 14:22:11 -0500 | [diff] [blame] | 230 | if (!memb) |
| 231 | return -ENOMEM; |
| 232 | memb->nodeid = rv->new[i]; |
| 233 | list_add_tail(&memb->list, &ls->ls_nodes_gone); |
| 234 | neg++; |
| 235 | } |
| 236 | |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 237 | /* add new members to ls_nodes */ |
| 238 | |
| 239 | for (i = 0; i < rv->node_count; i++) { |
| 240 | if (dlm_is_member(ls, rv->nodeids[i])) |
| 241 | continue; |
| 242 | dlm_add_member(ls, rv->nodeids[i]); |
| 243 | pos++; |
| 244 | log_debug(ls, "add member %d", rv->nodeids[i]); |
| 245 | } |
| 246 | |
| 247 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 248 | if (low == -1 || memb->nodeid < low) |
| 249 | low = memb->nodeid; |
| 250 | } |
| 251 | ls->ls_low_nodeid = low; |
| 252 | |
| 253 | make_member_array(ls); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 254 | *neg_out = neg; |
| 255 | |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 256 | error = ping_members(ls); |
David Teigland | 8b0e7b2 | 2007-05-18 09:03:35 -0500 | [diff] [blame] | 257 | if (!error || error == -EPROTO) { |
| 258 | /* new_lockspace() may be waiting to know if the config |
| 259 | is good or bad */ |
| 260 | ls->ls_members_result = error; |
| 261 | complete(&ls->ls_members_done); |
| 262 | } |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 263 | |
David Teigland | f95a34c | 2011-10-14 12:34:58 -0500 | [diff] [blame^] | 264 | log_debug(ls, "dlm_recover_members %d nodes", ls->ls_num_nodes); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 265 | return error; |
| 266 | } |
| 267 | |
David Teigland | c36258b | 2007-09-27 15:53:38 -0500 | [diff] [blame] | 268 | /* Userspace guarantees that dlm_ls_stop() has completed on all nodes before |
| 269 | dlm_ls_start() is called on any of them to start the new recovery. */ |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 270 | |
| 271 | int dlm_ls_stop(struct dlm_ls *ls) |
| 272 | { |
| 273 | int new; |
| 274 | |
| 275 | /* |
David Teigland | c36258b | 2007-09-27 15:53:38 -0500 | [diff] [blame] | 276 | * Prevent dlm_recv from being in the middle of something when we do |
| 277 | * the stop. This includes ensuring dlm_recv isn't processing a |
| 278 | * recovery message (rcom), while dlm_recoverd is aborting and |
| 279 | * resetting things from an in-progress recovery. i.e. we want |
| 280 | * dlm_recoverd to abort its recovery without worrying about dlm_recv |
| 281 | * processing an rcom at the same time. Stopping dlm_recv also makes |
| 282 | * it easy for dlm_receive_message() to check locking stopped and add a |
| 283 | * message to the requestqueue without races. |
| 284 | */ |
| 285 | |
| 286 | down_write(&ls->ls_recv_active); |
| 287 | |
| 288 | /* |
| 289 | * Abort any recovery that's in progress (see RECOVERY_STOP, |
| 290 | * dlm_recovery_stopped()) and tell any other threads running in the |
| 291 | * dlm to quit any processing (see RUNNING, dlm_locking_stopped()). |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 292 | */ |
| 293 | |
| 294 | spin_lock(&ls->ls_recover_lock); |
| 295 | set_bit(LSFL_RECOVERY_STOP, &ls->ls_flags); |
| 296 | new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags); |
| 297 | ls->ls_recover_seq++; |
| 298 | spin_unlock(&ls->ls_recover_lock); |
| 299 | |
| 300 | /* |
David Teigland | c36258b | 2007-09-27 15:53:38 -0500 | [diff] [blame] | 301 | * Let dlm_recv run again, now any normal messages will be saved on the |
| 302 | * requestqueue for later. |
| 303 | */ |
| 304 | |
| 305 | up_write(&ls->ls_recv_active); |
| 306 | |
| 307 | /* |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 308 | * This in_recovery lock does two things: |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 309 | * 1) Keeps this function from returning until all threads are out |
Adam Buchbinder | c41b20e | 2009-12-11 16:35:39 -0500 | [diff] [blame] | 310 | * of locking routines and locking is truly stopped. |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 311 | * 2) Keeps any new requests from being processed until it's unlocked |
| 312 | * when recovery is complete. |
| 313 | */ |
| 314 | |
| 315 | if (new) |
| 316 | down_write(&ls->ls_in_recovery); |
| 317 | |
| 318 | /* |
| 319 | * The recoverd suspend/resume makes sure that dlm_recoverd (if |
David Teigland | c36258b | 2007-09-27 15:53:38 -0500 | [diff] [blame] | 320 | * running) has noticed RECOVERY_STOP above and quit processing the |
| 321 | * previous recovery. |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 322 | */ |
| 323 | |
| 324 | dlm_recoverd_suspend(ls); |
| 325 | ls->ls_recover_status = 0; |
| 326 | dlm_recoverd_resume(ls); |
David Teigland | 3ae1acf | 2007-05-18 08:59:31 -0500 | [diff] [blame] | 327 | |
| 328 | if (!ls->ls_recover_begin) |
| 329 | ls->ls_recover_begin = jiffies; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 330 | return 0; |
| 331 | } |
| 332 | |
| 333 | int dlm_ls_start(struct dlm_ls *ls) |
| 334 | { |
| 335 | struct dlm_recover *rv = NULL, *rv_old; |
David Teigland | d44e0fc | 2008-03-18 14:22:11 -0500 | [diff] [blame] | 336 | int *ids = NULL, *new = NULL; |
| 337 | int error, ids_count = 0, new_count = 0; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 338 | |
David Teigland | 573c24c | 2009-11-30 16:34:43 -0600 | [diff] [blame] | 339 | rv = kzalloc(sizeof(struct dlm_recover), GFP_NOFS); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 340 | if (!rv) |
| 341 | return -ENOMEM; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 342 | |
David Teigland | d44e0fc | 2008-03-18 14:22:11 -0500 | [diff] [blame] | 343 | error = dlm_nodeid_list(ls->ls_name, &ids, &ids_count, |
| 344 | &new, &new_count); |
| 345 | if (error < 0) |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 346 | goto fail; |
| 347 | |
| 348 | spin_lock(&ls->ls_recover_lock); |
| 349 | |
| 350 | /* the lockspace needs to be stopped before it can be started */ |
| 351 | |
| 352 | if (!dlm_locking_stopped(ls)) { |
| 353 | spin_unlock(&ls->ls_recover_lock); |
| 354 | log_error(ls, "start ignored: lockspace running"); |
| 355 | error = -EINVAL; |
| 356 | goto fail; |
| 357 | } |
| 358 | |
| 359 | rv->nodeids = ids; |
David Teigland | d44e0fc | 2008-03-18 14:22:11 -0500 | [diff] [blame] | 360 | rv->node_count = ids_count; |
| 361 | rv->new = new; |
| 362 | rv->new_count = new_count; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 363 | rv->seq = ++ls->ls_recover_seq; |
| 364 | rv_old = ls->ls_recover_args; |
| 365 | ls->ls_recover_args = rv; |
| 366 | spin_unlock(&ls->ls_recover_lock); |
| 367 | |
| 368 | if (rv_old) { |
David Teigland | d44e0fc | 2008-03-18 14:22:11 -0500 | [diff] [blame] | 369 | log_error(ls, "unused recovery %llx %d", |
| 370 | (unsigned long long)rv_old->seq, rv_old->node_count); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 371 | kfree(rv_old->nodeids); |
David Teigland | d44e0fc | 2008-03-18 14:22:11 -0500 | [diff] [blame] | 372 | kfree(rv_old->new); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 373 | kfree(rv_old); |
| 374 | } |
| 375 | |
| 376 | dlm_recoverd_kick(ls); |
| 377 | return 0; |
| 378 | |
| 379 | fail: |
| 380 | kfree(rv); |
| 381 | kfree(ids); |
David Teigland | d44e0fc | 2008-03-18 14:22:11 -0500 | [diff] [blame] | 382 | kfree(new); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 383 | return error; |
| 384 | } |
| 385 | |