David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 1 | /****************************************************************************** |
| 2 | ******************************************************************************* |
| 3 | ** |
David Teigland | 3ae1acf | 2007-05-18 08:59:31 -0500 | [diff] [blame] | 4 | ** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 5 | ** |
| 6 | ** This copyrighted material is made available to anyone wishing to use, |
| 7 | ** modify, copy, or redistribute it subject to the terms and conditions |
| 8 | ** of the GNU General Public License v.2. |
| 9 | ** |
| 10 | ******************************************************************************* |
| 11 | ******************************************************************************/ |
| 12 | |
| 13 | #include "dlm_internal.h" |
| 14 | #include "lockspace.h" |
| 15 | #include "member.h" |
| 16 | #include "recoverd.h" |
| 17 | #include "recover.h" |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 18 | #include "rcom.h" |
| 19 | #include "config.h" |
| 20 | |
| 21 | /* |
| 22 | * Following called by dlm_recoverd thread |
| 23 | */ |
| 24 | |
| 25 | static void add_ordered_member(struct dlm_ls *ls, struct dlm_member *new) |
| 26 | { |
| 27 | struct dlm_member *memb = NULL; |
| 28 | struct list_head *tmp; |
| 29 | struct list_head *newlist = &new->list; |
| 30 | struct list_head *head = &ls->ls_nodes; |
| 31 | |
| 32 | list_for_each(tmp, head) { |
| 33 | memb = list_entry(tmp, struct dlm_member, list); |
| 34 | if (new->nodeid < memb->nodeid) |
| 35 | break; |
| 36 | } |
| 37 | |
| 38 | if (!memb) |
| 39 | list_add_tail(newlist, head); |
| 40 | else { |
| 41 | /* FIXME: can use list macro here */ |
| 42 | newlist->prev = tmp->prev; |
| 43 | newlist->next = tmp; |
| 44 | tmp->prev->next = newlist; |
| 45 | tmp->prev = newlist; |
| 46 | } |
| 47 | } |
| 48 | |
| 49 | static int dlm_add_member(struct dlm_ls *ls, int nodeid) |
| 50 | { |
| 51 | struct dlm_member *memb; |
| 52 | int w; |
| 53 | |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 54 | memb = kzalloc(sizeof(struct dlm_member), GFP_KERNEL); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 55 | if (!memb) |
| 56 | return -ENOMEM; |
| 57 | |
| 58 | w = dlm_node_weight(ls->ls_name, nodeid); |
Jesper Juhl | 1a2bf2e | 2007-07-19 00:27:43 +0200 | [diff] [blame^] | 59 | if (w < 0) { |
| 60 | kfree(memb); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 61 | return w; |
Jesper Juhl | 1a2bf2e | 2007-07-19 00:27:43 +0200 | [diff] [blame^] | 62 | } |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 63 | |
| 64 | memb->nodeid = nodeid; |
| 65 | memb->weight = w; |
| 66 | add_ordered_member(ls, memb); |
| 67 | ls->ls_num_nodes++; |
| 68 | return 0; |
| 69 | } |
| 70 | |
| 71 | static void dlm_remove_member(struct dlm_ls *ls, struct dlm_member *memb) |
| 72 | { |
| 73 | list_move(&memb->list, &ls->ls_nodes_gone); |
| 74 | ls->ls_num_nodes--; |
| 75 | } |
| 76 | |
| 77 | static int dlm_is_member(struct dlm_ls *ls, int nodeid) |
| 78 | { |
| 79 | struct dlm_member *memb; |
| 80 | |
| 81 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 82 | if (memb->nodeid == nodeid) |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 83 | return 1; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 84 | } |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 85 | return 0; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 86 | } |
| 87 | |
| 88 | int dlm_is_removed(struct dlm_ls *ls, int nodeid) |
| 89 | { |
| 90 | struct dlm_member *memb; |
| 91 | |
| 92 | list_for_each_entry(memb, &ls->ls_nodes_gone, list) { |
| 93 | if (memb->nodeid == nodeid) |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 94 | return 1; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 95 | } |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 96 | return 0; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 97 | } |
| 98 | |
| 99 | static void clear_memb_list(struct list_head *head) |
| 100 | { |
| 101 | struct dlm_member *memb; |
| 102 | |
| 103 | while (!list_empty(head)) { |
| 104 | memb = list_entry(head->next, struct dlm_member, list); |
| 105 | list_del(&memb->list); |
| 106 | kfree(memb); |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | void dlm_clear_members(struct dlm_ls *ls) |
| 111 | { |
| 112 | clear_memb_list(&ls->ls_nodes); |
| 113 | ls->ls_num_nodes = 0; |
| 114 | } |
| 115 | |
| 116 | void dlm_clear_members_gone(struct dlm_ls *ls) |
| 117 | { |
| 118 | clear_memb_list(&ls->ls_nodes_gone); |
| 119 | } |
| 120 | |
| 121 | static void make_member_array(struct dlm_ls *ls) |
| 122 | { |
| 123 | struct dlm_member *memb; |
| 124 | int i, w, x = 0, total = 0, all_zero = 0, *array; |
| 125 | |
| 126 | kfree(ls->ls_node_array); |
| 127 | ls->ls_node_array = NULL; |
| 128 | |
| 129 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 130 | if (memb->weight) |
| 131 | total += memb->weight; |
| 132 | } |
| 133 | |
| 134 | /* all nodes revert to weight of 1 if all have weight 0 */ |
| 135 | |
| 136 | if (!total) { |
| 137 | total = ls->ls_num_nodes; |
| 138 | all_zero = 1; |
| 139 | } |
| 140 | |
| 141 | ls->ls_total_weight = total; |
| 142 | |
| 143 | array = kmalloc(sizeof(int) * total, GFP_KERNEL); |
| 144 | if (!array) |
| 145 | return; |
| 146 | |
| 147 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 148 | if (!all_zero && !memb->weight) |
| 149 | continue; |
| 150 | |
| 151 | if (all_zero) |
| 152 | w = 1; |
| 153 | else |
| 154 | w = memb->weight; |
| 155 | |
| 156 | DLM_ASSERT(x < total, printk("total %d x %d\n", total, x);); |
| 157 | |
| 158 | for (i = 0; i < w; i++) |
| 159 | array[x++] = memb->nodeid; |
| 160 | } |
| 161 | |
| 162 | ls->ls_node_array = array; |
| 163 | } |
| 164 | |
| 165 | /* send a status request to all members just to establish comms connections */ |
| 166 | |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 167 | static int ping_members(struct dlm_ls *ls) |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 168 | { |
| 169 | struct dlm_member *memb; |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 170 | int error = 0; |
| 171 | |
| 172 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 173 | error = dlm_recovery_stopped(ls); |
| 174 | if (error) |
| 175 | break; |
| 176 | error = dlm_rcom_status(ls, memb->nodeid); |
| 177 | if (error) |
| 178 | break; |
| 179 | } |
| 180 | if (error) |
David Teigland | faa0f26 | 2006-08-08 17:08:42 -0500 | [diff] [blame] | 181 | log_debug(ls, "ping_members aborted %d last nodeid %d", |
| 182 | error, ls->ls_recover_nodeid); |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 183 | return error; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 184 | } |
| 185 | |
| 186 | int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out) |
| 187 | { |
| 188 | struct dlm_member *memb, *safe; |
| 189 | int i, error, found, pos = 0, neg = 0, low = -1; |
| 190 | |
David Teigland | 91c0dc9 | 2006-10-31 11:56:01 -0600 | [diff] [blame] | 191 | /* previously removed members that we've not finished removing need to |
| 192 | count as a negative change so the "neg" recovery steps will happen */ |
| 193 | |
| 194 | list_for_each_entry(memb, &ls->ls_nodes_gone, list) { |
| 195 | log_debug(ls, "prev removed member %d", memb->nodeid); |
| 196 | neg++; |
| 197 | } |
| 198 | |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 199 | /* move departed members from ls_nodes to ls_nodes_gone */ |
| 200 | |
| 201 | list_for_each_entry_safe(memb, safe, &ls->ls_nodes, list) { |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 202 | found = 0; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 203 | for (i = 0; i < rv->node_count; i++) { |
| 204 | if (memb->nodeid == rv->nodeids[i]) { |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 205 | found = 1; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 206 | break; |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | if (!found) { |
| 211 | neg++; |
| 212 | dlm_remove_member(ls, memb); |
| 213 | log_debug(ls, "remove member %d", memb->nodeid); |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | /* add new members to ls_nodes */ |
| 218 | |
| 219 | for (i = 0; i < rv->node_count; i++) { |
| 220 | if (dlm_is_member(ls, rv->nodeids[i])) |
| 221 | continue; |
| 222 | dlm_add_member(ls, rv->nodeids[i]); |
| 223 | pos++; |
| 224 | log_debug(ls, "add member %d", rv->nodeids[i]); |
| 225 | } |
| 226 | |
| 227 | list_for_each_entry(memb, &ls->ls_nodes, list) { |
| 228 | if (low == -1 || memb->nodeid < low) |
| 229 | low = memb->nodeid; |
| 230 | } |
| 231 | ls->ls_low_nodeid = low; |
| 232 | |
| 233 | make_member_array(ls); |
| 234 | dlm_set_recover_status(ls, DLM_RS_NODES); |
| 235 | *neg_out = neg; |
| 236 | |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 237 | error = ping_members(ls); |
David Teigland | 8b0e7b2 | 2007-05-18 09:03:35 -0500 | [diff] [blame] | 238 | if (!error || error == -EPROTO) { |
| 239 | /* new_lockspace() may be waiting to know if the config |
| 240 | is good or bad */ |
| 241 | ls->ls_members_result = error; |
| 242 | complete(&ls->ls_members_done); |
| 243 | } |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 244 | if (error) |
| 245 | goto out; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 246 | |
| 247 | error = dlm_recover_members_wait(ls); |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 248 | out: |
| 249 | log_debug(ls, "total members %d error %d", ls->ls_num_nodes, error); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 250 | return error; |
| 251 | } |
| 252 | |
| 253 | /* |
| 254 | * Following called from lockspace.c |
| 255 | */ |
| 256 | |
| 257 | int dlm_ls_stop(struct dlm_ls *ls) |
| 258 | { |
| 259 | int new; |
| 260 | |
| 261 | /* |
| 262 | * A stop cancels any recovery that's in progress (see RECOVERY_STOP, |
| 263 | * dlm_recovery_stopped()) and prevents any new locks from being |
| 264 | * processed (see RUNNING, dlm_locking_stopped()). |
| 265 | */ |
| 266 | |
| 267 | spin_lock(&ls->ls_recover_lock); |
| 268 | set_bit(LSFL_RECOVERY_STOP, &ls->ls_flags); |
| 269 | new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags); |
| 270 | ls->ls_recover_seq++; |
| 271 | spin_unlock(&ls->ls_recover_lock); |
| 272 | |
| 273 | /* |
| 274 | * This in_recovery lock does two things: |
| 275 | * |
| 276 | * 1) Keeps this function from returning until all threads are out |
| 277 | * of locking routines and locking is truely stopped. |
| 278 | * 2) Keeps any new requests from being processed until it's unlocked |
| 279 | * when recovery is complete. |
| 280 | */ |
| 281 | |
| 282 | if (new) |
| 283 | down_write(&ls->ls_in_recovery); |
| 284 | |
| 285 | /* |
| 286 | * The recoverd suspend/resume makes sure that dlm_recoverd (if |
| 287 | * running) has noticed the clearing of RUNNING above and quit |
| 288 | * processing the previous recovery. This will be true for all nodes |
| 289 | * before any nodes start the new recovery. |
| 290 | */ |
| 291 | |
| 292 | dlm_recoverd_suspend(ls); |
| 293 | ls->ls_recover_status = 0; |
| 294 | dlm_recoverd_resume(ls); |
David Teigland | 3ae1acf | 2007-05-18 08:59:31 -0500 | [diff] [blame] | 295 | |
| 296 | if (!ls->ls_recover_begin) |
| 297 | ls->ls_recover_begin = jiffies; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 298 | return 0; |
| 299 | } |
| 300 | |
| 301 | int dlm_ls_start(struct dlm_ls *ls) |
| 302 | { |
| 303 | struct dlm_recover *rv = NULL, *rv_old; |
| 304 | int *ids = NULL; |
| 305 | int error, count; |
| 306 | |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 307 | rv = kzalloc(sizeof(struct dlm_recover), GFP_KERNEL); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 308 | if (!rv) |
| 309 | return -ENOMEM; |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 310 | |
| 311 | error = count = dlm_nodeid_list(ls->ls_name, &ids); |
| 312 | if (error <= 0) |
| 313 | goto fail; |
| 314 | |
| 315 | spin_lock(&ls->ls_recover_lock); |
| 316 | |
| 317 | /* the lockspace needs to be stopped before it can be started */ |
| 318 | |
| 319 | if (!dlm_locking_stopped(ls)) { |
| 320 | spin_unlock(&ls->ls_recover_lock); |
| 321 | log_error(ls, "start ignored: lockspace running"); |
| 322 | error = -EINVAL; |
| 323 | goto fail; |
| 324 | } |
| 325 | |
| 326 | rv->nodeids = ids; |
| 327 | rv->node_count = count; |
| 328 | rv->seq = ++ls->ls_recover_seq; |
| 329 | rv_old = ls->ls_recover_args; |
| 330 | ls->ls_recover_args = rv; |
| 331 | spin_unlock(&ls->ls_recover_lock); |
| 332 | |
| 333 | if (rv_old) { |
| 334 | kfree(rv_old->nodeids); |
| 335 | kfree(rv_old); |
| 336 | } |
| 337 | |
| 338 | dlm_recoverd_kick(ls); |
| 339 | return 0; |
| 340 | |
| 341 | fail: |
| 342 | kfree(rv); |
| 343 | kfree(ids); |
| 344 | return error; |
| 345 | } |
| 346 | |